static atomic_t nr_comm_tracking __read_mostly;
int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
-int sysctl_perf_counter_mlock __read_mostly = 128; /* 'free' kb per counter */
+int sysctl_perf_counter_mlock __read_mostly = 512; /* 'free' kb per user */
+int sysctl_perf_counter_limit __read_mostly = 100000; /* max NMIs per second */
/*
* Lock for (sysadmin-configurable) counter reservations:
__perf_disable();
hw_perf_disable();
}
-EXPORT_SYMBOL_GPL(perf_disable); /* ACPI idle */
void perf_enable(void)
{
if (__perf_enable())
hw_perf_enable();
}
-EXPORT_SYMBOL_GPL(perf_enable); /* ACPI idle */
+static void get_ctx(struct perf_counter_context *ctx)
+{
+ atomic_inc(&ctx->refcount);
+}
+
+static void put_ctx(struct perf_counter_context *ctx)
+{
+ if (atomic_dec_and_test(&ctx->refcount)) {
+ if (ctx->parent_ctx)
+ put_ctx(ctx->parent_ctx);
+ kfree(ctx);
+ }
+}
+
+/*
+ * Add a counter from the lists for its context.
+ * Must be called with ctx->mutex and ctx->lock held.
+ */
static void
list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
{
}
list_add_rcu(&counter->event_entry, &ctx->event_list);
+ ctx->nr_counters++;
}
+/*
+ * Remove a counter from the lists for its context.
+ * Must be called with ctx->mutex and ctx->lock held.
+ */
static void
list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
{
struct perf_counter *sibling, *tmp;
+ if (list_empty(&counter->list_entry))
+ return;
+ ctx->nr_counters--;
+
list_del_init(&counter->list_entry);
list_del_rcu(&counter->event_entry);
cpuctx->exclusive = 0;
}
+/*
+ * Mark this context as not being a clone of another.
+ * Called when counters are added to or removed from this context.
+ * We also increment our generation number so that anything that
+ * was cloned from this context before this will not match anything
+ * cloned from this context after this.
+ */
+static void unclone_ctx(struct perf_counter_context *ctx)
+{
+ ++ctx->generation;
+ if (!ctx->parent_ctx)
+ return;
+ put_ctx(ctx->parent_ctx);
+ ctx->parent_ctx = NULL;
+}
+
/*
* Cross CPU call to remove a performance counter
*
return;
spin_lock_irqsave(&ctx->lock, flags);
-
- counter_sched_out(counter, cpuctx, ctx);
-
- counter->task = NULL;
- ctx->nr_counters--;
-
/*
* Protect the list operation against NMI by disabling the
- * counters on a global level. NOP for non NMI based counters.
+ * counters on a global level.
*/
perf_disable();
+
+ counter_sched_out(counter, cpuctx, ctx);
+
list_del_counter(counter, ctx);
- perf_enable();
if (!ctx->task) {
/*
perf_max_counters - perf_reserved_percpu);
}
+ perf_enable();
spin_unlock_irqrestore(&ctx->lock, flags);
}
/*
* Remove the counter from a task's (or a CPU's) list of counters.
*
- * Must be called with counter->mutex and ctx->mutex held.
+ * Must be called with ctx->mutex held.
*
* CPU counters are removed with a smp call. For task counters we only
* call when the task is on a CPU.
struct perf_counter_context *ctx = counter->ctx;
struct task_struct *task = ctx->task;
+ unclone_ctx(ctx);
if (!task) {
/*
* Per cpu counters are removed via an smp call and
* succeed.
*/
if (!list_empty(&counter->list_entry)) {
- ctx->nr_counters--;
list_del_counter(counter, ctx);
- counter->task = NULL;
}
spin_unlock_irq(&ctx->lock);
}
struct perf_counter_context *ctx)
{
list_add_counter(counter, ctx);
- ctx->nr_counters++;
counter->prev_state = PERF_COUNTER_STATE_OFF;
counter->tstamp_enabled = ctx->time;
counter->tstamp_running = ctx->time;
/*
* Cross CPU call to install and enable a performance counter
+ *
+ * Must be called with ctx->mutex held
*/
static void __perf_install_in_context(void *info)
{
* If this is a task context, we need to check whether it is
* the current task context of this cpu. If not it has been
* scheduled out before the smp call arrived.
+ * Or possibly this is the right context but it isn't
+ * on this cpu because it had no counters.
*/
- if (ctx->task && cpuctx->task_ctx != ctx)
- return;
+ if (ctx->task && cpuctx->task_ctx != ctx) {
+ if (cpuctx->task_ctx || ctx->task != current)
+ return;
+ cpuctx->task_ctx = ctx;
+ }
spin_lock_irqsave(&ctx->lock, flags);
+ ctx->is_active = 1;
update_context_time(ctx);
/*
return;
}
- counter->task = task;
retry:
task_oncpu_function_call(task, __perf_install_in_context,
counter);
* If this is a per-task counter, need to check whether this
* counter's task is the current task on this cpu.
*/
- if (ctx->task && cpuctx->task_ctx != ctx)
- return;
+ if (ctx->task && cpuctx->task_ctx != ctx) {
+ if (cpuctx->task_ctx || ctx->task != current)
+ return;
+ cpuctx->task_ctx = ctx;
+ }
spin_lock_irqsave(&ctx->lock, flags);
+ ctx->is_active = 1;
update_context_time(ctx);
counter->prev_state = counter->state;
perf_disable();
if (ctx->nr_active) {
- list_for_each_entry(counter, &ctx->counter_list, list_entry)
- group_sched_out(counter, cpuctx, ctx);
+ list_for_each_entry(counter, &ctx->counter_list, list_entry) {
+ if (counter != counter->group_leader)
+ counter_sched_out(counter, cpuctx, ctx);
+ else
+ group_sched_out(counter, cpuctx, ctx);
+ }
}
perf_enable();
out:
spin_unlock(&ctx->lock);
}
+/*
+ * Test whether two contexts are equivalent, i.e. whether they
+ * have both been cloned from the same version of the same context
+ * and they both have the same number of enabled counters.
+ * If the number of enabled counters is the same, then the set
+ * of enabled counters should be the same, because these are both
+ * inherited contexts, therefore we can't access individual counters
+ * in them directly with an fd; we can only enable/disable all
+ * counters via prctl, or enable/disable all counters in a family
+ * via ioctl, which will have the same effect on both contexts.
+ */
+static int context_equiv(struct perf_counter_context *ctx1,
+ struct perf_counter_context *ctx2)
+{
+ return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
+ && ctx1->parent_gen == ctx2->parent_gen;
+}
+
/*
* Called from scheduler to remove the counters of the current task,
* with interrupts disabled.
* accessing the counter control register. If a NMI hits, then it will
* not restart the counter.
*/
-void perf_counter_task_sched_out(struct task_struct *task, int cpu)
+void perf_counter_task_sched_out(struct task_struct *task,
+ struct task_struct *next, int cpu)
{
struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
- struct perf_counter_context *ctx = &task->perf_counter_ctx;
+ struct perf_counter_context *ctx = task->perf_counter_ctxp;
+ struct perf_counter_context *next_ctx;
struct pt_regs *regs;
- if (likely(!cpuctx->task_ctx))
+ regs = task_pt_regs(task);
+ perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
+
+ if (likely(!ctx || !cpuctx->task_ctx))
return;
update_context_time(ctx);
+ next_ctx = next->perf_counter_ctxp;
+ if (next_ctx && context_equiv(ctx, next_ctx)) {
+ task->perf_counter_ctxp = next_ctx;
+ next->perf_counter_ctxp = ctx;
+ ctx->task = next;
+ next_ctx->task = task;
+ return;
+ }
- regs = task_pt_regs(task);
- perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
__perf_counter_sched_out(ctx, cpuctx);
cpuctx->task_ctx = NULL;
{
struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ if (!cpuctx->task_ctx)
+ return;
__perf_counter_sched_out(ctx, cpuctx);
cpuctx->task_ctx = NULL;
}
if (counter->cpu != -1 && counter->cpu != cpu)
continue;
- if (group_can_go_on(counter, cpuctx, 1))
- group_sched_in(counter, cpuctx, ctx, cpu);
+ if (counter != counter->group_leader)
+ counter_sched_in(counter, cpuctx, ctx, cpu);
+ else {
+ if (group_can_go_on(counter, cpuctx, 1))
+ group_sched_in(counter, cpuctx, ctx, cpu);
+ }
/*
* If this pinned group hasn't been scheduled,
if (counter->cpu != -1 && counter->cpu != cpu)
continue;
- if (group_can_go_on(counter, cpuctx, can_add_hw)) {
- if (group_sched_in(counter, cpuctx, ctx, cpu))
+ if (counter != counter->group_leader) {
+ if (counter_sched_in(counter, cpuctx, ctx, cpu))
can_add_hw = 0;
+ } else {
+ if (group_can_go_on(counter, cpuctx, can_add_hw)) {
+ if (group_sched_in(counter, cpuctx, ctx, cpu))
+ can_add_hw = 0;
+ }
}
}
perf_enable();
void perf_counter_task_sched_in(struct task_struct *task, int cpu)
{
struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
- struct perf_counter_context *ctx = &task->perf_counter_ctx;
+ struct perf_counter_context *ctx = task->perf_counter_ctxp;
+ if (likely(!ctx))
+ return;
+ if (cpuctx->task_ctx == ctx)
+ return;
__perf_counter_sched_in(ctx, cpuctx, cpu);
cpuctx->task_ctx = ctx;
}
__perf_counter_sched_in(ctx, cpuctx, cpu);
}
-int perf_counter_task_disable(void)
-{
- struct task_struct *curr = current;
- struct perf_counter_context *ctx = &curr->perf_counter_ctx;
- struct perf_counter *counter;
- unsigned long flags;
+#define MAX_INTERRUPTS (~0ULL)
- if (likely(!ctx->nr_counters))
- return 0;
-
- local_irq_save(flags);
+static void perf_log_throttle(struct perf_counter *counter, int enable);
+static void perf_log_period(struct perf_counter *counter, u64 period);
- __perf_counter_task_sched_out(ctx);
+static void perf_adjust_freq(struct perf_counter_context *ctx)
+{
+ struct perf_counter *counter;
+ u64 interrupts, irq_period;
+ u64 events, period;
+ s64 delta;
spin_lock(&ctx->lock);
-
- /*
- * Disable all the counters:
- */
- perf_disable();
-
list_for_each_entry(counter, &ctx->counter_list, list_entry) {
- if (counter->state != PERF_COUNTER_STATE_ERROR) {
- update_group_times(counter);
- counter->state = PERF_COUNTER_STATE_OFF;
- }
- }
+ if (counter->state != PERF_COUNTER_STATE_ACTIVE)
+ continue;
- perf_enable();
+ interrupts = counter->hw.interrupts;
+ counter->hw.interrupts = 0;
- spin_unlock_irqrestore(&ctx->lock, flags);
+ if (interrupts == MAX_INTERRUPTS) {
+ perf_log_throttle(counter, 1);
+ counter->pmu->unthrottle(counter);
+ interrupts = 2*sysctl_perf_counter_limit/HZ;
+ }
- return 0;
-}
+ if (!counter->hw_event.freq || !counter->hw_event.irq_freq)
+ continue;
-int perf_counter_task_enable(void)
-{
- struct task_struct *curr = current;
- struct perf_counter_context *ctx = &curr->perf_counter_ctx;
- struct perf_counter *counter;
- unsigned long flags;
- int cpu;
+ events = HZ * interrupts * counter->hw.irq_period;
+ period = div64_u64(events, counter->hw_event.irq_freq);
- if (likely(!ctx->nr_counters))
- return 0;
+ delta = (s64)(1 + period - counter->hw.irq_period);
+ delta >>= 1;
- local_irq_save(flags);
- cpu = smp_processor_id();
-
- __perf_counter_task_sched_out(ctx);
+ irq_period = counter->hw.irq_period + delta;
- spin_lock(&ctx->lock);
+ if (!irq_period)
+ irq_period = 1;
- /*
- * Disable all the counters:
- */
- perf_disable();
+ perf_log_period(counter, irq_period);
- list_for_each_entry(counter, &ctx->counter_list, list_entry) {
- if (counter->state > PERF_COUNTER_STATE_OFF)
- continue;
- counter->state = PERF_COUNTER_STATE_INACTIVE;
- counter->tstamp_enabled =
- ctx->time - counter->total_time_enabled;
- counter->hw_event.disabled = 0;
+ counter->hw.irq_period = irq_period;
}
- perf_enable();
-
spin_unlock(&ctx->lock);
-
- perf_counter_task_sched_in(curr, cpu);
-
- local_irq_restore(flags);
-
- return 0;
}
/*
return;
cpuctx = &per_cpu(perf_cpu_context, cpu);
- ctx = &curr->perf_counter_ctx;
+ ctx = curr->perf_counter_ctxp;
+
+ perf_adjust_freq(&cpuctx->ctx);
+ if (ctx)
+ perf_adjust_freq(ctx);
perf_counter_cpu_sched_out(cpuctx);
- __perf_counter_task_sched_out(ctx);
+ if (ctx)
+ __perf_counter_task_sched_out(ctx);
rotate_ctx(&cpuctx->ctx);
- rotate_ctx(ctx);
+ if (ctx)
+ rotate_ctx(ctx);
perf_counter_cpu_sched_in(cpuctx, cpu);
- perf_counter_task_sched_in(curr, cpu);
+ if (ctx)
+ perf_counter_task_sched_in(curr, cpu);
}
/*
return atomic64_read(&counter->count);
}
+/*
+ * Initialize the perf_counter context in a task_struct:
+ */
+static void
+__perf_counter_init_context(struct perf_counter_context *ctx,
+ struct task_struct *task)
+{
+ memset(ctx, 0, sizeof(*ctx));
+ spin_lock_init(&ctx->lock);
+ mutex_init(&ctx->mutex);
+ INIT_LIST_HEAD(&ctx->counter_list);
+ INIT_LIST_HEAD(&ctx->event_list);
+ atomic_set(&ctx->refcount, 1);
+ ctx->task = task;
+}
+
static void put_context(struct perf_counter_context *ctx)
{
if (ctx->task)
{
struct perf_cpu_context *cpuctx;
struct perf_counter_context *ctx;
+ struct perf_counter_context *tctx;
struct task_struct *task;
/*
if (!task)
return ERR_PTR(-ESRCH);
- ctx = &task->perf_counter_ctx;
- ctx->task = task;
-
/* Reuse ptrace permission checks for now. */
if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
- put_context(ctx);
+ put_task_struct(task);
return ERR_PTR(-EACCES);
}
+ ctx = task->perf_counter_ctxp;
+ if (!ctx) {
+ ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
+ if (!ctx) {
+ put_task_struct(task);
+ return ERR_PTR(-ENOMEM);
+ }
+ __perf_counter_init_context(ctx, task);
+ /*
+ * Make sure other cpus see correct values for *ctx
+ * once task->perf_counter_ctxp is visible to them.
+ */
+ smp_wmb();
+ tctx = cmpxchg(&task->perf_counter_ctxp, NULL, ctx);
+ if (tctx) {
+ /*
+ * We raced with some other task; use
+ * the context they set.
+ */
+ kfree(ctx);
+ ctx = tctx;
+ }
+ }
+
return ctx;
}
struct perf_counter *counter;
counter = container_of(head, struct perf_counter, rcu_head);
+ put_ctx(counter->ctx);
kfree(counter);
}
file->private_data = NULL;
mutex_lock(&ctx->mutex);
- mutex_lock(&counter->mutex);
-
perf_counter_remove_from_context(counter);
-
- mutex_unlock(&counter->mutex);
mutex_unlock(&ctx->mutex);
+ mutex_lock(&counter->owner->perf_counter_mutex);
+ list_del_init(&counter->owner_entry);
+ mutex_unlock(&counter->owner->perf_counter_mutex);
+ put_task_struct(counter->owner);
+
free_counter(counter);
put_context(ctx);
if (counter->state == PERF_COUNTER_STATE_ERROR)
return 0;
- mutex_lock(&counter->mutex);
+ mutex_lock(&counter->child_mutex);
values[0] = perf_counter_read(counter);
n = 1;
if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
values[n++] = counter->total_time_running +
atomic64_read(&counter->child_total_time_running);
- mutex_unlock(&counter->mutex);
+ mutex_unlock(&counter->child_mutex);
if (count < n * sizeof(u64))
return -EINVAL;
struct perf_counter_context *ctx = counter->ctx;
struct perf_counter *sibling;
- spin_lock_irq(&ctx->lock);
+ mutex_lock(&ctx->mutex);
counter = counter->group_leader;
func(counter);
list_for_each_entry(sibling, &counter->sibling_list, list_entry)
func(sibling);
- spin_unlock_irq(&ctx->lock);
+ mutex_unlock(&ctx->mutex);
}
static void perf_counter_for_each_child(struct perf_counter *counter,
{
struct perf_counter *child;
- mutex_lock(&counter->mutex);
+ mutex_lock(&counter->child_mutex);
func(counter);
list_for_each_entry(child, &counter->child_list, child_list)
func(child);
- mutex_unlock(&counter->mutex);
+ mutex_unlock(&counter->child_mutex);
}
static void perf_counter_for_each(struct perf_counter *counter,
{
struct perf_counter *child;
- mutex_lock(&counter->mutex);
+ mutex_lock(&counter->child_mutex);
perf_counter_for_each_sibling(counter, func);
list_for_each_entry(child, &counter->child_list, child_list)
perf_counter_for_each_sibling(child, func);
- mutex_unlock(&counter->mutex);
+ mutex_unlock(&counter->child_mutex);
}
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
return 0;
}
+int perf_counter_task_enable(void)
+{
+ struct perf_counter *counter;
+
+ mutex_lock(¤t->perf_counter_mutex);
+ list_for_each_entry(counter, ¤t->perf_counter_list, owner_entry)
+ perf_counter_for_each_child(counter, perf_counter_enable);
+ mutex_unlock(¤t->perf_counter_mutex);
+
+ return 0;
+}
+
+int perf_counter_task_disable(void)
+{
+ struct perf_counter *counter;
+
+ mutex_lock(¤t->perf_counter_mutex);
+ list_for_each_entry(counter, ¤t->perf_counter_list, owner_entry)
+ perf_counter_for_each_child(counter, perf_counter_disable);
+ mutex_unlock(¤t->perf_counter_mutex);
+
+ return 0;
+}
+
/*
* Callers need to ensure there can be no nesting of this function, otherwise
* the seqlock logic goes bad. We can not serialize this because the arch
if (atomic_dec_and_mutex_lock(&counter->mmap_count,
&counter->mmap_mutex)) {
+ struct user_struct *user = current_user();
+
+ atomic_long_sub(counter->data->nr_pages + 1, &user->locked_vm);
vma->vm_mm->locked_vm -= counter->data->nr_locked;
perf_mmap_data_free(counter);
mutex_unlock(&counter->mmap_mutex);
static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
struct perf_counter *counter = file->private_data;
+ struct user_struct *user = current_user();
unsigned long vma_size;
unsigned long nr_pages;
+ unsigned long user_locked, user_lock_limit;
unsigned long locked, lock_limit;
+ long user_extra, extra;
int ret = 0;
- long extra;
if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
return -EINVAL;
goto unlock;
}
- extra = nr_pages /* + 1 only account the data pages */;
- extra -= sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
- if (extra < 0)
- extra = 0;
+ user_extra = nr_pages + 1;
+ user_lock_limit = sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
- locked = vma->vm_mm->locked_vm + extra;
+ /*
+ * Increase the limit linearly with more CPUs:
+ */
+ user_lock_limit *= num_online_cpus();
+
+ user_locked = atomic_long_read(&user->locked_vm) + user_extra;
+
+ extra = 0;
+ if (user_locked > user_lock_limit)
+ extra = user_locked - user_lock_limit;
lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
lock_limit >>= PAGE_SHIFT;
+ locked = vma->vm_mm->locked_vm + extra;
if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
ret = -EPERM;
goto unlock;
atomic_set(&counter->mmap_count, 1);
+ atomic_long_add(user_extra, &user->locked_vm);
vma->vm_mm->locked_vm += extra;
counter->data->nr_locked = extra;
unlock:
header.size = sizeof(header);
header.misc = PERF_EVENT_MISC_OVERFLOW;
- header.misc |= user_mode(regs) ?
- PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
+ header.misc |= perf_misc_flags(regs);
if (record_type & PERF_RECORD_IP) {
- ip = instruction_pointer(regs);
+ ip = perf_instruction_pointer(regs);
header.type |= PERF_RECORD_IP;
header.size += sizeof(ip);
}
perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
put_cpu_var(perf_cpu_context);
- perf_counter_comm_ctx(¤t->perf_counter_ctx, comm_event);
+ perf_counter_comm_ctx(current->perf_counter_ctxp, comm_event);
}
void perf_counter_comm(struct task_struct *task)
if (!atomic_read(&nr_comm_tracking))
return;
-
+ if (!current->perf_counter_ctxp)
+ return;
+
comm_event = (struct perf_comm_event){
.task = task,
.event = {
perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
put_cpu_var(perf_cpu_context);
- perf_counter_mmap_ctx(¤t->perf_counter_ctx, mmap_event);
+ perf_counter_mmap_ctx(current->perf_counter_ctxp, mmap_event);
kfree(buf);
}
if (!atomic_read(&nr_mmap_tracking))
return;
+ if (!current->perf_counter_ctxp)
+ return;
mmap_event = (struct perf_mmap_event){
.file = file,
perf_counter_mmap_event(&mmap_event);
}
+/*
+ * Log irq_period changes so that analyzing tools can re-normalize the
+ * event flow.
+ */
+
+static void perf_log_period(struct perf_counter *counter, u64 period)
+{
+ struct perf_output_handle handle;
+ int ret;
+
+ struct {
+ struct perf_event_header header;
+ u64 time;
+ u64 period;
+ } freq_event = {
+ .header = {
+ .type = PERF_EVENT_PERIOD,
+ .misc = 0,
+ .size = sizeof(freq_event),
+ },
+ .time = sched_clock(),
+ .period = period,
+ };
+
+ if (counter->hw.irq_period == period)
+ return;
+
+ ret = perf_output_begin(&handle, counter, sizeof(freq_event), 0, 0);
+ if (ret)
+ return;
+
+ perf_output_put(&handle, freq_event);
+ perf_output_end(&handle);
+}
+
+/*
+ * IRQ throttle logging
+ */
+
+static void perf_log_throttle(struct perf_counter *counter, int enable)
+{
+ struct perf_output_handle handle;
+ int ret;
+
+ struct {
+ struct perf_event_header header;
+ u64 time;
+ } throttle_event = {
+ .header = {
+ .type = PERF_EVENT_THROTTLE + 1,
+ .misc = 0,
+ .size = sizeof(throttle_event),
+ },
+ .time = sched_clock(),
+ };
+
+ ret = perf_output_begin(&handle, counter, sizeof(throttle_event), 1, 0);
+ if (ret)
+ return;
+
+ perf_output_put(&handle, throttle_event);
+ perf_output_end(&handle);
+}
+
/*
* Generic counter overflow handling.
*/
int nmi, struct pt_regs *regs, u64 addr)
{
int events = atomic_read(&counter->event_limit);
+ int throttle = counter->pmu->unthrottle != NULL;
int ret = 0;
+ if (!throttle) {
+ counter->hw.interrupts++;
+ } else if (counter->hw.interrupts != MAX_INTERRUPTS) {
+ counter->hw.interrupts++;
+ if (HZ*counter->hw.interrupts > (u64)sysctl_perf_counter_limit) {
+ counter->hw.interrupts = MAX_INTERRUPTS;
+ perf_log_throttle(counter, 0);
+ ret = 1;
+ }
+ }
+
/*
* XXX event_limit might not quite work as expected on inherited
* counters
enum hrtimer_restart ret = HRTIMER_RESTART;
struct perf_counter *counter;
struct pt_regs *regs;
+ u64 period;
counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
counter->pmu->read(counter);
ret = HRTIMER_NORESTART;
}
- hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
+ period = max_t(u64, 10000, counter->hw.irq_period);
+ hrtimer_forward_now(hrtimer, ns_to_ktime(period));
return ret;
}
hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hwc->hrtimer.function = perf_swcounter_hrtimer;
if (hwc->irq_period) {
+ u64 period = max_t(u64, 10000, hwc->irq_period);
__hrtimer_start_range_ns(&hwc->hrtimer,
- ns_to_ktime(hwc->irq_period), 0,
+ ns_to_ktime(period), 0,
HRTIMER_MODE_REL, 0);
}
static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
{
- hrtimer_cancel(&counter->hw.hrtimer);
+ if (counter->hw.irq_period)
+ hrtimer_cancel(&counter->hw.hrtimer);
cpu_clock_perf_counter_update(counter);
}
hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hwc->hrtimer.function = perf_swcounter_hrtimer;
if (hwc->irq_period) {
+ u64 period = max_t(u64, 10000, hwc->irq_period);
__hrtimer_start_range_ns(&hwc->hrtimer,
- ns_to_ktime(hwc->irq_period), 0,
+ ns_to_ktime(period), 0,
HRTIMER_MODE_REL, 0);
}
static void task_clock_perf_counter_disable(struct perf_counter *counter)
{
- hrtimer_cancel(&counter->hw.hrtimer);
+ if (counter->hw.irq_period)
+ hrtimer_cancel(&counter->hw.hrtimer);
task_clock_perf_counter_update(counter, counter->ctx->time);
}
static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
{
- struct perf_counter_hw_event *hw_event = &counter->hw_event;
const struct pmu *pmu = NULL;
- struct hw_perf_counter *hwc = &counter->hw;
/*
* Software counters (currently) can't in general distinguish
case PERF_COUNT_CPU_CLOCK:
pmu = &perf_ops_cpu_clock;
- if (hw_event->irq_period && hw_event->irq_period < 10000)
- hw_event->irq_period = 10000;
break;
case PERF_COUNT_TASK_CLOCK:
/*
else
pmu = &perf_ops_cpu_clock;
- if (hw_event->irq_period && hw_event->irq_period < 10000)
- hw_event->irq_period = 10000;
break;
case PERF_COUNT_PAGE_FAULTS:
case PERF_COUNT_PAGE_FAULTS_MIN:
break;
}
- if (pmu)
- hwc->irq_period = hw_event->irq_period;
-
return pmu;
}
{
const struct pmu *pmu;
struct perf_counter *counter;
+ struct hw_perf_counter *hwc;
long err;
counter = kzalloc(sizeof(*counter), gfpflags);
if (!group_leader)
group_leader = counter;
- mutex_init(&counter->mutex);
+ mutex_init(&counter->child_mutex);
+ INIT_LIST_HEAD(&counter->child_list);
+
INIT_LIST_HEAD(&counter->list_entry);
INIT_LIST_HEAD(&counter->event_entry);
INIT_LIST_HEAD(&counter->sibling_list);
mutex_init(&counter->mmap_mutex);
- INIT_LIST_HEAD(&counter->child_list);
-
counter->cpu = cpu;
counter->hw_event = *hw_event;
counter->group_leader = group_leader;
counter->pmu = NULL;
counter->ctx = ctx;
+ counter->oncpu = -1;
+
+ get_ctx(ctx);
counter->state = PERF_COUNTER_STATE_INACTIVE;
if (hw_event->disabled)
pmu = NULL;
+ hwc = &counter->hw;
+ if (hw_event->freq && hw_event->irq_freq)
+ hwc->irq_period = div64_u64(TICK_NSEC, hw_event->irq_freq);
+ else
+ hwc->irq_period = hw_event->irq_period;
+
/*
* we currently do not support PERF_RECORD_GROUP on inherited counters
*/
perf_install_in_context(ctx, counter, cpu);
mutex_unlock(&ctx->mutex);
+ counter->owner = current;
+ get_task_struct(current);
+ mutex_lock(¤t->perf_counter_mutex);
+ list_add_tail(&counter->owner_entry, ¤t->perf_counter_list);
+ mutex_unlock(¤t->perf_counter_mutex);
+
fput_light(counter_file, fput_needed2);
out_fput:
goto out_fput;
}
-/*
- * Initialize the perf_counter context in a task_struct:
- */
-static void
-__perf_counter_init_context(struct perf_counter_context *ctx,
- struct task_struct *task)
-{
- memset(ctx, 0, sizeof(*ctx));
- spin_lock_init(&ctx->lock);
- mutex_init(&ctx->mutex);
- INIT_LIST_HEAD(&ctx->counter_list);
- INIT_LIST_HEAD(&ctx->event_list);
- ctx->task = task;
-}
-
/*
* inherit a counter from parent task to child task:
*/
if (IS_ERR(child_counter))
return child_counter;
+ /*
+ * Make the child state follow the state of the parent counter,
+ * not its hw_event.disabled bit. We hold the parent's mutex,
+ * so we won't race with perf_counter_{en,dis}able_family.
+ */
+ if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
+ child_counter->state = PERF_COUNTER_STATE_INACTIVE;
+ else
+ child_counter->state = PERF_COUNTER_STATE_OFF;
+
/*
* Link it up in the child's context:
*/
- child_counter->task = child;
add_counter_to_ctx(child_counter, child_ctx);
child_counter->parent = parent_counter;
/*
* Link this into the parent counter's child list
*/
- mutex_lock(&parent_counter->mutex);
+ mutex_lock(&parent_counter->child_mutex);
list_add_tail(&child_counter->child_list, &parent_counter->child_list);
-
- /*
- * Make the child state follow the state of the parent counter,
- * not its hw_event.disabled bit. We hold the parent's mutex,
- * so we won't race with perf_counter_{en,dis}able_family.
- */
- if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
- child_counter->state = PERF_COUNTER_STATE_INACTIVE;
- else
- child_counter->state = PERF_COUNTER_STATE_OFF;
-
- mutex_unlock(&parent_counter->mutex);
+ mutex_unlock(&parent_counter->child_mutex);
return child_counter;
}
static void sync_child_counter(struct perf_counter *child_counter,
struct perf_counter *parent_counter)
{
- u64 parent_val, child_val;
+ u64 child_val;
- parent_val = atomic64_read(&parent_counter->count);
child_val = atomic64_read(&child_counter->count);
/*
/*
* Remove this counter from the parent's list
*/
- mutex_lock(&parent_counter->mutex);
+ mutex_lock(&parent_counter->child_mutex);
list_del_init(&child_counter->child_list);
- mutex_unlock(&parent_counter->mutex);
+ mutex_unlock(&parent_counter->child_mutex);
/*
* Release the parent counter, if this was the last
struct perf_counter_context *child_ctx)
{
struct perf_counter *parent_counter;
- struct perf_counter *sub, *tmp;
-
- /*
- * If we do not self-reap then we have to wait for the
- * child task to unschedule (it will happen for sure),
- * so that its counter is at its final count. (This
- * condition triggers rarely - child tasks usually get
- * off their CPU before the parent has a chance to
- * get this far into the reaping action)
- */
- if (child != current) {
- wait_task_inactive(child, 0);
- list_del_init(&child_counter->list_entry);
- update_counter_times(child_counter);
- } else {
- struct perf_cpu_context *cpuctx;
- unsigned long flags;
-
- /*
- * Disable and unlink this counter.
- *
- * Be careful about zapping the list - IRQ/NMI context
- * could still be processing it:
- */
- local_irq_save(flags);
- perf_disable();
- cpuctx = &__get_cpu_var(perf_cpu_context);
-
- group_sched_out(child_counter, cpuctx, child_ctx);
- update_counter_times(child_counter);
-
- list_del_init(&child_counter->list_entry);
-
- child_ctx->nr_counters--;
-
- perf_enable();
- local_irq_restore(flags);
- }
+ update_counter_times(child_counter);
+ perf_counter_remove_from_context(child_counter);
parent_counter = child_counter->parent;
/*
*/
if (parent_counter) {
sync_child_counter(child_counter, parent_counter);
- list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
- list_entry) {
- if (sub->parent) {
- sync_child_counter(sub, sub->parent);
- free_counter(sub);
- }
- }
free_counter(child_counter);
}
}
*
* Note: we may be running in child context, but the PID is not hashed
* anymore so new counters will not be added.
+ * (XXX not sure that is true when we get called from flush_old_exec.
+ * -- paulus)
*/
void perf_counter_exit_task(struct task_struct *child)
{
struct perf_counter *child_counter, *tmp;
struct perf_counter_context *child_ctx;
+ unsigned long flags;
- child_ctx = &child->perf_counter_ctx;
+ child_ctx = child->perf_counter_ctxp;
- if (likely(!child_ctx->nr_counters))
+ if (likely(!child_ctx))
return;
+ local_irq_save(flags);
+ __perf_counter_task_sched_out(child_ctx);
+ child->perf_counter_ctxp = NULL;
+ local_irq_restore(flags);
+
+ mutex_lock(&child_ctx->mutex);
+
+again:
list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
list_entry)
__perf_counter_exit_task(child, child_counter, child_ctx);
+
+ /*
+ * If the last counter was a group counter, it will have appended all
+ * its siblings to the list, but we obtained 'tmp' before that which
+ * will still point to the list head terminating the iteration.
+ */
+ if (!list_empty(&child_ctx->counter_list))
+ goto again;
+
+ mutex_unlock(&child_ctx->mutex);
+
+ put_ctx(child_ctx);
}
/*
* Initialize the perf_counter context in task_struct
*/
-void perf_counter_init_task(struct task_struct *child)
+int perf_counter_init_task(struct task_struct *child)
{
struct perf_counter_context *child_ctx, *parent_ctx;
struct perf_counter *counter;
struct task_struct *parent = current;
+ int inherited_all = 1;
+ int ret = 0;
- child_ctx = &child->perf_counter_ctx;
- parent_ctx = &parent->perf_counter_ctx;
+ child->perf_counter_ctxp = NULL;
- __perf_counter_init_context(child_ctx, child);
+ mutex_init(&child->perf_counter_mutex);
+ INIT_LIST_HEAD(&child->perf_counter_list);
+
+ parent_ctx = parent->perf_counter_ctxp;
+ if (likely(!parent_ctx || !parent_ctx->nr_counters))
+ return 0;
/*
* This is executed from the parent task context, so inherit
- * counters that have been marked for cloning:
+ * counters that have been marked for cloning.
+ * First allocate and initialize a context for the child.
*/
- if (likely(!parent_ctx->nr_counters))
- return;
+ child_ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
+ if (!child_ctx)
+ return -ENOMEM;
+
+ __perf_counter_init_context(child_ctx, child);
+ child->perf_counter_ctxp = child_ctx;
/*
* Lock the parent list. No need to lock the child - not PID
* We dont have to disable NMIs - we are only looking at
* the list, not manipulating it:
*/
- list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
- if (!counter->hw_event.inherit)
+ list_for_each_entry_rcu(counter, &parent_ctx->event_list, event_entry) {
+ if (counter != counter->group_leader)
continue;
- if (inherit_group(counter, parent,
- parent_ctx, child, child_ctx))
+ if (!counter->hw_event.inherit) {
+ inherited_all = 0;
+ continue;
+ }
+
+ ret = inherit_group(counter, parent, parent_ctx,
+ child, child_ctx);
+ if (ret) {
+ inherited_all = 0;
break;
+ }
+ }
+
+ if (inherited_all) {
+ /*
+ * Mark the child context as a clone of the parent
+ * context, or of whatever the parent is a clone of.
+ */
+ if (parent_ctx->parent_ctx) {
+ child_ctx->parent_ctx = parent_ctx->parent_ctx;
+ child_ctx->parent_gen = parent_ctx->parent_gen;
+ } else {
+ child_ctx->parent_ctx = parent_ctx;
+ child_ctx->parent_gen = parent_ctx->generation;
+ }
+ get_ctx(child_ctx->parent_ctx);
}
mutex_unlock(&parent_ctx->mutex);
+
+ return ret;
}
static void __cpuinit perf_counter_init_cpu(int cpu)
git.openpandora.org / pandora-kernel.git / blobdiff