1 // SPDX-License-Identifier: GPL-2.0
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
75 	struct task_struct *p = tfc->p;  in remote_function()
78 		/* -EAGAIN */  in remote_function()
87 		tfc->ret = -ESRCH; /* No such (running) process */  in remote_function()
92 	tfc->ret = tfc->func(tfc->info);  in remote_function()
96  * task_function_call - call a function on the cpu on which a task runs
106  * returns @func return value or -ESRCH or -ENXIO when the process isn't running
115 		.ret	= -EAGAIN,  in task_function_call()
125 		if (ret != -EAGAIN)  in task_function_call()
135  * cpu_function_call - call a function on the cpu
142  * returns: @func return value or -ENXIO when the cpu is offline
150 		.ret	= -ENXIO, /* No such CPU */  in cpu_function_call()
174 	raw_spin_lock(&ctx->lock);  in __perf_ctx_lock()
175 	WARN_ON_ONCE(ctx->is_active & EVENT_FROZEN);  in __perf_ctx_lock()
181 	__perf_ctx_lock(&cpuctx->ctx);  in perf_ctx_lock()
192 	if (ctx->is_active & EVENT_FROZEN) {  in __perf_ctx_unlock()
193 		if (!(ctx->is_active & EVENT_ALL))  in __perf_ctx_unlock()
194 			ctx->is_active = 0;  in __perf_ctx_unlock()
196 			ctx->is_active &= ~EVENT_FROZEN;  in __perf_ctx_unlock()
198 	raw_spin_unlock(&ctx->lock);  in __perf_ctx_unlock()
206 	__perf_ctx_unlock(&cpuctx->ctx);  in perf_ctx_unlock()
209 #define TASK_TOMBSTONE ((void *)-1L)
213 	return READ_ONCE(event->owner) == TASK_TOMBSTONE;  in is_kernel_event()
221 	return this_cpu_ptr(&perf_cpu_context)->task_ctx;  in perf_cpu_task_ctx()
227  * When !ctx->nr_events a task context will not be scheduled. This means
233  *  - removing the last event from a task ctx; this is relatively straight
236  *  - adding the first event to a task ctx; this is tricky because we cannot
237  *    rely on ctx->is_active and therefore cannot use event_function_call().
240  * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
255 	struct perf_event *event = efs->event;  in event_function()
256 	struct perf_event_context *ctx = event->ctx;  in event_function()
258 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in event_function()
265 	 * Since we do the IPI call without holding ctx->lock things can have  in event_function()
268 	if (ctx->task) {  in event_function()
269 		if (ctx->task != current) {  in event_function()
270 			ret = -ESRCH;  in event_function()
278 		 * above ctx->task != current test), therefore we must have  in event_function()
279 		 * ctx->is_active here.  in event_function()
281 		WARN_ON_ONCE(!ctx->is_active);  in event_function()
283 		 * And since we have ctx->is_active, cpuctx->task_ctx must  in event_function()
288 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function()
291 	efs->func(event, cpuctx, ctx, efs->data);  in event_function()
300 	struct perf_event_context *ctx = event->ctx;  in event_function_call()
301 	struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */  in event_function_call()
309 	if (!event->parent) {  in event_function_call()
312 		 * stabilize the event->ctx relation. See  in event_function_call()
315 		lockdep_assert_held(&ctx->mutex);  in event_function_call()
319 		cpu_function_call(event->cpu, event_function, &efs);  in event_function_call()
337 	task = ctx->task;  in event_function_call()
340 	if (ctx->is_active) {  in event_function_call()
357 	struct perf_event_context *ctx = event->ctx;  in event_function_local()
359 	struct task_struct *task = READ_ONCE(ctx->task);  in event_function_local()
373 	task = ctx->task;  in event_function_local()
383 		if (ctx->is_active) {  in event_function_local()
387 			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))  in event_function_local()
391 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function_local()
448  *  -1 - not paranoid at all
449  *   0 - disallow raw tracepoint access for unpriv
450  *   1 - disallow cpu events for unpriv
451  *   2 - disallow kernel profiling for unpriv
496 		return -EINVAL;  in perf_event_max_sample_rate_handler()
566 	running_len -= running_len/NR_ACCUMULATED_SAMPLES;  in perf_sample_event_took()
620 	return event->clock();  in perf_event_clock()
626  * The basic idea is to use event->state to determine which (if any) time
648 	struct perf_event *leader = event->group_leader;  in __perf_effective_state()
650 	if (leader->state <= PERF_EVENT_STATE_OFF)  in __perf_effective_state()
651 		return leader->state;  in __perf_effective_state()
653 	return event->state;  in __perf_effective_state()
660 	u64 delta = now - event->tstamp;  in __perf_update_times()
662 	*enabled = event->total_time_enabled;  in __perf_update_times()
666 	*running = event->total_time_running;  in __perf_update_times()
675 	__perf_update_times(event, now, &event->total_time_enabled,  in perf_event_update_time()
676 					&event->total_time_running);  in perf_event_update_time()
677 	event->tstamp = now;  in perf_event_update_time()
691 	if (event->state == state)  in perf_event_set_state()
699 	if ((event->state < 0) ^ (state < 0))  in perf_event_set_state()
702 	WRITE_ONCE(event->state, state);  in perf_event_set_state()
706  * UP store-release, load-acquire
723 	list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
724 		if (_cgroup && !_epc->nr_cgroups)			\
726 		else if (_pmu && _epc->pmu != _pmu)			\
735 		perf_pmu_disable(pmu_ctx->pmu);  in perf_ctx_disable()
743 		perf_pmu_enable(pmu_ctx->pmu);  in perf_ctx_enable()
757 	if (!event->cgrp)  in perf_cgroup_match()
761 	if (!cpuctx->cgrp)  in perf_cgroup_match()
770 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,  in perf_cgroup_match()
771 				    event->cgrp->css.cgroup);  in perf_cgroup_match()
776 	css_put(&event->cgrp->css);  in perf_detach_cgroup()
777 	event->cgrp = NULL;  in perf_detach_cgroup()
782 	return event->cgrp != NULL;  in is_cgroup_event()
789 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time()
790 	return t->time;  in perf_cgroup_event_time()
797 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time_now()
798 	if (!__load_acquire(&t->active))  in perf_cgroup_event_time_now()
799 		return t->time;  in perf_cgroup_event_time_now()
800 	now += READ_ONCE(t->timeoffset);  in perf_cgroup_event_time_now()
807 		info->time += now - info->timestamp;  in __update_cgrp_time()
808 	info->timestamp = now;  in __update_cgrp_time()
812 	WRITE_ONCE(info->timeoffset, info->time - info->timestamp);  in __update_cgrp_time()
817 	struct perf_cgroup *cgrp = cpuctx->cgrp;  in update_cgrp_time_from_cpuctx()
824 		for (css = &cgrp->css; css; css = css->parent) {  in update_cgrp_time_from_cpuctx()
826 			info = this_cpu_ptr(cgrp->info);  in update_cgrp_time_from_cpuctx()
830 				__store_release(&info->active, 0);  in update_cgrp_time_from_cpuctx()
846 	info = this_cpu_ptr(event->cgrp->info);  in update_cgrp_time_from_event()
850 	if (info->active)  in update_cgrp_time_from_event()
857 	struct perf_event_context *ctx = &cpuctx->ctx;  in perf_cgroup_set_timestamp()
858 	struct perf_cgroup *cgrp = cpuctx->cgrp;  in perf_cgroup_set_timestamp()
863 	 * ctx->lock held by caller  in perf_cgroup_set_timestamp()
870 	WARN_ON_ONCE(!ctx->nr_cgroups);  in perf_cgroup_set_timestamp()
872 	for (css = &cgrp->css; css; css = css->parent) {  in perf_cgroup_set_timestamp()
874 		info = this_cpu_ptr(cgrp->info);  in perf_cgroup_set_timestamp()
875 		__update_cgrp_time(info, ctx->timestamp, false);  in perf_cgroup_set_timestamp()
876 		__store_release(&info->active, 1);  in perf_cgroup_set_timestamp()
889 	 * cpuctx->cgrp is set when the first cgroup event enabled,  in perf_cgroup_switch()
892 	if (READ_ONCE(cpuctx->cgrp) == NULL)  in perf_cgroup_switch()
895 	WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);  in perf_cgroup_switch()
898 	if (READ_ONCE(cpuctx->cgrp) == cgrp)  in perf_cgroup_switch()
901 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
902 	perf_ctx_disable(&cpuctx->ctx, true);  in perf_cgroup_switch()
904 	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);  in perf_cgroup_switch()
910 	cpuctx->cgrp = cgrp;  in perf_cgroup_switch()
916 	ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);  in perf_cgroup_switch()
918 	perf_ctx_enable(&cpuctx->ctx, true);  in perf_cgroup_switch()
919 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
933 	for (heap_size = 1; css; css = css->parent)  in perf_cgroup_ensure_storage()
938 		if (heap_size <= cpuctx->heap_size)  in perf_cgroup_ensure_storage()
944 			ret = -ENOMEM;  in perf_cgroup_ensure_storage()
948 		raw_spin_lock_irq(&cpuctx->ctx.lock);  in perf_cgroup_ensure_storage()
949 		if (cpuctx->heap_size < heap_size) {  in perf_cgroup_ensure_storage()
950 			swap(cpuctx->heap, storage);  in perf_cgroup_ensure_storage()
951 			if (storage == cpuctx->heap_default)  in perf_cgroup_ensure_storage()
953 			cpuctx->heap_size = heap_size;  in perf_cgroup_ensure_storage()
955 		raw_spin_unlock_irq(&cpuctx->ctx.lock);  in perf_cgroup_ensure_storage()
973 		return -EBADF;  in perf_cgroup_connect()
975 	css = css_tryget_online_from_dir(fd_file(f)->f_path.dentry,  in perf_cgroup_connect()
985 	event->cgrp = cgrp;  in perf_cgroup_connect()
992 	if (group_leader && group_leader->cgrp != cgrp) {  in perf_cgroup_connect()
994 		ret = -EINVAL;  in perf_cgroup_connect()
1007 	event->pmu_ctx->nr_cgroups++;  in perf_cgroup_event_enable()
1010 	 * Because cgroup events are always per-cpu events,  in perf_cgroup_event_enable()
1011 	 * @ctx == &cpuctx->ctx.  in perf_cgroup_event_enable()
1015 	if (ctx->nr_cgroups++)  in perf_cgroup_event_enable()
1018 	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);  in perf_cgroup_event_enable()
1029 	event->pmu_ctx->nr_cgroups--;  in perf_cgroup_event_disable()
1032 	 * Because cgroup events are always per-cpu events,  in perf_cgroup_event_disable()
1033 	 * @ctx == &cpuctx->ctx.  in perf_cgroup_event_disable()
1037 	if (--ctx->nr_cgroups)  in perf_cgroup_event_disable()
1040 	cpuctx->cgrp = NULL;  in perf_cgroup_event_disable()
1072 	return -EINVAL;  in perf_cgroup_connect()
1123 	raw_spin_lock(&cpc->hrtimer_lock);  in perf_mux_hrtimer_handler()
1125 		hrtimer_forward_now(hr, cpc->hrtimer_interval);  in perf_mux_hrtimer_handler()
1127 		cpc->hrtimer_active = 0;  in perf_mux_hrtimer_handler()
1128 	raw_spin_unlock(&cpc->hrtimer_lock);  in perf_mux_hrtimer_handler()
1135 	struct hrtimer *timer = &cpc->hrtimer;  in __perf_mux_hrtimer_init()
1136 	struct pmu *pmu = cpc->epc.pmu;  in __perf_mux_hrtimer_init()
1143 	interval = pmu->hrtimer_interval_ms;  in __perf_mux_hrtimer_init()
1145 		interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;  in __perf_mux_hrtimer_init()
1147 	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);  in __perf_mux_hrtimer_init()
1149 	raw_spin_lock_init(&cpc->hrtimer_lock);  in __perf_mux_hrtimer_init()
1151 	timer->function = perf_mux_hrtimer_handler;  in __perf_mux_hrtimer_init()
1156 	struct hrtimer *timer = &cpc->hrtimer;  in perf_mux_hrtimer_restart()
1159 	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1160 	if (!cpc->hrtimer_active) {  in perf_mux_hrtimer_restart()
1161 		cpc->hrtimer_active = 1;  in perf_mux_hrtimer_restart()
1162 		hrtimer_forward_now(timer, cpc->hrtimer_interval);  in perf_mux_hrtimer_restart()
1165 	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1177 	int *count = this_cpu_ptr(pmu->pmu_disable_count);  in perf_pmu_disable()
1179 		pmu->pmu_disable(pmu);  in perf_pmu_disable()
1184 	int *count = this_cpu_ptr(pmu->pmu_disable_count);  in perf_pmu_enable()
1185 	if (!--(*count))  in perf_pmu_enable()
1186 		pmu->pmu_enable(pmu);  in perf_pmu_enable()
1191 	WARN_ON_ONCE(*this_cpu_ptr(pmu->pmu_disable_count) == 0);  in perf_assert_pmu_disabled()
1196 	refcount_inc(&ctx->refcount);  in get_ctx()
1201 	if (pmu->task_ctx_cache)  in alloc_task_ctx_data()
1202 		return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL);  in alloc_task_ctx_data()
1209 	if (pmu->task_ctx_cache && task_ctx_data)  in free_task_ctx_data()
1210 		kmem_cache_free(pmu->task_ctx_cache, task_ctx_data);  in free_task_ctx_data()
1223 	if (refcount_dec_and_test(&ctx->refcount)) {  in put_ctx()
1224 		if (ctx->parent_ctx)  in put_ctx()
1225 			put_ctx(ctx->parent_ctx);  in put_ctx()
1226 		if (ctx->task && ctx->task != TASK_TOMBSTONE)  in put_ctx()
1227 			put_task_struct(ctx->task);  in put_ctx()
1228 		call_rcu(&ctx->rcu_head, free_ctx);  in put_ctx()
1242  *  - perf_event_exit_task_context()	[ child , 0 ]
1246  *  - perf_event_init_context()		[ parent, 0 ]
1254  * While it appears there is an obvious deadlock here -- the parent and child
1256  * life-time rules separate them. That is an exiting task cannot fork, and a
1259  * But remember that these are parent<->child context relations, and
1265  * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1280  * However; because event->ctx can change while we're waiting to acquire
1281  * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1297  *	  cpuctx->mutex / perf_event_context::mutex
1306 	ctx = READ_ONCE(event->ctx);  in perf_event_ctx_lock_nested()
1307 	if (!refcount_inc_not_zero(&ctx->refcount)) {  in perf_event_ctx_lock_nested()
1313 	mutex_lock_nested(&ctx->mutex, nesting);  in perf_event_ctx_lock_nested()
1314 	if (event->ctx != ctx) {  in perf_event_ctx_lock_nested()
1315 		mutex_unlock(&ctx->mutex);  in perf_event_ctx_lock_nested()
1332 	mutex_unlock(&ctx->mutex);  in perf_event_ctx_unlock()
1337  * This must be done under the ctx->lock, such as to serialize against
1339  * calling scheduler related locks and ctx->lock nests inside those.
1344 	struct perf_event_context *parent_ctx = ctx->parent_ctx;  in unclone_ctx()
1346 	lockdep_assert_held(&ctx->lock);  in unclone_ctx()
1349 		ctx->parent_ctx = NULL;  in unclone_ctx()
1350 	ctx->generation++;  in unclone_ctx()
1362 	if (event->parent)  in perf_event_pid_type()
1363 		event = event->parent;  in perf_event_pid_type()
1365 	nr = __task_pid_nr_ns(p, type, event->ns);  in perf_event_pid_type()
1366 	/* avoid -1 if it is idle thread or runs in another ns */  in perf_event_pid_type()
1368 		nr = -1;  in perf_event_pid_type()
1388 	u64 id = event->id;  in primary_event_id()
1390 	if (event->parent)  in primary_event_id()
1391 		id = event->parent->id;  in primary_event_id()
1411 	 * part of the read side critical section was irqs-enabled -- see  in perf_lock_task_context()
1414 	 * Since ctx->lock nests under rq->lock we must ensure the entire read  in perf_lock_task_context()
1419 	ctx = rcu_dereference(task->perf_event_ctxp);  in perf_lock_task_context()
1431 		raw_spin_lock(&ctx->lock);  in perf_lock_task_context()
1432 		if (ctx != rcu_dereference(task->perf_event_ctxp)) {  in perf_lock_task_context()
1433 			raw_spin_unlock(&ctx->lock);  in perf_lock_task_context()
1439 		if (ctx->task == TASK_TOMBSTONE ||  in perf_lock_task_context()
1440 		    !refcount_inc_not_zero(&ctx->refcount)) {  in perf_lock_task_context()
1441 			raw_spin_unlock(&ctx->lock);  in perf_lock_task_context()
1444 			WARN_ON_ONCE(ctx->task != task);  in perf_lock_task_context()
1466 		++ctx->pin_count;  in perf_pin_task_context()
1467 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_pin_task_context()
1476 	raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_unpin_context()
1477 	--ctx->pin_count;  in perf_unpin_context()
1478 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_unpin_context()
1488 	lockdep_assert_held(&ctx->lock);  in __update_context_time()
1491 		ctx->time += now - ctx->timestamp;  in __update_context_time()
1492 	ctx->timestamp = now;  in __update_context_time()
1495 	 * The above: time' = time + (now - timestamp), can be re-arranged  in __update_context_time()
1496 	 * into: time` = now + (time - timestamp), which gives a single value  in __update_context_time()
1500 	 * it's (obviously) not possible to acquire ctx->lock in order to read  in __update_context_time()
1503 	WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);  in __update_context_time()
1513 	struct perf_event_context *ctx = event->ctx;  in perf_event_time()
1521 	return ctx->time;  in perf_event_time()
1526 	struct perf_event_context *ctx = event->ctx;  in perf_event_time_now()
1534 	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))  in perf_event_time_now()
1535 		return ctx->time;  in perf_event_time_now()
1537 	now += READ_ONCE(ctx->timeoffset);  in perf_event_time_now()
1543 	struct perf_event_context *ctx = event->ctx;  in get_event_type()
1546 	lockdep_assert_held(&ctx->lock);  in get_event_type()
1552 	if (event->group_leader != event)  in get_event_type()
1553 		event = event->group_leader;  in get_event_type()
1555 	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;  in get_event_type()
1556 	if (!ctx->task)  in get_event_type()
1567 	RB_CLEAR_NODE(&event->group_node);  in init_event_group()
1568 	event->group_index = 0;  in init_event_group()
1578 	if (event->attr.pinned)  in get_event_groups()
1579 		return &ctx->pinned_groups;  in get_event_groups()
1581 		return &ctx->flexible_groups;  in get_event_groups()
1589 	groups->tree = RB_ROOT;  in perf_event_groups_init()
1590 	groups->index = 0;  in perf_event_groups_init()
1598 	if (event->cgrp)  in event_cgroup()
1599 		cgroup = event->cgrp->css.cgroup;  in event_cgroup()
1616 	if (left_cpu < right->cpu)  in perf_event_groups_cmp()
1617 		return -1;  in perf_event_groups_cmp()
1618 	if (left_cpu > right->cpu)  in perf_event_groups_cmp()
1622 		if (left_pmu < right->pmu_ctx->pmu)  in perf_event_groups_cmp()
1623 			return -1;  in perf_event_groups_cmp()
1624 		if (left_pmu > right->pmu_ctx->pmu)  in perf_event_groups_cmp()
1638 				return -1;  in perf_event_groups_cmp()
1649 				return -1;  in perf_event_groups_cmp()
1656 	if (left_group_index < right->group_index)  in perf_event_groups_cmp()
1657 		return -1;  in perf_event_groups_cmp()
1658 	if (left_group_index > right->group_index)  in perf_event_groups_cmp()
1670 	return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e),  in __group_less()
1671 				     e->group_index, __node_2_pe(b)) < 0;  in __group_less()
1686 	return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b);  in __group_cmp()
1696 	return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b),  in __group_cmp_ignore_cgroup()
1697 				     b->group_index, b);  in __group_cmp_ignore_cgroup()
1702  *   {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1709 	event->group_index = ++groups->index;  in perf_event_groups_insert()
1711 	rb_add(&event->group_node, &groups->tree, __group_less);  in perf_event_groups_insert()
1733 	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||  in perf_event_groups_delete()
1734 		     RB_EMPTY_ROOT(&groups->tree));  in perf_event_groups_delete()
1736 	rb_erase(&event->group_node, &groups->tree);  in perf_event_groups_delete()
1766 	node = rb_find_first(&key, &groups->tree, __group_cmp);  in perf_event_groups_first()
1777 		.cpu = event->cpu,  in perf_event_groups_next()
1783 	next = rb_next_match(&key, &event->group_node, __group_cmp);  in perf_event_groups_next()
1798 	for (event = rb_entry_safe(rb_first(&((groups)->tree)),		\
1800 		event = rb_entry_safe(rb_next(&event->group_node),	\
1808 	return attr->inherit && (attr->sample_type & PERF_SAMPLE_READ);  in has_inherit_and_sample_read()
1813  * Must be called with ctx->mutex and ctx->lock held.
1818 	lockdep_assert_held(&ctx->lock);  in list_add_event()
1820 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);  in list_add_event()
1821 	event->attach_state |= PERF_ATTACH_CONTEXT;  in list_add_event()
1823 	event->tstamp = perf_event_time(event);  in list_add_event()
1830 	if (event->group_leader == event) {  in list_add_event()
1831 		event->group_caps = event->event_caps;  in list_add_event()
1835 	list_add_rcu(&event->event_entry, &ctx->event_list);  in list_add_event()
1836 	ctx->nr_events++;  in list_add_event()
1837 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)  in list_add_event()
1838 		ctx->nr_user++;  in list_add_event()
1839 	if (event->attr.inherit_stat)  in list_add_event()
1840 		ctx->nr_stat++;  in list_add_event()
1841 	if (has_inherit_and_sample_read(&event->attr))  in list_add_event()
1842 		local_inc(&ctx->nr_no_switch_fast);  in list_add_event()
1844 	if (event->state > PERF_EVENT_STATE_OFF)  in list_add_event()
1847 	ctx->generation++;  in list_add_event()
1848 	event->pmu_ctx->nr_events++;  in list_add_event()
1856 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :  in perf_event__state_init()
1896 		size += sizeof(data->ip);  in __perf_event_header_size()
1899 		size += sizeof(data->addr);  in __perf_event_header_size()
1902 		size += sizeof(data->period);  in __perf_event_header_size()
1905 		size += sizeof(data->weight.full);  in __perf_event_header_size()
1908 		size += event->read_size;  in __perf_event_header_size()
1911 		size += sizeof(data->data_src.val);  in __perf_event_header_size()
1914 		size += sizeof(data->txn);  in __perf_event_header_size()
1917 		size += sizeof(data->phys_addr);  in __perf_event_header_size()
1920 		size += sizeof(data->cgroup);  in __perf_event_header_size()
1923 		size += sizeof(data->data_page_size);  in __perf_event_header_size()
1926 		size += sizeof(data->code_page_size);  in __perf_event_header_size()
1928 	event->header_size = size;  in __perf_event_header_size()
1937 	event->read_size =  in perf_event__header_size()
1938 		__perf_event_read_size(event->attr.read_format,  in perf_event__header_size()
1939 				       event->group_leader->nr_siblings);  in perf_event__header_size()
1940 	__perf_event_header_size(event, event->attr.sample_type);  in perf_event__header_size()
1946 	u64 sample_type = event->attr.sample_type;  in perf_event__id_header_size()
1950 		size += sizeof(data->tid_entry);  in perf_event__id_header_size()
1953 		size += sizeof(data->time);  in perf_event__id_header_size()
1956 		size += sizeof(data->id);  in perf_event__id_header_size()
1959 		size += sizeof(data->id);  in perf_event__id_header_size()
1962 		size += sizeof(data->stream_id);  in perf_event__id_header_size()
1965 		size += sizeof(data->cpu_entry);  in perf_event__id_header_size()
1967 	event->id_header_size = size;  in perf_event__id_header_size()
1976  * depends on per-event read_format, also (re)check the existing events.
1983 	struct perf_event *sibling, *group_leader = event->group_leader;  in perf_event_validate_size()
1985 	if (__perf_event_read_size(event->attr.read_format,  in perf_event_validate_size()
1986 				   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
1989 	if (__perf_event_read_size(group_leader->attr.read_format,  in perf_event_validate_size()
1990 				   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
1994 	 * When creating a new group leader, group_leader->ctx is initialized  in perf_event_validate_size()
1996 	 * for_each_sibling_event() until group_leader->ctx is set. A new group  in perf_event_validate_size()
1998 	 * the non-existent siblings.  in perf_event_validate_size()
2004 		if (__perf_event_read_size(sibling->attr.read_format,  in perf_event_validate_size()
2005 					   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
2014 	struct perf_event *group_leader = event->group_leader, *pos;  in perf_group_attach()
2016 	lockdep_assert_held(&event->ctx->lock);  in perf_group_attach()
2022 	if (event->attach_state & PERF_ATTACH_GROUP)  in perf_group_attach()
2025 	event->attach_state |= PERF_ATTACH_GROUP;  in perf_group_attach()
2030 	WARN_ON_ONCE(group_leader->ctx != event->ctx);  in perf_group_attach()
2032 	group_leader->group_caps &= event->event_caps;  in perf_group_attach()
2034 	list_add_tail(&event->sibling_list, &group_leader->sibling_list);  in perf_group_attach()
2035 	group_leader->nr_siblings++;  in perf_group_attach()
2036 	group_leader->group_generation++;  in perf_group_attach()
2046  * Must be called with ctx->mutex and ctx->lock held.
2051 	WARN_ON_ONCE(event->ctx != ctx);  in list_del_event()
2052 	lockdep_assert_held(&ctx->lock);  in list_del_event()
2055 	 * We can have double detach due to exit/hot-unplug + close.  in list_del_event()
2057 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))  in list_del_event()
2060 	event->attach_state &= ~PERF_ATTACH_CONTEXT;  in list_del_event()
2062 	ctx->nr_events--;  in list_del_event()
2063 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)  in list_del_event()
2064 		ctx->nr_user--;  in list_del_event()
2065 	if (event->attr.inherit_stat)  in list_del_event()
2066 		ctx->nr_stat--;  in list_del_event()
2067 	if (has_inherit_and_sample_read(&event->attr))  in list_del_event()
2068 		local_dec(&ctx->nr_no_switch_fast);  in list_del_event()
2070 	list_del_rcu(&event->event_entry);  in list_del_event()
2072 	if (event->group_leader == event)  in list_del_event()
2079 	 * of error state is by explicit re-enabling  in list_del_event()
2082 	if (event->state > PERF_EVENT_STATE_OFF) {  in list_del_event()
2087 	ctx->generation++;  in list_del_event()
2088 	event->pmu_ctx->nr_events--;  in list_del_event()
2097 	if (!event->pmu->aux_output_match)  in perf_aux_output_match()
2100 	return event->pmu->aux_output_match(aux_event);  in perf_aux_output_match()
2109 	struct perf_event_context *ctx = event->ctx;  in perf_put_aux_event()
2115 	if (event->aux_event) {  in perf_put_aux_event()
2116 		iter = event->aux_event;  in perf_put_aux_event()
2117 		event->aux_event = NULL;  in perf_put_aux_event()
2126 	for_each_sibling_event(iter, event->group_leader) {  in perf_put_aux_event()
2127 		if (iter->aux_event != event)  in perf_put_aux_event()
2130 		iter->aux_event = NULL;  in perf_put_aux_event()
2145 	return event->attr.aux_output || has_aux_action(event);  in perf_need_aux_event()
2163 	if (event->attr.aux_output && event->attr.aux_sample_size)  in perf_get_aux_event()
2166 	if (event->attr.aux_output &&  in perf_get_aux_event()
2170 	if ((event->attr.aux_pause || event->attr.aux_resume) &&  in perf_get_aux_event()
2171 	    !(group_leader->pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE))  in perf_get_aux_event()
2174 	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)  in perf_get_aux_event()
2177 	if (!atomic_long_inc_not_zero(&group_leader->refcount))  in perf_get_aux_event()
2186 	event->aux_event = group_leader;  in perf_get_aux_event()
2193 	return event->attr.pinned ? &event->pmu_ctx->pinned_active :  in get_event_list()
2194 				    &event->pmu_ctx->flexible_active;  in get_event_list()
2205 	event_sched_out(event, event->ctx);  in perf_remove_sibling_event()
2211 	struct perf_event *leader = event->group_leader;  in perf_group_detach()
2213 	struct perf_event_context *ctx = event->ctx;  in perf_group_detach()
2215 	lockdep_assert_held(&ctx->lock);  in perf_group_detach()
2218 	 * We can have double detach due to exit/hot-unplug + close.  in perf_group_detach()
2220 	if (!(event->attach_state & PERF_ATTACH_GROUP))  in perf_group_detach()
2223 	event->attach_state &= ~PERF_ATTACH_GROUP;  in perf_group_detach()
2231 		list_del_init(&event->sibling_list);  in perf_group_detach()
2232 		event->group_leader->nr_siblings--;  in perf_group_detach()
2233 		event->group_leader->group_generation++;  in perf_group_detach()
2242 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {  in perf_group_detach()
2244 		if (sibling->event_caps & PERF_EV_CAP_SIBLING)  in perf_group_detach()
2247 		sibling->group_leader = sibling;  in perf_group_detach()
2248 		list_del_init(&sibling->sibling_list);  in perf_group_detach()
2251 		sibling->group_caps = event->group_caps;  in perf_group_detach()
2253 		if (sibling->attach_state & PERF_ATTACH_CONTEXT) {  in perf_group_detach()
2254 			add_event_to_groups(sibling, event->ctx);  in perf_group_detach()
2256 			if (sibling->state == PERF_EVENT_STATE_ACTIVE)  in perf_group_detach()
2257 				list_add_tail(&sibling->active_list, get_event_list(sibling));  in perf_group_detach()
2260 		WARN_ON_ONCE(sibling->ctx != event->ctx);  in perf_group_detach()
2274 	struct perf_event *parent_event = event->parent;  in perf_child_detach()
2276 	if (!(event->attach_state & PERF_ATTACH_CHILD))  in perf_child_detach()
2279 	event->attach_state &= ~PERF_ATTACH_CHILD;  in perf_child_detach()
2284 	lockdep_assert_held(&parent_event->child_mutex);  in perf_child_detach()
2287 	list_del_init(&event->child_list);  in perf_child_detach()
2292 	return event->state == PERF_EVENT_STATE_DEAD;  in is_orphaned_event()
2298 	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&  in event_filter_match()
2305 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in event_sched_out()
2306 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in event_sched_out()
2309 	// XXX cpc serialization, probably per-cpu IRQ disabled  in event_sched_out()
2311 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_out()
2312 	lockdep_assert_held(&ctx->lock);  in event_sched_out()
2314 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in event_sched_out()
2322 	list_del_init(&event->active_list);  in event_sched_out()
2324 	perf_pmu_disable(event->pmu);  in event_sched_out()
2326 	event->pmu->del(event, 0);  in event_sched_out()
2327 	event->oncpu = -1;  in event_sched_out()
2329 	if (event->pending_disable) {  in event_sched_out()
2330 		event->pending_disable = 0;  in event_sched_out()
2338 		cpc->active_oncpu--;  in event_sched_out()
2339 	if (event->attr.freq && event->attr.sample_freq) {  in event_sched_out()
2340 		ctx->nr_freq--;  in event_sched_out()
2341 		epc->nr_freq--;  in event_sched_out()
2343 	if (event->attr.exclusive || !cpc->active_oncpu)  in event_sched_out()
2344 		cpc->exclusive = 0;  in event_sched_out()
2346 	perf_pmu_enable(event->pmu);  in event_sched_out()
2354 	if (group_event->state != PERF_EVENT_STATE_ACTIVE)  in group_sched_out()
2357 	perf_assert_pmu_disabled(group_event->pmu_ctx->pmu);  in group_sched_out()
2371 	if (ctx->is_active & EVENT_TIME) {  in __ctx_time_update()
2372 		if (ctx->is_active & EVENT_FROZEN)  in __ctx_time_update()
2392 	if (ctx->is_active & EVENT_TIME)  in ctx_time_freeze()
2393 		ctx->is_active |= EVENT_FROZEN;  in ctx_time_freeze()
2399 	if (ctx->is_active & EVENT_TIME) {  in ctx_time_update_event()
2400 		if (ctx->is_active & EVENT_FROZEN)  in ctx_time_update_event()
2424 	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;  in __perf_remove_from_context()
2437 		event->pending_disable = 1;  in __perf_remove_from_context()
2441 	perf_event_set_state(event, min(event->state, state));  in __perf_remove_from_context()
2448 	if (!pmu_ctx->nr_events) {  in __perf_remove_from_context()
2449 		pmu_ctx->rotate_necessary = 0;  in __perf_remove_from_context()
2451 		if (ctx->task && ctx->is_active) {  in __perf_remove_from_context()
2454 			cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in __perf_remove_from_context()
2455 			WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __perf_remove_from_context()
2456 			cpc->task_epc = NULL;  in __perf_remove_from_context()
2460 	if (!ctx->nr_events && ctx->is_active) {  in __perf_remove_from_context()
2461 		if (ctx == &cpuctx->ctx)  in __perf_remove_from_context()
2464 		ctx->is_active = 0;  in __perf_remove_from_context()
2465 		if (ctx->task) {  in __perf_remove_from_context()
2466 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in __perf_remove_from_context()
2467 			cpuctx->task_ctx = NULL;  in __perf_remove_from_context()
2475  * If event->ctx is a cloned context, callers must make sure that
2476  * every task struct that event->ctx->task could possibly point to
2478  * that only calls us on the top-level context, which can't be a clone.
2484 	struct perf_event_context *ctx = event->ctx;  in perf_remove_from_context()
2486 	lockdep_assert_held(&ctx->mutex);  in perf_remove_from_context()
2493 	raw_spin_lock_irq(&ctx->lock);  in perf_remove_from_context()
2494 	if (!ctx->is_active) {  in perf_remove_from_context()
2497 		raw_spin_unlock_irq(&ctx->lock);  in perf_remove_from_context()
2500 	raw_spin_unlock_irq(&ctx->lock);  in perf_remove_from_context()
2513 	if (event->state < PERF_EVENT_STATE_INACTIVE)  in __perf_event_disable()
2516 	perf_pmu_disable(event->pmu_ctx->pmu);  in __perf_event_disable()
2519 	if (event == event->group_leader)  in __perf_event_disable()
2527 	perf_pmu_enable(event->pmu_ctx->pmu);  in __perf_event_disable()
2533  * If event->ctx is a cloned context, callers must make sure that
2534  * every task struct that event->ctx->task could possibly point to
2537  * hold the top-level event's child_mutex, so any descendant that
2540  * When called from perf_pending_disable it's OK because event->ctx
2546 	struct perf_event_context *ctx = event->ctx;  in _perf_event_disable()
2548 	raw_spin_lock_irq(&ctx->lock);  in _perf_event_disable()
2549 	if (event->state <= PERF_EVENT_STATE_OFF) {  in _perf_event_disable()
2550 		raw_spin_unlock_irq(&ctx->lock);  in _perf_event_disable()
2553 	raw_spin_unlock_irq(&ctx->lock);  in _perf_event_disable()
2579 	event->pending_disable = 1;  in perf_event_disable_inatomic()
2580 	irq_work_queue(&event->pending_disable_irq);  in perf_event_disable_inatomic()
2591 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in event_sched_in()
2592 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in event_sched_in()
2595 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_in()
2597 	lockdep_assert_held(&ctx->lock);  in event_sched_in()
2599 	if (event->state <= PERF_EVENT_STATE_OFF)  in event_sched_in()
2602 	WRITE_ONCE(event->oncpu, smp_processor_id());  in event_sched_in()
2606 	 * ->oncpu if it sees ACTIVE.  in event_sched_in()
2616 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {  in event_sched_in()
2618 		event->hw.interrupts = 0;  in event_sched_in()
2621 	perf_pmu_disable(event->pmu);  in event_sched_in()
2625 	if (event->pmu->add(event, PERF_EF_START)) {  in event_sched_in()
2627 		event->oncpu = -1;  in event_sched_in()
2628 		ret = -EAGAIN;  in event_sched_in()
2633 		cpc->active_oncpu++;  in event_sched_in()
2634 	if (event->attr.freq && event->attr.sample_freq) {  in event_sched_in()
2635 		ctx->nr_freq++;  in event_sched_in()
2636 		epc->nr_freq++;  in event_sched_in()
2638 	if (event->attr.exclusive)  in event_sched_in()
2639 		cpc->exclusive = 1;  in event_sched_in()
2642 	perf_pmu_enable(event->pmu);  in event_sched_in()
2651 	struct pmu *pmu = group_event->pmu_ctx->pmu;  in group_sched_in()
2653 	if (group_event->state == PERF_EVENT_STATE_OFF)  in group_sched_in()
2656 	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);  in group_sched_in()
2671 	if (!pmu->commit_txn(pmu))  in group_sched_in()
2689 	pmu->cancel_txn(pmu);  in group_sched_in()
2690 	return -EAGAIN;  in group_sched_in()
2698 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in group_can_go_on()
2699 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in group_can_go_on()
2704 	if (event->group_caps & PERF_EV_CAP_SOFTWARE)  in group_can_go_on()
2710 	if (cpc->exclusive)  in group_can_go_on()
2716 	if (event->attr.exclusive && !list_empty(get_event_list(event)))  in group_can_go_on()
2738 	if (!cpuctx->task_ctx)  in task_ctx_sched_out()
2741 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))  in task_ctx_sched_out()
2751 	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED);  in perf_event_sched_in()
2754 	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);  in perf_event_sched_in()
2761  *  - CPU pinned (EVENT_CPU | EVENT_PINNED)
2762  *  - task pinned (EVENT_PINNED)
2763  *  - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2764  *  - task flexible (EVENT_FLEXIBLE).
2790 	for_each_epc(epc, &cpuctx->ctx, pmu, false)  in ctx_resched()
2791 		perf_pmu_disable(epc->pmu);  in ctx_resched()
2795 			perf_pmu_disable(epc->pmu);  in ctx_resched()
2803 	 *  - EVENT_CPU: schedule out corresponding groups;  in ctx_resched()
2804 	 *  - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;  in ctx_resched()
2805 	 *  - otherwise, do nothing more.  in ctx_resched()
2808 		ctx_sched_out(&cpuctx->ctx, pmu, event_type);  in ctx_resched()
2810 		ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);  in ctx_resched()
2814 	for_each_epc(epc, &cpuctx->ctx, pmu, false)  in ctx_resched()
2815 		perf_pmu_enable(epc->pmu);  in ctx_resched()
2819 			perf_pmu_enable(epc->pmu);  in ctx_resched()
2826 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in perf_pmu_resched()
2837  * things like ctx->is_active and cpuctx->task_ctx are set.
2842 	struct perf_event_context *ctx = event->ctx;  in __perf_install_in_context()
2844 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in __perf_install_in_context()
2848 	raw_spin_lock(&cpuctx->ctx.lock);  in __perf_install_in_context()
2849 	if (ctx->task) {  in __perf_install_in_context()
2850 		raw_spin_lock(&ctx->lock);  in __perf_install_in_context()
2853 		reprogram = (ctx->task == current);  in __perf_install_in_context()
2859 		 * If its not running, we don't care, ctx->lock will  in __perf_install_in_context()
2862 		if (task_curr(ctx->task) && !reprogram) {  in __perf_install_in_context()
2863 			ret = -ESRCH;  in __perf_install_in_context()
2867 		WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);  in __perf_install_in_context()
2869 		raw_spin_lock(&task_ctx->lock);  in __perf_install_in_context()
2873 	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {  in __perf_install_in_context()
2879 		reprogram = cgroup_is_descendant(cgrp->css.cgroup,  in __perf_install_in_context()
2880 					event->cgrp->css.cgroup);  in __perf_install_in_context()
2887 		ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu,  in __perf_install_in_context()
2912 	struct task_struct *task = READ_ONCE(ctx->task);  in perf_install_in_context()
2914 	lockdep_assert_held(&ctx->mutex);  in perf_install_in_context()
2918 	if (event->cpu != -1)  in perf_install_in_context()
2919 		WARN_ON_ONCE(event->cpu != cpu);  in perf_install_in_context()
2922 	 * Ensures that if we can observe event->ctx, both the event and ctx  in perf_install_in_context()
2925 	smp_store_release(&event->ctx, ctx);  in perf_install_in_context()
2930 	 * that case we need the magic of the IPI to set ctx->is_active.  in perf_install_in_context()
2936 	    ctx->nr_events && !is_cgroup_event(event)) {  in perf_install_in_context()
2937 		raw_spin_lock_irq(&ctx->lock);  in perf_install_in_context()
2938 		if (ctx->task == TASK_TOMBSTONE) {  in perf_install_in_context()
2939 			raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
2943 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
2959 	 * Installing events is tricky because we cannot rely on ctx->is_active  in perf_install_in_context()
2960 	 * to be set in case this is the nr_events 0 -> 1 transition.  in perf_install_in_context()
2974 	 * our task->perf_event_ctxp[] store, such that it will in fact take  in perf_install_in_context()
2983 	 * This smp_mb() orders the task->perf_event_ctxp[] store with the  in perf_install_in_context()
2993 	raw_spin_lock_irq(&ctx->lock);  in perf_install_in_context()
2994 	task = ctx->task;  in perf_install_in_context()
2998 		 * cannot happen), and we hold ctx->mutex, which serializes us  in perf_install_in_context()
3001 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3005 	 * If the task is not running, ctx->lock will avoid it becoming so,  in perf_install_in_context()
3009 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3013 	raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3024 	struct perf_event *leader = event->group_leader;  in __perf_event_enable()
3027 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in __perf_event_enable()
3028 	    event->state <= PERF_EVENT_STATE_ERROR)  in __perf_event_enable()
3036 	if (!ctx->is_active)  in __perf_event_enable()
3046 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)  in __perf_event_enable()
3049 	task_ctx = cpuctx->task_ctx;  in __perf_event_enable()
3050 	if (ctx->task)  in __perf_event_enable()
3053 	ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event));  in __perf_event_enable()
3059  * If event->ctx is a cloned context, callers must make sure that
3060  * every task struct that event->ctx->task could possibly point to
3067 	struct perf_event_context *ctx = event->ctx;  in _perf_event_enable()
3069 	raw_spin_lock_irq(&ctx->lock);  in _perf_event_enable()
3070 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in _perf_event_enable()
3071 	    event->state <  PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
3073 		raw_spin_unlock_irq(&ctx->lock);  in _perf_event_enable()
3082 	 * been scheduled away before the cross-call arrived.  in _perf_event_enable()
3084 	if (event->state == PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
3088 		if (event->event_caps & PERF_EV_CAP_SIBLING &&  in _perf_event_enable()
3089 		    event->group_leader == event)  in _perf_event_enable()
3092 		event->state = PERF_EVENT_STATE_OFF;  in _perf_event_enable()
3094 	raw_spin_unlock_irq(&ctx->lock);  in _perf_event_enable()
3120 	struct perf_event *event = sd->event;  in __perf_event_stop()
3123 	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in __perf_event_stop()
3133 	if (READ_ONCE(event->oncpu) != smp_processor_id())  in __perf_event_stop()
3134 		return -EAGAIN;  in __perf_event_stop()
3136 	event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_stop()
3144 	 * Since this is happening on an event-local CPU, no trace is lost  in __perf_event_stop()
3147 	if (sd->restart)  in __perf_event_stop()
3148 		event->pmu->start(event, 0);  in __perf_event_stop()
3162 		if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in perf_event_stop()
3170 		 * inactive here (event->oncpu==-1), there's nothing more to do;  in perf_event_stop()
3171 		 * fall through with ret==-ENXIO.  in perf_event_stop()
3173 		ret = cpu_function_call(READ_ONCE(event->oncpu),  in perf_event_stop()
3175 	} while (ret == -EAGAIN);  in perf_event_stop()
3194  *     pre-existing mappings, called once when new filters arrive via SET_FILTER
3209 	raw_spin_lock(&ifh->lock);  in perf_event_addr_filters_sync()
3210 	if (event->addr_filters_gen != event->hw.addr_filters_gen) {  in perf_event_addr_filters_sync()
3211 		event->pmu->addr_filters_sync(event);  in perf_event_addr_filters_sync()
3212 		event->hw.addr_filters_gen = event->addr_filters_gen;  in perf_event_addr_filters_sync()
3214 	raw_spin_unlock(&ifh->lock);  in perf_event_addr_filters_sync()
3223 	if (event->attr.inherit || !is_sampling_event(event))  in _perf_event_refresh()
3224 		return -EINVAL;  in _perf_event_refresh()
3226 	atomic_add(refresh, &event->event_limit);  in _perf_event_refresh()
3257 	if (!bp->attr.disabled)  in perf_event_modify_breakpoint()
3264  * Copy event-type-independent attributes that may be modified.
3269 	to->sig_data = from->sig_data;  in perf_event_modify_copy_attr()
3279 	if (event->attr.type != attr->type)  in perf_event_modify_attr()
3280 		return -EINVAL;  in perf_event_modify_attr()
3282 	switch (event->attr.type) {  in perf_event_modify_attr()
3288 		return -EOPNOTSUPP;  in perf_event_modify_attr()
3291 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_modify_attr()
3293 	mutex_lock(&event->child_mutex);  in perf_event_modify_attr()
3295 	 * Event-type-independent attributes must be copied before event-type  in perf_event_modify_attr()
3299 	perf_event_modify_copy_attr(&event->attr, attr);  in perf_event_modify_attr()
3303 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_modify_attr()
3304 		perf_event_modify_copy_attr(&child->attr, attr);  in perf_event_modify_attr()
3310 	mutex_unlock(&event->child_mutex);  in perf_event_modify_attr()
3317 	struct perf_event_context *ctx = pmu_ctx->ctx;  in __pmu_ctx_sched_out()
3319 	struct pmu *pmu = pmu_ctx->pmu;  in __pmu_ctx_sched_out()
3321 	if (ctx->task && !(ctx->is_active & EVENT_ALL)) {  in __pmu_ctx_sched_out()
3324 		cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in __pmu_ctx_sched_out()
3325 		WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __pmu_ctx_sched_out()
3326 		cpc->task_epc = NULL;  in __pmu_ctx_sched_out()
3335 					 &pmu_ctx->pinned_active,  in __pmu_ctx_sched_out()
3342 					 &pmu_ctx->flexible_active,  in __pmu_ctx_sched_out()
3350 		pmu_ctx->rotate_necessary = 0;  in __pmu_ctx_sched_out()
3361  * state, eg. ctx->is_active cleared even though most EPCs would still actually
3369 	int is_active = ctx->is_active;  in ctx_sched_out()
3374 	lockdep_assert_held(&ctx->lock);  in ctx_sched_out()
3376 	if (likely(!ctx->nr_events)) {  in ctx_sched_out()
3380 		WARN_ON_ONCE(ctx->is_active);  in ctx_sched_out()
3381 		if (ctx->task)  in ctx_sched_out()
3382 			WARN_ON_ONCE(cpuctx->task_ctx);  in ctx_sched_out()
3396 	__ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx);  in ctx_sched_out()
3399 	 * CPU-release for the below ->is_active store,  in ctx_sched_out()
3403 	ctx->is_active &= ~event_type;  in ctx_sched_out()
3405 	if (!(ctx->is_active & EVENT_ALL)) {  in ctx_sched_out()
3410 		if (ctx->is_active & EVENT_FROZEN)  in ctx_sched_out()
3411 			ctx->is_active &= EVENT_TIME_FROZEN;  in ctx_sched_out()
3413 			ctx->is_active = 0;  in ctx_sched_out()
3416 	if (ctx->task) {  in ctx_sched_out()
3417 		WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_out()
3418 		if (!(ctx->is_active & EVENT_ALL))  in ctx_sched_out()
3419 			cpuctx->task_ctx = NULL;  in ctx_sched_out()
3422 	is_active ^= ctx->is_active; /* changed bits */  in ctx_sched_out()
3439 	lockdep_assert_held(&ctx1->lock);  in context_equiv()
3440 	lockdep_assert_held(&ctx2->lock);  in context_equiv()
3443 	if (ctx1->pin_count || ctx2->pin_count)  in context_equiv()
3447 	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)  in context_equiv()
3451 	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)  in context_equiv()
3458 	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&  in context_equiv()
3459 			ctx1->parent_gen == ctx2->parent_gen)  in context_equiv()
3471 	if (!event->attr.inherit_stat)  in __perf_event_sync_stat()
3481 	if (event->state == PERF_EVENT_STATE_ACTIVE)  in __perf_event_sync_stat()
3482 		event->pmu->read(event);  in __perf_event_sync_stat()
3487 	 * In order to keep per-task stats reliable we need to flip the event  in __perf_event_sync_stat()
3490 	value = local64_read(&next_event->count);  in __perf_event_sync_stat()
3491 	value = local64_xchg(&event->count, value);  in __perf_event_sync_stat()
3492 	local64_set(&next_event->count, value);  in __perf_event_sync_stat()
3494 	swap(event->total_time_enabled, next_event->total_time_enabled);  in __perf_event_sync_stat()
3495 	swap(event->total_time_running, next_event->total_time_running);  in __perf_event_sync_stat()
3509 	if (!ctx->nr_stat)  in perf_event_sync_stat()
3514 	event = list_first_entry(&ctx->event_list,  in perf_event_sync_stat()
3517 	next_event = list_first_entry(&next_ctx->event_list,  in perf_event_sync_stat()
3520 	while (&event->event_entry != &ctx->event_list &&  in perf_event_sync_stat()
3521 	       &next_event->event_entry != &next_ctx->event_list) {  in perf_event_sync_stat()
3543 	if (!prev_ctx->nr_task_data)  in perf_event_swap_task_ctx_data()
3547 				   &prev_ctx->pmu_ctx_list, &next_ctx->pmu_ctx_list,  in perf_event_swap_task_ctx_data()
3550 		if (WARN_ON_ONCE(prev_epc->pmu != next_epc->pmu))  in perf_event_swap_task_ctx_data()
3559 		if (prev_epc->pmu->swap_task_ctx)  in perf_event_swap_task_ctx_data()
3560 			prev_epc->pmu->swap_task_ctx(prev_epc, next_epc);  in perf_event_swap_task_ctx_data()
3562 			swap(prev_epc->task_ctx_data, next_epc->task_ctx_data);  in perf_event_swap_task_ctx_data()
3572 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in perf_ctx_sched_task_cb()
3573 		cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in perf_ctx_sched_task_cb()
3575 		if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)  in perf_ctx_sched_task_cb()
3576 			pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in);  in perf_ctx_sched_task_cb()
3583 	struct perf_event_context *ctx = task->perf_event_ctxp;  in perf_event_context_sched_out()
3592 	next_ctx = rcu_dereference(next->perf_event_ctxp);  in perf_event_context_sched_out()
3596 	parent = rcu_dereference(ctx->parent_ctx);  in perf_event_context_sched_out()
3597 	next_parent = rcu_dereference(next_ctx->parent_ctx);  in perf_event_context_sched_out()
3608 		 * lock (including re-checking that neither has been  in perf_event_context_sched_out()
3613 		raw_spin_lock(&ctx->lock);  in perf_event_context_sched_out()
3614 		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);  in perf_event_context_sched_out()
3619 			/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */  in perf_event_context_sched_out()
3620 			if (local_read(&ctx->nr_no_switch_fast) ||  in perf_event_context_sched_out()
3621 			    local_read(&next_ctx->nr_no_switch_fast)) {  in perf_event_context_sched_out()
3624 				 * events that rely on the ctx->task relation.  in perf_event_context_sched_out()
3631 				raw_spin_unlock(&next_ctx->lock);  in perf_event_context_sched_out()
3636 			WRITE_ONCE(ctx->task, next);  in perf_event_context_sched_out()
3637 			WRITE_ONCE(next_ctx->task, task);  in perf_event_context_sched_out()
3647 			 * ctx->task and ctx->task_ctx_data are immaterial  in perf_event_context_sched_out()
3649 			 * ctx->lock which we're now holding.  in perf_event_context_sched_out()
3651 			RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);  in perf_event_context_sched_out()
3652 			RCU_INIT_POINTER(next->perf_event_ctxp, ctx);  in perf_event_context_sched_out()
3658 		raw_spin_unlock(&next_ctx->lock);  in perf_event_context_sched_out()
3659 		raw_spin_unlock(&ctx->lock);  in perf_event_context_sched_out()
3665 		raw_spin_lock(&ctx->lock);  in perf_event_context_sched_out()
3673 		raw_spin_unlock(&ctx->lock);  in perf_event_context_sched_out()
3682 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in perf_sched_cb_dec()
3687 	if (!--cpc->sched_cb_usage)  in perf_sched_cb_dec()
3688 		list_del(&cpc->sched_cb_entry);  in perf_sched_cb_dec()
3694 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in perf_sched_cb_inc()
3696 	if (!cpc->sched_cb_usage++)  in perf_sched_cb_inc()
3697 		list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));  in perf_sched_cb_inc()
3707  * This callback is relevant even to per-cpu events; for example multi event
3717 	pmu = cpc->epc.pmu;  in __perf_pmu_sched_task()
3720 	if (WARN_ON_ONCE(!pmu->sched_task))  in __perf_pmu_sched_task()
3723 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in __perf_pmu_sched_task()
3726 	pmu->sched_task(cpc->task_epc, task, sched_in);  in __perf_pmu_sched_task()
3729 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in __perf_pmu_sched_task()
3739 	/* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */  in perf_pmu_sched_task()
3740 	if (prev == next || cpuctx->task_ctx)  in perf_pmu_sched_task()
3754  * We stop each event and update the event value in event->count.
3775 	 * cgroup event are system-wide mode only  in __perf_event_task_sched_out()
3785 	return le->group_index < re->group_index;  in perf_less_group_idx()
3797 	struct perf_event **itrs = heap->data;  in __heap_add()
3800 		itrs[heap->nr] = event;  in __heap_add()
3801 		heap->nr++;  in __heap_add()
3809 	if (!pmu_ctx->ctx->task)  in __link_epc()
3812 	cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in __link_epc()
3813 	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __link_epc()
3814 	cpc->task_epc = pmu_ctx;  in __link_epc()
3833 	if (pmu->filter && pmu->filter(pmu, cpu))  in visit_groups_merge()
3836 	if (!ctx->task) {  in visit_groups_merge()
3839 			.data = cpuctx->heap,  in visit_groups_merge()
3841 			.size = cpuctx->heap_size,  in visit_groups_merge()
3844 		lockdep_assert_held(&cpuctx->ctx.lock);  in visit_groups_merge()
3847 		if (cpuctx->cgrp)  in visit_groups_merge()
3848 			css = &cpuctx->cgrp->css;  in visit_groups_merge()
3857 		__heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL));  in visit_groups_merge()
3864 	for (; css; css = css->parent)  in visit_groups_merge()
3865 		__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup));  in visit_groups_merge()
3869 		__link_epc((*evt)->pmu_ctx);  in visit_groups_merge()
3870 		perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu);  in visit_groups_merge()
3891  * Because the userpage is strictly per-event (there is no concept of context,
3893  * when context time starts :-(
3899 	if (likely(!atomic_read(&event->mmap_count)))  in event_update_userpage()
3921 	struct perf_event_context *ctx = event->ctx;  in merge_sched_in()
3924 	if (event->state <= PERF_EVENT_STATE_OFF)  in merge_sched_in()
3932 			list_add_tail(&event->active_list, get_event_list(event));  in merge_sched_in()
3935 	if (event->state == PERF_EVENT_STATE_INACTIVE) {  in merge_sched_in()
3937 		if (event->attr.pinned) {  in merge_sched_in()
3943 			event->pmu_ctx->rotate_necessary = 1;  in merge_sched_in()
3944 			cpc = this_cpu_ptr(event->pmu_ctx->pmu->cpu_pmu_context);  in merge_sched_in()
3965 	struct perf_event_context *ctx = pmu_ctx->ctx;  in __pmu_ctx_sched_in()
3968 		pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu);  in __pmu_ctx_sched_in()
3970 		pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu);  in __pmu_ctx_sched_in()
3978 	int is_active = ctx->is_active;  in ctx_sched_in()
3983 	lockdep_assert_held(&ctx->lock);  in ctx_sched_in()
3985 	if (likely(!ctx->nr_events))  in ctx_sched_in()
3993 		 * CPU-release for the below ->is_active store,  in ctx_sched_in()
3999 	ctx->is_active |= (event_type | EVENT_TIME);  in ctx_sched_in()
4000 	if (ctx->task) {  in ctx_sched_in()
4002 			cpuctx->task_ctx = ctx;  in ctx_sched_in()
4004 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_in()
4007 	is_active ^= ctx->is_active; /* changed bits */  in ctx_sched_in()
4031 	ctx = rcu_dereference(task->perf_event_ctxp);  in perf_event_context_sched_in()
4035 	if (cpuctx->task_ctx == ctx) {  in perf_event_context_sched_in()
4048 	 * We must check ctx->nr_events while holding ctx->lock, such  in perf_event_context_sched_in()
4051 	if (!ctx->nr_events)  in perf_event_context_sched_in()
4063 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {  in perf_event_context_sched_in()
4064 		perf_ctx_disable(&cpuctx->ctx, false);  in perf_event_context_sched_in()
4065 		ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE);  in perf_event_context_sched_in()
4070 	perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true);  in perf_event_context_sched_in()
4072 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))  in perf_event_context_sched_in()
4073 		perf_ctx_enable(&cpuctx->ctx, false);  in perf_event_context_sched_in()
4108 	u64 frequency = event->attr.sample_freq;  in perf_calculate_period()
4124 	 * period = -------------------  in perf_calculate_period()
4137 		a##_fls--;		\  in perf_calculate_period()
4140 		b##_fls--;		\  in perf_calculate_period()
4184 	struct hw_perf_event *hwc = &event->hw;  in perf_adjust_period()
4190 	delta = (s64)(period - hwc->sample_period);  in perf_adjust_period()
4194 		delta -= 7;  in perf_adjust_period()
4197 	sample_period = hwc->sample_period + delta;  in perf_adjust_period()
4202 	hwc->sample_period = sample_period;  in perf_adjust_period()
4204 	if (local64_read(&hwc->period_left) > 8*sample_period) {  in perf_adjust_period()
4206 			event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_period()
4208 		local64_set(&hwc->period_left, 0);  in perf_adjust_period()
4211 			event->pmu->start(event, PERF_EF_RELOAD);  in perf_adjust_period()
4223 		if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_adjust_freq_unthr_events()
4226 		// XXX use visit thingy to avoid the -1,cpu match  in perf_adjust_freq_unthr_events()
4230 		hwc = &event->hw;  in perf_adjust_freq_unthr_events()
4232 		if (hwc->interrupts == MAX_INTERRUPTS) {  in perf_adjust_freq_unthr_events()
4233 			hwc->interrupts = 0;  in perf_adjust_freq_unthr_events()
4235 			if (!event->attr.freq || !event->attr.sample_freq)  in perf_adjust_freq_unthr_events()
4236 				event->pmu->start(event, 0);  in perf_adjust_freq_unthr_events()
4239 		if (!event->attr.freq || !event->attr.sample_freq)  in perf_adjust_freq_unthr_events()
4243 		 * stop the event and update event->count  in perf_adjust_freq_unthr_events()
4245 		event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_freq_unthr_events()
4247 		now = local64_read(&event->count);  in perf_adjust_freq_unthr_events()
4248 		delta = now - hwc->freq_count_stamp;  in perf_adjust_freq_unthr_events()
4249 		hwc->freq_count_stamp = now;  in perf_adjust_freq_unthr_events()
4261 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);  in perf_adjust_freq_unthr_events()
4277 	 * - context have events in frequency mode (needs freq adjust)  in perf_adjust_freq_unthr_context()
4278 	 * - there are events to unthrottle on this cpu  in perf_adjust_freq_unthr_context()
4280 	if (!(ctx->nr_freq || unthrottle))  in perf_adjust_freq_unthr_context()
4283 	raw_spin_lock(&ctx->lock);  in perf_adjust_freq_unthr_context()
4285 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in perf_adjust_freq_unthr_context()
4286 		if (!(pmu_ctx->nr_freq || unthrottle))  in perf_adjust_freq_unthr_context()
4290 		if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)  in perf_adjust_freq_unthr_context()
4293 		perf_pmu_disable(pmu_ctx->pmu);  in perf_adjust_freq_unthr_context()
4294 		perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);  in perf_adjust_freq_unthr_context()
4295 		perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);  in perf_adjust_freq_unthr_context()
4296 		perf_pmu_enable(pmu_ctx->pmu);  in perf_adjust_freq_unthr_context()
4299 	raw_spin_unlock(&ctx->lock);  in perf_adjust_freq_unthr_context()
4308 	 * Rotate the first entry last of non-pinned groups. Rotation might be  in rotate_ctx()
4311 	if (ctx->rotate_disable)  in rotate_ctx()
4314 	perf_event_groups_delete(&ctx->flexible_groups, event);  in rotate_ctx()
4315 	perf_event_groups_insert(&ctx->flexible_groups, event);  in rotate_ctx()
4326 		.pmu = pmu_ctx->pmu,  in ctx_event_to_rotate()
4330 	event = list_first_entry_or_null(&pmu_ctx->flexible_active,  in ctx_event_to_rotate()
4336 	tree = &pmu_ctx->ctx->flexible_groups.tree;  in ctx_event_to_rotate()
4338 	if (!pmu_ctx->ctx->task) {  in ctx_event_to_rotate()
4347 	key.cpu = -1;  in ctx_event_to_rotate()
4364 	pmu_ctx->rotate_necessary = 0;  in ctx_event_to_rotate()
4382 	cpu_epc = &cpc->epc;  in perf_rotate_context()
4383 	pmu = cpu_epc->pmu;  in perf_rotate_context()
4384 	task_epc = cpc->task_epc;  in perf_rotate_context()
4386 	cpu_rotate = cpu_epc->rotate_necessary;  in perf_rotate_context()
4387 	task_rotate = task_epc ? task_epc->rotate_necessary : 0;  in perf_rotate_context()
4392 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
4405 		update_context_time(task_epc->ctx);  in perf_rotate_context()
4410 		update_context_time(&cpuctx->ctx);  in perf_rotate_context()
4412 		rotate_ctx(&cpuctx->ctx, cpu_event);  in perf_rotate_context()
4417 		rotate_ctx(task_epc->ctx, task_event);  in perf_rotate_context()
4423 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
4440 	perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled);  in perf_event_task_tick()
4443 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_event_task_tick()
4452 	if (!event->attr.enable_on_exec)  in event_enable_on_exec()
4455 	event->attr.enable_on_exec = 0;  in event_enable_on_exec()
4456 	if (event->state >= PERF_EVENT_STATE_INACTIVE)  in event_enable_on_exec()
4465  * Enable all of a task's events that have been marked enable-on-exec.
4478 	if (WARN_ON_ONCE(current->perf_event_ctxp != ctx))  in perf_event_enable_on_exec()
4481 	if (!ctx->nr_events)  in perf_event_enable_on_exec()
4488 	list_for_each_entry(event, &ctx->event_list, event_entry) {  in perf_event_enable_on_exec()
4515  * remove-on-exec, and feeds their values back to parent events.
4524 	mutex_lock(&ctx->mutex);  in perf_event_remove_on_exec()
4526 	if (WARN_ON_ONCE(ctx->task != current))  in perf_event_remove_on_exec()
4529 	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {  in perf_event_remove_on_exec()
4530 		if (!event->attr.remove_on_exec)  in perf_event_remove_on_exec()
4541 	raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_event_remove_on_exec()
4544 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_remove_on_exec()
4547 	mutex_unlock(&ctx->mutex);  in perf_event_remove_on_exec()
4569 	if (event->group_caps & PERF_EV_CAP_READ_SCOPE) {  in __perf_event_read_cpu()
4570 		const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(event->pmu->scope, event_cpu);  in __perf_event_read_cpu()
4576 	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {  in __perf_event_read_cpu()
4593 	struct perf_event *sub, *event = data->event;  in __perf_event_read()
4594 	struct perf_event_context *ctx = event->ctx;  in __perf_event_read()
4596 	struct pmu *pmu = event->pmu;  in __perf_event_read()
4602 	 * event->count would have been updated to a recent sample  in __perf_event_read()
4605 	if (ctx->task && cpuctx->task_ctx != ctx)  in __perf_event_read()
4608 	raw_spin_lock(&ctx->lock);  in __perf_event_read()
4612 	if (data->group)  in __perf_event_read()
4615 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in __perf_event_read()
4618 	if (!data->group) {  in __perf_event_read()
4619 		pmu->read(event);  in __perf_event_read()
4620 		data->ret = 0;  in __perf_event_read()
4624 	pmu->start_txn(pmu, PERF_PMU_TXN_READ);  in __perf_event_read()
4626 	pmu->read(event);  in __perf_event_read()
4629 		if (sub->state == PERF_EVENT_STATE_ACTIVE) {  in __perf_event_read()
4634 			sub->pmu->read(sub);  in __perf_event_read()
4638 	data->ret = pmu->commit_txn(pmu);  in __perf_event_read()
4641 	raw_spin_unlock(&ctx->lock);  in __perf_event_read()
4647 		return local64_read(&event->count);  in perf_event_count()
4649 	return local64_read(&event->count) + atomic64_read(&event->child_count);  in perf_event_count()
4665  * NMI-safe method to read a local event, that is an event that
4667  *   - either for the current task, or for this CPU
4668  *   - does not have inherit set, for inherited task events
4670  *   - must not have a pmu::count method
4690 	if (event->attr.inherit) {  in perf_event_read_local()
4691 		ret = -EOPNOTSUPP;  in perf_event_read_local()
4695 	/* If this is a per-task event, it must be for current */  in perf_event_read_local()
4696 	if ((event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
4697 	    event->hw.target != current) {  in perf_event_read_local()
4698 		ret = -EINVAL;  in perf_event_read_local()
4704 	 * a per-package event that can be read locally  in perf_event_read_local()
4706 	event_oncpu = __perf_event_read_cpu(event, event->oncpu);  in perf_event_read_local()
4707 	event_cpu = __perf_event_read_cpu(event, event->cpu);  in perf_event_read_local()
4709 	/* If this is a per-CPU event, it must be for this CPU */  in perf_event_read_local()
4710 	if (!(event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
4712 		ret = -EINVAL;  in perf_event_read_local()
4717 	if (event->attr.pinned && event_oncpu != smp_processor_id()) {  in perf_event_read_local()
4718 		ret = -EBUSY;  in perf_event_read_local()
4723 	 * If the event is currently on this CPU, its either a per-task event,  in perf_event_read_local()
4725 	 * oncpu == -1).  in perf_event_read_local()
4728 		event->pmu->read(event);  in perf_event_read_local()
4730 	*value = local64_read(&event->count);  in perf_event_read_local()
4748 	enum perf_event_state state = READ_ONCE(event->state);  in perf_event_read()
4760 		 * Orders the ->state and ->oncpu loads such that if we see  in perf_event_read()
4761 		 * ACTIVE we must also see the right ->oncpu.  in perf_event_read()
4767 		event_cpu = READ_ONCE(event->oncpu);  in perf_event_read()
4787 		 * Therefore, either way, we'll have an up-to-date event count  in perf_event_read()
4795 		struct perf_event_context *ctx = event->ctx;  in perf_event_read()
4798 		raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_event_read()
4799 		state = event->state;  in perf_event_read()
4801 			raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_read()
4814 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_read()
4825 	raw_spin_lock_init(&ctx->lock);  in __perf_event_init_context()
4826 	mutex_init(&ctx->mutex);  in __perf_event_init_context()
4827 	INIT_LIST_HEAD(&ctx->pmu_ctx_list);  in __perf_event_init_context()
4828 	perf_event_groups_init(&ctx->pinned_groups);  in __perf_event_init_context()
4829 	perf_event_groups_init(&ctx->flexible_groups);  in __perf_event_init_context()
4830 	INIT_LIST_HEAD(&ctx->event_list);  in __perf_event_init_context()
4831 	refcount_set(&ctx->refcount, 1);  in __perf_event_init_context()
4837 	epc->pmu = pmu;  in __perf_init_event_pmu_context()
4838 	INIT_LIST_HEAD(&epc->pmu_ctx_entry);  in __perf_init_event_pmu_context()
4839 	INIT_LIST_HEAD(&epc->pinned_active);  in __perf_init_event_pmu_context()
4840 	INIT_LIST_HEAD(&epc->flexible_active);  in __perf_init_event_pmu_context()
4841 	atomic_set(&epc->refcount, 1);  in __perf_init_event_pmu_context()
4855 		ctx->task = get_task_struct(task);  in alloc_perf_context()
4875 		return ERR_PTR(-ESRCH);  in find_lively_task_by_vpid()
4893 		err = perf_allow_cpu(&event->attr);  in find_get_context()
4897 		cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);  in find_get_context()
4898 		ctx = &cpuctx->ctx;  in find_get_context()
4900 		raw_spin_lock_irqsave(&ctx->lock, flags);  in find_get_context()
4901 		++ctx->pin_count;  in find_get_context()
4902 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in find_get_context()
4907 	err = -EINVAL;  in find_get_context()
4912 		++ctx->pin_count;  in find_get_context()
4914 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in find_get_context()
4920 		err = -ENOMEM;  in find_get_context()
4925 		mutex_lock(&task->perf_event_mutex);  in find_get_context()
4930 		if (task->flags & PF_EXITING)  in find_get_context()
4931 			err = -ESRCH;  in find_get_context()
4932 		else if (task->perf_event_ctxp)  in find_get_context()
4933 			err = -EAGAIN;  in find_get_context()
4936 			++ctx->pin_count;  in find_get_context()
4937 			rcu_assign_pointer(task->perf_event_ctxp, ctx);  in find_get_context()
4939 		mutex_unlock(&task->perf_event_mutex);  in find_get_context()
4944 			if (err == -EAGAIN)  in find_get_context()
4963 	if (!ctx->task) {  in find_get_pmu_context()
4971 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);  in find_get_pmu_context()
4972 		epc = &cpc->epc;  in find_get_pmu_context()
4973 		raw_spin_lock_irq(&ctx->lock);  in find_get_pmu_context()
4974 		if (!epc->ctx) {  in find_get_pmu_context()
4975 			atomic_set(&epc->refcount, 1);  in find_get_pmu_context()
4976 			epc->embedded = 1;  in find_get_pmu_context()
4977 			list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);  in find_get_pmu_context()
4978 			epc->ctx = ctx;  in find_get_pmu_context()
4980 			WARN_ON_ONCE(epc->ctx != ctx);  in find_get_pmu_context()
4981 			atomic_inc(&epc->refcount);  in find_get_pmu_context()
4983 		raw_spin_unlock_irq(&ctx->lock);  in find_get_pmu_context()
4989 		return ERR_PTR(-ENOMEM);  in find_get_pmu_context()
4991 	if (event->attach_state & PERF_ATTACH_TASK_DATA) {  in find_get_pmu_context()
4995 			return ERR_PTR(-ENOMEM);  in find_get_pmu_context()
5004 	 * lockdep_assert_held(&ctx->mutex);  in find_get_pmu_context()
5007 	 * child_ctx->mutex.  in find_get_pmu_context()
5010 	raw_spin_lock_irq(&ctx->lock);  in find_get_pmu_context()
5011 	list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in find_get_pmu_context()
5012 		if (epc->pmu == pmu) {  in find_get_pmu_context()
5013 			WARN_ON_ONCE(epc->ctx != ctx);  in find_get_pmu_context()
5014 			atomic_inc(&epc->refcount);  in find_get_pmu_context()
5018 		if (!pos && epc->pmu->type > pmu->type)  in find_get_pmu_context()
5026 		list_add_tail(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);  in find_get_pmu_context()
5028 		list_add(&epc->pmu_ctx_entry, pos->pmu_ctx_entry.prev);  in find_get_pmu_context()
5030 	epc->ctx = ctx;  in find_get_pmu_context()
5033 	if (task_ctx_data && !epc->task_ctx_data) {  in find_get_pmu_context()
5034 		epc->task_ctx_data = task_ctx_data;  in find_get_pmu_context()
5036 		ctx->nr_task_data++;  in find_get_pmu_context()
5038 	raw_spin_unlock_irq(&ctx->lock);  in find_get_pmu_context()
5048 	WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));  in get_pmu_ctx()
5055 	kfree(epc->task_ctx_data);  in free_epc_rcu()
5061 	struct perf_event_context *ctx = epc->ctx;  in put_pmu_ctx()
5067 	 * lockdep_assert_held(&ctx->mutex);  in put_pmu_ctx()
5069 	 * can't because of the call-site in _free_event()/put_event()  in put_pmu_ctx()
5070 	 * which isn't always called under ctx->mutex.  in put_pmu_ctx()
5072 	if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags))  in put_pmu_ctx()
5075 	WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry));  in put_pmu_ctx()
5077 	list_del_init(&epc->pmu_ctx_entry);  in put_pmu_ctx()
5078 	epc->ctx = NULL;  in put_pmu_ctx()
5080 	WARN_ON_ONCE(!list_empty(&epc->pinned_active));  in put_pmu_ctx()
5081 	WARN_ON_ONCE(!list_empty(&epc->flexible_active));  in put_pmu_ctx()
5083 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in put_pmu_ctx()
5085 	if (epc->embedded)  in put_pmu_ctx()
5088 	call_rcu(&epc->rcu_head, free_epc_rcu);  in put_pmu_ctx()
5097 	if (event->ns)  in free_event_rcu()
5098 		put_pid_ns(event->ns);  in free_event_rcu()
5108 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in detach_sb_event()
5110 	raw_spin_lock(&pel->lock);  in detach_sb_event()
5111 	list_del_rcu(&event->sb_list);  in detach_sb_event()
5112 	raw_spin_unlock(&pel->lock);  in detach_sb_event()
5117 	struct perf_event_attr *attr = &event->attr;  in is_sb_event()
5119 	if (event->parent)  in is_sb_event()
5122 	if (event->attach_state & PERF_ATTACH_TASK)  in is_sb_event()
5125 	if (attr->mmap || attr->mmap_data || attr->mmap2 ||  in is_sb_event()
5126 	    attr->comm || attr->comm_exec ||  in is_sb_event()
5127 	    attr->task || attr->ksymbol ||  in is_sb_event()
5128 	    attr->context_switch || attr->text_poke ||  in is_sb_event()
5129 	    attr->bpf_event)  in is_sb_event()
5166 	if (event->parent)  in unaccount_event()
5169 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))  in unaccount_event()
5171 	if (event->attr.mmap || event->attr.mmap_data)  in unaccount_event()
5173 	if (event->attr.build_id)  in unaccount_event()
5175 	if (event->attr.comm)  in unaccount_event()
5177 	if (event->attr.namespaces)  in unaccount_event()
5179 	if (event->attr.cgroup)  in unaccount_event()
5181 	if (event->attr.task)  in unaccount_event()
5183 	if (event->attr.freq)  in unaccount_event()
5185 	if (event->attr.context_switch) {  in unaccount_event()
5193 	if (event->attr.ksymbol)  in unaccount_event()
5195 	if (event->attr.bpf_event)  in unaccount_event()
5197 	if (event->attr.text_poke)  in unaccount_event()
5201 		if (!atomic_add_unless(&perf_sched_count, -1, 1))  in unaccount_event()
5221  *  1) cpu-wide events in the presence of per-task events,
5222  *  2) per-task events in the presence of cpu-wide events,
5226  * _free_event()), the latter -- before the first perf_install_in_context().
5230 	struct pmu *pmu = event->pmu;  in exclusive_event_init()
5236 	 * Prevent co-existence of per-task and cpu-wide events on the  in exclusive_event_init()
5239 	 * Negative pmu::exclusive_cnt means there are cpu-wide  in exclusive_event_init()
5241 	 * per-task events.  in exclusive_event_init()
5245 	 * to mean "per-task event", because unlike other attach states it  in exclusive_event_init()
5248 	if (event->attach_state & PERF_ATTACH_TASK) {  in exclusive_event_init()
5249 		if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))  in exclusive_event_init()
5250 			return -EBUSY;  in exclusive_event_init()
5252 		if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))  in exclusive_event_init()
5253 			return -EBUSY;  in exclusive_event_init()
5256 	event->attach_state |= PERF_ATTACH_EXCLUSIVE;  in exclusive_event_init()
5263 	struct pmu *pmu = event->pmu;  in exclusive_event_destroy()
5266 	if (event->attach_state & PERF_ATTACH_TASK)  in exclusive_event_destroy()
5267 		atomic_dec(&pmu->exclusive_cnt);  in exclusive_event_destroy()
5269 		atomic_inc(&pmu->exclusive_cnt);  in exclusive_event_destroy()
5271 	event->attach_state &= ~PERF_ATTACH_EXCLUSIVE;  in exclusive_event_destroy()
5276 	if ((e1->pmu == e2->pmu) &&  in exclusive_event_match()
5277 	    (e1->cpu == e2->cpu ||  in exclusive_event_match()
5278 	     e1->cpu == -1 ||  in exclusive_event_match()
5279 	     e2->cpu == -1))  in exclusive_event_match()
5288 	struct pmu *pmu = event->pmu;  in exclusive_event_installable()
5290 	lockdep_assert_held(&ctx->mutex);  in exclusive_event_installable()
5295 	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {  in exclusive_event_installable()
5309 	if (event->attach_state & PERF_ATTACH_CALLCHAIN)  in __free_event()
5312 	kfree(event->addr_filter_ranges);  in __free_event()
5314 	if (event->attach_state & PERF_ATTACH_EXCLUSIVE)  in __free_event()
5320 	if (event->destroy)  in __free_event()
5321 		event->destroy(event);  in __free_event()
5324 	 * Must be after ->destroy(), due to uprobe_perf_close() using  in __free_event()
5327 	if (event->hw.target)  in __free_event()
5328 		put_task_struct(event->hw.target);  in __free_event()
5330 	if (event->pmu_ctx) {  in __free_event()
5332 		 * put_pmu_ctx() needs an event->ctx reference, because of  in __free_event()
5333 		 * epc->ctx.  in __free_event()
5335 		WARN_ON_ONCE(!event->ctx);  in __free_event()
5336 		WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx);  in __free_event()
5337 		put_pmu_ctx(event->pmu_ctx);  in __free_event()
5344 	if (event->ctx)  in __free_event()
5345 		put_ctx(event->ctx);  in __free_event()
5347 	if (event->pmu)  in __free_event()
5348 		module_put(event->pmu->module);  in __free_event()
5350 	call_rcu(&event->rcu_head, free_event_rcu);  in __free_event()
5356 	irq_work_sync(&event->pending_irq);  in _free_event()
5357 	irq_work_sync(&event->pending_disable_irq);  in _free_event()
5363 	if (event->rb) {  in _free_event()
5365 		 * Can happen when we close an event with re-directed output.  in _free_event()
5370 		mutex_lock(&event->mmap_mutex);  in _free_event()
5372 		mutex_unlock(&event->mmap_mutex);  in _free_event()
5387 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,  in free_event()
5389 				atomic_long_read(&event->refcount), event)) {  in free_event()
5390 		/* leak to avoid use-after-free */  in free_event()
5409 	 * owner->perf_event_mutex.  in perf_remove_from_owner()
5411 	owner = READ_ONCE(event->owner);  in perf_remove_from_owner()
5425 		 * holding ctx->mutex which would be an inversion wrt. the  in perf_remove_from_owner()
5429 		 * ctx->mutex.  in perf_remove_from_owner()
5431 		mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);  in perf_remove_from_owner()
5434 		 * We have to re-check the event->owner field, if it is cleared  in perf_remove_from_owner()
5439 		if (event->owner) {  in perf_remove_from_owner()
5440 			list_del_init(&event->owner_entry);  in perf_remove_from_owner()
5441 			smp_store_release(&event->owner, NULL);  in perf_remove_from_owner()
5443 		mutex_unlock(&owner->perf_event_mutex);  in perf_remove_from_owner()
5452 	if (!atomic_long_dec_and_test(&event->refcount))  in put_event()
5455 	parent = event->parent;  in put_event()
5470 	struct perf_event_context *ctx = event->ctx;  in perf_event_release_kernel()
5479 		WARN_ON_ONCE(event->attach_state &  in perf_event_release_kernel()
5488 	WARN_ON_ONCE(ctx->parent_ctx);  in perf_event_release_kernel()
5494 	 * Anybody acquiring event->child_mutex after the below loop _must_  in perf_event_release_kernel()
5506 	mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
5507 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_release_kernel()
5514 		ctx = READ_ONCE(child->ctx);  in perf_event_release_kernel()
5528 		 * can re-acquire child_mutex.  in perf_event_release_kernel()
5530 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5531 		mutex_lock(&ctx->mutex);  in perf_event_release_kernel()
5532 		mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
5539 		tmp = list_first_entry_or_null(&event->child_list,  in perf_event_release_kernel()
5543 			list_move(&child->child_list, &free_list);  in perf_event_release_kernel()
5545 			var = &ctx->refcount;  in perf_event_release_kernel()
5548 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5549 		mutex_unlock(&ctx->mutex);  in perf_event_release_kernel()
5563 	mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5566 		void *var = &child->ctx->refcount;  in perf_event_release_kernel()
5568 		list_del(&child->child_list);  in perf_event_release_kernel()
5569 		/* Last reference unless ->pending_task work is pending */  in perf_event_release_kernel()
5582 	 * Last reference unless ->pending_task work is pending on this event  in perf_event_release_kernel()
5595 	perf_event_release_kernel(file->private_data);  in perf_release()
5607 	mutex_lock(&event->child_mutex);  in __perf_event_read_value()
5612 	*enabled += event->total_time_enabled +  in __perf_event_read_value()
5613 			atomic64_read(&event->child_total_time_enabled);  in __perf_event_read_value()
5614 	*running += event->total_time_running +  in __perf_event_read_value()
5615 			atomic64_read(&event->child_total_time_running);  in __perf_event_read_value()
5617 	list_for_each_entry(child, &event->child_list, child_list) {  in __perf_event_read_value()
5620 		*enabled += child->total_time_enabled;  in __perf_event_read_value()
5621 		*running += child->total_time_running;  in __perf_event_read_value()
5623 	mutex_unlock(&event->child_mutex);  in __perf_event_read_value()
5644 	struct perf_event_context *ctx = leader->ctx;  in __perf_read_group_add()
5654 	raw_spin_lock_irqsave(&ctx->lock, flags);  in __perf_read_group_add()
5660 	 *  - leader->ctx->lock pins leader->sibling_list  in __perf_read_group_add()
5661 	 *  - parent->child_mutex pins parent->child_list  in __perf_read_group_add()
5662 	 *  - parent->ctx->mutex pins parent->sibling_list  in __perf_read_group_add()
5664 	 * Because parent->ctx != leader->ctx (and child_list nests inside  in __perf_read_group_add()
5665 	 * ctx->mutex), group destruction is not atomic between children, also  in __perf_read_group_add()
5675 	parent = leader->parent;  in __perf_read_group_add()
5677 	    (parent->group_generation != leader->group_generation ||  in __perf_read_group_add()
5678 	     parent->nr_siblings != leader->nr_siblings)) {  in __perf_read_group_add()
5679 		ret = -ECHILD;  in __perf_read_group_add()
5684 	 * Since we co-schedule groups, {enabled,running} times of siblings  in __perf_read_group_add()
5689 		values[n++] += leader->total_time_enabled +  in __perf_read_group_add()
5690 			atomic64_read(&leader->child_total_time_enabled);  in __perf_read_group_add()
5694 		values[n++] += leader->total_time_running +  in __perf_read_group_add()
5695 			atomic64_read(&leader->child_total_time_running);  in __perf_read_group_add()
5705 		values[n++] = atomic64_read(&leader->lost_samples);  in __perf_read_group_add()
5712 			values[n++] = atomic64_read(&sub->lost_samples);  in __perf_read_group_add()
5716 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in __perf_read_group_add()
5723 	struct perf_event *leader = event->group_leader, *child;  in perf_read_group()
5724 	struct perf_event_context *ctx = leader->ctx;  in perf_read_group()
5728 	lockdep_assert_held(&ctx->mutex);  in perf_read_group()
5730 	values = kzalloc(event->read_size, GFP_KERNEL);  in perf_read_group()
5732 		return -ENOMEM;  in perf_read_group()
5734 	values[0] = 1 + leader->nr_siblings;  in perf_read_group()
5736 	mutex_lock(&leader->child_mutex);  in perf_read_group()
5742 	list_for_each_entry(child, &leader->child_list, child_list) {  in perf_read_group()
5748 	mutex_unlock(&leader->child_mutex);  in perf_read_group()
5750 	ret = event->read_size;  in perf_read_group()
5751 	if (copy_to_user(buf, values, event->read_size))  in perf_read_group()
5752 		ret = -EFAULT;  in perf_read_group()
5756 	mutex_unlock(&leader->child_mutex);  in perf_read_group()
5777 		values[n++] = atomic64_read(&event->lost_samples);  in perf_read_one()
5780 		return -EFAULT;  in perf_read_one()
5789 	if (event->state > PERF_EVENT_STATE_EXIT)  in is_event_hup()
5792 	mutex_lock(&event->child_mutex);  in is_event_hup()
5793 	no_children = list_empty(&event->child_list);  in is_event_hup()
5794 	mutex_unlock(&event->child_mutex);  in is_event_hup()
5799  * Read the performance event - simple non blocking version for now
5804 	u64 read_format = event->attr.read_format;  in __perf_read()
5808 	 * Return end-of-file for a read on an event that is in  in __perf_read()
5812 	if (event->state == PERF_EVENT_STATE_ERROR)  in __perf_read()
5815 	if (count < event->read_size)  in __perf_read()
5816 		return -ENOSPC;  in __perf_read()
5818 	WARN_ON_ONCE(event->ctx->parent_ctx);  in __perf_read()
5830 	struct perf_event *event = file->private_data;  in perf_read()
5847 	struct perf_event *event = file->private_data;  in perf_poll()
5851 	poll_wait(file, &event->waitq, wait);  in perf_poll()
5857 	 * Pin the event->rb by taking event->mmap_mutex; otherwise  in perf_poll()
5860 	mutex_lock(&event->mmap_mutex);  in perf_poll()
5861 	rb = event->rb;  in perf_poll()
5863 		events = atomic_xchg(&rb->poll, 0);  in perf_poll()
5864 	mutex_unlock(&event->mmap_mutex);  in perf_poll()
5871 	local64_set(&event->count, 0);  in _perf_event_reset()
5882 	WARN_ON_ONCE(event->attr.inherit);  in perf_event_pause()
5884 	count = local64_read(&event->count);  in perf_event_pause()
5886 		local64_set(&event->count, 0);  in perf_event_pause()
5894  * Holding the top-level event's child_mutex means that any
5904 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_for_each_child()
5906 	mutex_lock(&event->child_mutex);  in perf_event_for_each_child()
5908 	list_for_each_entry(child, &event->child_list, child_list)  in perf_event_for_each_child()
5910 	mutex_unlock(&event->child_mutex);  in perf_event_for_each_child()
5916 	struct perf_event_context *ctx = event->ctx;  in perf_event_for_each()
5919 	lockdep_assert_held(&ctx->mutex);  in perf_event_for_each()
5921 	event = event->group_leader;  in perf_event_for_each()
5936 	if (event->attr.freq) {  in __perf_event_period()
5937 		event->attr.sample_freq = value;  in __perf_event_period()
5939 		event->attr.sample_period = value;  in __perf_event_period()
5940 		event->hw.sample_period = value;  in __perf_event_period()
5943 	active = (event->state == PERF_EVENT_STATE_ACTIVE);  in __perf_event_period()
5945 		perf_pmu_disable(event->pmu);  in __perf_event_period()
5948 		 * trying to unthrottle while we already re-started the event.  in __perf_event_period()
5950 		if (event->hw.interrupts == MAX_INTERRUPTS) {  in __perf_event_period()
5951 			event->hw.interrupts = 0;  in __perf_event_period()
5954 		event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_period()
5957 	local64_set(&event->hw.period_left, 0);  in __perf_event_period()
5960 		event->pmu->start(event, PERF_EF_RELOAD);  in __perf_event_period()
5961 		perf_pmu_enable(event->pmu);  in __perf_event_period()
5967 	return event->pmu->check_period(event, value);  in perf_event_check_period()
5973 		return -EINVAL;  in _perf_event_period()
5976 		return -EINVAL;  in _perf_event_period()
5978 	if (event->attr.freq) {  in _perf_event_period()
5980 			return -EINVAL;  in _perf_event_period()
5983 			return -EINVAL;  in _perf_event_period()
5985 			return -EINVAL;  in _perf_event_period()
6010 	return !fd_empty(f) && fd_file(f)->f_op == &perf_fops;  in is_perf_file()
6043 			return -EFAULT;  in _perf_ioctl()
6052 			return -EFAULT;  in _perf_ioctl()
6058 		CLASS(fd, output)(arg);	     // arg == -1 => empty  in _perf_ioctl()
6060 		if (arg != -1) {  in _perf_ioctl()
6062 				return -EBADF;  in _perf_ioctl()
6063 			output_event = fd_file(output)->private_data;  in _perf_ioctl()
6093 		rb = rcu_dereference(event->rb);  in _perf_ioctl()
6094 		if (!rb || !rb->nr_pages) {  in _perf_ioctl()
6096 			return -EINVAL;  in _perf_ioctl()
6117 		return -ENOTTY;  in _perf_ioctl()
6130 	struct perf_event *event = file->private_data;  in perf_ioctl()
6155 		/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */  in perf_compat_ioctl()
6173 	mutex_lock(&current->perf_event_mutex);  in perf_event_task_enable()
6174 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_enable()
6179 	mutex_unlock(&current->perf_event_mutex);  in perf_event_task_enable()
6189 	mutex_lock(&current->perf_event_mutex);  in perf_event_task_disable()
6190 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_disable()
6195 	mutex_unlock(&current->perf_event_mutex);  in perf_event_task_disable()
6202 	if (event->hw.state & PERF_HES_STOPPED)  in perf_event_index()
6205 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_event_index()
6208 	return event->pmu->event_idx(event);  in perf_event_index()
6217 	rb = rcu_dereference(event->rb);  in perf_event_init_userpage()
6221 	userpg = rb->user_page;  in perf_event_init_userpage()
6224 	userpg->cap_bit0_is_deprecated = 1;  in perf_event_init_userpage()
6225 	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);  in perf_event_init_userpage()
6226 	userpg->data_offset = PAGE_SIZE;  in perf_event_init_userpage()
6227 	userpg->data_size = perf_data_size(rb);  in perf_event_init_userpage()
6250 	rb = rcu_dereference(event->rb);  in perf_event_update_userpage()
6265 	userpg = rb->user_page;  in perf_event_update_userpage()
6271 	++userpg->lock;  in perf_event_update_userpage()
6273 	userpg->index = perf_event_index(event);  in perf_event_update_userpage()
6274 	userpg->offset = perf_event_count(event, false);  in perf_event_update_userpage()
6275 	if (userpg->index)  in perf_event_update_userpage()
6276 		userpg->offset -= local64_read(&event->hw.prev_count);  in perf_event_update_userpage()
6278 	userpg->time_enabled = enabled +  in perf_event_update_userpage()
6279 			atomic64_read(&event->child_total_time_enabled);  in perf_event_update_userpage()
6281 	userpg->time_running = running +  in perf_event_update_userpage()
6282 			atomic64_read(&event->child_total_time_running);  in perf_event_update_userpage()
6287 	++userpg->lock;  in perf_event_update_userpage()
6300 	WARN_ON_ONCE(event->parent);  in ring_buffer_attach()
6302 	if (event->rb) {  in ring_buffer_attach()
6305 		 * event->rb_entry and wait/clear when adding event->rb_entry.  in ring_buffer_attach()
6307 		WARN_ON_ONCE(event->rcu_pending);  in ring_buffer_attach()
6309 		old_rb = event->rb;  in ring_buffer_attach()
6310 		spin_lock_irqsave(&old_rb->event_lock, flags);  in ring_buffer_attach()
6311 		list_del_rcu(&event->rb_entry);  in ring_buffer_attach()
6312 		spin_unlock_irqrestore(&old_rb->event_lock, flags);  in ring_buffer_attach()
6314 		event->rcu_batches = get_state_synchronize_rcu();  in ring_buffer_attach()
6315 		event->rcu_pending = 1;  in ring_buffer_attach()
6319 		if (event->rcu_pending) {  in ring_buffer_attach()
6320 			cond_synchronize_rcu(event->rcu_batches);  in ring_buffer_attach()
6321 			event->rcu_pending = 0;  in ring_buffer_attach()
6324 		spin_lock_irqsave(&rb->event_lock, flags);  in ring_buffer_attach()
6325 		list_add_rcu(&event->rb_entry, &rb->event_list);  in ring_buffer_attach()
6326 		spin_unlock_irqrestore(&rb->event_lock, flags);  in ring_buffer_attach()
6336 	 * mid-air, but then again, whoever does it like this is  in ring_buffer_attach()
6342 	rcu_assign_pointer(event->rb, rb);  in ring_buffer_attach()
6351 		wake_up_all(&event->waitq);  in ring_buffer_attach()
6359 	if (event->parent)  in ring_buffer_wakeup()
6360 		event = event->parent;  in ring_buffer_wakeup()
6363 	rb = rcu_dereference(event->rb);  in ring_buffer_wakeup()
6365 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)  in ring_buffer_wakeup()
6366 			wake_up_all(&event->waitq);  in ring_buffer_wakeup()
6375 	if (event->parent)  in ring_buffer_get()
6376 		event = event->parent;  in ring_buffer_get()
6379 	rb = rcu_dereference(event->rb);  in ring_buffer_get()
6381 		if (!refcount_inc_not_zero(&rb->refcount))  in ring_buffer_get()
6391 	if (!refcount_dec_and_test(&rb->refcount))  in ring_buffer_put()
6394 	WARN_ON_ONCE(!list_empty(&rb->event_list));  in ring_buffer_put()
6396 	call_rcu(&rb->rcu_head, rb_free_rcu);  in ring_buffer_put()
6401 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_open()
6403 	atomic_inc(&event->mmap_count);  in perf_mmap_open()
6404 	atomic_inc(&event->rb->mmap_count);  in perf_mmap_open()
6406 	if (vma->vm_pgoff)  in perf_mmap_open()
6407 		atomic_inc(&event->rb->aux_mmap_count);  in perf_mmap_open()
6409 	if (event->pmu->event_mapped)  in perf_mmap_open()
6410 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap_open()
6425 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_close()
6427 	struct user_struct *mmap_user = rb->mmap_user;  in perf_mmap_close()
6428 	int mmap_locked = rb->mmap_locked;  in perf_mmap_close()
6432 	if (event->pmu->event_unmapped)  in perf_mmap_close()
6433 		event->pmu->event_unmapped(event, vma->vm_mm);  in perf_mmap_close()
6436 	 * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex  in perf_mmap_close()
6439 	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&  in perf_mmap_close()
6440 	    atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {  in perf_mmap_close()
6450 		atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm);  in perf_mmap_close()
6451 		atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);  in perf_mmap_close()
6455 		WARN_ON_ONCE(refcount_read(&rb->aux_refcount));  in perf_mmap_close()
6457 		mutex_unlock(&rb->aux_mutex);  in perf_mmap_close()
6460 	if (atomic_dec_and_test(&rb->mmap_count))  in perf_mmap_close()
6463 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))  in perf_mmap_close()
6467 	mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6480 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {  in perf_mmap_close()
6481 		if (!atomic_long_inc_not_zero(&event->refcount)) {  in perf_mmap_close()
6483 			 * This event is en-route to free_event() which will  in perf_mmap_close()
6490 		mutex_lock(&event->mmap_mutex);  in perf_mmap_close()
6501 		if (event->rb == rb)  in perf_mmap_close()
6504 		mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6516 	 * It could be there's still a few 0-ref events on the list; they'll  in perf_mmap_close()
6517 	 * get cleaned up by free_event() -- they'll also still have their  in perf_mmap_close()
6524 	atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked,  in perf_mmap_close()
6525 			&mmap_user->locked_vm);  in perf_mmap_close()
6526 	atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);  in perf_mmap_close()
6535 	/* The first page is the user control page, others are read-only. */  in perf_mmap_pfn_mkwrite()
6536 	return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS;  in perf_mmap_pfn_mkwrite()
6555 	 * referencing memory-mapped I/O ranges or non-system RAM i.e. for which  in map_range()
6558 	 * We are mapping kernel-allocated memory (memory we manage ourselves)  in map_range()
6564 	 * 1. It uses vma->vm_page_prot, but this field has not been completely  in map_range()
6565 	 *    setup at the point of the f_op->mmp() hook, so we are unable to  in map_range()
6572 	 *    vm_ops->page_mkwrite() will be invoked rather than  in map_range()
6573 	 *    vm_ops->pfn_mkwrite(), and this means we have to set page->mapping  in map_range()
6590 		unsigned long va = vma->vm_start + PAGE_SIZE * pagenum;  in map_range()
6591 		struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum);  in map_range()
6594 			err = -EINVAL;  in map_range()
6600 				      vm_get_page_prot(vma->vm_flags & ~VM_SHARED));  in map_range()
6608 		zap_page_range_single(vma, vma->vm_start, nr_pages * PAGE_SIZE, NULL);  in map_range()
6616 	struct perf_event *event = file->private_data;  in perf_mmap()
6628 	 * Don't allow mmap() of inherited per-task counters. This would  in perf_mmap()
6632 	if (event->cpu == -1 && event->attr.inherit)  in perf_mmap()
6633 		return -EINVAL;  in perf_mmap()
6635 	if (!(vma->vm_flags & VM_SHARED))  in perf_mmap()
6636 		return -EINVAL;  in perf_mmap()
6642 	vma_size = vma->vm_end - vma->vm_start;  in perf_mmap()
6644 	if (vma->vm_pgoff == 0) {  in perf_mmap()
6645 		nr_pages = (vma_size / PAGE_SIZE) - 1;  in perf_mmap()
6648 		 * AUX area mapping: if rb->aux_nr_pages != 0, it's already  in perf_mmap()
6654 		if (!event->rb)  in perf_mmap()
6655 			return -EINVAL;  in perf_mmap()
6659 			return -ENOMEM;  in perf_mmap()
6661 		mutex_lock(&event->mmap_mutex);  in perf_mmap()
6662 		ret = -EINVAL;  in perf_mmap()
6664 		rb = event->rb;  in perf_mmap()
6668 		aux_mutex = &rb->aux_mutex;  in perf_mmap()
6671 		aux_offset = READ_ONCE(rb->user_page->aux_offset);  in perf_mmap()
6672 		aux_size = READ_ONCE(rb->user_page->aux_size);  in perf_mmap()
6677 		if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)  in perf_mmap()
6681 		if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)  in perf_mmap()
6688 		if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)  in perf_mmap()
6694 		if (!atomic_inc_not_zero(&rb->mmap_count))  in perf_mmap()
6698 			atomic_inc(&rb->aux_mmap_count);  in perf_mmap()
6703 		atomic_set(&rb->aux_mmap_count, 1);  in perf_mmap()
6710 	 * If we have rb pages ensure they're a power-of-two number, so we  in perf_mmap()
6714 		return -EINVAL;  in perf_mmap()
6717 		return -EINVAL;  in perf_mmap()
6719 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_mmap()
6721 	mutex_lock(&event->mmap_mutex);  in perf_mmap()
6722 	if (event->rb) {  in perf_mmap()
6723 		if (data_page_nr(event->rb) != nr_pages) {  in perf_mmap()
6724 			ret = -EINVAL;  in perf_mmap()
6728 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {  in perf_mmap()
6734 			mutex_unlock(&event->mmap_mutex);  in perf_mmap()
6739 		rb = event->rb;  in perf_mmap()
6746 	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);  in perf_mmap()
6753 	user_locked = atomic_long_read(&user->locked_vm);  in perf_mmap()
6757 	 *     user->locked_vm > user_lock_limit  in perf_mmap()
6768 		extra = user_locked - user_lock_limit;  in perf_mmap()
6769 		user_extra -= extra;  in perf_mmap()
6774 	locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;  in perf_mmap()
6778 		ret = -EPERM;  in perf_mmap()
6782 	WARN_ON(!rb && event->rb);  in perf_mmap()
6784 	if (vma->vm_flags & VM_WRITE)  in perf_mmap()
6789 			      event->attr.watermark ? event->attr.wakeup_watermark : 0,  in perf_mmap()
6790 			      event->cpu, flags);  in perf_mmap()
6793 			ret = -ENOMEM;  in perf_mmap()
6797 		atomic_set(&rb->mmap_count, 1);  in perf_mmap()
6798 		rb->mmap_user = get_current_user();  in perf_mmap()
6799 		rb->mmap_locked = extra;  in perf_mmap()
6807 		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,  in perf_mmap()
6808 				   event->attr.aux_watermark, flags);  in perf_mmap()
6810 			rb->aux_mmap_locked = extra;  in perf_mmap()
6815 		atomic_long_add(user_extra, &user->locked_vm);  in perf_mmap()
6816 		atomic64_add(extra, &vma->vm_mm->pinned_vm);  in perf_mmap()
6818 		atomic_inc(&event->mmap_count);  in perf_mmap()
6820 		atomic_dec(&rb->mmap_count);  in perf_mmap()
6825 	mutex_unlock(&event->mmap_mutex);  in perf_mmap()
6832 	vma->vm_ops = &perf_mmap_vmops;  in perf_mmap()
6837 	if (event->pmu->event_mapped)  in perf_mmap()
6838 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap()
6846 	struct perf_event *event = filp->private_data;  in perf_fasync()
6850 	retval = fasync_helper(fd, filp, on, &event->fasync);  in perf_fasync()
6873  * to user-space before waking everybody up.
6880 	if (event->pending_kill) {  in perf_event_wakeup()
6881 		kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);  in perf_event_wakeup()
6882 		event->pending_kill = 0;  in perf_event_wakeup()
6890 	 * ctx->task or current has changed in the meantime. This can be the  in perf_sigtrap()
6893 	if (WARN_ON_ONCE(event->ctx->task != current))  in perf_sigtrap()
6900 	if (current->flags & PF_EXITING)  in perf_sigtrap()
6903 	send_sig_perf((void __user *)event->pending_addr,  in perf_sigtrap()
6904 		      event->orig_type, event->attr.sig_data);  in perf_sigtrap()
6908  * Deliver the pending work in-event-context or follow the context.
6912 	int cpu = READ_ONCE(event->oncpu);  in __perf_pending_disable()
6925 		if (event->pending_disable) {  in __perf_pending_disable()
6926 			event->pending_disable = 0;  in __perf_pending_disable()
6933 	 *  CPU-A			CPU-B  in __perf_pending_disable()
6936 	 *    @pending_disable = CPU-A;  in __perf_pending_disable()
6939 	 *  sched-out  in __perf_pending_disable()
6940 	 *    @pending_disable = -1;  in __perf_pending_disable()
6942 	 *				sched-in  in __perf_pending_disable()
6944 	 *				  @pending_disable = CPU-B;  in __perf_pending_disable()
6950 	 * But the event runs on CPU-B and wants disabling there.  in __perf_pending_disable()
6952 	irq_work_queue_on(&event->pending_disable_irq, cpu);  in __perf_pending_disable()
6982 	 * The wakeup isn't bound to the context of the event -- it can happen  in perf_pending_irq()
6985 	if (event->pending_wakeup) {  in perf_pending_irq()
6986 		event->pending_wakeup = 0;  in perf_pending_irq()
7005 	if (event->pending_work) {  in perf_pending_task()
7006 		event->pending_work = 0;  in perf_pending_task()
7008 		local_dec(&event->ctx->nr_no_switch_fast);  in perf_pending_task()
7019 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
7020 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
7021 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
7029 	static_call_update(__perf_guest_state, cbs->state);  in perf_register_guest_info_callbacks()
7030 	static_call_update(__perf_guest_get_ip, cbs->get_ip);  in perf_register_guest_info_callbacks()
7032 	/* Implementing ->handle_intel_pt_intr is optional. */  in perf_register_guest_info_callbacks()
7033 	if (cbs->handle_intel_pt_intr)  in perf_register_guest_info_callbacks()
7035 				   cbs->handle_intel_pt_intr);  in perf_register_guest_info_callbacks()
7056 	return !event->attr.exclude_guest && perf_guest_state();  in should_sample_guest()
7097 		regs_user->abi = perf_reg_abi(current);  in perf_sample_regs_user()
7098 		regs_user->regs = regs;  in perf_sample_regs_user()
7099 	} else if (!(current->flags & PF_KTHREAD)) {  in perf_sample_regs_user()
7102 		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;  in perf_sample_regs_user()
7103 		regs_user->regs = NULL;  in perf_sample_regs_user()
7110 	regs_intr->regs = regs;  in perf_sample_regs_intr()
7111 	regs_intr->abi  = perf_reg_abi(current);  in perf_sample_regs_intr()
7129 	return TASK_SIZE - addr;  in perf_ustack_task_size()
7144 	 * - TASK_SIZE  in perf_sample_ustack_size()
7147 	 * - remaining sample size  in perf_sample_ustack_size()
7164 		stack_size = USHRT_MAX - header_size - sizeof(u64);  in perf_sample_ustack_size()
7187 		 *   - the size requested by user or the best one we can fit  in perf_output_sample_ustack()
7190 		 *   - user stack dump data  in perf_output_sample_ustack()
7192 		 *   - the actual dumped size  in perf_output_sample_ustack()
7201 		dyn_size = dump_size - rem;  in perf_output_sample_ustack()
7214 	struct perf_event *sampler = event->aux_event;  in perf_prepare_sample_aux()
7217 	data->aux_size = 0;  in perf_prepare_sample_aux()
7222 	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))  in perf_prepare_sample_aux()
7225 	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))  in perf_prepare_sample_aux()
7236 	if (READ_ONCE(rb->aux_in_sampling)) {  in perf_prepare_sample_aux()
7237 		data->aux_size = 0;  in perf_prepare_sample_aux()
7240 		data->aux_size = ALIGN(size, sizeof(u64));  in perf_prepare_sample_aux()
7245 	return data->aux_size;  in perf_prepare_sample_aux()
7257 	 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler  in perf_pmu_snapshot_aux()
7259 	 * the IRQ ones, that is, for example, re-starting an event that's just  in perf_pmu_snapshot_aux()
7270 	WRITE_ONCE(rb->aux_in_sampling, 1);  in perf_pmu_snapshot_aux()
7273 	ret = event->pmu->snapshot_aux(event, handle, size);  in perf_pmu_snapshot_aux()
7276 	WRITE_ONCE(rb->aux_in_sampling, 0);  in perf_pmu_snapshot_aux()
7286 	struct perf_event *sampler = event->aux_event;  in perf_aux_sample_output()
7291 	if (WARN_ON_ONCE(!sampler || !data->aux_size))  in perf_aux_sample_output()
7298 	size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);  in perf_aux_sample_output()
7302 	 * non-zero surplus that it didn't copy), which in its current  in perf_aux_sample_output()
7310 	 * The pad comes from ALIGN()ing data->aux_size up to u64 in  in perf_aux_sample_output()
7313 	pad = data->aux_size - size;  in perf_aux_sample_output()
7327  * A set of common sample data types saved even for non-sample records
7328  * when event->attr.sample_id_all is set.
7338 	data->type = event->attr.sample_type;  in __perf_event_header__init_id()
7339 	data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL;  in __perf_event_header__init_id()
7343 		data->tid_entry.pid = perf_event_pid(event, current);  in __perf_event_header__init_id()
7344 		data->tid_entry.tid = perf_event_tid(event, current);  in __perf_event_header__init_id()
7348 		data->time = perf_event_clock(event);  in __perf_event_header__init_id()
7351 		data->id = primary_event_id(event);  in __perf_event_header__init_id()
7354 		data->stream_id = event->id;  in __perf_event_header__init_id()
7357 		data->cpu_entry.cpu	 = raw_smp_processor_id();  in __perf_event_header__init_id()
7358 		data->cpu_entry.reserved = 0;  in __perf_event_header__init_id()
7366 	if (event->attr.sample_id_all) {  in perf_event_header__init_id()
7367 		header->size += event->id_header_size;  in perf_event_header__init_id()
7368 		__perf_event_header__init_id(data, event, event->attr.sample_type);  in perf_event_header__init_id()
7375 	u64 sample_type = data->type;  in __perf_event__output_id_sample()
7378 		perf_output_put(handle, data->tid_entry);  in __perf_event__output_id_sample()
7381 		perf_output_put(handle, data->time);  in __perf_event__output_id_sample()
7384 		perf_output_put(handle, data->id);  in __perf_event__output_id_sample()
7387 		perf_output_put(handle, data->stream_id);  in __perf_event__output_id_sample()
7390 		perf_output_put(handle, data->cpu_entry);  in __perf_event__output_id_sample()
7393 		perf_output_put(handle, data->id);  in __perf_event__output_id_sample()
7400 	if (event->attr.sample_id_all)  in perf_event__output_id_sample()
7408 	u64 read_format = event->attr.read_format;  in perf_output_read_one()
7412 	values[n++] = perf_event_count(event, has_inherit_and_sample_read(&event->attr));  in perf_output_read_one()
7415 			atomic64_read(&event->child_total_time_enabled);  in perf_output_read_one()
7419 			atomic64_read(&event->child_total_time_running);  in perf_output_read_one()
7424 		values[n++] = atomic64_read(&event->lost_samples);  in perf_output_read_one()
7433 	struct perf_event *leader = event->group_leader, *sub;  in perf_output_read_group()
7434 	u64 read_format = event->attr.read_format;  in perf_output_read_group()
7438 	bool self = has_inherit_and_sample_read(&event->attr);  in perf_output_read_group()
7446 	values[n++] = 1 + leader->nr_siblings;  in perf_output_read_group()
7455 	    (leader->state == PERF_EVENT_STATE_ACTIVE))  in perf_output_read_group()
7456 		leader->pmu->read(leader);  in perf_output_read_group()
7462 		values[n++] = atomic64_read(&leader->lost_samples);  in perf_output_read_group()
7470 		    (sub->state == PERF_EVENT_STATE_ACTIVE))  in perf_output_read_group()
7471 			sub->pmu->read(sub);  in perf_output_read_group()
7477 			values[n++] = atomic64_read(&sub->lost_samples);  in perf_output_read_group()
7495  * Instead the combination of PERF_SAMPLE_READ and inherit will track per-thread
7503 	u64 read_format = event->attr.read_format;  in perf_output_read()
7517 	if (event->attr.read_format & PERF_FORMAT_GROUP)  in perf_output_read()
7528 	u64 sample_type = data->type;  in perf_output_sample()
7533 		perf_output_put(handle, data->id);  in perf_output_sample()
7536 		perf_output_put(handle, data->ip);  in perf_output_sample()
7539 		perf_output_put(handle, data->tid_entry);  in perf_output_sample()
7542 		perf_output_put(handle, data->time);  in perf_output_sample()
7545 		perf_output_put(handle, data->addr);  in perf_output_sample()
7548 		perf_output_put(handle, data->id);  in perf_output_sample()
7551 		perf_output_put(handle, data->stream_id);  in perf_output_sample()
7554 		perf_output_put(handle, data->cpu_entry);  in perf_output_sample()
7557 		perf_output_put(handle, data->period);  in perf_output_sample()
7565 		size += data->callchain->nr;  in perf_output_sample()
7567 		__output_copy(handle, data->callchain, size);  in perf_output_sample()
7571 		struct perf_raw_record *raw = data->raw;  in perf_output_sample()
7574 			struct perf_raw_frag *frag = &raw->frag;  in perf_output_sample()
7576 			perf_output_put(handle, raw->size);  in perf_output_sample()
7578 				if (frag->copy) {  in perf_output_sample()
7579 					__output_custom(handle, frag->copy,  in perf_output_sample()
7580 							frag->data, frag->size);  in perf_output_sample()
7582 					__output_copy(handle, frag->data,  in perf_output_sample()
7583 						      frag->size);  in perf_output_sample()
7587 				frag = frag->next;  in perf_output_sample()
7589 			if (frag->pad)  in perf_output_sample()
7590 				__output_skip(handle, NULL, frag->pad);  in perf_output_sample()
7604 		if (data->br_stack) {  in perf_output_sample()
7607 			size = data->br_stack->nr  in perf_output_sample()
7610 			perf_output_put(handle, data->br_stack->nr);  in perf_output_sample()
7612 				perf_output_put(handle, data->br_stack->hw_idx);  in perf_output_sample()
7613 			perf_output_copy(handle, data->br_stack->entries, size);  in perf_output_sample()
7615 			 * Add the extension space which is appended  in perf_output_sample()
7618 			if (data->br_stack_cntr) {  in perf_output_sample()
7619 				size = data->br_stack->nr * sizeof(u64);  in perf_output_sample()
7620 				perf_output_copy(handle, data->br_stack_cntr, size);  in perf_output_sample()
7632 		u64 abi = data->regs_user.abi;  in perf_output_sample()
7641 			u64 mask = event->attr.sample_regs_user;  in perf_output_sample()
7643 						data->regs_user.regs,  in perf_output_sample()
7650 					  data->stack_user_size,  in perf_output_sample()
7651 					  data->regs_user.regs);  in perf_output_sample()
7655 		perf_output_put(handle, data->weight.full);  in perf_output_sample()
7658 		perf_output_put(handle, data->data_src.val);  in perf_output_sample()
7661 		perf_output_put(handle, data->txn);  in perf_output_sample()
7664 		u64 abi = data->regs_intr.abi;  in perf_output_sample()
7672 			u64 mask = event->attr.sample_regs_intr;  in perf_output_sample()
7675 						data->regs_intr.regs,  in perf_output_sample()
7681 		perf_output_put(handle, data->phys_addr);  in perf_output_sample()
7684 		perf_output_put(handle, data->cgroup);  in perf_output_sample()
7687 		perf_output_put(handle, data->data_page_size);  in perf_output_sample()
7690 		perf_output_put(handle, data->code_page_size);  in perf_output_sample()
7693 		perf_output_put(handle, data->aux_size);  in perf_output_sample()
7695 		if (data->aux_size)  in perf_output_sample()
7699 	if (!event->attr.watermark) {  in perf_output_sample()
7700 		int wakeup_events = event->attr.wakeup_events;  in perf_output_sample()
7703 			struct perf_buffer *rb = handle->rb;  in perf_output_sample()
7704 			int events = local_inc_return(&rb->events);  in perf_output_sample()
7707 				local_sub(wakeup_events, &rb->events);  in perf_output_sample()
7708 				local_inc(&rb->wakeup);  in perf_output_sample()
7731 		 * Try IRQ-safe get_user_page_fast_only first.  in perf_virt_to_phys()
7734 		if (current->mm != NULL) {  in perf_virt_to_phys()
7819 	 * Software page-table walkers must disable IRQs,  in perf_get_page_size()
7824 	mm = current->mm;  in perf_get_page_size()
7845 	bool kernel = !event->attr.exclude_callchain_kernel;  in perf_callchain()
7846 	bool user   = !event->attr.exclude_callchain_user;  in perf_callchain()
7847 	/* Disallow cross-task user callchains. */  in perf_callchain()
7848 	bool crosstask = event->ctx->task && event->ctx->task != current;  in perf_callchain()
7849 	const u32 max_stack = event->attr.sample_max_stack;  in perf_callchain()
7869 	u64 sample_type = event->attr.sample_type;  in perf_prepare_sample()
7883 	filtered_sample_type &= ~data->sample_flags;  in perf_prepare_sample()
7886 		/* Make sure it has the correct data->type for output */  in perf_prepare_sample()
7887 		data->type = event->attr.sample_type;  in perf_prepare_sample()
7894 		data->ip = perf_instruction_pointer(event, regs);  in perf_prepare_sample()
7895 		data->sample_flags |= PERF_SAMPLE_IP;  in perf_prepare_sample()
7902 		data->raw = NULL;  in perf_prepare_sample()
7903 		data->dyn_size += sizeof(u64);  in perf_prepare_sample()
7904 		data->sample_flags |= PERF_SAMPLE_RAW;  in perf_prepare_sample()
7908 		data->br_stack = NULL;  in perf_prepare_sample()
7909 		data->dyn_size += sizeof(u64);  in perf_prepare_sample()
7910 		data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;  in perf_prepare_sample()
7914 		perf_sample_regs_user(&data->regs_user, regs);  in perf_prepare_sample()
7921 	if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {  in perf_prepare_sample()
7925 		if (data->regs_user.regs) {  in perf_prepare_sample()
7926 			u64 mask = event->attr.sample_regs_user;  in perf_prepare_sample()
7930 		data->dyn_size += size;  in perf_prepare_sample()
7931 		data->sample_flags |= PERF_SAMPLE_REGS_USER;  in perf_prepare_sample()
7941 		u16 stack_size = event->attr.sample_stack_user;  in perf_prepare_sample()
7946 						     data->regs_user.regs);  in perf_prepare_sample()
7956 		data->stack_user_size = stack_size;  in perf_prepare_sample()
7957 		data->dyn_size += size;  in perf_prepare_sample()
7958 		data->sample_flags |= PERF_SAMPLE_STACK_USER;  in perf_prepare_sample()
7962 		data->weight.full = 0;  in perf_prepare_sample()
7963 		data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;  in perf_prepare_sample()
7967 		data->data_src.val = PERF_MEM_NA;  in perf_prepare_sample()
7968 		data->sample_flags |= PERF_SAMPLE_DATA_SRC;  in perf_prepare_sample()
7972 		data->txn = 0;  in perf_prepare_sample()
7973 		data->sample_flags |= PERF_SAMPLE_TRANSACTION;  in perf_prepare_sample()
7977 		data->addr = 0;  in perf_prepare_sample()
7978 		data->sample_flags |= PERF_SAMPLE_ADDR;  in perf_prepare_sample()
7985 		perf_sample_regs_intr(&data->regs_intr, regs);  in perf_prepare_sample()
7987 		if (data->regs_intr.regs) {  in perf_prepare_sample()
7988 			u64 mask = event->attr.sample_regs_intr;  in perf_prepare_sample()
7993 		data->dyn_size += size;  in perf_prepare_sample()
7994 		data->sample_flags |= PERF_SAMPLE_REGS_INTR;  in perf_prepare_sample()
7998 		data->phys_addr = perf_virt_to_phys(data->addr);  in perf_prepare_sample()
7999 		data->sample_flags |= PERF_SAMPLE_PHYS_ADDR;  in perf_prepare_sample()
8007 		cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;  in perf_prepare_sample()
8008 		data->cgroup = cgroup_id(cgrp);  in perf_prepare_sample()
8009 		data->sample_flags |= PERF_SAMPLE_CGROUP;  in perf_prepare_sample()
8015 	 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,  in perf_prepare_sample()
8019 		data->data_page_size = perf_get_page_size(data->addr);  in perf_prepare_sample()
8020 		data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE;  in perf_prepare_sample()
8024 		data->code_page_size = perf_get_page_size(data->ip);  in perf_prepare_sample()
8025 		data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE;  in perf_prepare_sample()
8040 		size = min_t(size_t, U16_MAX - header_size,  in perf_prepare_sample()
8041 			     event->attr.aux_sample_size);  in perf_prepare_sample()
8046 		data->dyn_size += size + sizeof(u64); /* size above */  in perf_prepare_sample()
8047 		data->sample_flags |= PERF_SAMPLE_AUX;  in perf_prepare_sample()
8056 	header->type = PERF_RECORD_SAMPLE;  in perf_prepare_header()
8057 	header->size = perf_sample_data_size(data, event);  in perf_prepare_header()
8058 	header->misc = perf_misc_flags(event, regs);  in perf_prepare_header()
8068 	WARN_ON_ONCE(header->size & 7);  in perf_prepare_header()
8074 		if (!event->hw.aux_paused) {  in __perf_event_aux_pause()
8075 			event->hw.aux_paused = 1;  in __perf_event_aux_pause()
8076 			event->pmu->stop(event, PERF_EF_PAUSE);  in __perf_event_aux_pause()
8079 		if (event->hw.aux_paused) {  in __perf_event_aux_pause()
8080 			event->hw.aux_paused = 0;  in __perf_event_aux_pause()
8081 			event->pmu->start(event, PERF_EF_RESUME);  in __perf_event_aux_pause()
8099 		 * Guard against self-recursion here. Another event could trip  in perf_event_aux_pause()
8102 		if (READ_ONCE(rb->aux_in_pause_resume))  in perf_event_aux_pause()
8105 		WRITE_ONCE(rb->aux_in_pause_resume, 1);  in perf_event_aux_pause()
8109 		WRITE_ONCE(rb->aux_in_pause_resume, 0);  in perf_event_aux_pause()
8191 			.size = sizeof(read_event) + event->read_size,  in perf_event_read_event()
8219 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {  in perf_iterate_ctx()
8221 			if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_ctx()
8236 	list_for_each_entry_rcu(event, &pel->list, sb_list) {  in perf_iterate_sb_cpu()
8239 		 * if we observe event->ctx, both event and ctx will be  in perf_iterate_sb_cpu()
8242 		if (!smp_load_acquire(&event->ctx))  in perf_iterate_sb_cpu()
8245 		if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_sb_cpu()
8254  * Iterate all events that need to receive side-band events.
8280 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_iterate_sb()
8289  * Clear all file-based filters at exec, they'll have to be
8290  * re-instated when/if these objects are mmapped again.
8302 	raw_spin_lock_irqsave(&ifh->lock, flags);  in perf_event_addr_filters_exec()
8303 	list_for_each_entry(filter, &ifh->list, entry) {  in perf_event_addr_filters_exec()
8304 		if (filter->path.dentry) {  in perf_event_addr_filters_exec()
8305 			event->addr_filter_ranges[count].start = 0;  in perf_event_addr_filters_exec()
8306 			event->addr_filter_ranges[count].size = 0;  in perf_event_addr_filters_exec()
8314 		event->addr_filters_gen++;  in perf_event_addr_filters_exec()
8315 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in perf_event_addr_filters_exec()
8345 	struct perf_event *parent = event->parent;  in __perf_event_output_stop()
8347 	struct perf_buffer *rb = ro->rb;  in __perf_event_output_stop()
8360 	 * ring-buffer, but it will be the child that's actually using it.  in __perf_event_output_stop()
8365 	 * So ring_buffer_attach() has to stop an aux event before re-assigning  in __perf_event_output_stop()
8368 	if (rcu_dereference(parent->rb) == rb)  in __perf_event_output_stop()
8369 		ro->err = __perf_event_stop(&sd);  in __perf_event_output_stop()
8377 		.rb	= event->rb,  in __perf_pmu_output_stop()
8381 	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);  in __perf_pmu_output_stop()
8382 	if (cpuctx->task_ctx)  in __perf_pmu_output_stop()
8383 		perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,  in __perf_pmu_output_stop()
8397 	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {  in perf_pmu_output_stop()
8399 		 * For per-CPU events, we need to make sure that neither they  in perf_pmu_output_stop()
8400 		 * nor their children are running; for cpu==-1 events it's  in perf_pmu_output_stop()
8404 		cpu = iter->cpu;  in perf_pmu_output_stop()
8405 		if (cpu == -1)  in perf_pmu_output_stop()
8406 			cpu = READ_ONCE(iter->oncpu);  in perf_pmu_output_stop()
8408 		if (cpu == -1)  in perf_pmu_output_stop()
8412 		if (err == -EAGAIN) {  in perf_pmu_output_stop()
8421  * task tracking -- fork/exit
8443 	return event->attr.comm  || event->attr.mmap ||  in perf_event_task_match()
8444 	       event->attr.mmap2 || event->attr.mmap_data ||  in perf_event_task_match()
8445 	       event->attr.task;  in perf_event_task_match()
8454 	struct task_struct *task = task_event->task;  in perf_event_task_output()
8455 	int ret, size = task_event->event_id.header.size;  in perf_event_task_output()
8460 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);  in perf_event_task_output()
8463 				task_event->event_id.header.size);  in perf_event_task_output()
8467 	task_event->event_id.pid = perf_event_pid(event, task);  in perf_event_task_output()
8468 	task_event->event_id.tid = perf_event_tid(event, task);  in perf_event_task_output()
8470 	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {  in perf_event_task_output()
8471 		task_event->event_id.ppid = perf_event_pid(event,  in perf_event_task_output()
8472 							task->real_parent);  in perf_event_task_output()
8473 		task_event->event_id.ptid = perf_event_pid(event,  in perf_event_task_output()
8474 							task->real_parent);  in perf_event_task_output()
8476 		task_event->event_id.ppid = perf_event_pid(event, current);  in perf_event_task_output()
8477 		task_event->event_id.ptid = perf_event_tid(event, current);  in perf_event_task_output()
8480 	task_event->event_id.time = perf_event_clock(event);  in perf_event_task_output()
8482 	perf_output_put(&handle, task_event->event_id);  in perf_event_task_output()
8488 	task_event->event_id.header.size = size;  in perf_event_task_output()
8549 	return event->attr.comm;  in perf_event_comm_match()
8558 	int size = comm_event->event_id.header.size;  in perf_event_comm_output()
8564 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);  in perf_event_comm_output()
8566 				comm_event->event_id.header.size);  in perf_event_comm_output()
8571 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);  in perf_event_comm_output()
8572 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);  in perf_event_comm_output()
8574 	perf_output_put(&handle, comm_event->event_id);  in perf_event_comm_output()
8575 	__output_copy(&handle, comm_event->comm,  in perf_event_comm_output()
8576 				   comm_event->comm_size);  in perf_event_comm_output()
8582 	comm_event->event_id.header.size = size;  in perf_event_comm_output()
8591 	strscpy(comm, comm_event->task->comm, sizeof(comm));  in perf_event_comm_event()
8594 	comm_event->comm = comm;  in perf_event_comm_event()
8595 	comm_event->comm_size = size;  in perf_event_comm_event()
8597 	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;  in perf_event_comm_event()
8648 	return event->attr.namespaces;  in perf_event_namespaces_match()
8657 	u16 header_size = namespaces_event->event_id.header.size;  in perf_event_namespaces_output()
8663 	perf_event_header__init_id(&namespaces_event->event_id.header,  in perf_event_namespaces_output()
8666 				namespaces_event->event_id.header.size);  in perf_event_namespaces_output()
8670 	namespaces_event->event_id.pid = perf_event_pid(event,  in perf_event_namespaces_output()
8671 							namespaces_event->task);  in perf_event_namespaces_output()
8672 	namespaces_event->event_id.tid = perf_event_tid(event,  in perf_event_namespaces_output()
8673 							namespaces_event->task);  in perf_event_namespaces_output()
8675 	perf_output_put(&handle, namespaces_event->event_id);  in perf_event_namespaces_output()
8681 	namespaces_event->event_id.header.size = header_size;  in perf_event_namespaces_output()
8694 		ns_inode = ns_path.dentry->d_inode;  in perf_fill_ns_link_info()
8695 		ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);  in perf_fill_ns_link_info()
8696 		ns_link_info->ino = ns_inode->i_ino;  in perf_fill_ns_link_info()
8776 	return event->attr.cgroup;  in perf_event_cgroup_match()
8784 	u16 header_size = cgroup_event->event_id.header.size;  in perf_event_cgroup_output()
8790 	perf_event_header__init_id(&cgroup_event->event_id.header,  in perf_event_cgroup_output()
8793 				cgroup_event->event_id.header.size);  in perf_event_cgroup_output()
8797 	perf_output_put(&handle, cgroup_event->event_id);  in perf_event_cgroup_output()
8798 	__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);  in perf_event_cgroup_output()
8804 	cgroup_event->event_id.header.size = header_size;  in perf_event_cgroup_output()
8833 		cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));  in perf_event_cgroup()
8889 	struct vm_area_struct *vma = mmap_event->vma;  in perf_event_mmap_match()
8890 	int executable = vma->vm_flags & VM_EXEC;  in perf_event_mmap_match()
8892 	return (!executable && event->attr.mmap_data) ||  in perf_event_mmap_match()
8893 	       (executable && (event->attr.mmap || event->attr.mmap2));  in perf_event_mmap_match()
8902 	int size = mmap_event->event_id.header.size;  in perf_event_mmap_output()
8903 	u32 type = mmap_event->event_id.header.type;  in perf_event_mmap_output()
8910 	if (event->attr.mmap2) {  in perf_event_mmap_output()
8911 		mmap_event->event_id.header.type = PERF_RECORD_MMAP2;  in perf_event_mmap_output()
8912 		mmap_event->event_id.header.size += sizeof(mmap_event->maj);  in perf_event_mmap_output()
8913 		mmap_event->event_id.header.size += sizeof(mmap_event->min);  in perf_event_mmap_output()
8914 		mmap_event->event_id.header.size += sizeof(mmap_event->ino);  in perf_event_mmap_output()
8915 		mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);  in perf_event_mmap_output()
8916 		mmap_event->event_id.header.size += sizeof(mmap_event->prot);  in perf_event_mmap_output()
8917 		mmap_event->event_id.header.size += sizeof(mmap_event->flags);  in perf_event_mmap_output()
8920 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);  in perf_event_mmap_output()
8922 				mmap_event->event_id.header.size);  in perf_event_mmap_output()
8926 	mmap_event->event_id.pid = perf_event_pid(event, current);  in perf_event_mmap_output()
8927 	mmap_event->event_id.tid = perf_event_tid(event, current);  in perf_event_mmap_output()
8929 	use_build_id = event->attr.build_id && mmap_event->build_id_size;  in perf_event_mmap_output()
8931 	if (event->attr.mmap2 && use_build_id)  in perf_event_mmap_output()
8932 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;  in perf_event_mmap_output()
8934 	perf_output_put(&handle, mmap_event->event_id);  in perf_event_mmap_output()
8936 	if (event->attr.mmap2) {  in perf_event_mmap_output()
8938 			u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };  in perf_event_mmap_output()
8941 			__output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX);  in perf_event_mmap_output()
8943 			perf_output_put(&handle, mmap_event->maj);  in perf_event_mmap_output()
8944 			perf_output_put(&handle, mmap_event->min);  in perf_event_mmap_output()
8945 			perf_output_put(&handle, mmap_event->ino);  in perf_event_mmap_output()
8946 			perf_output_put(&handle, mmap_event->ino_generation);  in perf_event_mmap_output()
8948 		perf_output_put(&handle, mmap_event->prot);  in perf_event_mmap_output()
8949 		perf_output_put(&handle, mmap_event->flags);  in perf_event_mmap_output()
8952 	__output_copy(&handle, mmap_event->file_name,  in perf_event_mmap_output()
8953 				   mmap_event->file_size);  in perf_event_mmap_output()
8959 	mmap_event->event_id.header.size = size;  in perf_event_mmap_output()
8960 	mmap_event->event_id.header.type = type;  in perf_event_mmap_output()
8965 	struct vm_area_struct *vma = mmap_event->vma;  in perf_event_mmap_event()
8966 	struct file *file = vma->vm_file;  in perf_event_mmap_event()
8975 	if (vma->vm_flags & VM_READ)  in perf_event_mmap_event()
8977 	if (vma->vm_flags & VM_WRITE)  in perf_event_mmap_event()
8979 	if (vma->vm_flags & VM_EXEC)  in perf_event_mmap_event()
8982 	if (vma->vm_flags & VM_MAYSHARE)  in perf_event_mmap_event()
8987 	if (vma->vm_flags & VM_LOCKED)  in perf_event_mmap_event()
9006 		name = file_path(file, buf, PATH_MAX - sizeof(u64));  in perf_event_mmap_event()
9011 		inode = file_inode(vma->vm_file);  in perf_event_mmap_event()
9012 		dev = inode->i_sb->s_dev;  in perf_event_mmap_event()
9013 		ino = inode->i_ino;  in perf_event_mmap_event()
9014 		gen = inode->i_generation;  in perf_event_mmap_event()
9020 		if (vma->vm_ops && vma->vm_ops->name)  in perf_event_mmap_event()
9021 			name = (char *) vma->vm_ops->name(vma);  in perf_event_mmap_event()
9047 	mmap_event->file_name = name;  in perf_event_mmap_event()
9048 	mmap_event->file_size = size;  in perf_event_mmap_event()
9049 	mmap_event->maj = maj;  in perf_event_mmap_event()
9050 	mmap_event->min = min;  in perf_event_mmap_event()
9051 	mmap_event->ino = ino;  in perf_event_mmap_event()
9052 	mmap_event->ino_generation = gen;  in perf_event_mmap_event()
9053 	mmap_event->prot = prot;  in perf_event_mmap_event()
9054 	mmap_event->flags = flags;  in perf_event_mmap_event()
9056 	if (!(vma->vm_flags & VM_EXEC))  in perf_event_mmap_event()
9057 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;  in perf_event_mmap_event()
9059 	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;  in perf_event_mmap_event()
9062 		build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size);  in perf_event_mmap_event()
9078 	/* d_inode(NULL) won't be equal to any mapped user-space file */  in perf_addr_filter_match()
9079 	if (!filter->path.dentry)  in perf_addr_filter_match()
9082 	if (d_inode(filter->path.dentry) != file_inode(file))  in perf_addr_filter_match()
9085 	if (filter->offset > offset + size)  in perf_addr_filter_match()
9088 	if (filter->offset + filter->size < offset)  in perf_addr_filter_match()
9098 	unsigned long vma_size = vma->vm_end - vma->vm_start;  in perf_addr_filter_vma_adjust()
9099 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;  in perf_addr_filter_vma_adjust()
9100 	struct file *file = vma->vm_file;  in perf_addr_filter_vma_adjust()
9105 	if (filter->offset < off) {  in perf_addr_filter_vma_adjust()
9106 		fr->start = vma->vm_start;  in perf_addr_filter_vma_adjust()
9107 		fr->size = min(vma_size, filter->size - (off - filter->offset));  in perf_addr_filter_vma_adjust()
9109 		fr->start = vma->vm_start + filter->offset - off;  in perf_addr_filter_vma_adjust()
9110 		fr->size = min(vma->vm_end - fr->start, filter->size);  in perf_addr_filter_vma_adjust()
9127 	if (!vma->vm_file)  in __perf_addr_filters_adjust()
9130 	raw_spin_lock_irqsave(&ifh->lock, flags);  in __perf_addr_filters_adjust()
9131 	list_for_each_entry(filter, &ifh->list, entry) {  in __perf_addr_filters_adjust()
9133 						&event->addr_filter_ranges[count]))  in __perf_addr_filters_adjust()
9140 		event->addr_filters_gen++;  in __perf_addr_filters_adjust()
9141 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in __perf_addr_filters_adjust()
9158 	if (!(vma->vm_flags & VM_EXEC))  in perf_addr_filters_adjust()
9162 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_addr_filters_adjust()
9187 			.start  = vma->vm_start,  in perf_event_mmap()
9188 			.len    = vma->vm_end - vma->vm_start,  in perf_event_mmap()
9189 			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,  in perf_event_mmap()
9287 	return event->attr.context_switch;  in perf_event_switch_match()
9300 	/* Only CPU-wide events are allowed to see next/prev pid/tid */  in perf_event_switch_output()
9301 	if (event->ctx->task) {  in perf_event_switch_output()
9302 		se->event_id.header.type = PERF_RECORD_SWITCH;  in perf_event_switch_output()
9303 		se->event_id.header.size = sizeof(se->event_id.header);  in perf_event_switch_output()
9305 		se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;  in perf_event_switch_output()
9306 		se->event_id.header.size = sizeof(se->event_id);  in perf_event_switch_output()
9307 		se->event_id.next_prev_pid =  in perf_event_switch_output()
9308 					perf_event_pid(event, se->next_prev);  in perf_event_switch_output()
9309 		se->event_id.next_prev_tid =  in perf_event_switch_output()
9310 					perf_event_tid(event, se->next_prev);  in perf_event_switch_output()
9313 	perf_event_header__init_id(&se->event_id.header, &sample, event);  in perf_event_switch_output()
9315 	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);  in perf_event_switch_output()
9319 	if (event->ctx->task)  in perf_event_switch_output()
9320 		perf_output_put(&handle, se->event_id.header);  in perf_event_switch_output()
9322 		perf_output_put(&handle, se->event_id);  in perf_event_switch_output()
9381 		.stream_id	= event->id,  in perf_log_throttle()
9417 	return event->attr.ksymbol;  in perf_event_ksymbol_match()
9430 	perf_event_header__init_id(&ksymbol_event->event_id.header,  in perf_event_ksymbol_output()
9433 				ksymbol_event->event_id.header.size);  in perf_event_ksymbol_output()
9437 	perf_output_put(&handle, ksymbol_event->event_id);  in perf_event_ksymbol_output()
9438 	__output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);  in perf_event_ksymbol_output()
9507 	return event->attr.bpf_event;  in perf_event_bpf_match()
9520 	perf_event_header__init_id(&bpf_event->event_id.header,  in perf_event_bpf_output()
9523 				bpf_event->event_id.header.size);  in perf_event_bpf_output()
9527 	perf_output_put(&handle, bpf_event->event_id);  in perf_event_bpf_output()
9540 			   (u64)(unsigned long)prog->bpf_func,  in perf_event_bpf_emit_ksymbols()
9541 			   prog->jited_len, unregister,  in perf_event_bpf_emit_ksymbols()
9542 			   prog->aux->ksym.name);  in perf_event_bpf_emit_ksymbols()
9544 	for (i = 1; i < prog->aux->func_cnt; i++) {  in perf_event_bpf_emit_ksymbols()
9545 		struct bpf_prog *subprog = prog->aux->func[i];  in perf_event_bpf_emit_ksymbols()
9549 			(u64)(unsigned long)subprog->bpf_func,  in perf_event_bpf_emit_ksymbols()
9550 			subprog->jited_len, unregister,  in perf_event_bpf_emit_ksymbols()
9551 			subprog->aux->ksym.name);  in perf_event_bpf_emit_ksymbols()
9583 			.id = prog->aux->id,  in perf_event_bpf_event()
9589 	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);  in perf_event_bpf_event()
9609 	return event->attr.text_poke;  in perf_event_text_poke_match()
9623 	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);  in perf_event_text_poke_output()
9626 				text_poke_event->event_id.header.size);  in perf_event_text_poke_output()
9630 	perf_output_put(&handle, text_poke_event->event_id);  in perf_event_text_poke_output()
9631 	perf_output_put(&handle, text_poke_event->old_len);  in perf_event_text_poke_output()
9632 	perf_output_put(&handle, text_poke_event->new_len);  in perf_event_text_poke_output()
9634 	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);  in perf_event_text_poke_output()
9635 	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);  in perf_event_text_poke_output()
9637 	if (text_poke_event->pad)  in perf_event_text_poke_output()
9638 		__output_copy(&handle, &padding, text_poke_event->pad);  in perf_event_text_poke_output()
9656 	pad  = ALIGN(tot, sizeof(u64)) - tot;  in perf_event_text_poke()
9679 	event->attach_state |= PERF_ATTACH_ITRACE;  in perf_event_itrace_started()
9693 	if (event->parent)  in perf_log_itrace_start()
9694 		event = event->parent;  in perf_log_itrace_start()
9696 	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||  in perf_log_itrace_start()
9697 	    event->attach_state & PERF_ATTACH_ITRACE)  in perf_log_itrace_start()
9728 	if (event->parent)  in perf_report_aux_output_id()
9729 		event = event->parent;  in perf_report_aux_output_id()
9752 	struct hw_perf_event *hwc = &event->hw;  in __perf_event_account_interrupt()
9757 	if (seq != hwc->interrupts_seq) {  in __perf_event_account_interrupt()
9758 		hwc->interrupts_seq = seq;  in __perf_event_account_interrupt()
9759 		hwc->interrupts = 1;  in __perf_event_account_interrupt()
9761 		hwc->interrupts++;  in __perf_event_account_interrupt()
9763 			     hwc->interrupts > max_samples_per_tick)) {  in __perf_event_account_interrupt()
9766 			hwc->interrupts = MAX_INTERRUPTS;  in __perf_event_account_interrupt()
9772 	if (event->attr.freq) {  in __perf_event_account_interrupt()
9774 		s64 delta = now - hwc->freq_time_stamp;  in __perf_event_account_interrupt()
9776 		hwc->freq_time_stamp = now;  in __perf_event_account_interrupt()
9779 			perf_adjust_period(event, delta, hwc->last_period, true);  in __perf_event_account_interrupt()
9797 	if (event->attr.exclude_kernel && !user_mode(regs))  in sample_is_allowed()
9819 	prog = READ_ONCE(event->prog);  in bpf_overflow_handler()
9835 	if (event->overflow_handler_context)  in perf_event_set_bpf_handler()
9837 		return -EINVAL;  in perf_event_set_bpf_handler()
9839 	if (event->prog)  in perf_event_set_bpf_handler()
9840 		return -EEXIST;  in perf_event_set_bpf_handler()
9842 	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)  in perf_event_set_bpf_handler()
9843 		return -EINVAL;  in perf_event_set_bpf_handler()
9845 	if (event->attr.precise_ip &&  in perf_event_set_bpf_handler()
9846 	    prog->call_get_stack &&  in perf_event_set_bpf_handler()
9847 	    (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||  in perf_event_set_bpf_handler()
9848 	     event->attr.exclude_callchain_kernel ||  in perf_event_set_bpf_handler()
9849 	     event->attr.exclude_callchain_user)) {  in perf_event_set_bpf_handler()
9859 		return -EPROTO;  in perf_event_set_bpf_handler()
9862 	event->prog = prog;  in perf_event_set_bpf_handler()
9863 	event->bpf_cookie = bpf_cookie;  in perf_event_set_bpf_handler()
9869 	struct bpf_prog *prog = event->prog;  in perf_event_free_bpf_handler()
9874 	event->prog = NULL;  in perf_event_free_bpf_handler()
9889 	return -EOPNOTSUPP;  in perf_event_set_bpf_handler()
9905 	int events = atomic_read(&event->event_limit);  in __perf_event_overflow()
9909 	 * Non-sampling counters might still use the PMI to fold short  in __perf_event_overflow()
9917 	if (event->attr.aux_pause)  in __perf_event_overflow()
9918 		perf_event_aux_pause(event->aux_event, true);  in __perf_event_overflow()
9920 	if (event->prog && event->prog->type == BPF_PROG_TYPE_PERF_EVENT &&  in __perf_event_overflow()
9929 	event->pending_kill = POLL_IN;  in __perf_event_overflow()
9930 	if (events && atomic_dec_and_test(&event->event_limit)) {  in __perf_event_overflow()
9932 		event->pending_kill = POLL_HUP;  in __perf_event_overflow()
9936 	if (event->attr.sigtrap) {  in __perf_event_overflow()
9952 		if (!event->pending_work &&  in __perf_event_overflow()
9953 		    !task_work_add(current, &event->pending_task, notify_mode)) {  in __perf_event_overflow()
9954 			event->pending_work = pending_id;  in __perf_event_overflow()
9955 			local_inc(&event->ctx->nr_no_switch_fast);  in __perf_event_overflow()
9956 			WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));  in __perf_event_overflow()
9958 			event->pending_addr = 0;  in __perf_event_overflow()
9959 			if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))  in __perf_event_overflow()
9960 				event->pending_addr = data->addr;  in __perf_event_overflow()
9962 		} else if (event->attr.exclude_kernel && valid_sample) {  in __perf_event_overflow()
9970 			 *  2. Events that can overflow again before the IRQ-  in __perf_event_overflow()
9973 			 *     check 32-bit hash of the current IP.  in __perf_event_overflow()
9975 			WARN_ON_ONCE(event->pending_work != pending_id);  in __perf_event_overflow()
9979 	READ_ONCE(event->overflow_handler)(event, data, regs);  in __perf_event_overflow()
9981 	if (*perf_event_fasync(event) && event->pending_kill) {  in __perf_event_overflow()
9982 		event->pending_wakeup = 1;  in __perf_event_overflow()
9983 		irq_work_queue(&event->pending_irq);  in __perf_event_overflow()
9986 	if (event->attr.aux_resume)  in __perf_event_overflow()
9987 		perf_event_aux_pause(event->aux_event, false);  in __perf_event_overflow()
10011  * We directly increment event->count and keep a second value in
10012  * event->hw.period_left to count intervals. This period event
10013  * is kept in the range [-sample_period, 0] so that we can use the
10019 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_set_period()
10020 	u64 period = hwc->last_period;  in perf_swevent_set_period()
10024 	hwc->last_period = hwc->sample_period;  in perf_swevent_set_period()
10026 	old = local64_read(&hwc->period_left);  in perf_swevent_set_period()
10034 		val -= offset;  in perf_swevent_set_period()
10035 	} while (!local64_try_cmpxchg(&hwc->period_left, &old, val));  in perf_swevent_set_period()
10044 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_overflow()
10050 	if (hwc->interrupts == MAX_INTERRUPTS)  in perf_swevent_overflow()
10053 	for (; overflow; overflow--) {  in perf_swevent_overflow()
10058 			 * hwc->interrupts == MAX_INTERRUPTS.  in perf_swevent_overflow()
10070 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_event()
10072 	local64_add(nr, &event->count);  in perf_swevent_event()
10080 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {  in perf_swevent_event()
10081 		data->period = nr;  in perf_swevent_event()
10084 		data->period = event->hw.last_period;  in perf_swevent_event()
10086 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)  in perf_swevent_event()
10089 	if (local64_add_negative(nr, &hwc->period_left))  in perf_swevent_event()
10097 	if (event->hw.state & PERF_HES_STOPPED)  in perf_exclude_event()
10101 		if (event->attr.exclude_user && user_mode(regs))  in perf_exclude_event()
10104 		if (event->attr.exclude_kernel && !user_mode(regs))  in perf_exclude_event()
10117 	if (event->attr.type != type)  in perf_swevent_match()
10120 	if (event->attr.config != event_id)  in perf_swevent_match()
10141 	return &hlist->heads[hash];  in __find_swevent_head()
10150 	hlist = rcu_dereference(swhash->swevent_hlist);  in find_swevent_head_rcu()
10162 	u32 event_id = event->attr.config;  in find_swevent_head()
10163 	u64 type = event->attr.type;  in find_swevent_head()
10170 	hlist = rcu_dereference_protected(swhash->swevent_hlist,  in find_swevent_head()
10171 					  lockdep_is_held(&event->ctx->lock));  in find_swevent_head()
10204 	return get_recursion_context(current->perf_recursion);  in perf_swevent_get_recursion_context()
10210 	put_recursion_context(current->perf_recursion, rctx);  in perf_swevent_put_recursion_context()
10247 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_add()
10251 		hwc->last_period = hwc->sample_period;  in perf_swevent_add()
10255 	hwc->state = !(flags & PERF_EF_START);  in perf_swevent_add()
10259 		return -EINVAL;  in perf_swevent_add()
10261 	hlist_add_head_rcu(&event->hlist_entry, head);  in perf_swevent_add()
10269 	hlist_del_rcu(&event->hlist_entry);  in perf_swevent_del()
10274 	event->hw.state = 0;  in perf_swevent_start()
10279 	event->hw.state = PERF_HES_STOPPED;  in perf_swevent_stop()
10286 	return rcu_dereference_protected(swhash->swevent_hlist,  in swevent_hlist_deref()
10287 					 lockdep_is_held(&swhash->hlist_mutex));  in swevent_hlist_deref()
10297 	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);  in swevent_hlist_release()
10305 	mutex_lock(&swhash->hlist_mutex);  in swevent_hlist_put_cpu()
10307 	if (!--swhash->hlist_refcount)  in swevent_hlist_put_cpu()
10310 	mutex_unlock(&swhash->hlist_mutex);  in swevent_hlist_put_cpu()
10326 	mutex_lock(&swhash->hlist_mutex);  in swevent_hlist_get_cpu()
10333 			err = -ENOMEM;  in swevent_hlist_get_cpu()
10336 		rcu_assign_pointer(swhash->swevent_hlist, hlist);  in swevent_hlist_get_cpu()
10338 	swhash->hlist_refcount++;  in swevent_hlist_get_cpu()
10340 	mutex_unlock(&swhash->hlist_mutex);  in swevent_hlist_get_cpu()
10373 	u64 event_id = event->attr.config;  in sw_perf_event_destroy()
10375 	WARN_ON(event->parent);  in sw_perf_event_destroy()
10386 	u64 event_id = event->attr.config;  in perf_swevent_init()
10388 	if (event->attr.type != PERF_TYPE_SOFTWARE)  in perf_swevent_init()
10389 		return -ENOENT;  in perf_swevent_init()
10395 		return -EOPNOTSUPP;  in perf_swevent_init()
10399 		event->attr.type = perf_cpu_clock.type;  in perf_swevent_init()
10400 		return -ENOENT;  in perf_swevent_init()
10402 		event->attr.type = perf_task_clock.type;  in perf_swevent_init()
10403 		return -ENOENT;  in perf_swevent_init()
10410 		return -ENOENT;  in perf_swevent_init()
10412 	if (!event->parent) {  in perf_swevent_init()
10420 		event->destroy = sw_perf_event_destroy;  in perf_swevent_init()
10450 	if (event->attr.type != PERF_TYPE_TRACEPOINT)  in perf_tp_event_init()
10451 		return -ENOENT;  in perf_tp_event_init()
10457 		return -EOPNOTSUPP;  in perf_tp_event_init()
10463 	event->destroy = tp_perf_event_destroy;  in perf_tp_event_init()
10482 	void *record = raw->frag.data;  in perf_tp_filter_match()
10485 	if (event->parent)  in perf_tp_filter_match()
10486 		event = event->parent;  in perf_tp_filter_match()
10488 	if (likely(!event->filter) || filter_match_preds(event->filter, record))  in perf_tp_filter_match()
10497 	if (event->hw.state & PERF_HES_STOPPED)  in perf_tp_event_match()
10500 	 * If exclude_kernel, only trace user-space tracepoints (uprobes)  in perf_tp_event_match()
10502 	if (event->attr.exclude_kernel && !user_mode(regs))  in perf_tp_event_match()
10523 	perf_tp_event(call->event.type, count, raw_data, size, regs, head,  in perf_trace_run_bpf_submit()
10536 	if (event->attr.config != entry->type)  in __perf_tp_event_target_task()
10539 	if (event->attr.sigtrap)  in __perf_tp_event_target_task()
10558 	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {  in perf_tp_event_target_task()
10564 	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {  in perf_tp_event_target_task()
10590 			 * Here use the same on-stack perf_sample_data,  in perf_tp_event()
10591 			 * some members in data are event-specific and  in perf_tp_event()
10592 			 * need to be re-computed for different sweveents.  in perf_tp_event()
10593 			 * Re-initialize data->sample_flags safely to avoid  in perf_tp_event()
10595 			 * because data->sample_flags is set.  in perf_tp_event()
10611 		ctx = rcu_dereference(task->perf_event_ctxp);  in perf_tp_event()
10615 		raw_spin_lock(&ctx->lock);  in perf_tp_event()
10617 		raw_spin_unlock(&ctx->lock);  in perf_tp_event()
10644 	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
10683 	if (event->attr.type != perf_kprobe.type)  in perf_kprobe_event_init()
10684 		return -ENOENT;  in perf_kprobe_event_init()
10687 		return -EACCES;  in perf_kprobe_event_init()
10693 		return -EOPNOTSUPP;  in perf_kprobe_event_init()
10695 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_kprobe_event_init()
10700 	event->destroy = perf_kprobe_destroy;  in perf_kprobe_event_init()
10707 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
10743 	if (event->attr.type != perf_uprobe.type)  in perf_uprobe_event_init()
10744 		return -ENOENT;  in perf_uprobe_event_init()
10747 		return -EACCES;  in perf_uprobe_event_init()
10753 		return -EOPNOTSUPP;  in perf_uprobe_event_init()
10755 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_uprobe_event_init()
10756 	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;  in perf_uprobe_event_init()
10761 	event->destroy = perf_uprobe_destroy;  in perf_uprobe_event_init()
10771 	perf_pmu_register(&perf_kprobe, "kprobe", -1);  in perf_tp_register()
10774 	perf_pmu_register(&perf_uprobe, "uprobe", -1);  in perf_tp_register()
10789 	if (event->pmu == &perf_tracepoint)  in perf_event_is_tracing()
10792 	if (event->pmu == &perf_kprobe)  in perf_event_is_tracing()
10796 	if (event->pmu == &perf_uprobe)  in perf_event_is_tracing()
10810 	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;  in perf_event_set_bpf_prog()
10811 	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;  in perf_event_set_bpf_prog()
10812 	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;  in perf_event_set_bpf_prog()
10813 	is_syscall_tp = is_syscall_trace_event(event->tp_event);  in perf_event_set_bpf_prog()
10816 		return -EINVAL;  in perf_event_set_bpf_prog()
10818 	if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) ||  in perf_event_set_bpf_prog()
10819 	    (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||  in perf_event_set_bpf_prog()
10820 	    (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))  in perf_event_set_bpf_prog()
10821 		return -EINVAL;  in perf_event_set_bpf_prog()
10823 	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe)  in perf_event_set_bpf_prog()
10825 		return -EINVAL;  in perf_event_set_bpf_prog()
10828 	if (prog->kprobe_override && !is_kprobe)  in perf_event_set_bpf_prog()
10829 		return -EINVAL;  in perf_event_set_bpf_prog()
10832 		int off = trace_event_get_offsets(event->tp_event);  in perf_event_set_bpf_prog()
10834 		if (prog->aux->max_ctx_offset > off)  in perf_event_set_bpf_prog()
10835 			return -EACCES;  in perf_event_set_bpf_prog()
10863 	return -ENOENT;  in perf_event_set_bpf_prog()
10877 	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);  in perf_bp_event()
10879 	if (!bp->hw.state && !perf_exclude_event(bp, regs))  in perf_bp_event()
10890 	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);  in perf_addr_filter_new()
10897 	INIT_LIST_HEAD(&filter->entry);  in perf_addr_filter_new()
10898 	list_add_tail(&filter->entry, filters);  in perf_addr_filter_new()
10908 		path_put(&filter->path);  in free_filters_list()
10909 		list_del(&filter->entry);  in free_filters_list()
10927 	if (event->parent)  in perf_addr_filters_splice()
10930 	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
10932 	list_splice_init(&event->addr_filters.list, &list);  in perf_addr_filters_splice()
10934 		list_splice(head, &event->addr_filters.list);  in perf_addr_filters_splice()
10936 	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
10954 		if (!vma->vm_file)  in perf_addr_filter_apply()
10969 	struct task_struct *task = READ_ONCE(event->ctx->task);  in perf_event_addr_filters_apply()
10976 	 * We may observe TASK_TOMBSTONE, which means that the event tear-down  in perf_event_addr_filters_apply()
10982 	if (ifh->nr_file_filters) {  in perf_event_addr_filters_apply()
10990 	raw_spin_lock_irqsave(&ifh->lock, flags);  in perf_event_addr_filters_apply()
10991 	list_for_each_entry(filter, &ifh->list, entry) {  in perf_event_addr_filters_apply()
10992 		if (filter->path.dentry) {  in perf_event_addr_filters_apply()
10997 			event->addr_filter_ranges[count].start = 0;  in perf_event_addr_filters_apply()
10998 			event->addr_filter_ranges[count].size = 0;  in perf_event_addr_filters_apply()
11000 			perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);  in perf_event_addr_filters_apply()
11002 			event->addr_filter_ranges[count].start = filter->offset;  in perf_event_addr_filters_apply()
11003 			event->addr_filter_ranges[count].size  = filter->size;  in perf_event_addr_filters_apply()
11009 	event->addr_filters_gen++;  in perf_event_addr_filters_apply()
11010 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in perf_event_addr_filters_apply()
11012 	if (ifh->nr_file_filters) {  in perf_event_addr_filters_apply()
11031  *  * "filter": limit the trace to this region
11042 	IF_ACT_NONE = -1,
11081 	int ret = -EINVAL;  in perf_event_parse_addr_filter()
11085 		return -ENOMEM;  in perf_event_parse_addr_filter()
11093 		ret = -EINVAL;  in perf_event_parse_addr_filter()
11113 			filter->action = actions[token];  in perf_event_parse_addr_filter()
11128 			ret = kstrtoul(args[0].from, 0, &filter->offset);  in perf_event_parse_addr_filter()
11134 				ret = kstrtoul(args[1].from, 0, &filter->size);  in perf_event_parse_addr_filter()
11145 					ret = -ENOMEM;  in perf_event_parse_addr_filter()
11163 			ret = -EINVAL;  in perf_event_parse_addr_filter()
11166 			 * ACTION "filter" must have a non-zero length region  in perf_event_parse_addr_filter()
11169 			if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&  in perf_event_parse_addr_filter()
11170 			    !filter->size)  in perf_event_parse_addr_filter()
11178 				 * For now, we only support file-based filters  in perf_event_parse_addr_filter()
11179 				 * in per-task events; doing so for CPU-wide  in perf_event_parse_addr_filter()
11185 				ret = -EOPNOTSUPP;  in perf_event_parse_addr_filter()
11186 				if (!event->ctx->task)  in perf_event_parse_addr_filter()
11191 						&filter->path);  in perf_event_parse_addr_filter()
11195 				ret = -EINVAL;  in perf_event_parse_addr_filter()
11196 				if (!filter->path.dentry ||  in perf_event_parse_addr_filter()
11197 				    !S_ISREG(d_inode(filter->path.dentry)  in perf_event_parse_addr_filter()
11198 					     ->i_mode))  in perf_event_parse_addr_filter()
11201 				event->addr_filters.nr_file_filters++;  in perf_event_parse_addr_filter()
11239 	lockdep_assert_held(&event->ctx->mutex);  in perf_event_set_addr_filter()
11241 	if (WARN_ON_ONCE(event->parent))  in perf_event_set_addr_filter()
11242 		return -EINVAL;  in perf_event_set_addr_filter()
11248 	ret = event->pmu->addr_filters_validate(&filters);  in perf_event_set_addr_filter()
11264 	event->addr_filters.nr_file_filters = 0;  in perf_event_set_addr_filter()
11271 	int ret = -EINVAL;  in perf_event_set_filter()
11280 		struct perf_event_context *ctx = event->ctx;  in perf_event_set_filter()
11285 		 * the tracepoint muck will deadlock against ctx->mutex, but  in perf_event_set_filter()
11287 		 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we  in perf_event_set_filter()
11293 		mutex_unlock(&ctx->mutex);  in perf_event_set_filter()
11294 		ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);  in perf_event_set_filter()
11295 		mutex_lock(&ctx->mutex);  in perf_event_set_filter()
11319 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_swevent_hrtimer()
11322 	event->pmu->read(event);  in perf_swevent_hrtimer()
11324 	perf_sample_data_init(&data, 0, event->hw.last_period);  in perf_swevent_hrtimer()
11328 		if (!(event->attr.exclude_idle && is_idle_task(current)))  in perf_swevent_hrtimer()
11333 	period = max_t(u64, 10000, event->hw.sample_period);  in perf_swevent_hrtimer()
11341 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_start_hrtimer()
11347 	period = local64_read(&hwc->period_left);  in perf_swevent_start_hrtimer()
11352 		local64_set(&hwc->period_left, 0);  in perf_swevent_start_hrtimer()
11354 		period = max_t(u64, 10000, hwc->sample_period);  in perf_swevent_start_hrtimer()
11356 	hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),  in perf_swevent_start_hrtimer()
11362 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_cancel_hrtimer()
11365 		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);  in perf_swevent_cancel_hrtimer()
11366 		local64_set(&hwc->period_left, ktime_to_ns(remaining));  in perf_swevent_cancel_hrtimer()
11368 		hrtimer_cancel(&hwc->hrtimer);  in perf_swevent_cancel_hrtimer()
11374 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_init_hrtimer()
11379 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);  in perf_swevent_init_hrtimer()
11380 	hwc->hrtimer.function = perf_swevent_hrtimer;  in perf_swevent_init_hrtimer()
11383 	 * Since hrtimers have a fixed rate, we can do a static freq->period  in perf_swevent_init_hrtimer()
11386 	if (event->attr.freq) {  in perf_swevent_init_hrtimer()
11387 		long freq = event->attr.sample_freq;  in perf_swevent_init_hrtimer()
11389 		event->attr.sample_period = NSEC_PER_SEC / freq;  in perf_swevent_init_hrtimer()
11390 		hwc->sample_period = event->attr.sample_period;  in perf_swevent_init_hrtimer()
11391 		local64_set(&hwc->period_left, hwc->sample_period);  in perf_swevent_init_hrtimer()
11392 		hwc->last_period = hwc->sample_period;  in perf_swevent_init_hrtimer()
11393 		event->attr.freq = 0;  in perf_swevent_init_hrtimer()
11407 	prev = local64_xchg(&event->hw.prev_count, now);  in cpu_clock_event_update()
11408 	local64_add(now - prev, &event->count);  in cpu_clock_event_update()
11413 	local64_set(&event->hw.prev_count, local_clock());  in cpu_clock_event_start()
11444 	if (event->attr.type != perf_cpu_clock.type)  in cpu_clock_event_init()
11445 		return -ENOENT;  in cpu_clock_event_init()
11447 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)  in cpu_clock_event_init()
11448 		return -ENOENT;  in cpu_clock_event_init()
11454 		return -EOPNOTSUPP;  in cpu_clock_event_init()
11484 	prev = local64_xchg(&event->hw.prev_count, now);  in task_clock_event_update()
11485 	delta = now - prev;  in task_clock_event_update()
11486 	local64_add(delta, &event->count);  in task_clock_event_update()
11491 	local64_set(&event->hw.prev_count, event->ctx->time);  in task_clock_event_start()
11498 	task_clock_event_update(event, event->ctx->time);  in task_clock_event_stop()
11518 	u64 delta = now - event->ctx->timestamp;  in task_clock_event_read()
11519 	u64 time = event->ctx->time + delta;  in task_clock_event_read()
11526 	if (event->attr.type != perf_task_clock.type)  in task_clock_event_init()
11527 		return -ENOENT;  in task_clock_event_init()
11529 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)  in task_clock_event_init()
11530 		return -ENOENT;  in task_clock_event_init()
11536 		return -EOPNOTSUPP;  in task_clock_event_init()
11619 	free_percpu(pmu->cpu_pmu_context);  in free_pmu_context()
11631 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters);  in nr_addr_filters_show()
11642 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->type);  in type_show()
11653 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->hrtimer_interval_ms);  in perf_event_mux_interval_ms_show()
11671 		return -EINVAL;  in perf_event_mux_interval_ms_store()
11674 	if (timer == pmu->hrtimer_interval_ms)  in perf_event_mux_interval_ms_store()
11678 	pmu->hrtimer_interval_ms = timer;  in perf_event_mux_interval_ms_store()
11684 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);  in perf_event_mux_interval_ms_store()
11685 		cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);  in perf_event_mux_interval_ms_store()
11736 	struct cpumask *mask = perf_scope_cpumask(pmu->scope);  in cpumask_show()
11758 	if (n == 2 && !pmu->nr_addr_filters)  in pmu_dev_is_visible()
11762 	if (n == 3 && pmu->scope == PERF_PMU_SCOPE_NONE)  in pmu_dev_is_visible()
11765 	return a->mode;  in pmu_dev_is_visible()
11791 	int ret = -ENOMEM;  in pmu_dev_alloc()
11793 	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);  in pmu_dev_alloc()
11794 	if (!pmu->dev)  in pmu_dev_alloc()
11797 	pmu->dev->groups = pmu->attr_groups;  in pmu_dev_alloc()
11798 	device_initialize(pmu->dev);  in pmu_dev_alloc()
11800 	dev_set_drvdata(pmu->dev, pmu);  in pmu_dev_alloc()
11801 	pmu->dev->bus = &pmu_bus;  in pmu_dev_alloc()
11802 	pmu->dev->parent = pmu->parent;  in pmu_dev_alloc()
11803 	pmu->dev->release = pmu_dev_release;  in pmu_dev_alloc()
11805 	ret = dev_set_name(pmu->dev, "%s", pmu->name);  in pmu_dev_alloc()
11809 	ret = device_add(pmu->dev);  in pmu_dev_alloc()
11813 	if (pmu->attr_update) {  in pmu_dev_alloc()
11814 		ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);  in pmu_dev_alloc()
11823 	device_del(pmu->dev);  in pmu_dev_alloc()
11826 	put_device(pmu->dev);  in pmu_dev_alloc()
11853 	ret = -ENOMEM;  in perf_pmu_register()
11854 	pmu->pmu_disable_count = alloc_percpu(int);  in perf_pmu_register()
11855 	if (!pmu->pmu_disable_count)  in perf_pmu_register()
11858 	pmu->type = -1;  in perf_pmu_register()
11860 		ret = -EINVAL;  in perf_pmu_register()
11864 …if (WARN_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE, "Can not register a pmu with an invalid scope.\n")…  in perf_pmu_register()
11865 		ret = -EINVAL;  in perf_pmu_register()
11869 	pmu->name = name;  in perf_pmu_register()
11881 	pmu->type = type;  in perf_pmu_register()
11882 	atomic_set(&pmu->exclusive_cnt, 0);  in perf_pmu_register()
11884 	if (pmu_bus_running && !pmu->dev) {  in perf_pmu_register()
11890 	ret = -ENOMEM;  in perf_pmu_register()
11891 	pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context);  in perf_pmu_register()
11892 	if (!pmu->cpu_pmu_context)  in perf_pmu_register()
11898 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);  in perf_pmu_register()
11899 		__perf_init_event_pmu_context(&cpc->epc, pmu);  in perf_pmu_register()
11903 	if (!pmu->start_txn) {  in perf_pmu_register()
11904 		if (pmu->pmu_enable) {  in perf_pmu_register()
11910 			pmu->start_txn  = perf_pmu_start_txn;  in perf_pmu_register()
11911 			pmu->commit_txn = perf_pmu_commit_txn;  in perf_pmu_register()
11912 			pmu->cancel_txn = perf_pmu_cancel_txn;  in perf_pmu_register()
11914 			pmu->start_txn  = perf_pmu_nop_txn;  in perf_pmu_register()
11915 			pmu->commit_txn = perf_pmu_nop_int;  in perf_pmu_register()
11916 			pmu->cancel_txn = perf_pmu_nop_void;  in perf_pmu_register()
11920 	if (!pmu->pmu_enable) {  in perf_pmu_register()
11921 		pmu->pmu_enable  = perf_pmu_nop_void;  in perf_pmu_register()
11922 		pmu->pmu_disable = perf_pmu_nop_void;  in perf_pmu_register()
11925 	if (!pmu->check_period)  in perf_pmu_register()
11926 		pmu->check_period = perf_event_nop_int;  in perf_pmu_register()
11928 	if (!pmu->event_idx)  in perf_pmu_register()
11929 		pmu->event_idx = perf_event_idx_default;  in perf_pmu_register()
11934 	if (!idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu))  in perf_pmu_register()
11936 	list_add_rcu(&pmu->entry, &pmus);  in perf_pmu_register()
11945 	free_percpu(pmu->cpu_pmu_context);  in perf_pmu_register()
11948 	if (pmu->dev && pmu->dev != PMU_NULL_DEV) {  in perf_pmu_register()
11949 		device_del(pmu->dev);  in perf_pmu_register()
11950 		put_device(pmu->dev);  in perf_pmu_register()
11954 	idr_remove(&pmu_idr, pmu->type);  in perf_pmu_register()
11957 	free_percpu(pmu->pmu_disable_count);  in perf_pmu_register()
11965 	list_del_rcu(&pmu->entry);  in perf_pmu_unregister()
11966 	idr_remove(&pmu_idr, pmu->type);  in perf_pmu_unregister()
11976 	free_percpu(pmu->pmu_disable_count);  in perf_pmu_unregister()
11977 	if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) {  in perf_pmu_unregister()
11978 		if (pmu->nr_addr_filters)  in perf_pmu_unregister()
11979 			device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);  in perf_pmu_unregister()
11980 		device_del(pmu->dev);  in perf_pmu_unregister()
11981 		put_device(pmu->dev);  in perf_pmu_unregister()
11989 	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||  in has_extended_regs()
11990 	       (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);  in has_extended_regs()
11998 	if (!try_module_get(pmu->module))  in perf_try_init_event()
11999 		return -ENODEV;  in perf_try_init_event()
12002 	 * A number of pmu->event_init() methods iterate the sibling_list to,  in perf_try_init_event()
12004 	 * if this is a sibling event, acquire the ctx->mutex to protect  in perf_try_init_event()
12007 	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {  in perf_try_init_event()
12009 		 * This ctx->mutex can nest when we're called through  in perf_try_init_event()
12012 		ctx = perf_event_ctx_lock_nested(event->group_leader,  in perf_try_init_event()
12017 	event->pmu = pmu;  in perf_try_init_event()
12018 	ret = pmu->event_init(event);  in perf_try_init_event()
12021 		perf_event_ctx_unlock(event->group_leader, ctx);  in perf_try_init_event()
12024 		if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&  in perf_try_init_event()
12026 			ret = -EOPNOTSUPP;  in perf_try_init_event()
12028 		if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&  in perf_try_init_event()
12030 			ret = -EINVAL;  in perf_try_init_event()
12032 		if (pmu->scope != PERF_PMU_SCOPE_NONE && event->cpu >= 0) {  in perf_try_init_event()
12033 			const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(pmu->scope, event->cpu);  in perf_try_init_event()
12034 			struct cpumask *pmu_cpumask = perf_scope_cpumask(pmu->scope);  in perf_try_init_event()
12040 					ret = -ENODEV;  in perf_try_init_event()
12042 					event->event_caps |= PERF_EV_CAP_READ_SCOPE;  in perf_try_init_event()
12044 				ret = -ENODEV;  in perf_try_init_event()
12048 		if (ret && event->destroy)  in perf_try_init_event()
12049 			event->destroy(event);  in perf_try_init_event()
12053 		event->pmu = NULL;  in perf_try_init_event()
12054 		module_put(pmu->module);  in perf_try_init_event()
12069 	 * Save original type before calling pmu->event_init() since certain  in perf_init_event()
12070 	 * pmus overwrites event->attr.type to forward event to another pmu.  in perf_init_event()
12072 	event->orig_type = event->attr.type;  in perf_init_event()
12075 	if (event->parent && event->parent->pmu) {  in perf_init_event()
12076 		pmu = event->parent->pmu;  in perf_init_event()
12086 	type = event->attr.type;  in perf_init_event()
12088 		type = event->attr.config >> PERF_PMU_TYPE_SHIFT;  in perf_init_event()
12093 			event->attr.config &= PERF_HW_EVENT_MASK;  in perf_init_event()
12102 		if (event->attr.type != type && type != PERF_TYPE_RAW &&  in perf_init_event()
12103 		    !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))  in perf_init_event()
12107 		if (ret == -ENOENT && event->attr.type != type && !extended_type) {  in perf_init_event()
12108 			type = event->attr.type;  in perf_init_event()
12123 		if (ret != -ENOENT) {  in perf_init_event()
12129 	pmu = ERR_PTR(-ENOENT);  in perf_init_event()
12138 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in attach_sb_event()
12140 	raw_spin_lock(&pel->lock);  in attach_sb_event()
12141 	list_add_rcu(&event->sb_list, &pel->list);  in attach_sb_event()
12142 	raw_spin_unlock(&pel->lock);  in attach_sb_event()
12146  * We keep a list of all !task (and therefore per-cpu) events
12147  * that need to receive side-band records.
12149  * This avoids having to scan all the various PMU per-cpu contexts
12183 	if (event->parent)  in account_event()
12186 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))  in account_event()
12188 	if (event->attr.mmap || event->attr.mmap_data)  in account_event()
12190 	if (event->attr.build_id)  in account_event()
12192 	if (event->attr.comm)  in account_event()
12194 	if (event->attr.namespaces)  in account_event()
12196 	if (event->attr.cgroup)  in account_event()
12198 	if (event->attr.task)  in account_event()
12200 	if (event->attr.freq)  in account_event()
12202 	if (event->attr.context_switch) {  in account_event()
12210 	if (event->attr.ksymbol)  in account_event()
12212 	if (event->attr.bpf_event)  in account_event()
12214 	if (event->attr.text_poke)  in account_event()
12238 		 * increments to by-pass the mutex.  in account_event()
12262 	long err = -EINVAL;  in perf_event_alloc()
12266 		if (!task || cpu != -1)  in perf_event_alloc()
12267 			return ERR_PTR(-EINVAL);  in perf_event_alloc()
12269 	if (attr->sigtrap && !task) {  in perf_event_alloc()
12271 		return ERR_PTR(-EINVAL);  in perf_event_alloc()
12274 	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;  in perf_event_alloc()
12278 		return ERR_PTR(-ENOMEM);  in perf_event_alloc()
12287 	mutex_init(&event->child_mutex);  in perf_event_alloc()
12288 	INIT_LIST_HEAD(&event->child_list);  in perf_event_alloc()
12290 	INIT_LIST_HEAD(&event->event_entry);  in perf_event_alloc()
12291 	INIT_LIST_HEAD(&event->sibling_list);  in perf_event_alloc()
12292 	INIT_LIST_HEAD(&event->active_list);  in perf_event_alloc()
12294 	INIT_LIST_HEAD(&event->rb_entry);  in perf_event_alloc()
12295 	INIT_LIST_HEAD(&event->active_entry);  in perf_event_alloc()
12296 	INIT_LIST_HEAD(&event->addr_filters.list);  in perf_event_alloc()
12297 	INIT_HLIST_NODE(&event->hlist_entry);  in perf_event_alloc()
12300 	init_waitqueue_head(&event->waitq);  in perf_event_alloc()
12301 	init_irq_work(&event->pending_irq, perf_pending_irq);  in perf_event_alloc()
12302 	event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);  in perf_event_alloc()
12303 	init_task_work(&event->pending_task, perf_pending_task);  in perf_event_alloc()
12305 	mutex_init(&event->mmap_mutex);  in perf_event_alloc()
12306 	raw_spin_lock_init(&event->addr_filters.lock);  in perf_event_alloc()
12308 	atomic_long_set(&event->refcount, 1);  in perf_event_alloc()
12309 	event->cpu		= cpu;  in perf_event_alloc()
12310 	event->attr		= *attr;  in perf_event_alloc()
12311 	event->group_leader	= group_leader;  in perf_event_alloc()
12312 	event->pmu		= NULL;  in perf_event_alloc()
12313 	event->oncpu		= -1;  in perf_event_alloc()
12315 	event->parent		= parent_event;  in perf_event_alloc()
12317 	event->ns		= get_pid_ns(task_active_pid_ns(current));  in perf_event_alloc()
12318 	event->id		= atomic64_inc_return(&perf_event_id);  in perf_event_alloc()
12320 	event->state		= PERF_EVENT_STATE_INACTIVE;  in perf_event_alloc()
12323 		event->event_caps = parent_event->event_caps;  in perf_event_alloc()
12326 		event->attach_state = PERF_ATTACH_TASK;  in perf_event_alloc()
12332 		event->hw.target = get_task_struct(task);  in perf_event_alloc()
12335 	event->clock = &local_clock;  in perf_event_alloc()
12337 		event->clock = parent_event->clock;  in perf_event_alloc()
12340 		overflow_handler = parent_event->overflow_handler;  in perf_event_alloc()
12341 		context = parent_event->overflow_handler_context;  in perf_event_alloc()
12343 		if (parent_event->prog) {  in perf_event_alloc()
12344 			struct bpf_prog *prog = parent_event->prog;  in perf_event_alloc()
12347 			event->prog = prog;  in perf_event_alloc()
12353 		event->overflow_handler	= overflow_handler;  in perf_event_alloc()
12354 		event->overflow_handler_context = context;  in perf_event_alloc()
12356 		event->overflow_handler = perf_event_output_backward;  in perf_event_alloc()
12357 		event->overflow_handler_context = NULL;  in perf_event_alloc()
12359 		event->overflow_handler = perf_event_output_forward;  in perf_event_alloc()
12360 		event->overflow_handler_context = NULL;  in perf_event_alloc()
12367 	hwc = &event->hw;  in perf_event_alloc()
12368 	hwc->sample_period = attr->sample_period;  in perf_event_alloc()
12369 	if (attr->freq && attr->sample_freq)  in perf_event_alloc()
12370 		hwc->sample_period = 1;  in perf_event_alloc()
12371 	hwc->last_period = hwc->sample_period;  in perf_event_alloc()
12373 	local64_set(&hwc->period_left, hwc->sample_period);  in perf_event_alloc()
12378 	 * collect per-thread samples.  in perf_event_alloc()
12381 	if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID))  in perf_event_alloc()
12385 		event->attr.branch_sample_type = 0;  in perf_event_alloc()
12394 	 * Disallow uncore-task events. Similarly, disallow uncore-cgroup  in perf_event_alloc()
12398 	if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1)) {  in perf_event_alloc()
12399 		err = -EINVAL;  in perf_event_alloc()
12403 	if (event->attr.aux_output &&  in perf_event_alloc()
12404 	    (!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT) ||  in perf_event_alloc()
12405 	     event->attr.aux_pause || event->attr.aux_resume)) {  in perf_event_alloc()
12406 		err = -EOPNOTSUPP;  in perf_event_alloc()
12410 	if (event->attr.aux_pause && event->attr.aux_resume) {  in perf_event_alloc()
12411 		err = -EINVAL;  in perf_event_alloc()
12415 	if (event->attr.aux_start_paused) {  in perf_event_alloc()
12416 		if (!(pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE)) {  in perf_event_alloc()
12417 			err = -EOPNOTSUPP;  in perf_event_alloc()
12420 		event->hw.aux_paused = 1;  in perf_event_alloc()
12423 	if (cgroup_fd != -1) {  in perf_event_alloc()
12434 		event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,  in perf_event_alloc()
12437 		if (!event->addr_filter_ranges) {  in perf_event_alloc()
12438 			err = -ENOMEM;  in perf_event_alloc()
12446 		if (event->parent) {  in perf_event_alloc()
12449 			raw_spin_lock_irq(&ifh->lock);  in perf_event_alloc()
12450 			memcpy(event->addr_filter_ranges,  in perf_event_alloc()
12451 			       event->parent->addr_filter_ranges,  in perf_event_alloc()
12452 			       pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));  in perf_event_alloc()
12453 			raw_spin_unlock_irq(&ifh->lock);  in perf_event_alloc()
12457 		event->addr_filters_gen = 1;  in perf_event_alloc()
12460 	if (!event->parent) {  in perf_event_alloc()
12461 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {  in perf_event_alloc()
12462 			err = get_callchain_buffers(attr->sample_max_stack);  in perf_event_alloc()
12465 			event->attach_state |= PERF_ATTACH_CALLCHAIN;  in perf_event_alloc()
12492 	ret = get_user(size, &uattr->size);  in perf_copy_attr()
12504 		if (ret == -E2BIG)  in perf_copy_attr()
12509 	attr->size = size;  in perf_copy_attr()
12511 	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)  in perf_copy_attr()
12512 		return -EINVAL;  in perf_copy_attr()
12514 	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))  in perf_copy_attr()
12515 		return -EINVAL;  in perf_copy_attr()
12517 	if (attr->read_format & ~(PERF_FORMAT_MAX-1))  in perf_copy_attr()
12518 		return -EINVAL;  in perf_copy_attr()
12520 	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {  in perf_copy_attr()
12521 		u64 mask = attr->branch_sample_type;  in perf_copy_attr()
12524 		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))  in perf_copy_attr()
12525 			return -EINVAL;  in perf_copy_attr()
12529 			return -EINVAL;  in perf_copy_attr()
12535 			if (!attr->exclude_kernel)  in perf_copy_attr()
12538 			if (!attr->exclude_user)  in perf_copy_attr()
12541 			if (!attr->exclude_hv)  in perf_copy_attr()
12546 			attr->branch_sample_type = mask;  in perf_copy_attr()
12556 	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {  in perf_copy_attr()
12557 		ret = perf_reg_validate(attr->sample_regs_user);  in perf_copy_attr()
12562 	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {  in perf_copy_attr()
12564 			return -ENOSYS;  in perf_copy_attr()
12571 		if (attr->sample_stack_user >= USHRT_MAX)  in perf_copy_attr()
12572 			return -EINVAL;  in perf_copy_attr()
12573 		else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))  in perf_copy_attr()
12574 			return -EINVAL;  in perf_copy_attr()
12577 	if (!attr->sample_max_stack)  in perf_copy_attr()
12578 		attr->sample_max_stack = sysctl_perf_event_max_stack;  in perf_copy_attr()
12580 	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)  in perf_copy_attr()
12581 		ret = perf_reg_validate(attr->sample_regs_intr);  in perf_copy_attr()
12584 	if (attr->sample_type & PERF_SAMPLE_CGROUP)  in perf_copy_attr()
12585 		return -EINVAL;  in perf_copy_attr()
12587 	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&  in perf_copy_attr()
12588 	    (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))  in perf_copy_attr()
12589 		return -EINVAL;  in perf_copy_attr()
12591 	if (!attr->inherit && attr->inherit_thread)  in perf_copy_attr()
12592 		return -EINVAL;  in perf_copy_attr()
12594 	if (attr->remove_on_exec && attr->enable_on_exec)  in perf_copy_attr()
12595 		return -EINVAL;  in perf_copy_attr()
12597 	if (attr->sigtrap && !attr->remove_on_exec)  in perf_copy_attr()
12598 		return -EINVAL;  in perf_copy_attr()
12604 	put_user(sizeof(*attr), &uattr->size);  in perf_copy_attr()
12605 	ret = -E2BIG;  in perf_copy_attr()
12622 	int ret = -EINVAL;  in perf_event_set_output()
12625 		mutex_lock(&event->mmap_mutex);  in perf_event_set_output()
12634 	 * Don't allow cross-cpu buffers  in perf_event_set_output()
12636 	if (output_event->cpu != event->cpu)  in perf_event_set_output()
12640 	 * If its not a per-cpu rb, it must be the same task.  in perf_event_set_output()
12642 	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)  in perf_event_set_output()
12648 	if (output_event->clock != event->clock)  in perf_event_set_output()
12662 	    event->pmu != output_event->pmu)  in perf_event_set_output()
12667 	 * output_event is already on rb->event_list, and the list iteration  in perf_event_set_output()
12670 	 * observe !rb->mmap_count.  in perf_event_set_output()
12672 	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);  in perf_event_set_output()
12675 	if (atomic_read(&event->mmap_count))  in perf_event_set_output()
12685 		if (!atomic_read(&rb->mmap_count)) {  in perf_event_set_output()
12695 	mutex_unlock(&event->mmap_mutex);  in perf_event_set_output()
12697 		mutex_unlock(&output_event->mmap_mutex);  in perf_event_set_output()
12709 		event->clock = &ktime_get_mono_fast_ns;  in perf_event_set_clock()
12714 		event->clock = &ktime_get_raw_fast_ns;  in perf_event_set_clock()
12719 		event->clock = &ktime_get_real_ns;  in perf_event_set_clock()
12723 		event->clock = &ktime_get_boottime_ns;  in perf_event_set_clock()
12727 		event->clock = &ktime_get_clocktai_ns;  in perf_event_set_clock()
12731 		return -EINVAL;  in perf_event_set_clock()
12734 	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))  in perf_event_set_clock()
12735 		return -EINVAL;  in perf_event_set_clock()
12746 	if (attr->sigtrap) {  in perf_check_permission()
12752 		is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);  in perf_check_permission()
12772  * sys_perf_event_open - open a performance event, associate it to a task/cpu
12796 	int cgroup_fd = -1;  in SYSCALL_DEFINE5()
12800 		return -EINVAL;  in SYSCALL_DEFINE5()
12819 			return -EACCES;  in SYSCALL_DEFINE5()
12824 			return -EINVAL;  in SYSCALL_DEFINE5()
12827 			return -EINVAL;  in SYSCALL_DEFINE5()
12850 	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))  in SYSCALL_DEFINE5()
12851 		return -EINVAL;  in SYSCALL_DEFINE5()
12860 	CLASS(fd, group)(group_fd);     // group_fd == -1 => empty  in SYSCALL_DEFINE5()
12861 	if (group_fd != -1) {  in SYSCALL_DEFINE5()
12863 			err = -EBADF;  in SYSCALL_DEFINE5()
12866 		group_leader = fd_file(group)->private_data;  in SYSCALL_DEFINE5()
12873 	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {  in SYSCALL_DEFINE5()
12882 	    group_leader->attr.inherit != attr.inherit) {  in SYSCALL_DEFINE5()
12883 		err = -EINVAL;  in SYSCALL_DEFINE5()
12898 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {  in SYSCALL_DEFINE5()
12899 			err = -EOPNOTSUPP;  in SYSCALL_DEFINE5()
12908 	pmu = event->pmu;  in SYSCALL_DEFINE5()
12916 	if (pmu->task_ctx_nr == perf_sw_context)  in SYSCALL_DEFINE5()
12917 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in SYSCALL_DEFINE5()
12920 		err = down_read_interruptible(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
12930 		err = -EACCES;  in SYSCALL_DEFINE5()
12944 	mutex_lock(&ctx->mutex);  in SYSCALL_DEFINE5()
12946 	if (ctx->task == TASK_TOMBSTONE) {  in SYSCALL_DEFINE5()
12947 		err = -ESRCH;  in SYSCALL_DEFINE5()
12958 		struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);  in SYSCALL_DEFINE5()
12960 		if (!cpuctx->online) {  in SYSCALL_DEFINE5()
12961 			err = -ENODEV;  in SYSCALL_DEFINE5()
12967 		err = -EINVAL;  in SYSCALL_DEFINE5()
12971 		 * becoming part of another group-sibling):  in SYSCALL_DEFINE5()
12973 		if (group_leader->group_leader != group_leader)  in SYSCALL_DEFINE5()
12977 		if (group_leader->clock != event->clock)  in SYSCALL_DEFINE5()
12985 		if (group_leader->cpu != event->cpu)  in SYSCALL_DEFINE5()
12991 		if (group_leader->ctx != ctx)  in SYSCALL_DEFINE5()
13013 			pmu = group_leader->pmu_ctx->pmu;  in SYSCALL_DEFINE5()
13016 			    (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {  in SYSCALL_DEFINE5()
13027 			    group_leader->pmu_ctx->pmu != pmu)  in SYSCALL_DEFINE5()
13040 	event->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
13049 		err = -E2BIG;  in SYSCALL_DEFINE5()
13054 		err = -EINVAL;  in SYSCALL_DEFINE5()
13063 		err = -EBUSY;  in SYSCALL_DEFINE5()
13067 	WARN_ON_ONCE(ctx->parent_ctx);  in SYSCALL_DEFINE5()
13083 		put_pmu_ctx(group_leader->pmu_ctx);  in SYSCALL_DEFINE5()
13087 			put_pmu_ctx(sibling->pmu_ctx);  in SYSCALL_DEFINE5()
13094 		 * (through the sibling list, which is still in-tact), we can  in SYSCALL_DEFINE5()
13097 		 * By installing siblings first we NO-OP because they're not  in SYSCALL_DEFINE5()
13101 			sibling->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
13104 			perf_install_in_context(ctx, sibling, sibling->cpu);  in SYSCALL_DEFINE5()
13112 		group_leader->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
13115 		perf_install_in_context(ctx, group_leader, group_leader->cpu);  in SYSCALL_DEFINE5()
13127 	event->owner = current;  in SYSCALL_DEFINE5()
13129 	perf_install_in_context(ctx, event, event->cpu);  in SYSCALL_DEFINE5()
13132 	mutex_unlock(&ctx->mutex);  in SYSCALL_DEFINE5()
13135 		up_read(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
13139 	mutex_lock(&current->perf_event_mutex);  in SYSCALL_DEFINE5()
13140 	list_add_tail(&event->owner_entry, &current->perf_event_list);  in SYSCALL_DEFINE5()
13141 	mutex_unlock(&current->perf_event_mutex);  in SYSCALL_DEFINE5()
13153 	put_pmu_ctx(event->pmu_ctx);  in SYSCALL_DEFINE5()
13154 	event->pmu_ctx = NULL; /* _free_event() */  in SYSCALL_DEFINE5()
13156 	mutex_unlock(&ctx->mutex);  in SYSCALL_DEFINE5()
13161 		up_read(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
13197 	if (attr->aux_output || attr->aux_action)  in perf_event_create_kernel_counter()
13198 		return ERR_PTR(-EINVAL);  in perf_event_create_kernel_counter()
13201 				 overflow_handler, context, -1);  in perf_event_create_kernel_counter()
13208 	event->owner = TASK_TOMBSTONE;  in perf_event_create_kernel_counter()
13209 	pmu = event->pmu;  in perf_event_create_kernel_counter()
13211 	if (pmu->task_ctx_nr == perf_sw_context)  in perf_event_create_kernel_counter()
13212 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in perf_event_create_kernel_counter()
13223 	WARN_ON_ONCE(ctx->parent_ctx);  in perf_event_create_kernel_counter()
13224 	mutex_lock(&ctx->mutex);  in perf_event_create_kernel_counter()
13225 	if (ctx->task == TASK_TOMBSTONE) {  in perf_event_create_kernel_counter()
13226 		err = -ESRCH;  in perf_event_create_kernel_counter()
13235 	event->pmu_ctx = pmu_ctx;  in perf_event_create_kernel_counter()
13246 		if (!cpuctx->online) {  in perf_event_create_kernel_counter()
13247 			err = -ENODEV;  in perf_event_create_kernel_counter()
13253 		err = -EBUSY;  in perf_event_create_kernel_counter()
13257 	perf_install_in_context(ctx, event, event->cpu);  in perf_event_create_kernel_counter()
13259 	mutex_unlock(&ctx->mutex);  in perf_event_create_kernel_counter()
13265 	event->pmu_ctx = NULL; /* _free_event() */  in perf_event_create_kernel_counter()
13267 	mutex_unlock(&ctx->mutex);  in perf_event_create_kernel_counter()
13286 		put_pmu_ctx(event->pmu_ctx);  in __perf_pmu_remove()
13287 		list_add(&event->migrate_entry, events);  in __perf_pmu_remove()
13291 			put_pmu_ctx(sibling->pmu_ctx);  in __perf_pmu_remove()
13292 			list_add(&sibling->migrate_entry, events);  in __perf_pmu_remove()
13302 	struct perf_event_context *old_ctx = event->ctx;  in __perf_pmu_install_event()
13306 	event->cpu = cpu;  in __perf_pmu_install_event()
13308 	event->pmu_ctx = epc;  in __perf_pmu_install_event()
13310 	if (event->state >= PERF_EVENT_STATE_OFF)  in __perf_pmu_install_event()
13311 		event->state = PERF_EVENT_STATE_INACTIVE;  in __perf_pmu_install_event()
13315 	 * Now that event->ctx is updated and visible, put the old ctx.  in __perf_pmu_install_event()
13326 	 * Re-instate events in 2 passes.  in __perf_pmu_install()
13334 		if (event->group_leader == event)  in __perf_pmu_install()
13337 		list_del(&event->migrate_entry);  in __perf_pmu_install()
13346 		list_del(&event->migrate_entry);  in __perf_pmu_install()
13357 	 * Since per-cpu context is persistent, no need to grab an extra  in perf_pmu_migrate_context()
13360 	src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;  in perf_pmu_migrate_context()
13361 	dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;  in perf_pmu_migrate_context()
13367 	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);  in perf_pmu_migrate_context()
13369 	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->pinned_groups, &events);  in perf_pmu_migrate_context()
13370 	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->flexible_groups, &events);  in perf_pmu_migrate_context()
13374 		 * Wait for the events to quiesce before re-instating them.  in perf_pmu_migrate_context()
13381 	mutex_unlock(&dst_ctx->mutex);  in perf_pmu_migrate_context()
13382 	mutex_unlock(&src_ctx->mutex);  in perf_pmu_migrate_context()
13388 	struct perf_event *parent_event = child_event->parent;  in sync_child_event()
13391 	if (child_event->attr.inherit_stat) {  in sync_child_event()
13392 		struct task_struct *task = child_event->ctx->task;  in sync_child_event()
13403 	atomic64_add(child_val, &parent_event->child_count);  in sync_child_event()
13404 	atomic64_add(child_event->total_time_enabled,  in sync_child_event()
13405 		     &parent_event->child_total_time_enabled);  in sync_child_event()
13406 	atomic64_add(child_event->total_time_running,  in sync_child_event()
13407 		     &parent_event->child_total_time_running);  in sync_child_event()
13413 	struct perf_event *parent_event = event->parent;  in perf_event_exit_event()
13430 		mutex_lock(&parent_event->child_mutex);  in perf_event_exit_event()
13439 		mutex_unlock(&parent_event->child_mutex);  in perf_event_exit_event()
13466 	 * In order to reduce the amount of tricky in ctx tear-down, we hold  in perf_event_exit_task_context()
13475 	mutex_lock(&child_ctx->mutex);  in perf_event_exit_task_context()
13478 	 * In a single ctx::lock section, de-schedule the events and detach the  in perf_event_exit_task_context()
13482 	raw_spin_lock_irq(&child_ctx->lock);  in perf_event_exit_task_context()
13486 	 * Now that the context is inactive, destroy the task <-> ctx relation  in perf_event_exit_task_context()
13489 	RCU_INIT_POINTER(child->perf_event_ctxp, NULL);  in perf_event_exit_task_context()
13491 	WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);  in perf_event_exit_task_context()
13495 	raw_spin_unlock_irq(&child_ctx->lock);  in perf_event_exit_task_context()
13507 	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)  in perf_event_exit_task_context()
13510 	mutex_unlock(&child_ctx->mutex);  in perf_event_exit_task_context()
13525 	mutex_lock(&child->perf_event_mutex);  in perf_event_exit_task()
13526 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,  in perf_event_exit_task()
13528 		list_del_init(&event->owner_entry);  in perf_event_exit_task()
13533 		 * we need to serialize on the owner->perf_event_mutex.  in perf_event_exit_task()
13535 		smp_store_release(&event->owner, NULL);  in perf_event_exit_task()
13537 	mutex_unlock(&child->perf_event_mutex);  in perf_event_exit_task()
13544 	 * child contexts and sets child->perf_event_ctxp[] to NULL.  in perf_event_exit_task()
13553 	struct perf_event *parent = event->parent;  in perf_free_event()
13558 	mutex_lock(&parent->child_mutex);  in perf_free_event()
13559 	list_del_init(&event->child_list);  in perf_free_event()
13560 	mutex_unlock(&parent->child_mutex);  in perf_free_event()
13562 	raw_spin_lock_irq(&ctx->lock);  in perf_free_event()
13565 	raw_spin_unlock_irq(&ctx->lock);  in perf_free_event()
13581 	ctx = rcu_access_pointer(task->perf_event_ctxp);  in perf_event_free_task()
13585 	mutex_lock(&ctx->mutex);  in perf_event_free_task()
13586 	raw_spin_lock_irq(&ctx->lock);  in perf_event_free_task()
13588 	 * Destroy the task <-> ctx relation and mark the context dead.  in perf_event_free_task()
13593 	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);  in perf_event_free_task()
13594 	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);  in perf_event_free_task()
13596 	raw_spin_unlock_irq(&ctx->lock);  in perf_event_free_task()
13599 	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)  in perf_event_free_task()
13602 	mutex_unlock(&ctx->mutex);  in perf_event_free_task()
13613 	 * _free_event()'s put_task_struct(event->hw.target) will be a  in perf_event_free_task()
13614 	 * use-after-free.  in perf_event_free_task()
13618 	wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);  in perf_event_free_task()
13624 	WARN_ON_ONCE(task->perf_event_ctxp);  in perf_event_delayed_put()
13631 		return ERR_PTR(-EBADF);  in perf_event_get()
13633 	if (file->f_op != &perf_fops) {  in perf_event_get()
13635 		return ERR_PTR(-EBADF);  in perf_event_get()
13643 	if (file->f_op != &perf_fops)  in perf_get_event()
13644 		return ERR_PTR(-EINVAL);  in perf_get_event()
13646 	return file->private_data;  in perf_get_event()
13652 		return ERR_PTR(-EINVAL);  in perf_event_attrs()
13654 	return &event->attr;  in perf_event_attrs()
13660 		return -EACCES;  in perf_allow_kernel()
13670  *  - valid pointer on success
13671  *  - NULL for orphaned events
13672  *  - IS_ERR() on error
13682 	enum perf_event_state parent_state = parent_event->state;  in inherit_event()
13693 	if (parent_event->parent)  in inherit_event()
13694 		parent_event = parent_event->parent;  in inherit_event()
13696 	child_event = perf_event_alloc(&parent_event->attr,  in inherit_event()
13697 					   parent_event->cpu,  in inherit_event()
13700 					   NULL, NULL, -1);  in inherit_event()
13704 	pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event);  in inherit_event()
13709 	child_event->pmu_ctx = pmu_ctx;  in inherit_event()
13712 	 * is_orphaned_event() and list_add_tail(&parent_event->child_list)  in inherit_event()
13717 	mutex_lock(&parent_event->child_mutex);  in inherit_event()
13719 	    !atomic_long_inc_not_zero(&parent_event->refcount)) {  in inherit_event()
13720 		mutex_unlock(&parent_event->child_mutex);  in inherit_event()
13734 		child_event->state = PERF_EVENT_STATE_INACTIVE;  in inherit_event()
13736 		child_event->state = PERF_EVENT_STATE_OFF;  in inherit_event()
13738 	if (parent_event->attr.freq) {  in inherit_event()
13739 		u64 sample_period = parent_event->hw.sample_period;  in inherit_event()
13740 		struct hw_perf_event *hwc = &child_event->hw;  in inherit_event()
13742 		hwc->sample_period = sample_period;  in inherit_event()
13743 		hwc->last_period   = sample_period;  in inherit_event()
13745 		local64_set(&hwc->period_left, sample_period);  in inherit_event()
13748 	child_event->ctx = child_ctx;  in inherit_event()
13749 	child_event->overflow_handler = parent_event->overflow_handler;  in inherit_event()
13750 	child_event->overflow_handler_context  in inherit_event()
13751 		= parent_event->overflow_handler_context;  in inherit_event()
13762 	raw_spin_lock_irqsave(&child_ctx->lock, flags);  in inherit_event()
13764 	child_event->attach_state |= PERF_ATTACH_CHILD;  in inherit_event()
13765 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);  in inherit_event()
13770 	list_add_tail(&child_event->child_list, &parent_event->child_list);  in inherit_event()
13771 	mutex_unlock(&parent_event->child_mutex);  in inherit_event()
13783  *  - 0 on success
13784  *  - <0 on error
13811 		if (sub->aux_event == parent_event && child_ctr &&  in inherit_group()
13813 			return -EINVAL;  in inherit_group()
13816 		leader->group_generation = parent_event->group_generation;  in inherit_group()
13821  * Creates the child task context and tries to inherit the event-group.
13828  *  - 0 on success
13829  *  - <0 on error
13840 	if (!event->attr.inherit ||  in inherit_task_group()
13841 	    (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) ||  in inherit_task_group()
13843 	    (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {  in inherit_task_group()
13848 	child_ctx = child->perf_event_ctxp;  in inherit_task_group()
13858 			return -ENOMEM;  in inherit_task_group()
13860 		child->perf_event_ctxp = child_ctx;  in inherit_task_group()
13883 	if (likely(!parent->perf_event_ctxp))  in perf_event_init_context()
13896 	 * it non-NULL earlier, the only reason for it to become NULL  in perf_event_init_context()
13902 	 * Lock the parent list. No need to lock the child - not PID  in perf_event_init_context()
13905 	mutex_lock(&parent_ctx->mutex);  in perf_event_init_context()
13908 	 * We dont have to disable NMIs - we are only looking at  in perf_event_init_context()
13911 	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {  in perf_event_init_context()
13919 	 * We can't hold ctx->lock when iterating the ->flexible_group list due  in perf_event_init_context()
13923 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);  in perf_event_init_context()
13924 	parent_ctx->rotate_disable = 1;  in perf_event_init_context()
13925 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);  in perf_event_init_context()
13927 	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {  in perf_event_init_context()
13934 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);  in perf_event_init_context()
13935 	parent_ctx->rotate_disable = 0;  in perf_event_init_context()
13937 	child_ctx = child->perf_event_ctxp;  in perf_event_init_context()
13945 		 * parent_ctx->lock avoids it from being uncloned.  in perf_event_init_context()
13947 		cloned_ctx = parent_ctx->parent_ctx;  in perf_event_init_context()
13949 			child_ctx->parent_ctx = cloned_ctx;  in perf_event_init_context()
13950 			child_ctx->parent_gen = parent_ctx->parent_gen;  in perf_event_init_context()
13952 			child_ctx->parent_ctx = parent_ctx;  in perf_event_init_context()
13953 			child_ctx->parent_gen = parent_ctx->generation;  in perf_event_init_context()
13955 		get_ctx(child_ctx->parent_ctx);  in perf_event_init_context()
13958 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);  in perf_event_init_context()
13960 	mutex_unlock(&parent_ctx->mutex);  in perf_event_init_context()
13975 	memset(child->perf_recursion, 0, sizeof(child->perf_recursion));  in perf_event_init_task()
13976 	child->perf_event_ctxp = NULL;  in perf_event_init_task()
13977 	mutex_init(&child->perf_event_mutex);  in perf_event_init_task()
13978 	INIT_LIST_HEAD(&child->perf_event_list);  in perf_event_init_task()
13979 	child->perf_ctx_data = NULL;  in perf_event_init_task()
14006 		mutex_init(&swhash->hlist_mutex);  in perf_event_init_all_cpus()
14014 		__perf_event_init_context(&cpuctx->ctx);  in perf_event_init_all_cpus()
14015 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);  in perf_event_init_all_cpus()
14016 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);  in perf_event_init_all_cpus()
14017 		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);  in perf_event_init_all_cpus()
14018 		cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);  in perf_event_init_all_cpus()
14019 		cpuctx->heap = cpuctx->heap_default;  in perf_event_init_all_cpus()
14027 	mutex_lock(&swhash->hlist_mutex);  in perf_swevent_init_cpu()
14028 	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {  in perf_swevent_init_cpu()
14033 		rcu_assign_pointer(swhash->swevent_hlist, hlist);  in perf_swevent_init_cpu()
14035 	mutex_unlock(&swhash->hlist_mutex);  in perf_swevent_init_cpu()
14045 	raw_spin_lock(&ctx->lock);  in __perf_event_exit_context()
14047 	list_for_each_entry(event, &ctx->event_list, event_entry)  in __perf_event_exit_context()
14049 	raw_spin_unlock(&ctx->lock);  in __perf_event_exit_context()
14064 		target[scope] = -1;  in perf_event_clear_cpumask()
14077 		if (pmu->scope == PERF_PMU_SCOPE_NONE ||  in perf_event_clear_cpumask()
14078 		    WARN_ON_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE))  in perf_event_clear_cpumask()
14081 		if (target[pmu->scope] >= 0 && target[pmu->scope] < nr_cpu_ids)  in perf_event_clear_cpumask()
14082 			perf_pmu_migrate_context(pmu, cpu, target[pmu->scope]);  in perf_event_clear_cpumask()
14091 	// XXX simplify cpuctx->online  in perf_event_exit_cpu_context()
14099 	ctx = &cpuctx->ctx;  in perf_event_exit_cpu_context()
14101 	mutex_lock(&ctx->mutex);  in perf_event_exit_cpu_context()
14103 	cpuctx->online = 0;  in perf_event_exit_cpu_context()
14104 	mutex_unlock(&ctx->mutex);  in perf_event_exit_cpu_context()
14159 	ctx = &cpuctx->ctx;  in perf_event_init_cpu()
14161 	mutex_lock(&ctx->mutex);  in perf_event_init_cpu()
14162 	cpuctx->online = 1;  in perf_event_init_cpu()
14163 	mutex_unlock(&ctx->mutex);  in perf_event_init_cpu()
14204 	perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1);  in perf_event_init()
14205 	perf_pmu_register(&perf_task_clock, "task_clock", -1);  in perf_event_init()
14229 	if (pmu_attr->event_str)  in perf_event_sysfs_show()
14230 		return sprintf(page, "%s\n", pmu_attr->event_str);  in perf_event_sysfs_show()
14248 		if (pmu->dev)  in perf_event_sysfs_init()
14252 		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);  in perf_event_sysfs_init()
14272 		return ERR_PTR(-ENOMEM);  in perf_cgroup_css_alloc()
14274 	jc->info = alloc_percpu(struct perf_cgroup_info);  in perf_cgroup_css_alloc()
14275 	if (!jc->info) {  in perf_cgroup_css_alloc()
14277 		return ERR_PTR(-ENOMEM);  in perf_cgroup_css_alloc()
14280 	return &jc->css;  in perf_cgroup_css_alloc()
14287 	free_percpu(jc->info);  in perf_cgroup_css_free()
14293 	perf_event_cgroup(css->cgroup);  in perf_cgroup_css_online()