1 // SPDX-License-Identifier: GPL-2.0-only
3  * intel-bts.c: Intel Processor Trace support
4  * Copyright (c) 2013-2015, Intel Corporation.
25 #include "thread-stack.h"
29 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
30 #include "intel-bts.h"
31 #include "util/synthetic-events.h"
75 	u64			time;  member
110 			color_fprintf(stdout, color, " %"PRIx64" -> %"PRIx64" %s\n",  in intel_bts_dump()
111 				      le64_to_cpu(branch->from),  in intel_bts_dump()
112 				      le64_to_cpu(branch->to),  in intel_bts_dump()
113 				      le64_to_cpu(branch->misc) & 0x10 ?  in intel_bts_dump()
120 		len -= sz;  in intel_bts_dump()
131 static int intel_bts_lost(struct intel_bts *bts, struct perf_sample *sample)  in intel_bts_lost()  argument
137 			     INTEL_BTS_ERR_LOST, sample->cpu, sample->pid,  in intel_bts_lost()
138 			     sample->tid, 0, "Lost trace data", sample->time);  in intel_bts_lost()
140 	err = perf_session__deliver_synth_event(bts->session, &event, NULL);  in intel_bts_lost()
157 	btsq->bts = bts;  in intel_bts_alloc_queue()
158 	btsq->queue_nr = queue_nr;  in intel_bts_alloc_queue()
159 	btsq->pid = -1;  in intel_bts_alloc_queue()
160 	btsq->tid = -1;  in intel_bts_alloc_queue()
161 	btsq->cpu = -1;  in intel_bts_alloc_queue()
170 	struct intel_bts_queue *btsq = queue->priv;  in intel_bts_setup_queue()
172 	if (list_empty(&queue->head))  in intel_bts_setup_queue()
178 			return -ENOMEM;  in intel_bts_setup_queue()
179 		queue->priv = btsq;  in intel_bts_setup_queue()
181 		if (queue->cpu != -1)  in intel_bts_setup_queue()
182 			btsq->cpu = queue->cpu;  in intel_bts_setup_queue()
183 		btsq->tid = queue->tid;  in intel_bts_setup_queue()
186 	if (bts->sampling_mode)  in intel_bts_setup_queue()
189 	if (!btsq->on_heap && !btsq->buffer) {  in intel_bts_setup_queue()
192 		btsq->buffer = auxtrace_buffer__next(queue, NULL);  in intel_bts_setup_queue()
193 		if (!btsq->buffer)  in intel_bts_setup_queue()
196 		ret = auxtrace_heap__add(&bts->heap, queue_nr,  in intel_bts_setup_queue()
197 					 btsq->buffer->reference);  in intel_bts_setup_queue()
200 		btsq->on_heap = true;  in intel_bts_setup_queue()
211 	for (i = 0; i < bts->queues.nr_queues; i++) {  in intel_bts_setup_queues()
212 		ret = intel_bts_setup_queue(bts, &bts->queues.queue_array[i],  in intel_bts_setup_queues()
222 	if (bts->queues.new_data) {  in intel_bts_update_queues()
223 		bts->queues.new_data = false;  in intel_bts_update_queues()
235 		offs = len_a - len_b;  in intel_bts_find_overlap()
240 		len = len_a - offs;  in intel_bts_find_overlap()
254 	if (b->list.prev == &queue->head)  in intel_bts_do_fix_overlap()
256 	a = list_entry(b->list.prev, struct auxtrace_buffer, list);  in intel_bts_do_fix_overlap()
257 	start = intel_bts_find_overlap(a->data, a->size, b->data, b->size);  in intel_bts_do_fix_overlap()
259 		return -EINVAL;  in intel_bts_do_fix_overlap()
260 	b->use_size = b->data + b->size - start;  in intel_bts_do_fix_overlap()
261 	b->use_data = start;  in intel_bts_do_fix_overlap()
267 	return machine__kernel_ip(bts->machine, ip) ?  in intel_bts_cpumode()
276 	struct intel_bts *bts = btsq->bts;  in intel_bts_synth_branch_sample()
278 	struct perf_sample sample = { .ip = 0, };  in intel_bts_synth_branch_sample()  local
280 	if (bts->synth_opts.initial_skip &&  in intel_bts_synth_branch_sample()
281 	    bts->num_events++ <= bts->synth_opts.initial_skip)  in intel_bts_synth_branch_sample()
284 	sample.ip = le64_to_cpu(branch->from);  in intel_bts_synth_branch_sample()
285 	sample.cpumode = intel_bts_cpumode(bts, sample.ip);  in intel_bts_synth_branch_sample()
286 	sample.pid = btsq->pid;  in intel_bts_synth_branch_sample()
287 	sample.tid = btsq->tid;  in intel_bts_synth_branch_sample()
288 	sample.addr = le64_to_cpu(branch->to);  in intel_bts_synth_branch_sample()
289 	sample.id = btsq->bts->branches_id;  in intel_bts_synth_branch_sample()
290 	sample.stream_id = btsq->bts->branches_id;  in intel_bts_synth_branch_sample()
291 	sample.period = 1;  in intel_bts_synth_branch_sample()
292 	sample.cpu = btsq->cpu;  in intel_bts_synth_branch_sample()
293 	sample.flags = btsq->sample_flags;  in intel_bts_synth_branch_sample()
294 	sample.insn_len = btsq->intel_pt_insn.length;  in intel_bts_synth_branch_sample()
295 	memcpy(sample.insn, btsq->intel_pt_insn.buf, INTEL_PT_INSN_BUF_SZ);  in intel_bts_synth_branch_sample()
297 	event.sample.header.type = PERF_RECORD_SAMPLE;  in intel_bts_synth_branch_sample()
298 	event.sample.header.misc = sample.cpumode;  in intel_bts_synth_branch_sample()
299 	event.sample.header.size = sizeof(struct perf_event_header);  in intel_bts_synth_branch_sample()
301 	if (bts->synth_opts.inject) {  in intel_bts_synth_branch_sample()
302 		event.sample.header.size = bts->branches_event_size;  in intel_bts_synth_branch_sample()
304 						    bts->branches_sample_type,  in intel_bts_synth_branch_sample()
305 						    0, &sample);  in intel_bts_synth_branch_sample()
310 	ret = perf_session__deliver_synth_event(bts->session, &event, &sample);  in intel_bts_synth_branch_sample()
320 	struct machine *machine = btsq->bts->machine;  in intel_bts_get_next_insn()
325 	int err = -1;  in intel_bts_get_next_insn()
327 	thread = machine__find_thread(machine, -1, btsq->tid);  in intel_bts_get_next_insn()
329 		return -1;  in intel_bts_get_next_insn()
335 	if (intel_pt_get_insn(buf, len, x86_64, &btsq->intel_pt_insn))  in intel_bts_get_next_insn()
354 	err = perf_session__deliver_synth_event(bts->session, &event, NULL);  in intel_bts_synth_error()
367 	if (!branch->from) {  in intel_bts_get_branch_type()
368 		if (branch->to)  in intel_bts_get_branch_type()
369 			btsq->sample_flags = PERF_IP_FLAG_BRANCH |  in intel_bts_get_branch_type()
372 			btsq->sample_flags = 0;  in intel_bts_get_branch_type()
373 		btsq->intel_pt_insn.length = 0;  in intel_bts_get_branch_type()
374 	} else if (!branch->to) {  in intel_bts_get_branch_type()
375 		btsq->sample_flags = PERF_IP_FLAG_BRANCH |  in intel_bts_get_branch_type()
377 		btsq->intel_pt_insn.length = 0;  in intel_bts_get_branch_type()
379 		err = intel_bts_get_next_insn(btsq, branch->from);  in intel_bts_get_branch_type()
381 			btsq->sample_flags = 0;  in intel_bts_get_branch_type()
382 			btsq->intel_pt_insn.length = 0;  in intel_bts_get_branch_type()
383 			if (!btsq->bts->synth_opts.errors)  in intel_bts_get_branch_type()
385 			err = intel_bts_synth_error(btsq->bts, btsq->cpu,  in intel_bts_get_branch_type()
386 						    btsq->pid, btsq->tid,  in intel_bts_get_branch_type()
387 						    branch->from);  in intel_bts_get_branch_type()
390 		btsq->sample_flags = intel_pt_insn_type(btsq->intel_pt_insn.op);  in intel_bts_get_branch_type()
392 		if (!machine__kernel_ip(btsq->bts->machine, branch->from) &&  in intel_bts_get_branch_type()
393 		    machine__kernel_ip(btsq->bts->machine, branch->to) &&  in intel_bts_get_branch_type()
394 		    btsq->sample_flags != (PERF_IP_FLAG_BRANCH |  in intel_bts_get_branch_type()
397 			btsq->sample_flags = PERF_IP_FLAG_BRANCH |  in intel_bts_get_branch_type()
412 	u32 filter = btsq->bts->branches_filter;  in intel_bts_process_buffer()
415 	if (buffer->use_data) {  in intel_bts_process_buffer()
416 		sz = buffer->use_size;  in intel_bts_process_buffer()
417 		branch = buffer->use_data;  in intel_bts_process_buffer()
419 		sz = buffer->size;  in intel_bts_process_buffer()
420 		branch = buffer->data;  in intel_bts_process_buffer()
423 	if (!btsq->bts->sample_branches)  in intel_bts_process_buffer()
426 	for (; sz > bsz; branch += 1, sz -= bsz) {  in intel_bts_process_buffer()
427 		if (!branch->from && !branch->to)  in intel_bts_process_buffer()
430 		if (btsq->bts->synth_opts.thread_stack)  in intel_bts_process_buffer()
431 			thread_stack__event(thread, btsq->cpu, btsq->sample_flags,  in intel_bts_process_buffer()
432 					    le64_to_cpu(branch->from),  in intel_bts_process_buffer()
433 					    le64_to_cpu(branch->to),  in intel_bts_process_buffer()
434 					    btsq->intel_pt_insn.length,  in intel_bts_process_buffer()
435 					    buffer->buffer_nr + 1, true, 0, 0);  in intel_bts_process_buffer()
436 		if (filter && !(filter & btsq->sample_flags))  in intel_bts_process_buffer()
447 	struct auxtrace_buffer *buffer = btsq->buffer, *old_buffer = buffer;  in intel_bts_process_queue()
452 	if (btsq->done)  in intel_bts_process_queue()
455 	if (btsq->pid == -1) {  in intel_bts_process_queue()
456 		thread = machine__find_thread(btsq->bts->machine, -1,  in intel_bts_process_queue()
457 					      btsq->tid);  in intel_bts_process_queue()
459 			btsq->pid = thread__pid(thread);  in intel_bts_process_queue()
461 		thread = machine__findnew_thread(btsq->bts->machine, btsq->pid,  in intel_bts_process_queue()
462 						 btsq->tid);  in intel_bts_process_queue()
465 	queue = &btsq->bts->queues.queue_array[btsq->queue_nr];  in intel_bts_process_queue()
471 		if (!btsq->bts->sampling_mode)  in intel_bts_process_queue()
472 			btsq->done = 1;  in intel_bts_process_queue()
478 	if (buffer->consecutive) {  in intel_bts_process_queue()
479 		err = -EINVAL;  in intel_bts_process_queue()
483 	if (!buffer->data) {  in intel_bts_process_queue()
484 		int fd = perf_data__fd(btsq->bts->session->data);  in intel_bts_process_queue()
486 		buffer->data = auxtrace_buffer__get_data(buffer, fd);  in intel_bts_process_queue()
487 		if (!buffer->data) {  in intel_bts_process_queue()
488 			err = -ENOMEM;  in intel_bts_process_queue()
493 	if (btsq->bts->snapshot_mode && !buffer->consecutive &&  in intel_bts_process_queue()
495 		err = -ENOMEM;  in intel_bts_process_queue()
499 	if (!btsq->bts->synth_opts.callchain &&  in intel_bts_process_queue()
500 	    !btsq->bts->synth_opts.thread_stack && thread &&  in intel_bts_process_queue()
501 	    (!old_buffer || btsq->bts->sampling_mode ||  in intel_bts_process_queue()
502 	     (btsq->bts->snapshot_mode && !buffer->consecutive)))  in intel_bts_process_queue()
503 		thread_stack__set_trace_nr(thread, btsq->cpu, buffer->buffer_nr + 1);  in intel_bts_process_queue()
509 	btsq->buffer = auxtrace_buffer__next(queue, buffer);  in intel_bts_process_queue()
510 	if (btsq->buffer) {  in intel_bts_process_queue()
512 			*timestamp = btsq->buffer->reference;  in intel_bts_process_queue()
514 		if (!btsq->bts->sampling_mode)  in intel_bts_process_queue()
515 			btsq->done = 1;  in intel_bts_process_queue()
539 	struct auxtrace_queues *queues = &bts->queues;  in intel_bts_process_tid_exit()
542 	for (i = 0; i < queues->nr_queues; i++) {  in intel_bts_process_tid_exit()
543 		struct auxtrace_queue *queue = &bts->queues.queue_array[i];  in intel_bts_process_tid_exit()
544 		struct intel_bts_queue *btsq = queue->priv;  in intel_bts_process_tid_exit()
546 		if (btsq && btsq->tid == tid)  in intel_bts_process_tid_exit()
561 		if (!bts->heap.heap_cnt)  in intel_bts_process_queues()
564 		if (bts->heap.heap_array[0].ordinal > timestamp)  in intel_bts_process_queues()
567 		queue_nr = bts->heap.heap_array[0].queue_nr;  in intel_bts_process_queues()
568 		queue = &bts->queues.queue_array[queue_nr];  in intel_bts_process_queues()
569 		btsq = queue->priv;  in intel_bts_process_queues()
571 		auxtrace_heap__pop(&bts->heap);  in intel_bts_process_queues()
575 			auxtrace_heap__add(&bts->heap, queue_nr, ts);  in intel_bts_process_queues()
580 			ret = auxtrace_heap__add(&bts->heap, queue_nr, ts);  in intel_bts_process_queues()
584 			btsq->on_heap = false;  in intel_bts_process_queues()
593 				   struct perf_sample *sample,  in intel_bts_process_event()  argument
596 	struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,  in intel_bts_process_event()
604 	if (!tool->ordered_events) {  in intel_bts_process_event()
606 		return -EINVAL;  in intel_bts_process_event()
609 	if (sample->time && sample->time != (u64)-1)  in intel_bts_process_event()
610 		timestamp = perf_time_to_tsc(sample->time, &bts->tc);  in intel_bts_process_event()
621 	if (event->header.type == PERF_RECORD_EXIT) {  in intel_bts_process_event()
622 		err = intel_bts_process_tid_exit(bts, event->fork.tid);  in intel_bts_process_event()
627 	if (event->header.type == PERF_RECORD_AUX &&  in intel_bts_process_event()
628 	    (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&  in intel_bts_process_event()
629 	    bts->synth_opts.errors)  in intel_bts_process_event()
630 		err = intel_bts_lost(bts, sample);  in intel_bts_process_event()
639 	struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,  in intel_bts_process_auxtrace_event()
642 	if (bts->sampling_mode)  in intel_bts_process_auxtrace_event()
645 	if (!bts->data_queued) {  in intel_bts_process_auxtrace_event()
648 		int fd = perf_data__fd(session->data);  in intel_bts_process_auxtrace_event()
651 		if (perf_data__is_pipe(session->data)) {  in intel_bts_process_auxtrace_event()
655 			if (data_offset == -1)  in intel_bts_process_auxtrace_event()
656 				return -errno;  in intel_bts_process_auxtrace_event()
659 		err = auxtrace_queues__add_event(&bts->queues, session, event,  in intel_bts_process_auxtrace_event()
667 				intel_bts_dump_event(bts, buffer->data,  in intel_bts_process_auxtrace_event()
668 						     buffer->size);  in intel_bts_process_auxtrace_event()
680 	struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,  in intel_bts_flush()
684 	if (dump_trace || bts->sampling_mode)  in intel_bts_flush()
687 	if (!tool->ordered_events)  in intel_bts_flush()
688 		return -EINVAL;  in intel_bts_flush()
708 	struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,  in intel_bts_free_events()
710 	struct auxtrace_queues *queues = &bts->queues;  in intel_bts_free_events()
713 	for (i = 0; i < queues->nr_queues; i++) {  in intel_bts_free_events()
714 		intel_bts_free_queue(queues->queue_array[i].priv);  in intel_bts_free_events()
715 		queues->queue_array[i].priv = NULL;  in intel_bts_free_events()
722 	struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,  in intel_bts_free()
725 	auxtrace_heap__free(&bts->heap);  in intel_bts_free()
727 	session->auxtrace = NULL;  in intel_bts_free()
734 	struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,  in intel_bts_evsel_is_auxtrace()
737 	return evsel->core.attr.type == bts->pmu_type;  in intel_bts_evsel_is_auxtrace()
743 	struct evlist *evlist = session->evlist;  in intel_bts_synth_events()
751 		if (evsel->core.attr.type == bts->pmu_type && evsel->core.ids) {  in intel_bts_synth_events()
765 	attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;  in intel_bts_synth_events()
770 	attr.exclude_user = evsel->core.attr.exclude_user;  in intel_bts_synth_events()
771 	attr.exclude_kernel = evsel->core.attr.exclude_kernel;  in intel_bts_synth_events()
772 	attr.exclude_hv = evsel->core.attr.exclude_hv;  in intel_bts_synth_events()
773 	attr.exclude_host = evsel->core.attr.exclude_host;  in intel_bts_synth_events()
774 	attr.exclude_guest = evsel->core.attr.exclude_guest;  in intel_bts_synth_events()
775 	attr.sample_id_all = evsel->core.attr.sample_id_all;  in intel_bts_synth_events()
776 	attr.read_format = evsel->core.attr.read_format;  in intel_bts_synth_events()
778 	id = evsel->core.id[0] + 1000000000;  in intel_bts_synth_events()
782 	if (bts->synth_opts.branches) {  in intel_bts_synth_events()
786 		pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",  in intel_bts_synth_events()
794 		bts->sample_branches = true;  in intel_bts_synth_events()
795 		bts->branches_sample_type = attr.sample_type;  in intel_bts_synth_events()
796 		bts->branches_id = id;  in intel_bts_synth_events()
798 		 * We only use sample types from PERF_SAMPLE_MASK so we can use  in intel_bts_synth_events()
801 		bts->branches_event_size = sizeof(struct perf_record_sample) +  in intel_bts_synth_events()
810 	[INTEL_BTS_TIME_SHIFT]		= "  Time Shift         %"PRIu64"\n",
811 	[INTEL_BTS_TIME_MULT]		= "  Time Multiplier    %"PRIu64"\n",
812 	[INTEL_BTS_TIME_ZERO]		= "  Time Zero          %"PRIu64"\n",
813 	[INTEL_BTS_CAP_USER_TIME_ZERO]	= "  Cap Time Zero      %"PRId64"\n",
831 	struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;  in intel_bts_process_auxtrace_info()
836 	if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +  in intel_bts_process_auxtrace_info()
838 		return -EINVAL;  in intel_bts_process_auxtrace_info()
842 		return -ENOMEM;  in intel_bts_process_auxtrace_info()
844 	err = auxtrace_queues__init(&bts->queues);  in intel_bts_process_auxtrace_info()
848 	bts->session = session;  in intel_bts_process_auxtrace_info()
849 	bts->machine = &session->machines.host; /* No kvm support */  in intel_bts_process_auxtrace_info()
850 	bts->auxtrace_type = auxtrace_info->type;  in intel_bts_process_auxtrace_info()
851 	bts->pmu_type = auxtrace_info->priv[INTEL_BTS_PMU_TYPE];  in intel_bts_process_auxtrace_info()
852 	bts->tc.time_shift = auxtrace_info->priv[INTEL_BTS_TIME_SHIFT];  in intel_bts_process_auxtrace_info()
853 	bts->tc.time_mult = auxtrace_info->priv[INTEL_BTS_TIME_MULT];  in intel_bts_process_auxtrace_info()
854 	bts->tc.time_zero = auxtrace_info->priv[INTEL_BTS_TIME_ZERO];  in intel_bts_process_auxtrace_info()
855 	bts->cap_user_time_zero =  in intel_bts_process_auxtrace_info()
856 			auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO];  in intel_bts_process_auxtrace_info()
857 	bts->snapshot_mode = auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE];  in intel_bts_process_auxtrace_info()
859 	bts->sampling_mode = false;  in intel_bts_process_auxtrace_info()
861 	bts->auxtrace.process_event = intel_bts_process_event;  in intel_bts_process_auxtrace_info()
862 	bts->auxtrace.process_auxtrace_event = intel_bts_process_auxtrace_event;  in intel_bts_process_auxtrace_info()
863 	bts->auxtrace.flush_events = intel_bts_flush;  in intel_bts_process_auxtrace_info()
864 	bts->auxtrace.free_events = intel_bts_free_events;  in intel_bts_process_auxtrace_info()
865 	bts->auxtrace.free = intel_bts_free;  in intel_bts_process_auxtrace_info()
866 	bts->auxtrace.evsel_is_auxtrace = intel_bts_evsel_is_auxtrace;  in intel_bts_process_auxtrace_info()
867 	session->auxtrace = &bts->auxtrace;  in intel_bts_process_auxtrace_info()
869 	intel_bts_print_info(&auxtrace_info->priv[0], INTEL_BTS_PMU_TYPE,  in intel_bts_process_auxtrace_info()
875 	if (session->itrace_synth_opts->set) {  in intel_bts_process_auxtrace_info()
876 		bts->synth_opts = *session->itrace_synth_opts;  in intel_bts_process_auxtrace_info()
878 		itrace_synth_opts__set_default(&bts->synth_opts,  in intel_bts_process_auxtrace_info()
879 				session->itrace_synth_opts->default_no_sample);  in intel_bts_process_auxtrace_info()
880 		bts->synth_opts.thread_stack =  in intel_bts_process_auxtrace_info()
881 				session->itrace_synth_opts->thread_stack;  in intel_bts_process_auxtrace_info()
884 	if (bts->synth_opts.calls)  in intel_bts_process_auxtrace_info()
885 		bts->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |  in intel_bts_process_auxtrace_info()
887 	if (bts->synth_opts.returns)  in intel_bts_process_auxtrace_info()
888 		bts->branches_filter |= PERF_IP_FLAG_RETURN |  in intel_bts_process_auxtrace_info()
895 	err = auxtrace_queues__process_index(&bts->queues, session);  in intel_bts_process_auxtrace_info()
899 	if (bts->queues.populated)  in intel_bts_process_auxtrace_info()
900 		bts->data_queued = true;  in intel_bts_process_auxtrace_info()
905 	auxtrace_queues__free(&bts->queues);  in intel_bts_process_auxtrace_info()
906 	session->auxtrace = NULL;  in intel_bts_process_auxtrace_info()