1 // SPDX-License-Identifier: GPL-2.0-only
53 	if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && !perf_allow_kernel(&event->attr))  in set_spe_event_has_cx()
54 		event->hw.flags |= SPE_PMU_HW_FLAGS_CX;  in set_spe_event_has_cx()
59 	return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX);  in get_spe_event_has_cx()
71 	struct pmu				pmu;  member
97 #define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu))
99 /* Convert a free-running index from perf into an SPE buffer offset */
100 #define PERF_IDX2OFF(idx, buf)	((idx) % ((buf)->nr_pages << PAGE_SHIFT))
128 		return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]);  in arm_spe_pmu_cap_get()
132 		return spe_pmu->counter_sz;  in arm_spe_pmu_cap_get()
134 		return spe_pmu->min_period;  in arm_spe_pmu_cap_get()
149 	int cap = (long)ea->var;  in arm_spe_pmu_cap_show()
247 	if (attr == &format_attr_discard.attr && !(spe_pmu->features & SPE_PMU_FEAT_DISCARD))  in arm_spe_pmu_format_attr_is_visible()
250 	if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT))  in arm_spe_pmu_format_attr_is_visible()
253 	return attr->mode;  in arm_spe_pmu_format_attr_is_visible()
267 	return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);  in cpumask_show()
290 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmscr()
297 	if (!attr->exclude_user)  in arm_spe_event_to_pmscr()
300 	if (!attr->exclude_kernel)  in arm_spe_event_to_pmscr()
311 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_event_sanitise_period()
312 	u64 period = event->hw.sample_period;  in arm_spe_event_sanitise_period()
315 	if (period < spe_pmu->min_period)  in arm_spe_event_sanitise_period()
316 		period = spe_pmu->min_period;  in arm_spe_event_sanitise_period()
322 	event->hw.sample_period = period;  in arm_spe_event_sanitise_period()
327 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsirr()
333 	reg |= event->hw.sample_period;  in arm_spe_event_to_pmsirr()
340 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsfcr()
364 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsevfr()
370 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmsnevfr()
376 	struct perf_event_attr *attr = &event->attr;  in arm_spe_event_to_pmslatfr()
383 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_pad_buf()
385 	memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len);  in arm_spe_pmu_pad_buf()
386 	if (!buf->snapshot)  in arm_spe_pmu_pad_buf()
393 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_snapshot_off()
394 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_snapshot_off()
395 	u64 limit = buf->nr_pages * PAGE_SIZE;  in arm_spe_pmu_next_snapshot_off()
410 	if (limit - head < spe_pmu->max_record_sz) {  in arm_spe_pmu_next_snapshot_off()
411 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_snapshot_off()
412 		handle->head = PERF_IDX2OFF(limit, buf);  in arm_spe_pmu_next_snapshot_off()
413 		limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;  in arm_spe_pmu_next_snapshot_off()
421 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in __arm_spe_pmu_next_off()
423 	const u64 bufsize = buf->nr_pages * PAGE_SIZE;  in __arm_spe_pmu_next_off()
433 	 * 2. We used perf_aux_output_skip to consume handle->size bytes  in __arm_spe_pmu_next_off()
439 	 * reduce handle->size to zero and end up reporting truncation.  in __arm_spe_pmu_next_off()
441 	head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
442 	if (!IS_ALIGNED(head, spe_pmu->align)) {  in __arm_spe_pmu_next_off()
443 		unsigned long delta = roundup(head, spe_pmu->align) - head;  in __arm_spe_pmu_next_off()
445 		delta = min(delta, handle->size);  in __arm_spe_pmu_next_off()
447 		head = PERF_IDX2OFF(handle->head, buf);  in __arm_spe_pmu_next_off()
451 	if (!handle->size)  in __arm_spe_pmu_next_off()
455 	tail = PERF_IDX2OFF(handle->head + handle->size, buf);  in __arm_spe_pmu_next_off()
456 	wakeup = PERF_IDX2OFF(handle->wakeup, buf);  in __arm_spe_pmu_next_off()
476 	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)  in __arm_spe_pmu_next_off()
482 	arm_spe_pmu_pad_buf(handle, handle->size);  in __arm_spe_pmu_next_off()
492 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_off()
494 	u64 head = PERF_IDX2OFF(handle->head, buf);  in arm_spe_pmu_next_off()
500 	if (limit && (limit - head < spe_pmu->max_record_sz)) {  in arm_spe_pmu_next_off()
501 		arm_spe_pmu_pad_buf(handle, limit - head);  in arm_spe_pmu_next_off()
514 	if (ATTR_CFG_GET_FLD(&event->attr, discard)) {  in arm_spe_perf_aux_output_begin()
523 		event->hw.state |= PERF_HES_STOPPED;  in arm_spe_perf_aux_output_begin()
532 	limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)  in arm_spe_perf_aux_output_begin()
537 	limit += (u64)buf->base;  in arm_spe_perf_aux_output_begin()
538 	base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_begin()
550 	offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base;  in arm_spe_perf_aux_output_end()
551 	size = offset - PERF_IDX2OFF(handle->head, buf);  in arm_spe_perf_aux_output_end()
553 	if (buf->snapshot)  in arm_spe_perf_aux_output_end()
554 		handle->head = offset;  in arm_spe_perf_aux_output_end()
647 	struct perf_event *event = handle->event;  in arm_spe_pmu_irq_handler()
681 		if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {  in arm_spe_pmu_irq_handler()
714 	struct perf_event_attr *attr = &event->attr;  in arm_spe_pmu_event_init()
715 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_event_init()
717 	/* This is, of course, deeply driver-specific */  in arm_spe_pmu_event_init()
718 	if (attr->type != event->pmu->type)  in arm_spe_pmu_event_init()
719 		return -ENOENT;  in arm_spe_pmu_event_init()
721 	if (event->cpu >= 0 &&  in arm_spe_pmu_event_init()
722 	    !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_event_init()
723 		return -ENOENT;  in arm_spe_pmu_event_init()
725 	if (arm_spe_event_to_pmsevfr(event) & arm_spe_pmsevfr_res0(spe_pmu->pmsver))  in arm_spe_pmu_event_init()
726 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
728 	if (arm_spe_event_to_pmsnevfr(event) & arm_spe_pmsevfr_res0(spe_pmu->pmsver))  in arm_spe_pmu_event_init()
729 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
731 	if (attr->exclude_idle)  in arm_spe_pmu_event_init()
732 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
735 	 * Feedback-directed frequency throttling doesn't work when we  in arm_spe_pmu_event_init()
741 	if (attr->freq)  in arm_spe_pmu_event_init()
742 		return -EINVAL;  in arm_spe_pmu_event_init()
746 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT))  in arm_spe_pmu_event_init()
747 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
750 	    !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT))  in arm_spe_pmu_event_init()
751 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
754 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP))  in arm_spe_pmu_event_init()
755 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
758 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT))  in arm_spe_pmu_event_init()
759 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
761 	if (ATTR_CFG_GET_FLD(&event->attr, discard) &&  in arm_spe_pmu_event_init()
762 	    !(spe_pmu->features & SPE_PMU_FEAT_DISCARD))  in arm_spe_pmu_event_init()
763 		return -EOPNOTSUPP;  in arm_spe_pmu_event_init()
768 		return perf_allow_kernel(&event->attr);  in arm_spe_pmu_event_init()
776 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_start()
777 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_start()
778 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_start()
780 	hwc->state = 0;  in arm_spe_pmu_start()
782 	if (hwc->state)  in arm_spe_pmu_start()
791 	if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) {  in arm_spe_pmu_start()
803 		reg = local64_read(&hwc->period_left);  in arm_spe_pmu_start()
814 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_stop()
815 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_stop()
816 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_stop()
819 	if (hwc->state & PERF_HES_STOPPED)  in arm_spe_pmu_stop()
828 		 * to this buffer, since we might be on the context-switch  in arm_spe_pmu_stop()
846 		local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1));  in arm_spe_pmu_stop()
847 		hwc->state |= PERF_HES_UPTODATE;  in arm_spe_pmu_stop()
850 	hwc->state |= PERF_HES_STOPPED;  in arm_spe_pmu_stop()
856 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_add()
857 	struct hw_perf_event *hwc = &event->hw;  in arm_spe_pmu_add()
858 	int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu;  in arm_spe_pmu_add()
860 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_add()
861 		return -ENOENT;  in arm_spe_pmu_add()
863 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;  in arm_spe_pmu_add()
867 		if (hwc->state & PERF_HES_STOPPED)  in arm_spe_pmu_add()
868 			ret = -EINVAL;  in arm_spe_pmu_add()
886 	int i, cpu = event->cpu;  in arm_spe_pmu_setup_aux()
903 	if (cpu == -1)  in arm_spe_pmu_setup_aux()
917 	buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);  in arm_spe_pmu_setup_aux()
918 	if (!buf->base)  in arm_spe_pmu_setup_aux()
921 	buf->nr_pages	= nr_pages;  in arm_spe_pmu_setup_aux()
922 	buf->snapshot	= snapshot;  in arm_spe_pmu_setup_aux()
938 	vunmap(buf->base);  in arm_spe_pmu_free_aux()
945 	static atomic_t pmu_idx = ATOMIC_INIT(-1);  in arm_spe_pmu_perf_init()
949 	struct device *dev = &spe_pmu->pdev->dev;  in arm_spe_pmu_perf_init()
951 	spe_pmu->pmu = (struct pmu) {  in arm_spe_pmu_perf_init()
953 		.parent		= &spe_pmu->pdev->dev,  in arm_spe_pmu_perf_init()
958 		 * we can support per-task profiling (which is not possible  in arm_spe_pmu_perf_init()
980 		dev_err(dev, "failed to allocate name for pmu %d\n", idx);  in arm_spe_pmu_perf_init()
981 		return -ENOMEM;  in arm_spe_pmu_perf_init()
984 	return perf_pmu_register(&spe_pmu->pmu, name, -1);  in arm_spe_pmu_perf_init()
989 	perf_pmu_unregister(&spe_pmu->pmu);  in arm_spe_pmu_perf_destroy()
997 	struct device *dev = &spe_pmu->pdev->dev;  in __arm_spe_pmu_dev_probe()
1007 	spe_pmu->pmsver = (u16)fld;  in __arm_spe_pmu_dev_probe()
1017 	/* Minimum alignment. If it's out-of-range, then fail the probe */  in __arm_spe_pmu_dev_probe()
1019 	spe_pmu->align = 1 << fld;  in __arm_spe_pmu_dev_probe()
1020 	if (spe_pmu->align > SZ_2K) {  in __arm_spe_pmu_dev_probe()
1029 		spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT;  in __arm_spe_pmu_dev_probe()
1032 		spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT;  in __arm_spe_pmu_dev_probe()
1035 		spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP;  in __arm_spe_pmu_dev_probe()
1038 		spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT;  in __arm_spe_pmu_dev_probe()
1041 		spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST;  in __arm_spe_pmu_dev_probe()
1044 		spe_pmu->features |= SPE_PMU_FEAT_LDS;  in __arm_spe_pmu_dev_probe()
1047 		spe_pmu->features |= SPE_PMU_FEAT_ERND;  in __arm_spe_pmu_dev_probe()
1049 	if (spe_pmu->pmsver >= ID_AA64DFR0_EL1_PMSVer_V1P2)  in __arm_spe_pmu_dev_probe()
1050 		spe_pmu->features |= SPE_PMU_FEAT_DISCARD;  in __arm_spe_pmu_dev_probe()
1052 	/* This field has a spaced out encoding, so just use a look-up */  in __arm_spe_pmu_dev_probe()
1056 		spe_pmu->min_period = 256;  in __arm_spe_pmu_dev_probe()
1059 		spe_pmu->min_period = 512;  in __arm_spe_pmu_dev_probe()
1062 		spe_pmu->min_period = 768;  in __arm_spe_pmu_dev_probe()
1065 		spe_pmu->min_period = 1024;  in __arm_spe_pmu_dev_probe()
1068 		spe_pmu->min_period = 1536;  in __arm_spe_pmu_dev_probe()
1071 		spe_pmu->min_period = 2048;  in __arm_spe_pmu_dev_probe()
1074 		spe_pmu->min_period = 3072;  in __arm_spe_pmu_dev_probe()
1081 		spe_pmu->min_period = 4096;  in __arm_spe_pmu_dev_probe()
1084 	/* Maximum record size. If it's out-of-range, then fail the probe */  in __arm_spe_pmu_dev_probe()
1086 	spe_pmu->max_record_sz = 1 << fld;  in __arm_spe_pmu_dev_probe()
1087 	if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) {  in __arm_spe_pmu_dev_probe()
1100 		spe_pmu->counter_sz = 12;  in __arm_spe_pmu_dev_probe()
1103 		spe_pmu->counter_sz = 16;  in __arm_spe_pmu_dev_probe()
1108 		 spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus),  in __arm_spe_pmu_dev_probe()
1109 		 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);  in __arm_spe_pmu_dev_probe()
1111 	spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED;  in __arm_spe_pmu_dev_probe()
1136 	enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE);  in __arm_spe_pmu_setup_one()
1143 	disable_percpu_irq(spe_pmu->irq);  in __arm_spe_pmu_stop_one()
1152 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_cpu_startup()
1164 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_cpu_teardown()
1174 	cpumask_t *mask = &spe_pmu->supported_cpus;  in arm_spe_pmu_dev_init()
1178 	if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED))  in arm_spe_pmu_dev_init()
1179 		return -ENXIO;  in arm_spe_pmu_dev_init()
1182 	ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME,  in arm_spe_pmu_dev_init()
1183 				 spe_pmu->handle);  in arm_spe_pmu_dev_init()
1193 				       &spe_pmu->hotplug_node);  in arm_spe_pmu_dev_init()
1195 		free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_init()
1202 	cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node);  in arm_spe_pmu_dev_teardown()
1203 	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_teardown()
1209 	struct platform_device *pdev = spe_pmu->pdev;  in arm_spe_pmu_irq_probe()
1213 		return -ENXIO;  in arm_spe_pmu_irq_probe()
1216 		dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq);  in arm_spe_pmu_irq_probe()
1217 		return -EINVAL;  in arm_spe_pmu_irq_probe()
1220 	if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) {  in arm_spe_pmu_irq_probe()
1221 		dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq);  in arm_spe_pmu_irq_probe()
1222 		return -EINVAL;  in arm_spe_pmu_irq_probe()
1225 	spe_pmu->irq = irq;  in arm_spe_pmu_irq_probe()
1230 	{ .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 },
1245 	struct device *dev = &pdev->dev;  in arm_spe_pmu_device_probe()
1248 	 * If kernelspace is unmapped when running at EL0, then the SPE  in arm_spe_pmu_device_probe()
1253 		return -EPERM;  in arm_spe_pmu_device_probe()
1258 		return -ENOMEM;  in arm_spe_pmu_device_probe()
1260 	spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));  in arm_spe_pmu_device_probe()
1261 	if (!spe_pmu->handle)  in arm_spe_pmu_device_probe()
1262 		return -ENOMEM;  in arm_spe_pmu_device_probe()
1264 	spe_pmu->pdev = pdev;  in arm_spe_pmu_device_probe()
1284 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_probe()
1294 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_remove()