1 // SPDX-License-Identifier: GPL-2.0-only
3  * ARM DMC-620 memory controller PMU driver
40  * The PMU registers start at 0xA00 in the DMC-620 memory map, and these
50 		(DMC620_PMU_CLKDIV2_MAX_COUNTERS - 1)
53 		(DMC620_PMU_CLK_MAX_COUNTERS - 1)
121 	return sysfs_emit(page, "event=0x%x,clkdiv2=0x%x\n", eattr->eventid, eattr->clkdiv2);  in dmc620_pmu_event_show()
168 	.name = "events",
193 	(lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
198 #define GEN_PMU_FORMAT_ATTR(name)				\  argument
199 	PMU_FORMAT_ATTR(name,					\
200 	_GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG,		\
201 			     ATTR_CFG_FLD_##name##_LO,		\
202 			     ATTR_CFG_FLD_##name##_HI))
205 	((((attr)->cfg) >> lo) & GENMASK_ULL(hi - lo, 0))
207 #define ATTR_CFG_GET_FLD(attr, name)				\  argument
209 			  ATTR_CFG_FLD_##name##_CFG,		\
210 			  ATTR_CFG_FLD_##name##_LO,		\
211 			  ATTR_CFG_FLD_##name##_HI)
217 GEN_PMU_FORMAT_ATTR(event);
231 	.name	= "format",
241 				       cpumask_of(dmc620_pmu->irq->cpu));  in dmc620_pmu_cpumask_show()
267 	return readl(dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg);  in dmc620_pmu_creg_read()
274 	writel(val, dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg);  in dmc620_pmu_creg_write()
278 unsigned int dmc620_event_to_counter_control(struct perf_event *event)  in dmc620_event_to_counter_control()  argument
280 	struct perf_event_attr *attr = &event->attr;  in dmc620_event_to_counter_control()
286 			ATTR_CFG_GET_FLD(attr, event));  in dmc620_event_to_counter_control()
293 static int dmc620_get_event_idx(struct perf_event *event)  in dmc620_get_event_idx()  argument
295 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_get_event_idx()
298 	if (ATTR_CFG_GET_FLD(&event->attr, clkdiv2)) {  in dmc620_get_event_idx()
307 		if (!test_and_set_bit(idx, dmc620_pmu->used_mask))  in dmc620_get_event_idx()
312 	return -EAGAIN;  in dmc620_get_event_idx()
316 u64 dmc620_pmu_read_counter(struct perf_event *event)  in dmc620_pmu_read_counter()  argument
318 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_read_counter()
321 				    event->hw.idx, DMC620_PMU_COUNTERn_VALUE);  in dmc620_pmu_read_counter()
324 static void dmc620_pmu_event_update(struct perf_event *event)  in dmc620_pmu_event_update()  argument
326 	struct hw_perf_event *hwc = &event->hw;  in dmc620_pmu_event_update()
331 		prev_count = local64_read(&hwc->prev_count);  in dmc620_pmu_event_update()
332 		new_count = dmc620_pmu_read_counter(event);  in dmc620_pmu_event_update()
333 	} while (local64_cmpxchg(&hwc->prev_count,  in dmc620_pmu_event_update()
335 	delta = (new_count - prev_count) & DMC620_CNT_MAX_PERIOD;  in dmc620_pmu_event_update()
336 	local64_add(delta, &event->count);  in dmc620_pmu_event_update()
339 static void dmc620_pmu_event_set_period(struct perf_event *event)  in dmc620_pmu_event_set_period()  argument
341 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_event_set_period()
343 	local64_set(&event->hw.prev_count, DMC620_CNT_INIT);  in dmc620_pmu_event_set_period()
345 			      event->hw.idx, DMC620_PMU_COUNTERn_VALUE, DMC620_CNT_INIT);  in dmc620_pmu_event_set_period()
348 static void dmc620_pmu_enable_counter(struct perf_event *event)  in dmc620_pmu_enable_counter()  argument
350 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_enable_counter()
353 	reg = dmc620_event_to_counter_control(event) | DMC620_PMU_COUNTERn_CONTROL_ENABLE;  in dmc620_pmu_enable_counter()
355 			      event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, reg);  in dmc620_pmu_enable_counter()
358 static void dmc620_pmu_disable_counter(struct perf_event *event)  in dmc620_pmu_disable_counter()  argument
360 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_disable_counter()
363 			      event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, 0);  in dmc620_pmu_disable_counter()
373 	list_for_each_entry_rcu(dmc620_pmu, &irq->pmus_node, pmus_node) {  in dmc620_pmu_handle_irq()
375 		struct perf_event *event;  in dmc620_pmu_handle_irq()  local
384 			event = dmc620_pmu->events[idx];  in dmc620_pmu_handle_irq()
385 			if (!event)  in dmc620_pmu_handle_irq()
387 			dmc620_pmu_disable_counter(event);  in dmc620_pmu_handle_irq()
390 		status = readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);  in dmc620_pmu_handle_irq()
391 		status |= (readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK) <<  in dmc620_pmu_handle_irq()
396 				event = dmc620_pmu->events[idx];  in dmc620_pmu_handle_irq()
397 				if (WARN_ON_ONCE(!event))  in dmc620_pmu_handle_irq()
399 				dmc620_pmu_event_update(event);  in dmc620_pmu_handle_irq()
400 				dmc620_pmu_event_set_period(event);  in dmc620_pmu_handle_irq()
404 				writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);  in dmc620_pmu_handle_irq()
408 				writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK);  in dmc620_pmu_handle_irq()
412 			event = dmc620_pmu->events[idx];  in dmc620_pmu_handle_irq()
413 			if (!event)  in dmc620_pmu_handle_irq()
415 			if (!(event->hw.state & PERF_HES_STOPPED))  in dmc620_pmu_handle_irq()
416 				dmc620_pmu_enable_counter(event);  in dmc620_pmu_handle_irq()
432 		if (irq->irq_num == irq_num && refcount_inc_not_zero(&irq->refcount))  in __dmc620_pmu_get_irq()
437 		return ERR_PTR(-ENOMEM);  in __dmc620_pmu_get_irq()
439 	INIT_LIST_HEAD(&irq->pmus_node);  in __dmc620_pmu_get_irq()
442 	irq->cpu = raw_smp_processor_id();  in __dmc620_pmu_get_irq()
443 	refcount_set(&irq->refcount, 1);  in __dmc620_pmu_get_irq()
447 			  "dmc620-pmu", irq);  in __dmc620_pmu_get_irq()
451 	ret = irq_set_affinity(irq_num, cpumask_of(irq->cpu));  in __dmc620_pmu_get_irq()
455 	ret = cpuhp_state_add_instance_nocalls(cpuhp_state_num, &irq->node);  in __dmc620_pmu_get_irq()
459 	irq->irq_num = irq_num;  in __dmc620_pmu_get_irq()
460 	list_add(&irq->irqs_node, &dmc620_pmu_irqs);  in __dmc620_pmu_get_irq()
482 	dmc620_pmu->irq = irq;  in dmc620_pmu_get_irq()
484 	list_add_rcu(&dmc620_pmu->pmus_node, &irq->pmus_node);  in dmc620_pmu_get_irq()
492 	struct dmc620_pmu_irq *irq = dmc620_pmu->irq;  in dmc620_pmu_put_irq()
495 	list_del_rcu(&dmc620_pmu->pmus_node);  in dmc620_pmu_put_irq()
499 	if (!refcount_dec_and_test(&irq->refcount)) {  in dmc620_pmu_put_irq()
504 	list_del(&irq->irqs_node);  in dmc620_pmu_put_irq()
507 	free_irq(irq->irq_num, irq);  in dmc620_pmu_put_irq()
508 	cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &irq->node);  in dmc620_pmu_put_irq()
512 static int dmc620_pmu_event_init(struct perf_event *event)  in dmc620_pmu_event_init()  argument
514 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_event_init()
515 	struct hw_perf_event *hwc = &event->hw;  in dmc620_pmu_event_init()
518 	if (event->attr.type != event->pmu->type)  in dmc620_pmu_event_init()
519 		return -ENOENT;  in dmc620_pmu_event_init()
525 	if (is_sampling_event(event) ||  in dmc620_pmu_event_init()
526 		event->attach_state & PERF_ATTACH_TASK) {  in dmc620_pmu_event_init()
527 		dev_dbg(dmc620_pmu->pmu.dev,  in dmc620_pmu_event_init()
528 			"Can't support per-task counters\n");  in dmc620_pmu_event_init()
529 		return -EOPNOTSUPP;  in dmc620_pmu_event_init()
536 	 * but can lead to issues for off-core PMUs, where each  in dmc620_pmu_event_init()
537 	 * event could be theoretically assigned to a different CPU. To  in dmc620_pmu_event_init()
541 	event->cpu = dmc620_pmu->irq->cpu;  in dmc620_pmu_event_init()
542 	if (event->cpu < 0)  in dmc620_pmu_event_init()
543 		return -EINVAL;  in dmc620_pmu_event_init()
545 	hwc->idx = -1;  in dmc620_pmu_event_init()
547 	if (event->group_leader == event)  in dmc620_pmu_event_init()
551 	 * We can't atomically disable all HW counters so only one event allowed,  in dmc620_pmu_event_init()
554 	if (!is_software_event(event->group_leader))  in dmc620_pmu_event_init()
555 		return -EINVAL;  in dmc620_pmu_event_init()
557 	for_each_sibling_event(sibling, event->group_leader) {  in dmc620_pmu_event_init()
558 		if (sibling != event &&  in dmc620_pmu_event_init()
560 			return -EINVAL;  in dmc620_pmu_event_init()
566 static void dmc620_pmu_read(struct perf_event *event)  in dmc620_pmu_read()  argument
568 	dmc620_pmu_event_update(event);  in dmc620_pmu_read()
571 static void dmc620_pmu_start(struct perf_event *event, int flags)  in dmc620_pmu_start()  argument
573 	event->hw.state = 0;  in dmc620_pmu_start()
574 	dmc620_pmu_event_set_period(event);  in dmc620_pmu_start()
575 	dmc620_pmu_enable_counter(event);  in dmc620_pmu_start()
578 static void dmc620_pmu_stop(struct perf_event *event, int flags)  in dmc620_pmu_stop()  argument
580 	if (event->hw.state & PERF_HES_STOPPED)  in dmc620_pmu_stop()
583 	dmc620_pmu_disable_counter(event);  in dmc620_pmu_stop()
584 	dmc620_pmu_event_update(event);  in dmc620_pmu_stop()
585 	event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;  in dmc620_pmu_stop()
588 static int dmc620_pmu_add(struct perf_event *event, int flags)  in dmc620_pmu_add()  argument
590 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_add()
591 	struct perf_event_attr *attr = &event->attr;  in dmc620_pmu_add()
592 	struct hw_perf_event *hwc = &event->hw;  in dmc620_pmu_add()
596 	idx = dmc620_get_event_idx(event);  in dmc620_pmu_add()
600 	hwc->idx = idx;  in dmc620_pmu_add()
601 	dmc620_pmu->events[idx] = event;  in dmc620_pmu_add()
602 	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;  in dmc620_pmu_add()
617 		dmc620_pmu_start(event, PERF_EF_RELOAD);  in dmc620_pmu_add()
619 	perf_event_update_userpage(event);  in dmc620_pmu_add()
623 static void dmc620_pmu_del(struct perf_event *event, int flags)  in dmc620_pmu_del()  argument
625 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);  in dmc620_pmu_del()
626 	struct hw_perf_event *hwc = &event->hw;  in dmc620_pmu_del()
627 	int idx = hwc->idx;  in dmc620_pmu_del()
629 	dmc620_pmu_stop(event, PERF_EF_UPDATE);  in dmc620_pmu_del()
630 	dmc620_pmu->events[idx] = NULL;  in dmc620_pmu_del()
631 	clear_bit(idx, dmc620_pmu->used_mask);  in dmc620_pmu_del()
632 	perf_event_update_userpage(event);  in dmc620_pmu_del()
643 	if (cpu != irq->cpu)  in dmc620_pmu_cpu_teardown()
652 	list_for_each_entry(dmc620_pmu, &irq->pmus_node, pmus_node)  in dmc620_pmu_cpu_teardown()
653 		perf_pmu_migrate_context(&dmc620_pmu->pmu, irq->cpu, target);  in dmc620_pmu_cpu_teardown()
656 	WARN_ON(irq_set_affinity(irq->irq_num, cpumask_of(target)));  in dmc620_pmu_cpu_teardown()
657 	irq->cpu = target;  in dmc620_pmu_cpu_teardown()
666 	char *name;  in dmc620_pmu_device_probe()  local
670 	dmc620_pmu = devm_kzalloc(&pdev->dev,  in dmc620_pmu_device_probe()
673 		return -ENOMEM;  in dmc620_pmu_device_probe()
677 	dmc620_pmu->pmu = (struct pmu) {  in dmc620_pmu_device_probe()
679 		.parent		= &pdev->dev,  in dmc620_pmu_device_probe()
691 	dmc620_pmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);  in dmc620_pmu_device_probe()
692 	if (IS_ERR(dmc620_pmu->base))  in dmc620_pmu_device_probe()
693 		return PTR_ERR(dmc620_pmu->base);  in dmc620_pmu_device_probe()
698 	writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);  in dmc620_pmu_device_probe()
699 	writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK);  in dmc620_pmu_device_probe()
709 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL,  in dmc620_pmu_device_probe()
711 				  (u64)(res->start >> DMC620_PA_SHIFT));  in dmc620_pmu_device_probe()
712 	if (!name) {  in dmc620_pmu_device_probe()
713 		dev_err(&pdev->dev,  in dmc620_pmu_device_probe()
714 			  "Create name failed, PMU @%pa\n", &res->start);  in dmc620_pmu_device_probe()
715 		ret = -ENOMEM;  in dmc620_pmu_device_probe()
719 	ret = perf_pmu_register(&dmc620_pmu->pmu, name, -1);  in dmc620_pmu_device_probe()
738 	perf_pmu_unregister(&dmc620_pmu->pmu);  in dmc620_pmu_device_remove()
748 		.name		= DMC620_DRVNAME,
783 MODULE_DESCRIPTION("Perf driver for the ARM DMC-620 memory controller");