1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell CN10K LLC-TAD perf driver
3  *
4  * Copyright (C) 2021 Marvell
5  */
6 
7 #define pr_fmt(fmt) "tad_pmu: " fmt
8 
9 #include <linux/io.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/cpuhotplug.h>
13 #include <linux/perf_event.h>
14 #include <linux/platform_device.h>
15 #include <linux/acpi.h>
16 
17 #define TAD_PFC_OFFSET		0x800
18 #define TAD_PFC(counter)	(TAD_PFC_OFFSET | (counter << 3))
19 #define TAD_PRF_OFFSET		0x900
20 #define TAD_PRF(counter)	(TAD_PRF_OFFSET | (counter << 3))
21 #define TAD_PRF_CNTSEL_MASK	0xFF
22 #define TAD_MAX_COUNTERS	8
23 
24 #define to_tad_pmu(p) (container_of(p, struct tad_pmu, pmu))
25 
26 struct tad_region {
27 	void __iomem	*base;
28 };
29 
30 struct tad_pmu {
31 	struct pmu pmu;
32 	struct tad_region *regions;
33 	u32 region_cnt;
34 	unsigned int cpu;
35 	struct hlist_node node;
36 	struct perf_event *events[TAD_MAX_COUNTERS];
37 	DECLARE_BITMAP(counters_map, TAD_MAX_COUNTERS);
38 };
39 
40 enum mrvl_tad_pmu_version {
41 	TAD_PMU_V1 = 1,
42 	TAD_PMU_V2,
43 };
44 
45 struct tad_pmu_data {
46 	int id;
47 };
48 
49 static int tad_pmu_cpuhp_state;
50 
tad_pmu_event_counter_read(struct perf_event * event)51 static void tad_pmu_event_counter_read(struct perf_event *event)
52 {
53 	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
54 	struct hw_perf_event *hwc = &event->hw;
55 	u32 counter_idx = hwc->idx;
56 	u64 prev, new;
57 	int i;
58 
59 	do {
60 		prev = local64_read(&hwc->prev_count);
61 		for (i = 0, new = 0; i < tad_pmu->region_cnt; i++)
62 			new += readq(tad_pmu->regions[i].base +
63 				     TAD_PFC(counter_idx));
64 	} while (local64_cmpxchg(&hwc->prev_count, prev, new) != prev);
65 
66 	local64_add(new - prev, &event->count);
67 }
68 
tad_pmu_event_counter_stop(struct perf_event * event,int flags)69 static void tad_pmu_event_counter_stop(struct perf_event *event, int flags)
70 {
71 	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
72 	struct hw_perf_event *hwc = &event->hw;
73 	u32 counter_idx = hwc->idx;
74 	int i;
75 
76 	/* TAD()_PFC() stop counting on the write
77 	 * which sets TAD()_PRF()[CNTSEL] == 0
78 	 */
79 	for (i = 0; i < tad_pmu->region_cnt; i++) {
80 		writeq_relaxed(0, tad_pmu->regions[i].base +
81 			       TAD_PRF(counter_idx));
82 	}
83 
84 	tad_pmu_event_counter_read(event);
85 	hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
86 }
87 
tad_pmu_event_counter_start(struct perf_event * event,int flags)88 static void tad_pmu_event_counter_start(struct perf_event *event, int flags)
89 {
90 	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
91 	struct hw_perf_event *hwc = &event->hw;
92 	u32 event_idx = event->attr.config;
93 	u32 counter_idx = hwc->idx;
94 	u64 reg_val;
95 	int i;
96 
97 	hwc->state = 0;
98 
99 	/* Typically TAD_PFC() are zeroed to start counting */
100 	for (i = 0; i < tad_pmu->region_cnt; i++)
101 		writeq_relaxed(0, tad_pmu->regions[i].base +
102 			       TAD_PFC(counter_idx));
103 
104 	/* TAD()_PFC() start counting on the write
105 	 * which sets TAD()_PRF()[CNTSEL] != 0
106 	 */
107 	for (i = 0; i < tad_pmu->region_cnt; i++) {
108 		reg_val = event_idx & 0xFF;
109 		writeq_relaxed(reg_val,	tad_pmu->regions[i].base +
110 			       TAD_PRF(counter_idx));
111 	}
112 }
113 
tad_pmu_event_counter_del(struct perf_event * event,int flags)114 static void tad_pmu_event_counter_del(struct perf_event *event, int flags)
115 {
116 	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
117 	struct hw_perf_event *hwc = &event->hw;
118 	int idx = hwc->idx;
119 
120 	tad_pmu_event_counter_stop(event, flags | PERF_EF_UPDATE);
121 	tad_pmu->events[idx] = NULL;
122 	clear_bit(idx, tad_pmu->counters_map);
123 }
124 
tad_pmu_event_counter_add(struct perf_event * event,int flags)125 static int tad_pmu_event_counter_add(struct perf_event *event, int flags)
126 {
127 	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
128 	struct hw_perf_event *hwc = &event->hw;
129 	int idx;
130 
131 	/* Get a free counter for this event */
132 	idx = find_first_zero_bit(tad_pmu->counters_map, TAD_MAX_COUNTERS);
133 	if (idx == TAD_MAX_COUNTERS)
134 		return -EAGAIN;
135 
136 	set_bit(idx, tad_pmu->counters_map);
137 
138 	hwc->idx = idx;
139 	hwc->state = PERF_HES_STOPPED;
140 	tad_pmu->events[idx] = event;
141 
142 	if (flags & PERF_EF_START)
143 		tad_pmu_event_counter_start(event, flags);
144 
145 	return 0;
146 }
147 
tad_pmu_event_init(struct perf_event * event)148 static int tad_pmu_event_init(struct perf_event *event)
149 {
150 	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
151 
152 	if (event->attr.type != event->pmu->type)
153 		return -ENOENT;
154 
155 	if (!event->attr.disabled)
156 		return -EINVAL;
157 
158 	if (event->state != PERF_EVENT_STATE_OFF)
159 		return -EINVAL;
160 
161 	event->cpu = tad_pmu->cpu;
162 	event->hw.idx = -1;
163 	event->hw.config_base = event->attr.config;
164 
165 	return 0;
166 }
167 
tad_pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)168 static ssize_t tad_pmu_event_show(struct device *dev,
169 				struct device_attribute *attr, char *page)
170 {
171 	struct perf_pmu_events_attr *pmu_attr;
172 
173 	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
174 	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
175 }
176 
177 #define TAD_PMU_EVENT_ATTR(name, config)			\
178 	PMU_EVENT_ATTR_ID(name, tad_pmu_event_show, config)
179 
180 static struct attribute *tad_pmu_event_attrs[] = {
181 	TAD_PMU_EVENT_ATTR(tad_none, 0x0),
182 	TAD_PMU_EVENT_ATTR(tad_req_msh_in_any, 0x1),
183 	TAD_PMU_EVENT_ATTR(tad_req_msh_in_mn, 0x2),
184 	TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3),
185 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_any, 0x4),
186 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_mn, 0x5),
187 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_exlmn, 0x6),
188 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_dss, 0x7),
189 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_retry_dss, 0x8),
190 	TAD_PMU_EVENT_ATTR(tad_dat_msh_in_any, 0x9),
191 	TAD_PMU_EVENT_ATTR(tad_dat_msh_in_dss, 0xa),
192 	TAD_PMU_EVENT_ATTR(tad_req_msh_out_any, 0xb),
193 	TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_rd, 0xc),
194 	TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_wr, 0xd),
195 	TAD_PMU_EVENT_ATTR(tad_req_msh_out_evict, 0xe),
196 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_any, 0xf),
197 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_exlmn, 0x10),
198 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_mn, 0x11),
199 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_exlmn, 0x12),
200 	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_mn, 0x13),
201 	TAD_PMU_EVENT_ATTR(tad_snp_msh_out_any, 0x14),
202 	TAD_PMU_EVENT_ATTR(tad_snp_msh_out_mn, 0x15),
203 	TAD_PMU_EVENT_ATTR(tad_snp_msh_out_exlmn, 0x16),
204 	TAD_PMU_EVENT_ATTR(tad_dat_msh_out_any, 0x17),
205 	TAD_PMU_EVENT_ATTR(tad_dat_msh_out_fill, 0x18),
206 	TAD_PMU_EVENT_ATTR(tad_dat_msh_out_dss, 0x19),
207 	TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a),
208 	TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b),
209 	TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c),
210 	TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d),
211 	TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e),
212 	TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f),
213 	TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20),
214 	TAD_PMU_EVENT_ATTR(tad_dat_rd, 0x21),
215 	TAD_PMU_EVENT_ATTR(tad_dat_rd_byp, 0x22),
216 	TAD_PMU_EVENT_ATTR(tad_ifb_occ, 0x23),
217 	TAD_PMU_EVENT_ATTR(tad_req_occ, 0x24),
218 	NULL
219 };
220 
221 static const struct attribute_group tad_pmu_events_attr_group = {
222 	.name = "events",
223 	.attrs = tad_pmu_event_attrs,
224 };
225 
226 static struct attribute *ody_tad_pmu_event_attrs[] = {
227 	TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3),
228 	TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a),
229 	TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b),
230 	TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c),
231 	TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d),
232 	TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e),
233 	TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f),
234 	TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20),
235 	TAD_PMU_EVENT_ATTR(tad_tot_cycle, 0xFF),
236 	NULL
237 };
238 
239 static const struct attribute_group ody_tad_pmu_events_attr_group = {
240 	.name = "events",
241 	.attrs = ody_tad_pmu_event_attrs,
242 };
243 
244 PMU_FORMAT_ATTR(event, "config:0-7");
245 
246 static struct attribute *tad_pmu_format_attrs[] = {
247 	&format_attr_event.attr,
248 	NULL
249 };
250 
251 static struct attribute_group tad_pmu_format_attr_group = {
252 	.name = "format",
253 	.attrs = tad_pmu_format_attrs,
254 };
255 
tad_pmu_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)256 static ssize_t tad_pmu_cpumask_show(struct device *dev,
257 				struct device_attribute *attr, char *buf)
258 {
259 	struct tad_pmu *tad_pmu = to_tad_pmu(dev_get_drvdata(dev));
260 
261 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(tad_pmu->cpu));
262 }
263 
264 static DEVICE_ATTR(cpumask, 0444, tad_pmu_cpumask_show, NULL);
265 
266 static struct attribute *tad_pmu_cpumask_attrs[] = {
267 	&dev_attr_cpumask.attr,
268 	NULL
269 };
270 
271 static struct attribute_group tad_pmu_cpumask_attr_group = {
272 	.attrs = tad_pmu_cpumask_attrs,
273 };
274 
275 static const struct attribute_group *tad_pmu_attr_groups[] = {
276 	&tad_pmu_events_attr_group,
277 	&tad_pmu_format_attr_group,
278 	&tad_pmu_cpumask_attr_group,
279 	NULL
280 };
281 
282 static const struct attribute_group *ody_tad_pmu_attr_groups[] = {
283 	&ody_tad_pmu_events_attr_group,
284 	&tad_pmu_format_attr_group,
285 	&tad_pmu_cpumask_attr_group,
286 	NULL
287 };
288 
tad_pmu_probe(struct platform_device * pdev)289 static int tad_pmu_probe(struct platform_device *pdev)
290 {
291 	const struct tad_pmu_data *dev_data;
292 	struct device *dev = &pdev->dev;
293 	struct tad_region *regions;
294 	struct tad_pmu *tad_pmu;
295 	struct resource *res;
296 	u32 tad_pmu_page_size;
297 	u32 tad_page_size;
298 	u32 tad_cnt;
299 	int version;
300 	int i, ret;
301 	char *name;
302 
303 	tad_pmu = devm_kzalloc(&pdev->dev, sizeof(*tad_pmu), GFP_KERNEL);
304 	if (!tad_pmu)
305 		return -ENOMEM;
306 
307 	platform_set_drvdata(pdev, tad_pmu);
308 
309 	dev_data = device_get_match_data(&pdev->dev);
310 	if (!dev_data) {
311 		dev_err(&pdev->dev, "Error: No device match data found\n");
312 		return -ENODEV;
313 	}
314 	version = dev_data->id;
315 
316 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
317 	if (!res) {
318 		dev_err(&pdev->dev, "Mem resource not found\n");
319 		return -ENODEV;
320 	}
321 
322 	ret = device_property_read_u32(dev, "marvell,tad-page-size",
323 				       &tad_page_size);
324 	if (ret) {
325 		dev_err(&pdev->dev, "Can't find tad-page-size property\n");
326 		return ret;
327 	}
328 
329 	ret = device_property_read_u32(dev, "marvell,tad-pmu-page-size",
330 				       &tad_pmu_page_size);
331 	if (ret) {
332 		dev_err(&pdev->dev, "Can't find tad-pmu-page-size property\n");
333 		return ret;
334 	}
335 
336 	ret = device_property_read_u32(dev, "marvell,tad-cnt", &tad_cnt);
337 	if (ret) {
338 		dev_err(&pdev->dev, "Can't find tad-cnt property\n");
339 		return ret;
340 	}
341 
342 	regions = devm_kcalloc(&pdev->dev, tad_cnt,
343 			       sizeof(*regions), GFP_KERNEL);
344 	if (!regions)
345 		return -ENOMEM;
346 
347 	/* ioremap the distributed TAD pmu regions */
348 	for (i = 0; i < tad_cnt && res->start < res->end; i++) {
349 		regions[i].base = devm_ioremap(&pdev->dev,
350 					       res->start,
351 					       tad_pmu_page_size);
352 		if (!regions[i].base) {
353 			dev_err(&pdev->dev, "TAD%d ioremap fail\n", i);
354 			return -ENOMEM;
355 		}
356 		res->start += tad_page_size;
357 	}
358 
359 	tad_pmu->regions = regions;
360 	tad_pmu->region_cnt = tad_cnt;
361 
362 	tad_pmu->pmu = (struct pmu) {
363 
364 		.module		= THIS_MODULE,
365 		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE |
366 				  PERF_PMU_CAP_NO_INTERRUPT,
367 		.task_ctx_nr	= perf_invalid_context,
368 
369 		.event_init	= tad_pmu_event_init,
370 		.add		= tad_pmu_event_counter_add,
371 		.del		= tad_pmu_event_counter_del,
372 		.start		= tad_pmu_event_counter_start,
373 		.stop		= tad_pmu_event_counter_stop,
374 		.read		= tad_pmu_event_counter_read,
375 	};
376 
377 	if (version == TAD_PMU_V1)
378 		tad_pmu->pmu.attr_groups = tad_pmu_attr_groups;
379 	else
380 		tad_pmu->pmu.attr_groups = ody_tad_pmu_attr_groups;
381 
382 	tad_pmu->cpu = raw_smp_processor_id();
383 
384 	/* Register pmu instance for cpu hotplug */
385 	ret = cpuhp_state_add_instance_nocalls(tad_pmu_cpuhp_state,
386 					       &tad_pmu->node);
387 	if (ret) {
388 		dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
389 		return ret;
390 	}
391 
392 	name = "tad";
393 	ret = perf_pmu_register(&tad_pmu->pmu, name, -1);
394 	if (ret)
395 		cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state,
396 						    &tad_pmu->node);
397 
398 	return ret;
399 }
400 
tad_pmu_remove(struct platform_device * pdev)401 static void tad_pmu_remove(struct platform_device *pdev)
402 {
403 	struct tad_pmu *pmu = platform_get_drvdata(pdev);
404 
405 	cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state,
406 						&pmu->node);
407 	perf_pmu_unregister(&pmu->pmu);
408 }
409 
410 #if defined(CONFIG_OF) || defined(CONFIG_ACPI)
411 static const struct tad_pmu_data tad_pmu_data = {
412 	.id   = TAD_PMU_V1,
413 };
414 #endif
415 
416 #ifdef CONFIG_ACPI
417 static const struct tad_pmu_data tad_pmu_v2_data = {
418 	.id   = TAD_PMU_V2,
419 };
420 #endif
421 
422 #ifdef CONFIG_OF
423 static const struct of_device_id tad_pmu_of_match[] = {
424 	{ .compatible = "marvell,cn10k-tad-pmu", .data = &tad_pmu_data },
425 	{},
426 };
427 #endif
428 
429 #ifdef CONFIG_ACPI
430 static const struct acpi_device_id tad_pmu_acpi_match[] = {
431 	{"MRVL000B", (kernel_ulong_t)&tad_pmu_data},
432 	{"MRVL000D", (kernel_ulong_t)&tad_pmu_v2_data},
433 	{},
434 };
435 MODULE_DEVICE_TABLE(acpi, tad_pmu_acpi_match);
436 #endif
437 
438 static struct platform_driver tad_pmu_driver = {
439 	.driver         = {
440 		.name   = "cn10k_tad_pmu",
441 		.of_match_table = of_match_ptr(tad_pmu_of_match),
442 		.acpi_match_table = ACPI_PTR(tad_pmu_acpi_match),
443 		.suppress_bind_attrs = true,
444 	},
445 	.probe          = tad_pmu_probe,
446 	.remove         = tad_pmu_remove,
447 };
448 
tad_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)449 static int tad_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
450 {
451 	struct tad_pmu *pmu = hlist_entry_safe(node, struct tad_pmu, node);
452 	unsigned int target;
453 
454 	if (cpu != pmu->cpu)
455 		return 0;
456 
457 	target = cpumask_any_but(cpu_online_mask, cpu);
458 	if (target >= nr_cpu_ids)
459 		return 0;
460 
461 	perf_pmu_migrate_context(&pmu->pmu, cpu, target);
462 	pmu->cpu = target;
463 
464 	return 0;
465 }
466 
tad_pmu_init(void)467 static int __init tad_pmu_init(void)
468 {
469 	int ret;
470 
471 	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
472 				      "perf/cn10k/tadpmu:online",
473 				      NULL,
474 				      tad_pmu_offline_cpu);
475 	if (ret < 0)
476 		return ret;
477 	tad_pmu_cpuhp_state = ret;
478 	ret = platform_driver_register(&tad_pmu_driver);
479 	if (ret)
480 		cpuhp_remove_multi_state(tad_pmu_cpuhp_state);
481 
482 	return ret;
483 }
484 
tad_pmu_exit(void)485 static void __exit tad_pmu_exit(void)
486 {
487 	platform_driver_unregister(&tad_pmu_driver);
488 	cpuhp_remove_multi_state(tad_pmu_cpuhp_state);
489 }
490 
491 module_init(tad_pmu_init);
492 module_exit(tad_pmu_exit);
493 
494 MODULE_DESCRIPTION("Marvell CN10K LLC-TAD Perf driver");
495 MODULE_AUTHOR("Bhaskara Budiredla <[email protected]>");
496 MODULE_LICENSE("GPL v2");
497