1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * CPU PMU driver for the Apple M1 and derivatives
4 *
5 * Copyright (C) 2021 Google LLC
6 *
7 * Author: Marc Zyngier <[email protected]>
8 *
9 * Most of the information used in this driver was provided by the
10 * Asahi Linux project. The rest was experimentally discovered.
11 */
12
13 #include <linux/of.h>
14 #include <linux/perf/arm_pmu.h>
15 #include <linux/platform_device.h>
16
17 #include <asm/apple_m1_pmu.h>
18 #include <asm/irq_regs.h>
19 #include <asm/perf_event.h>
20
21 #define M1_PMU_NR_COUNTERS 10
22
23 #define M1_PMU_CFG_EVENT GENMASK(7, 0)
24
25 #define ANY_BUT_0_1 GENMASK(9, 2)
26 #define ONLY_2_TO_7 GENMASK(7, 2)
27 #define ONLY_2_4_6 (BIT(2) | BIT(4) | BIT(6))
28 #define ONLY_5_6_7 (BIT(5) | BIT(6) | BIT(7))
29
30 /*
31 * Description of the events we actually know about, as well as those with
32 * a specific counter affinity. Yes, this is a grand total of two known
33 * counters, and the rest is anybody's guess.
34 *
35 * Not all counters can count all events. Counters #0 and #1 are wired to
36 * count cycles and instructions respectively, and some events have
37 * bizarre mappings (every other counter, or even *one* counter). These
38 * restrictions equally apply to both P and E cores.
39 *
40 * It is worth noting that the PMUs attached to P and E cores are likely
41 * to be different because the underlying uarches are different. At the
42 * moment, we don't really need to distinguish between the two because we
43 * know next to nothing about the events themselves, and we already have
44 * per cpu-type PMU abstractions.
45 *
46 * If we eventually find out that the events are different across
47 * implementations, we'll have to introduce per cpu-type tables.
48 */
49 enum m1_pmu_events {
50 M1_PMU_PERFCTR_RETIRE_UOP = 0x1,
51 M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE = 0x2,
52 M1_PMU_PERFCTR_L1I_TLB_FILL = 0x4,
53 M1_PMU_PERFCTR_L1D_TLB_FILL = 0x5,
54 M1_PMU_PERFCTR_MMU_TABLE_WALK_INSTRUCTION = 0x7,
55 M1_PMU_PERFCTR_MMU_TABLE_WALK_DATA = 0x8,
56 M1_PMU_PERFCTR_L2_TLB_MISS_INSTRUCTION = 0xa,
57 M1_PMU_PERFCTR_L2_TLB_MISS_DATA = 0xb,
58 M1_PMU_PERFCTR_MMU_VIRTUAL_MEMORY_FAULT_NONSPEC = 0xd,
59 M1_PMU_PERFCTR_SCHEDULE_UOP = 0x52,
60 M1_PMU_PERFCTR_INTERRUPT_PENDING = 0x6c,
61 M1_PMU_PERFCTR_MAP_STALL_DISPATCH = 0x70,
62 M1_PMU_PERFCTR_MAP_REWIND = 0x75,
63 M1_PMU_PERFCTR_MAP_STALL = 0x76,
64 M1_PMU_PERFCTR_MAP_INT_UOP = 0x7c,
65 M1_PMU_PERFCTR_MAP_LDST_UOP = 0x7d,
66 M1_PMU_PERFCTR_MAP_SIMD_UOP = 0x7e,
67 M1_PMU_PERFCTR_FLUSH_RESTART_OTHER_NONSPEC = 0x84,
68 M1_PMU_PERFCTR_INST_ALL = 0x8c,
69 M1_PMU_PERFCTR_INST_BRANCH = 0x8d,
70 M1_PMU_PERFCTR_INST_BRANCH_CALL = 0x8e,
71 M1_PMU_PERFCTR_INST_BRANCH_RET = 0x8f,
72 M1_PMU_PERFCTR_INST_BRANCH_TAKEN = 0x90,
73 M1_PMU_PERFCTR_INST_BRANCH_INDIR = 0x93,
74 M1_PMU_PERFCTR_INST_BRANCH_COND = 0x94,
75 M1_PMU_PERFCTR_INST_INT_LD = 0x95,
76 M1_PMU_PERFCTR_INST_INT_ST = 0x96,
77 M1_PMU_PERFCTR_INST_INT_ALU = 0x97,
78 M1_PMU_PERFCTR_INST_SIMD_LD = 0x98,
79 M1_PMU_PERFCTR_INST_SIMD_ST = 0x99,
80 M1_PMU_PERFCTR_INST_SIMD_ALU = 0x9a,
81 M1_PMU_PERFCTR_INST_LDST = 0x9b,
82 M1_PMU_PERFCTR_INST_BARRIER = 0x9c,
83 M1_PMU_PERFCTR_UNKNOWN_9f = 0x9f,
84 M1_PMU_PERFCTR_L1D_TLB_ACCESS = 0xa0,
85 M1_PMU_PERFCTR_L1D_TLB_MISS = 0xa1,
86 M1_PMU_PERFCTR_L1D_CACHE_MISS_ST = 0xa2,
87 M1_PMU_PERFCTR_L1D_CACHE_MISS_LD = 0xa3,
88 M1_PMU_PERFCTR_LD_UNIT_UOP = 0xa6,
89 M1_PMU_PERFCTR_ST_UNIT_UOP = 0xa7,
90 M1_PMU_PERFCTR_L1D_CACHE_WRITEBACK = 0xa8,
91 M1_PMU_PERFCTR_LDST_X64_UOP = 0xb1,
92 M1_PMU_PERFCTR_LDST_XPG_UOP = 0xb2,
93 M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_SUCC = 0xb3,
94 M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_FAIL = 0xb4,
95 M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC = 0xbf,
96 M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC = 0xc0,
97 M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC = 0xc1,
98 M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC = 0xc4,
99 M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC = 0xc5,
100 M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC = 0xc6,
101 M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC = 0xc8,
102 M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC = 0xca,
103 M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC = 0xcb,
104 M1_PMU_PERFCTR_L1I_TLB_MISS_DEMAND = 0xd4,
105 M1_PMU_PERFCTR_MAP_DISPATCH_BUBBLE = 0xd6,
106 M1_PMU_PERFCTR_L1I_CACHE_MISS_DEMAND = 0xdb,
107 M1_PMU_PERFCTR_FETCH_RESTART = 0xde,
108 M1_PMU_PERFCTR_ST_NT_UOP = 0xe5,
109 M1_PMU_PERFCTR_LD_NT_UOP = 0xe6,
110 M1_PMU_PERFCTR_UNKNOWN_f5 = 0xf5,
111 M1_PMU_PERFCTR_UNKNOWN_f6 = 0xf6,
112 M1_PMU_PERFCTR_UNKNOWN_f7 = 0xf7,
113 M1_PMU_PERFCTR_UNKNOWN_f8 = 0xf8,
114 M1_PMU_PERFCTR_UNKNOWN_fd = 0xfd,
115 M1_PMU_PERFCTR_LAST = M1_PMU_CFG_EVENT,
116
117 /*
118 * From this point onwards, these are not actual HW events,
119 * but attributes that get stored in hw->config_base.
120 */
121 M1_PMU_CFG_COUNT_USER = BIT(8),
122 M1_PMU_CFG_COUNT_KERNEL = BIT(9),
123 };
124
125 /*
126 * Per-event affinity table. Most events can be installed on counter
127 * 2-9, but there are a number of exceptions. Note that this table
128 * has been created experimentally, and I wouldn't be surprised if more
129 * counters had strange affinities.
130 */
131 static const u16 m1_pmu_event_affinity[M1_PMU_PERFCTR_LAST + 1] = {
132 [0 ... M1_PMU_PERFCTR_LAST] = ANY_BUT_0_1,
133 [M1_PMU_PERFCTR_RETIRE_UOP] = BIT(7),
134 [M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE] = ANY_BUT_0_1 | BIT(0),
135 [M1_PMU_PERFCTR_INST_ALL] = BIT(7) | BIT(1),
136 [M1_PMU_PERFCTR_INST_BRANCH] = ONLY_5_6_7,
137 [M1_PMU_PERFCTR_INST_BRANCH_CALL] = ONLY_5_6_7,
138 [M1_PMU_PERFCTR_INST_BRANCH_RET] = ONLY_5_6_7,
139 [M1_PMU_PERFCTR_INST_BRANCH_TAKEN] = ONLY_5_6_7,
140 [M1_PMU_PERFCTR_INST_BRANCH_INDIR] = ONLY_5_6_7,
141 [M1_PMU_PERFCTR_INST_BRANCH_COND] = ONLY_5_6_7,
142 [M1_PMU_PERFCTR_INST_INT_LD] = ONLY_5_6_7,
143 [M1_PMU_PERFCTR_INST_INT_ST] = BIT(7),
144 [M1_PMU_PERFCTR_INST_INT_ALU] = BIT(7),
145 [M1_PMU_PERFCTR_INST_SIMD_LD] = ONLY_5_6_7,
146 [M1_PMU_PERFCTR_INST_SIMD_ST] = ONLY_5_6_7,
147 [M1_PMU_PERFCTR_INST_SIMD_ALU] = BIT(7),
148 [M1_PMU_PERFCTR_INST_LDST] = BIT(7),
149 [M1_PMU_PERFCTR_INST_BARRIER] = ONLY_5_6_7,
150 [M1_PMU_PERFCTR_UNKNOWN_9f] = BIT(7),
151 [M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC] = ONLY_5_6_7,
152 [M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC] = ONLY_5_6_7,
153 [M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC] = ONLY_5_6_7,
154 [M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC] = ONLY_5_6_7,
155 [M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC] = ONLY_5_6_7,
156 [M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7,
157 [M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7,
158 [M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7,
159 [M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC] = ONLY_5_6_7,
160 [M1_PMU_PERFCTR_UNKNOWN_f5] = ONLY_2_4_6,
161 [M1_PMU_PERFCTR_UNKNOWN_f6] = ONLY_2_4_6,
162 [M1_PMU_PERFCTR_UNKNOWN_f7] = ONLY_2_4_6,
163 [M1_PMU_PERFCTR_UNKNOWN_f8] = ONLY_2_TO_7,
164 [M1_PMU_PERFCTR_UNKNOWN_fd] = ONLY_2_4_6,
165 };
166
167 static const unsigned m1_pmu_perf_map[PERF_COUNT_HW_MAX] = {
168 PERF_MAP_ALL_UNSUPPORTED,
169 [PERF_COUNT_HW_CPU_CYCLES] = M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE,
170 [PERF_COUNT_HW_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_ALL,
171 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_BRANCH,
172 [PERF_COUNT_HW_BRANCH_MISSES] = M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC,
173 };
174
175 /* sysfs definitions */
m1_pmu_events_sysfs_show(struct device * dev,struct device_attribute * attr,char * page)176 static ssize_t m1_pmu_events_sysfs_show(struct device *dev,
177 struct device_attribute *attr,
178 char *page)
179 {
180 struct perf_pmu_events_attr *pmu_attr;
181
182 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
183
184 return sprintf(page, "event=0x%04llx\n", pmu_attr->id);
185 }
186
187 #define M1_PMU_EVENT_ATTR(name, config) \
188 PMU_EVENT_ATTR_ID(name, m1_pmu_events_sysfs_show, config)
189
190 static struct attribute *m1_pmu_event_attrs[] = {
191 M1_PMU_EVENT_ATTR(cycles, M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE),
192 M1_PMU_EVENT_ATTR(instructions, M1_PMU_PERFCTR_INST_ALL),
193 NULL,
194 };
195
196 static const struct attribute_group m1_pmu_events_attr_group = {
197 .name = "events",
198 .attrs = m1_pmu_event_attrs,
199 };
200
201 PMU_FORMAT_ATTR(event, "config:0-7");
202
203 static struct attribute *m1_pmu_format_attrs[] = {
204 &format_attr_event.attr,
205 NULL,
206 };
207
208 static const struct attribute_group m1_pmu_format_attr_group = {
209 .name = "format",
210 .attrs = m1_pmu_format_attrs,
211 };
212
213 /* Low level accessors. No synchronisation. */
214 #define PMU_READ_COUNTER(_idx) \
215 case _idx: return read_sysreg_s(SYS_IMP_APL_PMC## _idx ##_EL1)
216
217 #define PMU_WRITE_COUNTER(_val, _idx) \
218 case _idx: \
219 write_sysreg_s(_val, SYS_IMP_APL_PMC## _idx ##_EL1); \
220 return
221
m1_pmu_read_hw_counter(unsigned int index)222 static u64 m1_pmu_read_hw_counter(unsigned int index)
223 {
224 switch (index) {
225 PMU_READ_COUNTER(0);
226 PMU_READ_COUNTER(1);
227 PMU_READ_COUNTER(2);
228 PMU_READ_COUNTER(3);
229 PMU_READ_COUNTER(4);
230 PMU_READ_COUNTER(5);
231 PMU_READ_COUNTER(6);
232 PMU_READ_COUNTER(7);
233 PMU_READ_COUNTER(8);
234 PMU_READ_COUNTER(9);
235 }
236
237 BUG();
238 }
239
m1_pmu_write_hw_counter(u64 val,unsigned int index)240 static void m1_pmu_write_hw_counter(u64 val, unsigned int index)
241 {
242 switch (index) {
243 PMU_WRITE_COUNTER(val, 0);
244 PMU_WRITE_COUNTER(val, 1);
245 PMU_WRITE_COUNTER(val, 2);
246 PMU_WRITE_COUNTER(val, 3);
247 PMU_WRITE_COUNTER(val, 4);
248 PMU_WRITE_COUNTER(val, 5);
249 PMU_WRITE_COUNTER(val, 6);
250 PMU_WRITE_COUNTER(val, 7);
251 PMU_WRITE_COUNTER(val, 8);
252 PMU_WRITE_COUNTER(val, 9);
253 }
254
255 BUG();
256 }
257
258 #define get_bit_offset(index, mask) (__ffs(mask) + (index))
259
__m1_pmu_enable_counter(unsigned int index,bool en)260 static void __m1_pmu_enable_counter(unsigned int index, bool en)
261 {
262 u64 val, bit;
263
264 switch (index) {
265 case 0 ... 7:
266 bit = BIT(get_bit_offset(index, PMCR0_CNT_ENABLE_0_7));
267 break;
268 case 8 ... 9:
269 bit = BIT(get_bit_offset(index - 8, PMCR0_CNT_ENABLE_8_9));
270 break;
271 default:
272 BUG();
273 }
274
275 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
276
277 if (en)
278 val |= bit;
279 else
280 val &= ~bit;
281
282 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
283 }
284
m1_pmu_enable_counter(unsigned int index)285 static void m1_pmu_enable_counter(unsigned int index)
286 {
287 __m1_pmu_enable_counter(index, true);
288 }
289
m1_pmu_disable_counter(unsigned int index)290 static void m1_pmu_disable_counter(unsigned int index)
291 {
292 __m1_pmu_enable_counter(index, false);
293 }
294
__m1_pmu_enable_counter_interrupt(unsigned int index,bool en)295 static void __m1_pmu_enable_counter_interrupt(unsigned int index, bool en)
296 {
297 u64 val, bit;
298
299 switch (index) {
300 case 0 ... 7:
301 bit = BIT(get_bit_offset(index, PMCR0_PMI_ENABLE_0_7));
302 break;
303 case 8 ... 9:
304 bit = BIT(get_bit_offset(index - 8, PMCR0_PMI_ENABLE_8_9));
305 break;
306 default:
307 BUG();
308 }
309
310 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
311
312 if (en)
313 val |= bit;
314 else
315 val &= ~bit;
316
317 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
318 }
319
m1_pmu_enable_counter_interrupt(unsigned int index)320 static void m1_pmu_enable_counter_interrupt(unsigned int index)
321 {
322 __m1_pmu_enable_counter_interrupt(index, true);
323 }
324
m1_pmu_disable_counter_interrupt(unsigned int index)325 static void m1_pmu_disable_counter_interrupt(unsigned int index)
326 {
327 __m1_pmu_enable_counter_interrupt(index, false);
328 }
329
m1_pmu_configure_counter(unsigned int index,u8 event,bool user,bool kernel)330 static void m1_pmu_configure_counter(unsigned int index, u8 event,
331 bool user, bool kernel)
332 {
333 u64 val, user_bit, kernel_bit;
334 int shift;
335
336 switch (index) {
337 case 0 ... 7:
338 user_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL0_0_7));
339 kernel_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL1_0_7));
340 break;
341 case 8 ... 9:
342 user_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL0_8_9));
343 kernel_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL1_8_9));
344 break;
345 default:
346 BUG();
347 }
348
349 val = read_sysreg_s(SYS_IMP_APL_PMCR1_EL1);
350
351 if (user)
352 val |= user_bit;
353 else
354 val &= ~user_bit;
355
356 if (kernel)
357 val |= kernel_bit;
358 else
359 val &= ~kernel_bit;
360
361 write_sysreg_s(val, SYS_IMP_APL_PMCR1_EL1);
362
363 /*
364 * Counters 0 and 1 have fixed events. For anything else,
365 * place the event at the expected location in the relevant
366 * register (PMESR0 holds the event configuration for counters
367 * 2-5, resp. PMESR1 for counters 6-9).
368 */
369 switch (index) {
370 case 0 ... 1:
371 break;
372 case 2 ... 5:
373 shift = (index - 2) * 8;
374 val = read_sysreg_s(SYS_IMP_APL_PMESR0_EL1);
375 val &= ~((u64)0xff << shift);
376 val |= (u64)event << shift;
377 write_sysreg_s(val, SYS_IMP_APL_PMESR0_EL1);
378 break;
379 case 6 ... 9:
380 shift = (index - 6) * 8;
381 val = read_sysreg_s(SYS_IMP_APL_PMESR1_EL1);
382 val &= ~((u64)0xff << shift);
383 val |= (u64)event << shift;
384 write_sysreg_s(val, SYS_IMP_APL_PMESR1_EL1);
385 break;
386 }
387 }
388
389 /* arm_pmu backend */
m1_pmu_enable_event(struct perf_event * event)390 static void m1_pmu_enable_event(struct perf_event *event)
391 {
392 bool user, kernel;
393 u8 evt;
394
395 evt = event->hw.config_base & M1_PMU_CFG_EVENT;
396 user = event->hw.config_base & M1_PMU_CFG_COUNT_USER;
397 kernel = event->hw.config_base & M1_PMU_CFG_COUNT_KERNEL;
398
399 m1_pmu_disable_counter_interrupt(event->hw.idx);
400 m1_pmu_disable_counter(event->hw.idx);
401 isb();
402
403 m1_pmu_configure_counter(event->hw.idx, evt, user, kernel);
404 m1_pmu_enable_counter(event->hw.idx);
405 m1_pmu_enable_counter_interrupt(event->hw.idx);
406 isb();
407 }
408
m1_pmu_disable_event(struct perf_event * event)409 static void m1_pmu_disable_event(struct perf_event *event)
410 {
411 m1_pmu_disable_counter_interrupt(event->hw.idx);
412 m1_pmu_disable_counter(event->hw.idx);
413 isb();
414 }
415
m1_pmu_handle_irq(struct arm_pmu * cpu_pmu)416 static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu)
417 {
418 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
419 struct pt_regs *regs;
420 u64 overflow, state;
421 int idx;
422
423 overflow = read_sysreg_s(SYS_IMP_APL_PMSR_EL1);
424 if (!overflow) {
425 /* Spurious interrupt? */
426 state = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
427 state &= ~PMCR0_IACT;
428 write_sysreg_s(state, SYS_IMP_APL_PMCR0_EL1);
429 isb();
430 return IRQ_NONE;
431 }
432
433 cpu_pmu->stop(cpu_pmu);
434
435 regs = get_irq_regs();
436
437 for_each_set_bit(idx, cpu_pmu->cntr_mask, M1_PMU_NR_COUNTERS) {
438 struct perf_event *event = cpuc->events[idx];
439 struct perf_sample_data data;
440
441 if (!event)
442 continue;
443
444 armpmu_event_update(event);
445 perf_sample_data_init(&data, 0, event->hw.last_period);
446 if (!armpmu_event_set_period(event))
447 continue;
448
449 if (perf_event_overflow(event, &data, regs))
450 m1_pmu_disable_event(event);
451 }
452
453 cpu_pmu->start(cpu_pmu);
454
455 return IRQ_HANDLED;
456 }
457
m1_pmu_read_counter(struct perf_event * event)458 static u64 m1_pmu_read_counter(struct perf_event *event)
459 {
460 return m1_pmu_read_hw_counter(event->hw.idx);
461 }
462
m1_pmu_write_counter(struct perf_event * event,u64 value)463 static void m1_pmu_write_counter(struct perf_event *event, u64 value)
464 {
465 m1_pmu_write_hw_counter(value, event->hw.idx);
466 isb();
467 }
468
m1_pmu_get_event_idx(struct pmu_hw_events * cpuc,struct perf_event * event)469 static int m1_pmu_get_event_idx(struct pmu_hw_events *cpuc,
470 struct perf_event *event)
471 {
472 unsigned long evtype = event->hw.config_base & M1_PMU_CFG_EVENT;
473 unsigned long affinity = m1_pmu_event_affinity[evtype];
474 int idx;
475
476 /*
477 * Place the event on the first free counter that can count
478 * this event.
479 *
480 * We could do a better job if we had a view of all the events
481 * counting on the PMU at any given time, and by placing the
482 * most constraining events first.
483 */
484 for_each_set_bit(idx, &affinity, M1_PMU_NR_COUNTERS) {
485 if (!test_and_set_bit(idx, cpuc->used_mask))
486 return idx;
487 }
488
489 return -EAGAIN;
490 }
491
m1_pmu_clear_event_idx(struct pmu_hw_events * cpuc,struct perf_event * event)492 static void m1_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
493 struct perf_event *event)
494 {
495 clear_bit(event->hw.idx, cpuc->used_mask);
496 }
497
__m1_pmu_set_mode(u8 mode)498 static void __m1_pmu_set_mode(u8 mode)
499 {
500 u64 val;
501
502 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
503 val &= ~(PMCR0_IMODE | PMCR0_IACT);
504 val |= FIELD_PREP(PMCR0_IMODE, mode);
505 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
506 isb();
507 }
508
m1_pmu_start(struct arm_pmu * cpu_pmu)509 static void m1_pmu_start(struct arm_pmu *cpu_pmu)
510 {
511 __m1_pmu_set_mode(PMCR0_IMODE_FIQ);
512 }
513
m1_pmu_stop(struct arm_pmu * cpu_pmu)514 static void m1_pmu_stop(struct arm_pmu *cpu_pmu)
515 {
516 __m1_pmu_set_mode(PMCR0_IMODE_OFF);
517 }
518
m1_pmu_map_event(struct perf_event * event)519 static int m1_pmu_map_event(struct perf_event *event)
520 {
521 /*
522 * Although the counters are 48bit wide, bit 47 is what
523 * triggers the overflow interrupt. Advertise the counters
524 * being 47bit wide to mimick the behaviour of the ARM PMU.
525 */
526 event->hw.flags |= ARMPMU_EVT_47BIT;
527 return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT);
528 }
529
m2_pmu_map_event(struct perf_event * event)530 static int m2_pmu_map_event(struct perf_event *event)
531 {
532 /*
533 * Same deal as the above, except that M2 has 64bit counters.
534 * Which, as far as we're concerned, actually means 63 bits.
535 * Yes, this is getting awkward.
536 */
537 event->hw.flags |= ARMPMU_EVT_63BIT;
538 return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT);
539 }
540
m1_pmu_reset(void * info)541 static void m1_pmu_reset(void *info)
542 {
543 int i;
544
545 __m1_pmu_set_mode(PMCR0_IMODE_OFF);
546
547 for (i = 0; i < M1_PMU_NR_COUNTERS; i++) {
548 m1_pmu_disable_counter(i);
549 m1_pmu_disable_counter_interrupt(i);
550 m1_pmu_write_hw_counter(0, i);
551 }
552
553 isb();
554 }
555
m1_pmu_set_event_filter(struct hw_perf_event * event,struct perf_event_attr * attr)556 static int m1_pmu_set_event_filter(struct hw_perf_event *event,
557 struct perf_event_attr *attr)
558 {
559 unsigned long config_base = 0;
560
561 if (!attr->exclude_guest) {
562 pr_debug("ARM performance counters do not support mode exclusion\n");
563 return -EOPNOTSUPP;
564 }
565 if (!attr->exclude_kernel)
566 config_base |= M1_PMU_CFG_COUNT_KERNEL;
567 if (!attr->exclude_user)
568 config_base |= M1_PMU_CFG_COUNT_USER;
569
570 event->config_base = config_base;
571
572 return 0;
573 }
574
m1_pmu_init(struct arm_pmu * cpu_pmu,u32 flags)575 static int m1_pmu_init(struct arm_pmu *cpu_pmu, u32 flags)
576 {
577 cpu_pmu->handle_irq = m1_pmu_handle_irq;
578 cpu_pmu->enable = m1_pmu_enable_event;
579 cpu_pmu->disable = m1_pmu_disable_event;
580 cpu_pmu->read_counter = m1_pmu_read_counter;
581 cpu_pmu->write_counter = m1_pmu_write_counter;
582 cpu_pmu->get_event_idx = m1_pmu_get_event_idx;
583 cpu_pmu->clear_event_idx = m1_pmu_clear_event_idx;
584 cpu_pmu->start = m1_pmu_start;
585 cpu_pmu->stop = m1_pmu_stop;
586
587 if (flags & ARMPMU_EVT_47BIT)
588 cpu_pmu->map_event = m1_pmu_map_event;
589 else if (flags & ARMPMU_EVT_63BIT)
590 cpu_pmu->map_event = m2_pmu_map_event;
591 else
592 return WARN_ON(-EINVAL);
593
594 cpu_pmu->reset = m1_pmu_reset;
595 cpu_pmu->set_event_filter = m1_pmu_set_event_filter;
596
597 bitmap_set(cpu_pmu->cntr_mask, 0, M1_PMU_NR_COUNTERS);
598 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &m1_pmu_events_attr_group;
599 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &m1_pmu_format_attr_group;
600 return 0;
601 }
602
603 /* Device driver gunk */
m1_pmu_ice_init(struct arm_pmu * cpu_pmu)604 static int m1_pmu_ice_init(struct arm_pmu *cpu_pmu)
605 {
606 cpu_pmu->name = "apple_icestorm_pmu";
607 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT);
608 }
609
m1_pmu_fire_init(struct arm_pmu * cpu_pmu)610 static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu)
611 {
612 cpu_pmu->name = "apple_firestorm_pmu";
613 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT);
614 }
615
m2_pmu_avalanche_init(struct arm_pmu * cpu_pmu)616 static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu)
617 {
618 cpu_pmu->name = "apple_avalanche_pmu";
619 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT);
620 }
621
m2_pmu_blizzard_init(struct arm_pmu * cpu_pmu)622 static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu)
623 {
624 cpu_pmu->name = "apple_blizzard_pmu";
625 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT);
626 }
627
628 static const struct of_device_id m1_pmu_of_device_ids[] = {
629 { .compatible = "apple,avalanche-pmu", .data = m2_pmu_avalanche_init, },
630 { .compatible = "apple,blizzard-pmu", .data = m2_pmu_blizzard_init, },
631 { .compatible = "apple,icestorm-pmu", .data = m1_pmu_ice_init, },
632 { .compatible = "apple,firestorm-pmu", .data = m1_pmu_fire_init, },
633 { },
634 };
635 MODULE_DEVICE_TABLE(of, m1_pmu_of_device_ids);
636
m1_pmu_device_probe(struct platform_device * pdev)637 static int m1_pmu_device_probe(struct platform_device *pdev)
638 {
639 return arm_pmu_device_probe(pdev, m1_pmu_of_device_ids, NULL);
640 }
641
642 static struct platform_driver m1_pmu_driver = {
643 .driver = {
644 .name = "apple-m1-cpu-pmu",
645 .of_match_table = m1_pmu_of_device_ids,
646 .suppress_bind_attrs = true,
647 },
648 .probe = m1_pmu_device_probe,
649 };
650
651 module_platform_driver(m1_pmu_driver);
652