1 /*
2 * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <cdefs.h>
9 #include <inttypes.h>
10 #include <stdbool.h>
11 #include <stdint.h>
12
13 #include "../amu_private.h"
14 #include <arch.h>
15 #include <arch_features.h>
16 #include <arch_helpers.h>
17 #include <common/debug.h>
18 #include <lib/el3_runtime/pubsub_events.h>
19 #include <lib/extensions/amu.h>
20
21 #include <plat/common/platform.h>
22
23 #if ENABLE_AMU_FCONF
24 # include <lib/fconf/fconf.h>
25 # include <lib/fconf/fconf_amu_getter.h>
26 #endif
27
28 #if ENABLE_MPMM
29 # include <lib/mpmm/mpmm.h>
30 #endif
31
32 struct amu_ctx {
33 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS];
34 #if ENABLE_AMU_AUXILIARY_COUNTERS
35 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS];
36 #endif
37
38 /* Architected event counter 1 does not have an offset register */
39 uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U];
40 #if ENABLE_AMU_AUXILIARY_COUNTERS
41 uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS];
42 #endif
43
44 uint16_t group0_enable;
45 #if ENABLE_AMU_AUXILIARY_COUNTERS
46 uint16_t group1_enable;
47 #endif
48 };
49
50 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT];
51
52 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS,
53 amu_ctx_group0_enable_cannot_represent_all_group0_counters);
54
55 #if ENABLE_AMU_AUXILIARY_COUNTERS
56 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS,
57 amu_ctx_group1_enable_cannot_represent_all_group1_counters);
58 #endif
59
read_hcr_el2_amvoffen(void)60 static inline __unused uint64_t read_hcr_el2_amvoffen(void)
61 {
62 return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >>
63 HCR_AMVOFFEN_SHIFT;
64 }
65
write_cptr_el2_tam(uint64_t value)66 static inline __unused void write_cptr_el2_tam(uint64_t value)
67 {
68 write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) |
69 ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT));
70 }
71
ctx_write_scr_el3_amvoffen(cpu_context_t * ctx,uint64_t amvoffen)72 static inline __unused void ctx_write_scr_el3_amvoffen(cpu_context_t *ctx, uint64_t amvoffen)
73 {
74 uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3);
75
76 value &= ~SCR_AMVOFFEN_BIT;
77 value |= (amvoffen << SCR_AMVOFFEN_SHIFT) & SCR_AMVOFFEN_BIT;
78
79 write_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3, value);
80 }
81
write_hcr_el2_amvoffen(uint64_t value)82 static inline __unused void write_hcr_el2_amvoffen(uint64_t value)
83 {
84 write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) |
85 ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT));
86 }
87
write_amcr_el0_cg1rz(uint64_t value)88 static inline __unused void write_amcr_el0_cg1rz(uint64_t value)
89 {
90 write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) |
91 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT));
92 }
93
read_amcfgr_el0_ncg(void)94 static inline __unused uint64_t read_amcfgr_el0_ncg(void)
95 {
96 return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) &
97 AMCFGR_EL0_NCG_MASK;
98 }
99
read_amcgcr_el0_cg0nc(void)100 static inline __unused uint64_t read_amcgcr_el0_cg0nc(void)
101 {
102 return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) &
103 AMCGCR_EL0_CG0NC_MASK;
104 }
105
read_amcg1idr_el0_voff(void)106 static inline __unused uint64_t read_amcg1idr_el0_voff(void)
107 {
108 return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) &
109 AMCG1IDR_VOFF_MASK;
110 }
111
read_amcgcr_el0_cg1nc(void)112 static inline __unused uint64_t read_amcgcr_el0_cg1nc(void)
113 {
114 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) &
115 AMCGCR_EL0_CG1NC_MASK;
116 }
117
read_amcntenset0_el0_px(void)118 static inline __unused uint64_t read_amcntenset0_el0_px(void)
119 {
120 return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) &
121 AMCNTENSET0_EL0_Pn_MASK;
122 }
123
read_amcntenset1_el0_px(void)124 static inline __unused uint64_t read_amcntenset1_el0_px(void)
125 {
126 return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) &
127 AMCNTENSET1_EL0_Pn_MASK;
128 }
129
write_amcntenset0_el0_px(uint64_t px)130 static inline __unused void write_amcntenset0_el0_px(uint64_t px)
131 {
132 uint64_t value = read_amcntenset0_el0();
133
134 value &= ~AMCNTENSET0_EL0_Pn_MASK;
135 value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK;
136
137 write_amcntenset0_el0(value);
138 }
139
write_amcntenset1_el0_px(uint64_t px)140 static inline __unused void write_amcntenset1_el0_px(uint64_t px)
141 {
142 uint64_t value = read_amcntenset1_el0();
143
144 value &= ~AMCNTENSET1_EL0_Pn_MASK;
145 value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK;
146
147 write_amcntenset1_el0(value);
148 }
149
write_amcntenclr0_el0_px(uint64_t px)150 static inline __unused void write_amcntenclr0_el0_px(uint64_t px)
151 {
152 uint64_t value = read_amcntenclr0_el0();
153
154 value &= ~AMCNTENCLR0_EL0_Pn_MASK;
155 value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK;
156
157 write_amcntenclr0_el0(value);
158 }
159
write_amcntenclr1_el0_px(uint64_t px)160 static inline __unused void write_amcntenclr1_el0_px(uint64_t px)
161 {
162 uint64_t value = read_amcntenclr1_el0();
163
164 value &= ~AMCNTENCLR1_EL0_Pn_MASK;
165 value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK;
166
167 write_amcntenclr1_el0(value);
168 }
169
170 #if ENABLE_AMU_AUXILIARY_COUNTERS
amu_group1_supported(void)171 static __unused bool amu_group1_supported(void)
172 {
173 return read_amcfgr_el0_ncg() > 0U;
174 }
175 #endif
176
177 /*
178 * Enable counters. This function is meant to be invoked by the context
179 * management library before exiting from EL3.
180 */
amu_enable(cpu_context_t * ctx)181 void amu_enable(cpu_context_t *ctx)
182 {
183 /* Initialize FEAT_AMUv1p1 features if present. */
184 if (is_feat_amuv1p1_supported()) {
185 /*
186 * Set SCR_EL3.AMVOFFEN to one so that accesses to virtual
187 * offset registers at EL2 do not trap to EL3
188 */
189 ctx_write_scr_el3_amvoffen(ctx, 1U);
190 }
191 }
192
amu_enable_per_world(per_world_context_t * per_world_ctx)193 void amu_enable_per_world(per_world_context_t *per_world_ctx)
194 {
195 /*
196 * Set CPTR_EL3.TAM to zero so that any accesses to the Activity Monitor
197 * registers do not trap to EL3.
198 */
199 uint64_t cptr_el3 = per_world_ctx->ctx_cptr_el3;
200
201 cptr_el3 &= ~TAM_BIT;
202 per_world_ctx->ctx_cptr_el3 = cptr_el3;
203 }
204
amu_init_el3(void)205 void amu_init_el3(void)
206 {
207 uint64_t group0_impl_ctr = read_amcgcr_el0_cg0nc();
208 uint64_t group0_en_mask = (1 << (group0_impl_ctr)) - 1U;
209 uint64_t num_ctr_groups = read_amcfgr_el0_ncg();
210
211 /* Enable all architected counters by default */
212 write_amcntenset0_el0_px(group0_en_mask);
213
214 #if ENABLE_AMU_AUXILIARY_COUNTERS
215 if (num_ctr_groups > 0U) {
216 uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */
217 const struct amu_topology *topology;
218
219 /*
220 * The platform may opt to enable specific auxiliary counters.
221 * This can be done via the common FCONF getter, or via the
222 * platform-implemented function.
223 */
224 #if ENABLE_AMU_FCONF
225 topology = FCONF_GET_PROPERTY(amu, config, topology);
226 #else
227 topology = plat_amu_topology();
228 #endif /* ENABLE_AMU_FCONF */
229
230 if (topology != NULL) {
231 unsigned int core_pos = plat_my_core_pos();
232
233 amcntenset1_el0_px = topology->cores[core_pos].enable;
234 } else {
235 ERROR("AMU: failed to generate AMU topology\n");
236 }
237
238 write_amcntenset1_el0_px(amcntenset1_el0_px);
239 }
240 #else /* ENABLE_AMU_AUXILIARY_COUNTERS */
241 if (num_ctr_groups > 0U) {
242 VERBOSE("AMU: auxiliary counters detected but support is disabled\n");
243 }
244 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */
245
246 if (is_feat_amuv1p1_supported()) {
247 #if AMU_RESTRICT_COUNTERS
248 /*
249 * FEAT_AMUv1p1 adds a register field to restrict access to
250 * group 1 counters at all but the highest implemented EL. This
251 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time
252 * flag, when set, system register reads at lower ELs return
253 * zero. Reads from the memory mapped view are unaffected.
254 */
255 VERBOSE("AMU group 1 counter access restricted.\n");
256 write_amcr_el0_cg1rz(1U);
257 #else
258 write_amcr_el0_cg1rz(0U);
259 #endif
260 }
261
262 #if ENABLE_MPMM
263 mpmm_enable();
264 #endif
265 }
266
amu_init_el2_unused(void)267 void amu_init_el2_unused(void)
268 {
269 /*
270 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity Monitor
271 * registers do not trap to EL2.
272 */
273 write_cptr_el2_tam(0U);
274
275 /* Initialize FEAT_AMUv1p1 features if present. */
276 if (is_feat_amuv1p1_supported()) {
277 /* Make sure virtual offsets are disabled if EL2 not used. */
278 write_hcr_el2_amvoffen(0U);
279 }
280 }
281
282 /* Read the group 0 counter identified by the given `idx`. */
amu_group0_cnt_read(unsigned int idx)283 static uint64_t amu_group0_cnt_read(unsigned int idx)
284 {
285 assert(is_feat_amu_supported());
286 assert(idx < read_amcgcr_el0_cg0nc());
287
288 return amu_group0_cnt_read_internal(idx);
289 }
290
291 /* Write the group 0 counter identified by the given `idx` with `val` */
amu_group0_cnt_write(unsigned int idx,uint64_t val)292 static void amu_group0_cnt_write(unsigned int idx, uint64_t val)
293 {
294 assert(is_feat_amu_supported());
295 assert(idx < read_amcgcr_el0_cg0nc());
296
297 amu_group0_cnt_write_internal(idx, val);
298 isb();
299 }
300
301 /*
302 * Unlike with auxiliary counters, we cannot detect at runtime whether an
303 * architected counter supports a virtual offset. These are instead fixed
304 * according to FEAT_AMUv1p1, but this switch will need to be updated if later
305 * revisions of FEAT_AMU add additional architected counters.
306 */
amu_group0_voffset_supported(uint64_t idx)307 static bool amu_group0_voffset_supported(uint64_t idx)
308 {
309 switch (idx) {
310 case 0U:
311 case 2U:
312 case 3U:
313 return true;
314
315 case 1U:
316 return false;
317
318 default:
319 ERROR("AMU: can't set up virtual offset for unknown "
320 "architected counter %" PRIu64 "!\n", idx);
321
322 panic();
323 }
324 }
325
326 /*
327 * Read the group 0 offset register for a given index. Index must be 0, 2,
328 * or 3, the register for 1 does not exist.
329 *
330 * Using this function requires FEAT_AMUv1p1 support.
331 */
amu_group0_voffset_read(unsigned int idx)332 static uint64_t amu_group0_voffset_read(unsigned int idx)
333 {
334 assert(is_feat_amuv1p1_supported());
335 assert(idx < read_amcgcr_el0_cg0nc());
336 assert(idx != 1U);
337
338 return amu_group0_voffset_read_internal(idx);
339 }
340
341 /*
342 * Write the group 0 offset register for a given index. Index must be 0, 2, or
343 * 3, the register for 1 does not exist.
344 *
345 * Using this function requires FEAT_AMUv1p1 support.
346 */
amu_group0_voffset_write(unsigned int idx,uint64_t val)347 static void amu_group0_voffset_write(unsigned int idx, uint64_t val)
348 {
349 assert(is_feat_amuv1p1_supported());
350 assert(idx < read_amcgcr_el0_cg0nc());
351 assert(idx != 1U);
352
353 amu_group0_voffset_write_internal(idx, val);
354 isb();
355 }
356
357 #if ENABLE_AMU_AUXILIARY_COUNTERS
358 /* Read the group 1 counter identified by the given `idx` */
amu_group1_cnt_read(unsigned int idx)359 static uint64_t amu_group1_cnt_read(unsigned int idx)
360 {
361 assert(is_feat_amu_supported());
362 assert(amu_group1_supported());
363 assert(idx < read_amcgcr_el0_cg1nc());
364
365 return amu_group1_cnt_read_internal(idx);
366 }
367
368 /* Write the group 1 counter identified by the given `idx` with `val` */
amu_group1_cnt_write(unsigned int idx,uint64_t val)369 static void amu_group1_cnt_write(unsigned int idx, uint64_t val)
370 {
371 assert(is_feat_amu_supported());
372 assert(amu_group1_supported());
373 assert(idx < read_amcgcr_el0_cg1nc());
374
375 amu_group1_cnt_write_internal(idx, val);
376 isb();
377 }
378
379 /*
380 * Read the group 1 offset register for a given index.
381 *
382 * Using this function requires FEAT_AMUv1p1 support.
383 */
amu_group1_voffset_read(unsigned int idx)384 static uint64_t amu_group1_voffset_read(unsigned int idx)
385 {
386 assert(is_feat_amuv1p1_supported());
387 assert(amu_group1_supported());
388 assert(idx < read_amcgcr_el0_cg1nc());
389 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U);
390
391 return amu_group1_voffset_read_internal(idx);
392 }
393
394 /*
395 * Write the group 1 offset register for a given index.
396 *
397 * Using this function requires FEAT_AMUv1p1 support.
398 */
amu_group1_voffset_write(unsigned int idx,uint64_t val)399 static void amu_group1_voffset_write(unsigned int idx, uint64_t val)
400 {
401 assert(is_feat_amuv1p1_supported());
402 assert(amu_group1_supported());
403 assert(idx < read_amcgcr_el0_cg1nc());
404 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U);
405
406 amu_group1_voffset_write_internal(idx, val);
407 isb();
408 }
409 #endif
410
amu_context_save(const void * arg)411 static void *amu_context_save(const void *arg)
412 {
413 uint64_t i, j;
414
415 unsigned int core_pos;
416 struct amu_ctx *ctx;
417
418 uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */
419 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */
420
421 #if ENABLE_AMU_AUXILIARY_COUNTERS
422 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */
423 uint64_t amcfgr_el0_ncg; /* Number of counter groups */
424 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */
425 #endif
426
427 if (!is_feat_amu_supported()) {
428 return (void *)0;
429 }
430
431 core_pos = plat_my_core_pos();
432 ctx = &amu_ctxs_[core_pos];
433
434 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc();
435 if (is_feat_amuv1p1_supported()) {
436 hcr_el2_amvoffen = read_hcr_el2_amvoffen();
437 }
438
439 #if ENABLE_AMU_AUXILIARY_COUNTERS
440 amcfgr_el0_ncg = read_amcfgr_el0_ncg();
441 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U;
442 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U;
443 #endif
444
445 /*
446 * Disable all AMU counters.
447 */
448
449 ctx->group0_enable = read_amcntenset0_el0_px();
450 write_amcntenclr0_el0_px(ctx->group0_enable);
451
452 #if ENABLE_AMU_AUXILIARY_COUNTERS
453 if (amcfgr_el0_ncg > 0U) {
454 ctx->group1_enable = read_amcntenset1_el0_px();
455 write_amcntenclr1_el0_px(ctx->group1_enable);
456 }
457 #endif
458
459 /*
460 * Save the counters to the local context.
461 */
462
463 isb(); /* Ensure counters have been stopped */
464
465 for (i = 0U; i < amcgcr_el0_cg0nc; i++) {
466 ctx->group0_cnts[i] = amu_group0_cnt_read(i);
467 }
468
469 #if ENABLE_AMU_AUXILIARY_COUNTERS
470 for (i = 0U; i < amcgcr_el0_cg1nc; i++) {
471 ctx->group1_cnts[i] = amu_group1_cnt_read(i);
472 }
473 #endif
474
475 /*
476 * Save virtual offsets for counters that offer them.
477 */
478
479 if (hcr_el2_amvoffen != 0U) {
480 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) {
481 if (!amu_group0_voffset_supported(i)) {
482 continue; /* No virtual offset */
483 }
484
485 ctx->group0_voffsets[j++] = amu_group0_voffset_read(i);
486 }
487
488 #if ENABLE_AMU_AUXILIARY_COUNTERS
489 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) {
490 if ((amcg1idr_el0_voff >> i) & 1U) {
491 continue; /* No virtual offset */
492 }
493
494 ctx->group1_voffsets[j++] = amu_group1_voffset_read(i);
495 }
496 #endif
497 }
498
499 return (void *)0;
500 }
501
amu_context_restore(const void * arg)502 static void *amu_context_restore(const void *arg)
503 {
504 uint64_t i, j;
505
506 unsigned int core_pos;
507 struct amu_ctx *ctx;
508
509 uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */
510
511 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */
512
513 #if ENABLE_AMU_AUXILIARY_COUNTERS
514 uint64_t amcfgr_el0_ncg; /* Number of counter groups */
515 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */
516 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */
517 #endif
518
519 if (!is_feat_amu_supported()) {
520 return (void *)0;
521 }
522
523 core_pos = plat_my_core_pos();
524 ctx = &amu_ctxs_[core_pos];
525
526 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc();
527
528 if (is_feat_amuv1p1_supported()) {
529 hcr_el2_amvoffen = read_hcr_el2_amvoffen();
530 }
531
532 #if ENABLE_AMU_AUXILIARY_COUNTERS
533 amcfgr_el0_ncg = read_amcfgr_el0_ncg();
534 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U;
535 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U;
536 #endif
537
538 /*
539 * Restore the counter values from the local context.
540 */
541
542 for (i = 0U; i < amcgcr_el0_cg0nc; i++) {
543 amu_group0_cnt_write(i, ctx->group0_cnts[i]);
544 }
545
546 #if ENABLE_AMU_AUXILIARY_COUNTERS
547 for (i = 0U; i < amcgcr_el0_cg1nc; i++) {
548 amu_group1_cnt_write(i, ctx->group1_cnts[i]);
549 }
550 #endif
551
552 /*
553 * Restore virtual offsets for counters that offer them.
554 */
555
556 if (hcr_el2_amvoffen != 0U) {
557 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) {
558 if (!amu_group0_voffset_supported(i)) {
559 continue; /* No virtual offset */
560 }
561
562 amu_group0_voffset_write(i, ctx->group0_voffsets[j++]);
563 }
564
565 #if ENABLE_AMU_AUXILIARY_COUNTERS
566 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) {
567 if ((amcg1idr_el0_voff >> i) & 1U) {
568 continue; /* No virtual offset */
569 }
570
571 amu_group1_voffset_write(i, ctx->group1_voffsets[j++]);
572 }
573 #endif
574 }
575
576 /*
577 * Re-enable counters that were disabled during context save.
578 */
579
580 write_amcntenset0_el0_px(ctx->group0_enable);
581
582 #if ENABLE_AMU_AUXILIARY_COUNTERS
583 if (amcfgr_el0_ncg > 0) {
584 write_amcntenset1_el0_px(ctx->group1_enable);
585 }
586 #endif
587
588 #if ENABLE_MPMM
589 mpmm_enable();
590 #endif
591
592 return (void *)0;
593 }
594
595 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save);
596 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore);
597