1 /*
2 * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <cdefs.h>
9 #include <stdbool.h>
10
11 #include "../amu_private.h"
12 #include <arch.h>
13 #include <arch_features.h>
14 #include <arch_helpers.h>
15 #include <common/debug.h>
16 #include <lib/el3_runtime/pubsub_events.h>
17 #include <lib/extensions/amu.h>
18
19 #include <plat/common/platform.h>
20
21 struct amu_ctx {
22 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS];
23 #if ENABLE_AMU_AUXILIARY_COUNTERS
24 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS];
25 #endif
26
27 uint16_t group0_enable;
28 #if ENABLE_AMU_AUXILIARY_COUNTERS
29 uint16_t group1_enable;
30 #endif
31 };
32
33 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT];
34
35 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS,
36 amu_ctx_group0_enable_cannot_represent_all_group0_counters);
37
38 #if ENABLE_AMU_AUXILIARY_COUNTERS
39 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS,
40 amu_ctx_group1_enable_cannot_represent_all_group1_counters);
41 #endif
42
write_hcptr_tam(uint32_t value)43 static inline __unused void write_hcptr_tam(uint32_t value)
44 {
45 write_hcptr((read_hcptr() & ~TAM_BIT) |
46 ((value << TAM_SHIFT) & TAM_BIT));
47 }
48
write_amcr_cg1rz(uint32_t value)49 static inline __unused void write_amcr_cg1rz(uint32_t value)
50 {
51 write_amcr((read_amcr() & ~AMCR_CG1RZ_BIT) |
52 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT));
53 }
54
read_amcfgr_ncg(void)55 static inline __unused uint32_t read_amcfgr_ncg(void)
56 {
57 return (read_amcfgr() >> AMCFGR_NCG_SHIFT) &
58 AMCFGR_NCG_MASK;
59 }
60
read_amcgcr_cg0nc(void)61 static inline __unused uint32_t read_amcgcr_cg0nc(void)
62 {
63 return (read_amcgcr() >> AMCGCR_CG0NC_SHIFT) &
64 AMCGCR_CG0NC_MASK;
65 }
66
read_amcgcr_cg1nc(void)67 static inline __unused uint32_t read_amcgcr_cg1nc(void)
68 {
69 return (read_amcgcr() >> AMCGCR_CG1NC_SHIFT) &
70 AMCGCR_CG1NC_MASK;
71 }
72
read_amcntenset0_px(void)73 static inline __unused uint32_t read_amcntenset0_px(void)
74 {
75 return (read_amcntenset0() >> AMCNTENSET0_Pn_SHIFT) &
76 AMCNTENSET0_Pn_MASK;
77 }
78
read_amcntenset1_px(void)79 static inline __unused uint32_t read_amcntenset1_px(void)
80 {
81 return (read_amcntenset1() >> AMCNTENSET1_Pn_SHIFT) &
82 AMCNTENSET1_Pn_MASK;
83 }
84
write_amcntenset0_px(uint32_t px)85 static inline __unused void write_amcntenset0_px(uint32_t px)
86 {
87 uint32_t value = read_amcntenset0();
88
89 value &= ~AMCNTENSET0_Pn_MASK;
90 value |= (px << AMCNTENSET0_Pn_SHIFT) &
91 AMCNTENSET0_Pn_MASK;
92
93 write_amcntenset0(value);
94 }
95
write_amcntenset1_px(uint32_t px)96 static inline __unused void write_amcntenset1_px(uint32_t px)
97 {
98 uint32_t value = read_amcntenset1();
99
100 value &= ~AMCNTENSET1_Pn_MASK;
101 value |= (px << AMCNTENSET1_Pn_SHIFT) &
102 AMCNTENSET1_Pn_MASK;
103
104 write_amcntenset1(value);
105 }
106
write_amcntenclr0_px(uint32_t px)107 static inline __unused void write_amcntenclr0_px(uint32_t px)
108 {
109 uint32_t value = read_amcntenclr0();
110
111 value &= ~AMCNTENCLR0_Pn_MASK;
112 value |= (px << AMCNTENCLR0_Pn_SHIFT) & AMCNTENCLR0_Pn_MASK;
113
114 write_amcntenclr0(value);
115 }
116
write_amcntenclr1_px(uint32_t px)117 static inline __unused void write_amcntenclr1_px(uint32_t px)
118 {
119 uint32_t value = read_amcntenclr1();
120
121 value &= ~AMCNTENCLR1_Pn_MASK;
122 value |= (px << AMCNTENCLR1_Pn_SHIFT) & AMCNTENCLR1_Pn_MASK;
123
124 write_amcntenclr1(value);
125 }
126
127 #if ENABLE_AMU_AUXILIARY_COUNTERS
amu_group1_supported(void)128 static __unused bool amu_group1_supported(void)
129 {
130 return read_amcfgr_ncg() > 0U;
131 }
132 #endif
133
134 /*
135 * Enable counters. This function is meant to be invoked by the context
136 * management library before exiting from EL3.
137 */
amu_enable(bool el2_unused)138 void amu_enable(bool el2_unused)
139 {
140 uint32_t amcfgr_ncg; /* Number of counter groups */
141 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */
142
143 uint32_t amcntenset0_px = 0x0; /* Group 0 enable mask */
144 uint32_t amcntenset1_px = 0x0; /* Group 1 enable mask */
145
146 if (el2_unused) {
147 /*
148 * HCPTR.TAM: Set to zero so any accesses to the Activity
149 * Monitor registers do not trap to EL2.
150 */
151 write_hcptr_tam(0U);
152 }
153
154 /*
155 * Retrieve the number of architected counters. All of these counters
156 * are enabled by default.
157 */
158
159 amcgcr_cg0nc = read_amcgcr_cg0nc();
160 amcntenset0_px = (UINT32_C(1) << (amcgcr_cg0nc)) - 1U;
161
162 assert(amcgcr_cg0nc <= AMU_AMCGCR_CG0NC_MAX);
163
164 /*
165 * The platform may opt to enable specific auxiliary counters. This can
166 * be done via the common FCONF getter, or via the platform-implemented
167 * function.
168 */
169
170 #if ENABLE_AMU_AUXILIARY_COUNTERS
171 const struct amu_topology *topology;
172
173 #if ENABLE_AMU_FCONF
174 topology = FCONF_GET_PROPERTY(amu, config, topology);
175 #else
176 topology = plat_amu_topology();
177 #endif /* ENABLE_AMU_FCONF */
178
179 if (topology != NULL) {
180 unsigned int core_pos = plat_my_core_pos();
181
182 amcntenset1_el0_px = topology->cores[core_pos].enable;
183 } else {
184 ERROR("AMU: failed to generate AMU topology\n");
185 }
186 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */
187
188 /*
189 * Enable the requested counters.
190 */
191
192 write_amcntenset0_px(amcntenset0_px);
193
194 amcfgr_ncg = read_amcfgr_ncg();
195 if (amcfgr_ncg > 0U) {
196 write_amcntenset1_px(amcntenset1_px);
197
198 #if !ENABLE_AMU_AUXILIARY_COUNTERS
199 VERBOSE("AMU: auxiliary counters detected but support is disabled\n");
200 #endif
201 }
202
203 /* Bail out if FEAT_AMUv1p1 features are not present. */
204 if (!is_feat_amuv1p1_supported()) {
205 return;
206 }
207
208 #if AMU_RESTRICT_COUNTERS
209 /*
210 * FEAT_AMUv1p1 adds a register field to restrict access to group 1
211 * counters at all but the highest implemented EL. This is controlled
212 * with the AMU_RESTRICT_COUNTERS compile time flag, when set, system
213 * register reads at lower ELs return zero. Reads from the memory
214 * mapped view are unaffected.
215 */
216 VERBOSE("AMU group 1 counter access restricted.\n");
217 write_amcr_cg1rz(1U);
218 #else
219 write_amcr_cg1rz(0U);
220 #endif
221 }
222
223 /* Read the group 0 counter identified by the given `idx`. */
amu_group0_cnt_read(unsigned int idx)224 static uint64_t amu_group0_cnt_read(unsigned int idx)
225 {
226 assert(is_feat_amu_supported());
227 assert(idx < read_amcgcr_cg0nc());
228
229 return amu_group0_cnt_read_internal(idx);
230 }
231
232 /* Write the group 0 counter identified by the given `idx` with `val` */
amu_group0_cnt_write(unsigned int idx,uint64_t val)233 static void amu_group0_cnt_write(unsigned int idx, uint64_t val)
234 {
235 assert(is_feat_amu_supported());
236 assert(idx < read_amcgcr_cg0nc());
237
238 amu_group0_cnt_write_internal(idx, val);
239 isb();
240 }
241
242 #if ENABLE_AMU_AUXILIARY_COUNTERS
243 /* Read the group 1 counter identified by the given `idx` */
amu_group1_cnt_read(unsigned int idx)244 static uint64_t amu_group1_cnt_read(unsigned int idx)
245 {
246 assert(is_feat_amu_supported());
247 assert(amu_group1_supported());
248 assert(idx < read_amcgcr_cg1nc());
249
250 return amu_group1_cnt_read_internal(idx);
251 }
252
253 /* Write the group 1 counter identified by the given `idx` with `val` */
amu_group1_cnt_write(unsigned int idx,uint64_t val)254 static void amu_group1_cnt_write(unsigned int idx, uint64_t val)
255 {
256 assert(is_feat_amu_supported());
257 assert(amu_group1_supported());
258 assert(idx < read_amcgcr_cg1nc());
259
260 amu_group1_cnt_write_internal(idx, val);
261 isb();
262 }
263 #endif
264
amu_context_save(const void * arg)265 static void *amu_context_save(const void *arg)
266 {
267 uint32_t i;
268
269 unsigned int core_pos;
270 struct amu_ctx *ctx;
271
272 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */
273
274 #if ENABLE_AMU_AUXILIARY_COUNTERS
275 uint32_t amcfgr_ncg; /* Number of counter groups */
276 uint32_t amcgcr_cg1nc; /* Number of group 1 counters */
277 #endif
278
279 if (!is_feat_amu_supported()) {
280 return (void *)0;
281 }
282
283 core_pos = plat_my_core_pos();
284 ctx = &amu_ctxs_[core_pos];
285
286 amcgcr_cg0nc = read_amcgcr_cg0nc();
287
288 #if ENABLE_AMU_AUXILIARY_COUNTERS
289 amcfgr_ncg = read_amcfgr_ncg();
290 amcgcr_cg1nc = (amcfgr_ncg > 0U) ? read_amcgcr_cg1nc() : 0U;
291 #endif
292
293 /*
294 * Disable all AMU counters.
295 */
296
297 ctx->group0_enable = read_amcntenset0_px();
298 write_amcntenclr0_px(ctx->group0_enable);
299
300 #if ENABLE_AMU_AUXILIARY_COUNTERS
301 if (amcfgr_ncg > 0U) {
302 ctx->group1_enable = read_amcntenset1_px();
303 write_amcntenclr1_px(ctx->group1_enable);
304 }
305 #endif
306
307 /*
308 * Save the counters to the local context.
309 */
310
311 isb(); /* Ensure counters have been stopped */
312
313 for (i = 0U; i < amcgcr_cg0nc; i++) {
314 ctx->group0_cnts[i] = amu_group0_cnt_read(i);
315 }
316
317 #if ENABLE_AMU_AUXILIARY_COUNTERS
318 for (i = 0U; i < amcgcr_cg1nc; i++) {
319 ctx->group1_cnts[i] = amu_group1_cnt_read(i);
320 }
321 #endif
322
323 return (void *)0;
324 }
325
amu_context_restore(const void * arg)326 static void *amu_context_restore(const void *arg)
327 {
328 uint32_t i;
329
330 unsigned int core_pos;
331 struct amu_ctx *ctx;
332
333 uint32_t amcfgr_ncg; /* Number of counter groups */
334 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */
335
336 #if ENABLE_AMU_AUXILIARY_COUNTERS
337 uint32_t amcgcr_cg1nc; /* Number of group 1 counters */
338 #endif
339
340 if (!is_feat_amu_supported()) {
341 return (void *)0;
342 }
343
344 core_pos = plat_my_core_pos();
345 ctx = &amu_ctxs_[core_pos];
346
347 amcfgr_ncg = read_amcfgr_ncg();
348 amcgcr_cg0nc = read_amcgcr_cg0nc();
349
350 #if ENABLE_AMU_AUXILIARY_COUNTERS
351 amcgcr_cg1nc = (amcfgr_ncg > 0U) ? read_amcgcr_cg1nc() : 0U;
352 #endif
353
354 /*
355 * Sanity check that all counters were disabled when the context was
356 * previously saved.
357 */
358
359 assert(read_amcntenset0_px() == 0U);
360
361 if (amcfgr_ncg > 0U) {
362 assert(read_amcntenset1_px() == 0U);
363 }
364
365 /*
366 * Restore the counter values from the local context.
367 */
368
369 for (i = 0U; i < amcgcr_cg0nc; i++) {
370 amu_group0_cnt_write(i, ctx->group0_cnts[i]);
371 }
372
373 #if ENABLE_AMU_AUXILIARY_COUNTERS
374 for (i = 0U; i < amcgcr_cg1nc; i++) {
375 amu_group1_cnt_write(i, ctx->group1_cnts[i]);
376 }
377 #endif
378
379 /*
380 * Re-enable counters that were disabled during context save.
381 */
382
383 write_amcntenset0_px(ctx->group0_enable);
384
385 #if ENABLE_AMU_AUXILIARY_COUNTERS
386 if (amcfgr_ncg > 0U) {
387 write_amcntenset1_px(ctx->group1_enable);
388 }
389 #endif
390
391 return (void *)0;
392 }
393
394 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save);
395 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore);
396