1 /*
2 * Copyright (c) 2013, Google Inc. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files
6 * (the "Software"), to deal in the Software without restriction,
7 * including without limitation the rights to use, copy, modify, merge,
8 * publish, distribute, sublicense, and/or sell copies of the Software,
9 * and to permit persons to whom the Software is furnished to do so,
10 * subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <arch/ops.h>
25 #include <assert.h>
26 #include <lk/init.h>
27 #include <platform.h>
28 #include <platform/interrupts.h>
29 #include <platform/timer.h>
30 #include <trace.h>
31 #include <inttypes.h>
32
33 #define LOCAL_TRACE 0
34
35 #include <lib/fixed_point.h>
36
37 #if ARCH_ARM64
38
39 /* CNTFRQ AArch64 register */
40 #define TIMER_REG_CNTFRQ cntfrq_el0
41
42 /* CNTP AArch64 registers */
43 #define TIMER_REG_CNTP_CTL cntp_ctl_el0
44 #define TIMER_REG_CNTP_CVAL cntp_cval_el0
45 #define TIMER_REG_CNTP_TVAL cntp_tval_el0
46 #define TIMER_REG_CNTPCT cntpct_el0
47
48 /* CNTPS AArch64 registers */
49 #define TIMER_REG_CNTPS_CTL cntps_ctl_el1
50 #define TIMER_REG_CNTPS_CVAL cntps_cval_el1
51 #define TIMER_REG_CNTPS_TVAL cntps_tval_el1
52 #define TIMER_REG_CNTPSCT cntpct_el0
53
54 /* CNTV AArch64 registers */
55 #define TIMER_REG_CNTV_CTL cntv_ctl_el0
56 #define TIMER_REG_CNTV_CVAL cntv_cval_el0
57 #define TIMER_REG_CNTV_TVAL cntv_tval_el0
58 #define TIMER_REG_CNTVCT cntvct_el0
59
60 #define READ_TIMER_REG32(reg) ARM64_READ_SYSREG(reg)
61 #define READ_TIMER_REG64(reg) ARM64_READ_SYSREG(reg)
62 #define WRITE_TIMER_REG32(reg, val) ARM64_WRITE_SYSREG(reg, (uint64_t)val)
63 #define WRITE_TIMER_REG64(reg, val) ARM64_WRITE_SYSREG(reg, val)
64
65 #else
66
67 /* CNTFRQ AArch32 register */
68 #define TIMER_REG_CNTFRQ "c0, 0"
69
70 /* CNTP AArch32 registers */
71 #define TIMER_REG_CNTP_CTL "c2, 1"
72 #define TIMER_REG_CNTP_CVAL "2"
73 #define TIMER_REG_CNTP_TVAL "c2, 0"
74 #define TIMER_REG_CNTPCT "0"
75
76 /* CNTPS AArch32 registers are banked and accessed though CNTP */
77 #define CNTPS CNTP
78
79 /* CNTV AArch32 registers */
80 #define TIMER_REG_CNTV_CTL "c3, 1"
81 #define TIMER_REG_CNTV_CVAL "3"
82 #define TIMER_REG_CNTV_TVAL "c3, 0"
83 #define TIMER_REG_CNTVCT "1"
84
85 #define READ_TIMER_REG32(reg) \
86 ({ \
87 uint32_t _val; \
88 __asm__ volatile("mrc p15, 0, %0, c14, " reg : "=r" (_val)); \
89 _val; \
90 })
91
92 #define READ_TIMER_REG64(reg) \
93 ({ \
94 uint64_t _val; \
95 __asm__ volatile("mrrc p15, " reg ", %0, %H0, c14" : "=r" (_val)); \
96 _val; \
97 })
98
99 #define WRITE_TIMER_REG32(reg, val) \
100 ({ \
101 __asm__ volatile("mcr p15, 0, %0, c14, " reg :: "r" (val)); \
102 ISB; \
103 })
104
105 #define WRITE_TIMER_REG64(reg, val) \
106 ({ \
107 __asm__ volatile("mcrr p15, " reg ", %0, %H0, c14" :: "r" (val)); \
108 ISB; \
109 })
110
111 #endif
112
113 #ifndef TIMER_ARM_GENERIC_SELECTED
114 #define TIMER_ARM_GENERIC_SELECTED CNTP
115 #endif
116
117 #define COMBINE3(a,b,c) a ## b ## c
118 #define XCOMBINE3(a,b,c) COMBINE3(a, b, c)
119
120 #define SELECTED_TIMER_REG(reg) XCOMBINE3(TIMER_REG_, TIMER_ARM_GENERIC_SELECTED, reg)
121 #define TIMER_REG_CTL SELECTED_TIMER_REG(_CTL)
122 #define TIMER_REG_CVAL SELECTED_TIMER_REG(_CVAL)
123 #define TIMER_REG_TVAL SELECTED_TIMER_REG(_TVAL)
124 #define TIMER_REG_CT SELECTED_TIMER_REG(CT)
125
126
127 static platform_timer_callback t_callback;
128 static int timer_irq;
129
130 struct fp_32_64 cntpct_per_ns;
131 struct fp_32_64 ms_per_cntpct;
132 struct fp_32_64 ns_per_cntpct;
133
lk_time_ns_to_cntpct(lk_time_ns_t lk_time_ns)134 static uint64_t lk_time_ns_to_cntpct(lk_time_ns_t lk_time_ns)
135 {
136 return u64_mul_u64_fp32_64(lk_time_ns, cntpct_per_ns);
137 }
138
cntpct_to_lk_time(uint64_t cntpct)139 static lk_time_t cntpct_to_lk_time(uint64_t cntpct)
140 {
141 return u32_mul_u64_fp32_64(cntpct, ms_per_cntpct);
142 }
143
cntpct_to_lk_time_ns(uint64_t cntpct)144 static lk_time_ns_t cntpct_to_lk_time_ns(uint64_t cntpct)
145 {
146 return u64_mul_u64_fp32_64(cntpct, ns_per_cntpct);
147 }
148
read_cntfrq(void)149 static uint32_t read_cntfrq(void)
150 {
151 uint32_t cntfrq;
152
153 cntfrq = READ_TIMER_REG32(TIMER_REG_CNTFRQ);
154 LTRACEF("cntfrq: 0x%08x, %u\n", cntfrq, cntfrq);
155 return cntfrq;
156 }
157
read_cntp_ctl(void)158 static uint32_t read_cntp_ctl(void)
159 {
160 uint32_t cntp_ctl;
161
162 cntp_ctl = READ_TIMER_REG32(TIMER_REG_CTL);
163 return cntp_ctl;
164 }
165
write_cntp_ctl(uint32_t cntp_ctl)166 static void write_cntp_ctl(uint32_t cntp_ctl)
167 {
168 LTRACEF_LEVEL(3, "cntp_ctl: 0x%x %x\n", cntp_ctl, read_cntp_ctl());
169 WRITE_TIMER_REG32(TIMER_REG_CTL, cntp_ctl);
170 }
171
read_cntp_cval(void)172 static uint64_t read_cntp_cval(void)
173 {
174 uint64_t cval;
175
176 cval = READ_TIMER_REG64(TIMER_REG_CVAL);
177 LTRACEF_LEVEL(3, "cntp cval: 0x%016" PRIx64 ", %" PRIu64 "\n", cval, cval);
178 return cval;
179 }
180
write_cntp_cval(uint64_t cntp_cval)181 static void write_cntp_cval(uint64_t cntp_cval)
182 {
183 LTRACEF_LEVEL(3, "cntp_cval: 0x%016" PRIx64 ", %" PRIu64 "\n", cntp_cval, cntp_cval);
184 WRITE_TIMER_REG64(TIMER_REG_CVAL, cntp_cval);
185 }
186
write_cntp_tval(int32_t cntp_tval)187 static void write_cntp_tval(int32_t cntp_tval)
188 {
189 LTRACEF_LEVEL(3, "cntp_tval: 0x%08x, %d\n", cntp_tval, cntp_tval);
190 WRITE_TIMER_REG32(TIMER_REG_TVAL, cntp_tval);
191 }
192
read_cntpct(void)193 static uint64_t read_cntpct(void)
194 {
195 uint64_t cntpct;
196
197 cntpct = READ_TIMER_REG64(TIMER_REG_CT);
198 LTRACEF_LEVEL(3, "cntpct: 0x%016" PRIx64 ", %" PRIu64 "\n", cntpct, cntpct);
199 return cntpct;
200 }
201
platform_tick(void * arg)202 static enum handler_return platform_tick(void *arg)
203 {
204 write_cntp_ctl(0);
205 if (t_callback) {
206 return t_callback(arg, current_time_ns());
207 } else {
208 return INT_NO_RESCHEDULE;
209 }
210 }
211
platform_set_oneshot_timer(platform_timer_callback callback,lk_time_ns_t time_ns)212 status_t platform_set_oneshot_timer(platform_timer_callback callback,
213 lk_time_ns_t time_ns)
214 {
215 uint64_t target_cntpct = lk_time_ns_to_cntpct(time_ns);
216
217 t_callback = callback;
218 write_cntp_cval(target_cntpct);
219 write_cntp_ctl(1);
220
221 return 0;
222 }
223
platform_stop_timer(void)224 void platform_stop_timer(void)
225 {
226 write_cntp_ctl(0);
227 }
228
current_time_ns(void)229 lk_time_ns_t current_time_ns(void)
230 {
231 return cntpct_to_lk_time_ns(read_cntpct());
232 }
233
current_time(void)234 lk_time_t current_time(void)
235 {
236 return cntpct_to_lk_time(read_cntpct());
237 }
238
uint64_sub_ignore_overflow(uint64_t a,uint64_t b)239 static uint64_t uint64_sub_ignore_overflow(uint64_t a, uint64_t b) {
240 uint64_t res;
241 __builtin_sub_overflow(a, b, &res);
242 return res;
243 }
244
test_time_conversion_check_result(uint64_t a,uint64_t b,uint64_t limit,bool is32)245 static void test_time_conversion_check_result(uint64_t a, uint64_t b, uint64_t limit, bool is32)
246 {
247 /* Check limit will not overflow if converted to signed type */
248 if (limit > (uint64_t)INT64_MAX) {
249 TRACEF("ERROR, limit too large\n");
250 } else if (a != b) {
251 const int64_t slimit = (int64_t)limit;
252 uint64_t a_minus_b = uint64_sub_ignore_overflow(a, b);
253 int64_t diff = is32 ? (int32_t)a_minus_b : (int64_t)a_minus_b;
254
255 if (diff >= -slimit && diff <= slimit)
256 LTRACEF("ROUNDED by %" PRId64 " (up to +/-%" PRIu64 " allowed)\n", diff, limit);
257 else
258 TRACEF("FAIL, off by %" PRId64 "\n", diff);
259 }
260 }
261
test_lk_time_ns_to_cntpct(uint32_t cntfrq,lk_time_ns_t lk_time_ns)262 static void test_lk_time_ns_to_cntpct(uint32_t cntfrq, lk_time_ns_t lk_time_ns)
263 {
264 uint64_t cntpct = lk_time_ns_to_cntpct(lk_time_ns);
265 uint64_t s = (1000 * 1000 * 1000);
266 uint64_t lk_time_s = lk_time_ns / s;
267 uint64_t lk_time_ns_rem = lk_time_ns % s;
268 uint64_t expected_cntpct = (lk_time_s * cntfrq) +
269 (lk_time_ns_rem * cntfrq + (s / 2)) / s;
270
271 test_time_conversion_check_result(cntpct, expected_cntpct, 1, false);
272 LTRACEF_LEVEL(2, "lk_time_ns_to_cntpct(%llu): got %" PRIu64 ", expect %" PRIu64 "\n",
273 lk_time_ns, cntpct, expected_cntpct);
274 }
275
test_cntpct_to_lk_time(uint32_t cntfrq,lk_time_t expected_lk_time,uint32_t wrap_count)276 static void test_cntpct_to_lk_time(uint32_t cntfrq, lk_time_t expected_lk_time, uint32_t wrap_count)
277 {
278 lk_time_t lk_time;
279 uint64_t cntpct;
280
281 cntpct = (uint64_t)cntfrq * expected_lk_time / 1000;
282 if ((uint64_t)cntfrq * wrap_count > UINT_MAX)
283 cntpct += (((uint64_t)cntfrq << 32) / 1000) * wrap_count;
284 else
285 cntpct += (((uint64_t)(cntfrq * wrap_count) << 32) / 1000);
286 lk_time = cntpct_to_lk_time(cntpct);
287
288 test_time_conversion_check_result(lk_time, expected_lk_time,
289 (1000ULL + cntfrq - 1) / cntfrq, true);
290 LTRACEF_LEVEL(2, "cntpct_to_lk_time(%" PRIu64 "): got %u, expect %u\n", cntpct, lk_time, expected_lk_time);
291 }
292
test_cntpct_to_lk_time_ns(uint32_t cntfrq,uint64_t expected_s)293 static void test_cntpct_to_lk_time_ns(uint32_t cntfrq, uint64_t expected_s)
294 {
295 lk_time_ns_t expected_lk_time_ns = expected_s * 1000 * 1000 * 1000;
296 uint64_t cntpct = (uint64_t)cntfrq * expected_s;
297 lk_time_ns_t lk_time_ns = cntpct_to_lk_time_ns(cntpct);
298
299 test_time_conversion_check_result(
300 lk_time_ns, expected_lk_time_ns,
301 (1000ULL * 1000 * 1000 + cntfrq - 1) / cntfrq, false);
302 LTRACEF_LEVEL(2, "cntpct_to_lk_time_ns(%" PRIu64 "): got %llu, expect %llu\n",
303 cntpct, lk_time_ns, expected_lk_time_ns);
304 }
305
test_time_conversions(uint32_t cntfrq)306 static void test_time_conversions(uint32_t cntfrq)
307 {
308 test_lk_time_ns_to_cntpct(cntfrq, 0);
309 test_lk_time_ns_to_cntpct(cntfrq, 1);
310 test_lk_time_ns_to_cntpct(cntfrq, INT_MAX);
311 test_lk_time_ns_to_cntpct(cntfrq, INT_MAX + 1U);
312 test_lk_time_ns_to_cntpct(cntfrq, ~0U);
313 test_lk_time_ns_to_cntpct(cntfrq, cntfrq <= 1000 * 1000 * 1000 ? ~0ULL :
314 cntpct_to_lk_time_ns(~0ULL));
315 test_cntpct_to_lk_time(cntfrq, 0, 0);
316 test_cntpct_to_lk_time(cntfrq, INT_MAX, 0);
317 test_cntpct_to_lk_time(cntfrq, INT_MAX + 1U, 0);
318 test_cntpct_to_lk_time(cntfrq, ~0U, 0);
319 test_cntpct_to_lk_time(cntfrq, 0, 1);
320 test_cntpct_to_lk_time(cntfrq, 0, 7);
321 test_cntpct_to_lk_time(cntfrq, 0, 70);
322 test_cntpct_to_lk_time(cntfrq, 0, 700);
323 test_cntpct_to_lk_time_ns(cntfrq, 0);
324 test_cntpct_to_lk_time_ns(cntfrq, 1);
325 test_cntpct_to_lk_time_ns(cntfrq, 60 * 60 * 24);
326 test_cntpct_to_lk_time_ns(cntfrq, 60 * 60 * 24 * 365);
327 test_cntpct_to_lk_time_ns(cntfrq, 60 * 60 * 24 * (365 * 10 + 2));
328 test_cntpct_to_lk_time_ns(cntfrq, 60ULL * 60 * 24 * (365 * 100 + 2));
329 }
330
arm_generic_timer_init_conversion_factors(uint32_t cntfrq)331 static void arm_generic_timer_init_conversion_factors(uint32_t cntfrq)
332 {
333 fp_32_64_div_32_32(&cntpct_per_ns, cntfrq, 1000 * 1000 * 1000);
334 fp_32_64_div_32_32(&ms_per_cntpct, 1000, cntfrq);
335 fp_32_64_div_32_32(&ns_per_cntpct, 1000 * 1000 * 1000, cntfrq);
336 LTRACEF("cntpct_per_ns: %08x.%08x%08x\n", cntpct_per_ns.l0, cntpct_per_ns.l32, cntpct_per_ns.l64);
337 LTRACEF("ms_per_cntpct: %08x.%08x%08x\n", ms_per_cntpct.l0, ms_per_cntpct.l32, ms_per_cntpct.l64);
338 LTRACEF("ns_per_cntpct: %08x.%08x%08x\n", ns_per_cntpct.l0, ns_per_cntpct.l32, ns_per_cntpct.l64);
339 }
340
arm_generic_timer_init(int irq,uint32_t freq_override)341 void arm_generic_timer_init(int irq, uint32_t freq_override)
342 {
343 uint32_t cntfrq;
344
345 // use frequency value from cntfrq reg by default
346 cntfrq = read_cntfrq();
347
348 if (!cntfrq) {
349 if (freq_override == 0) {
350 TRACEF("Failed to initialize timer, frequency is 0\n");
351 return;
352 } else {
353 cntfrq = freq_override;
354 }
355 }
356
357 #if LOCAL_TRACE
358 LTRACEF("Test min cntfrq\n");
359 arm_generic_timer_init_conversion_factors(1);
360 test_time_conversions(1);
361 LTRACEF("Test max cntfrq\n");
362 arm_generic_timer_init_conversion_factors(~0U);
363 test_time_conversions(~0U);
364 LTRACEF("Set actual cntfrq\n");
365 #endif
366 arm_generic_timer_init_conversion_factors(cntfrq);
367 test_time_conversions(cntfrq);
368
369 LTRACEF("register irq %d on cpu %d\n", irq, arch_curr_cpu_num());
370 register_int_handler(irq, &platform_tick, NULL);
371 unmask_interrupt(irq);
372
373 timer_irq = irq;
374 }
375
arm_generic_timer_init_secondary_cpu(uint level)376 static void arm_generic_timer_init_secondary_cpu(uint level)
377 {
378 LTRACEF("register irq %d on cpu %d\n", timer_irq, arch_curr_cpu_num());
379 register_int_handler(timer_irq, &platform_tick, NULL);
380 unmask_interrupt(timer_irq);
381 }
382
383 /* secondary cpu initialize the timer just before the kernel starts with interrupts enabled */
384 LK_INIT_HOOK_FLAGS(arm_generic_timer_init_secondary_cpu,
385 arm_generic_timer_init_secondary_cpu,
386 LK_INIT_LEVEL_THREADING - 1, LK_INIT_FLAG_SECONDARY_CPUS);
387
388
389 static struct platform_timer_state saved_state[SMP_MAX_CPUS];
390
platform_export_timer_state(struct platform_timer_state * ts,bool raw)391 void platform_export_timer_state(struct platform_timer_state *ts, bool raw)
392 {
393 ASSERT(ts);
394
395 uint32_t ctl = read_cntp_ctl();
396 if (ctl & 1) {
397 /* timer is set */
398 if (raw) {
399 ts->tval = read_cntpct();
400 ts->cval = read_cntp_cval();
401 } else {
402 ts->tval = u64_mul_u64_fp32_64(read_cntpct(), ns_per_cntpct);
403 ts->cval = u64_mul_u64_fp32_64(read_cntp_cval(), ns_per_cntpct);
404 }
405 } else {
406 /* timer is not set */
407 ts->tval = 0;
408 ts->cval = 0;
409 }
410 }
411
arm_generic_timer_suspend_cpu(uint level)412 static void arm_generic_timer_suspend_cpu(uint level)
413 {
414 unsigned int cpu = arch_curr_cpu_num();
415 platform_export_timer_state(&saved_state[cpu], true);
416 }
417
418 LK_INIT_HOOK_FLAGS(arm_generic_timer_suspend_cpu, arm_generic_timer_suspend_cpu,
419 LK_INIT_LEVEL_PLATFORM, LK_INIT_FLAG_CPU_SUSPEND);
420
arm_generic_timer_resume_cpu(uint level)421 static void arm_generic_timer_resume_cpu(uint level)
422 {
423 unsigned int cpu = arch_curr_cpu_num();
424 struct platform_timer_state *ts = &saved_state[cpu];
425
426 if (ts->tval) {
427 /* need to restart timer */
428 write_cntp_cval(ts->cval);
429 write_cntp_ctl(1);
430 }
431 }
432
433 LK_INIT_HOOK_FLAGS(arm_generic_timer_resume_cpu, arm_generic_timer_resume_cpu,
434 LK_INIT_LEVEL_PLATFORM, LK_INIT_FLAG_CPU_RESUME);
435