1 // SPDX-License-Identifier: GPL-2.0-only
42 #define KHZ	1000
83 		data->cyc2ns_offset = this_cpu_read(cyc2ns.data[idx].cyc2ns_offset);  in __cyc2ns_read()
84 		data->cyc2ns_mul    = this_cpu_read(cyc2ns.data[idx].cyc2ns_mul);  in __cyc2ns_read()
85 		data->cyc2ns_shift  = this_cpu_read(cyc2ns.data[idx].cyc2ns_shift);  in __cyc2ns_read()
116  *  cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication
117  *  (64-bit result) can be used.
122  *                      -[email protected] "math is hard, lets go shopping!"
166 	 * conversion algorithm shifting a 32-bit value (now specifies a 64-bit  in __set_cyc2ns_scale()
167 	 * value) - refer perf_event_mmap_page documentation in perf_event.h.  in __set_cyc2ns_scale()
174 	data.cyc2ns_offset = ns_now -  in __set_cyc2ns_scale()
179 	write_seqcount_latch_begin(&c2n->seq);  in __set_cyc2ns_scale()
180 	c2n->data[0] = data;  in __set_cyc2ns_scale()
181 	write_seqcount_latch(&c2n->seq);  in __set_cyc2ns_scale()
182 	c2n->data[1] = data;  in __set_cyc2ns_scale()
183 	write_seqcount_latch_end(&c2n->seq);  in __set_cyc2ns_scale()
207 	seqcount_latch_init(&c2n->seq);  in cyc2ns_init_boot_cpu()
220 	struct cyc2ns_data *data = c2n->data;  in cyc2ns_init_secondary_cpus()
224 			seqcount_latch_init(&c2n->seq);  in cyc2ns_init_secondary_cpus()
226 			c2n->data[0] = data[0];  in cyc2ns_init_secondary_cpus()
227 			c2n->data[1] = data[1];  in cyc2ns_init_secondary_cpus()
233  * Scheduler clock - returns current time in nanosec units.
254 	return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);  in native_sched_clock()
370 		if ((t2 - t1) < thresh)  in tsc_read_refs()
385 	hpet2 -= hpet1;  in calc_hpet_ref()
405 	pm2 -= pm1;  in calc_pmtimer_ref()
414 #define CAL_LATCH	(PIT_TICK_RATE / (1000 / CAL_MS))
415 #define CAL_PIT_LOOPS	1000
418 #define CAL2_LATCH	(PIT_TICK_RATE / (1000 / CAL2_MS))
468 		delta = t2 - tsc;  in pit_calibrate_tsc()
490 	delta = t2 - t1;  in pit_calibrate_tsc()
498  * non-virtualized hardware.
502  *  - the PIT is running at roughly 1.19MHz
504  *  - each IO is going to take about 1us on real hardware,
507  *    update - anything else implies a unacceptably slow CPU
510  *  - with 256 PIT ticks to read the value, we have 214us to
514  *  - We're doing 2 reads per loop (LSB, MSB), and we expect
519  *  - if the PIT is stuck, and we see *many* more reads, we
548 	*deltap = get_cycles() - prev_tsc;  in pit_expect_msb()
565 #define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
580 	 * Counter 2, mode 0 (one-shot), binary count  in quick_pit_calibrate()
584 	 * final output frequency as a decrement-by-one),  in quick_pit_calibrate()
597 	 * to do that is to just read back the 16-bit counter  in quick_pit_calibrate()
604 			if (!pit_expect_msb(0xff-i, &delta, &d2))  in quick_pit_calibrate()
607 			delta -= tsc;  in quick_pit_calibrate()
630 			if (!pit_verify_msb(0xfe - i))  in quick_pit_calibrate()
648 	 * kHz = ticks / time-in-seconds / 1000;  in quick_pit_calibrate()
649 	 * kHz = (t2 - t1) / (I * 256 / PIT_TICK_RATE) / 1000  in quick_pit_calibrate()
650 	 * kHz = ((t2 - t1) * PIT_TICK_RATE) / (I * 256 * 1000)  in quick_pit_calibrate()
653 	do_div(delta, i*256*1000);  in quick_pit_calibrate()
659  * native_calibrate_tsc - determine TSC frequency
681 	crystal_khz = ecx_hz / 1000;  in native_calibrate_tsc()
706 		unsigned int eax_base_mhz, ebx, ecx, edx;  in native_calibrate_tsc()  local
708 		cpuid(CPUID_LEAF_FREQ, &eax_base_mhz, &ebx, &ecx, &edx);  in native_calibrate_tsc()
709 		crystal_khz = eax_base_mhz * 1000 *  in native_calibrate_tsc()
730 	lapic_timer_period = crystal_khz * 1000 / HZ;  in native_calibrate_tsc()
750 	return eax_base_mhz * 1000;  in cpu_khz_from_cpuid()
820 		tsc2 = (tsc2 - tsc1) * 1000000LL;  in pit_hpet_ptimer_calibrate_cpu()
907  * native_calibrate_cpu_early - can calibrate the cpu early in boot
925  * native_calibrate_cpu - calibrate the cpu
949 	else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz)  in recalibrate_cpu_khz()
996 	offset = cyc2ns_suspend - sched_clock();  in tsc_restore_sched_clock_state()
1033 		ref_freq = freq->old;  in time_cpufreq_notifier()
1038 	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||  in time_cpufreq_notifier()
1039 	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) {  in time_cpufreq_notifier()
1041 			cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);  in time_cpufreq_notifier()
1043 		tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);  in time_cpufreq_notifier()
1044 		if (!(freq->flags & CPUFREQ_CONST_LOOPS))  in time_cpufreq_notifier()
1047 		set_cyc2ns_scale(tsc_khz, freq->policy->cpu, rdtsc());  in time_cpufreq_notifier()
1085 	 * Don't enable ART in a VM, non-stop TSC and TSC_ADJUST required,  in detect_art()
1117  * structure to avoid a nasty time-warp. This can be observed in a
1128  * checking the result of read_tsc() - cycle_last for being negative.
1167 	.name			= "tsc-early",
1244 		/* Geode_LX - the OLPC CPU has a very reliable TSC */  in check_system_tsc_reliable()
1254 	 *  - TSC running at constant frequency  in check_system_tsc_reliable()
1255 	 *  - TSC which does not stop in C-States  in check_system_tsc_reliable()
1256 	 *  - the TSC_ADJUST register which allows to detect even minimal  in check_system_tsc_reliable()
1258 	 *  - not more than four packages  in check_system_tsc_reliable()
1302  * tsc_refine_calibration_work - Further refine tsc freq calibration
1354 	delta = tsc_stop - tsc_start;  in tsc_refine_calibration_work()
1365 		if (abs(tsc_khz - freq) > (tsc_khz >> 11)) {  in tsc_refine_calibration_work()
1368 				(unsigned long)tsc_khz / 1000,  in tsc_refine_calibration_work()
1369 				(unsigned long)tsc_khz % 1000);  in tsc_refine_calibration_work()
1374 			(unsigned long)freq / 1000,  in tsc_refine_calibration_work()
1375 			(unsigned long)freq % 1000);  in tsc_refine_calibration_work()
1381 	if (abs(tsc_khz - freq) > tsc_khz/100)  in tsc_refine_calibration_work()
1386 		(unsigned long)tsc_khz / 1000,  in tsc_refine_calibration_work()
1387 		(unsigned long)tsc_khz % 1000);  in tsc_refine_calibration_work()
1462 		/* We should not be here with non-native cpu calibration */  in determine_cpu_tsc_frequencies()
1468 	 * Trust non-zero tsc_khz as authoritative,  in determine_cpu_tsc_frequencies()
1474 	else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz)  in determine_cpu_tsc_frequencies()
1515 	/* Don't change UV TSC multi-chassis synchronization */  in tsc_early_init()