1 // SPDX-License-Identifier: GPL-2.0
3  * CPUFreq governor based on scheduler-provided CPU utilization data.
52 	/* The field below is for single-CPU policies only: */
67 	 * Since cpufreq_update_util() is called with rq->lock held for  in sugov_should_update_freq()
68 	 * the @target_cpu, our per-CPU data is fully serialized.  in sugov_should_update_freq()
70 	 * However, drivers cannot in general deal with cross-CPU  in sugov_should_update_freq()
81 	if (!cpufreq_this_cpu_can_update(sg_policy->policy))  in sugov_should_update_freq()
84 	if (unlikely(READ_ONCE(sg_policy->limits_changed))) {  in sugov_should_update_freq()
85 		WRITE_ONCE(sg_policy->limits_changed, false);  in sugov_should_update_freq()
86 		sg_policy->need_freq_update = true;  in sugov_should_update_freq()
101 	delta_ns = time - sg_policy->last_freq_update_time;  in sugov_should_update_freq()
103 	return delta_ns >= sg_policy->freq_update_delay_ns;  in sugov_should_update_freq()
109 	if (sg_policy->need_freq_update) {  in sugov_update_next_freq()
110 		sg_policy->need_freq_update = false;  in sugov_update_next_freq()
119 		if (sg_policy->next_freq == next_freq &&  in sugov_update_next_freq()
122 	} else if (sg_policy->next_freq == next_freq) {  in sugov_update_next_freq()
126 	sg_policy->next_freq = next_freq;  in sugov_update_next_freq()
127 	sg_policy->last_freq_update_time = time;  in sugov_update_next_freq()
134 	if (!sg_policy->work_in_progress) {  in sugov_deferred_update()
135 		sg_policy->work_in_progress = true;  in sugov_deferred_update()
136 		irq_work_queue(&sg_policy->irq_work);  in sugov_deferred_update()
141  * get_capacity_ref_freq - get the reference frequency that has been used to
151 	unsigned int freq = arch_scale_freq_ref(policy->cpu);  in get_capacity_ref_freq()  local
153 	if (freq)  in get_capacity_ref_freq()
154 		return freq;  in get_capacity_ref_freq()
157 		return policy->cpuinfo.max_freq;  in get_capacity_ref_freq()
163 	return policy->cur + (policy->cur >> 2);  in get_capacity_ref_freq()
167  * get_next_freq - Compute a new frequency for a given cpufreq policy.
170  * @max: CPU capacity.
172  * If the utilization is frequency-invariant, choose the new frequency to be
175  * next_freq = C * max_freq * util / max
177  * Otherwise, approximate the would-be frequency-invariant utilization by
180  * next_freq = C * curr_freq * util_raw / max
182  * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
184  * The lowest driver-supported frequency which is equal or greater than the raw
185  * next_freq (as calculated above) is returned, subject to policy min/max and
189 				  unsigned long util, unsigned long max)  in get_next_freq()  argument
191 	struct cpufreq_policy *policy = sg_policy->policy;  in get_next_freq()
192 	unsigned int freq;  in get_next_freq()  local
194 	freq = get_capacity_ref_freq(policy);  in get_next_freq()
195 	freq = map_util_freq(util, freq, max);  in get_next_freq()
197 	if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)  in get_next_freq()
198 		return sg_policy->next_freq;  in get_next_freq()
200 	sg_policy->cached_raw_freq = freq;  in get_next_freq()
201 	return cpufreq_driver_resolve_freq(policy, freq);  in get_next_freq()
206 				 unsigned long max)  in sugov_effective_cpu_perf()  argument
210 	/* Actually we don't need to target the max performance */  in sugov_effective_cpu_perf()
211 	if (actual < max)  in sugov_effective_cpu_perf()
212 		max = actual;  in sugov_effective_cpu_perf()
218 	return max(min, max);  in sugov_effective_cpu_perf()
223 	unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu);  in sugov_get_util()  local
226 		util += cpu_util_cfs_boost(sg_cpu->cpu);  in sugov_get_util()
227 	util = effective_cpu_util(sg_cpu->cpu, util, &min, &max);  in sugov_get_util()
228 	util = max(util, boost);  in sugov_get_util()
229 	sg_cpu->bw_min = min;  in sugov_get_util()
230 	sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max);  in sugov_get_util()
234  * sugov_iowait_reset() - Reset the IO boost status of a CPU.
247 	s64 delta_ns = time - sg_cpu->last_update;  in sugov_iowait_reset()
253 	sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0;  in sugov_iowait_reset()
254 	sg_cpu->iowait_boost_pending = set_iowait_boost;  in sugov_iowait_reset()
260  * sugov_iowait_boost() - Updates the IO boost status of a CPU.
279 	if (sg_cpu->iowait_boost &&  in sugov_iowait_boost()
288 	if (sg_cpu->iowait_boost_pending)  in sugov_iowait_boost()
290 	sg_cpu->iowait_boost_pending = true;  in sugov_iowait_boost()
293 	if (sg_cpu->iowait_boost) {  in sugov_iowait_boost()
294 		sg_cpu->iowait_boost =  in sugov_iowait_boost()
295 			min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE);  in sugov_iowait_boost()
300 	sg_cpu->iowait_boost = IOWAIT_BOOST_MIN;  in sugov_iowait_boost()
304  * sugov_iowait_apply() - Apply the IO boost to a CPU.
307  * @max_cap: the max CPU capacity
325 	if (!sg_cpu->iowait_boost)  in sugov_iowait_apply()
332 	if (!sg_cpu->iowait_boost_pending) {  in sugov_iowait_apply()
336 		sg_cpu->iowait_boost >>= 1;  in sugov_iowait_apply()
337 		if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) {  in sugov_iowait_apply()
338 			sg_cpu->iowait_boost = 0;  in sugov_iowait_apply()
343 	sg_cpu->iowait_boost_pending = false;  in sugov_iowait_apply()
346 	 * sg_cpu->util is already in capacity scale; convert iowait_boost  in sugov_iowait_apply()
349 	return (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT;  in sugov_iowait_apply()
367 	if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)))  in sugov_hold_freq()
374 	idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);  in sugov_hold_freq()
375 	ret = idle_calls == sg_cpu->saved_idle_calls;  in sugov_hold_freq()
377 	sg_cpu->saved_idle_calls = idle_calls;  in sugov_hold_freq()
390 	if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min)  in ignore_dl_rate_limit()
391 		WRITE_ONCE(sg_cpu->sg_policy->limits_changed, true);  in ignore_dl_rate_limit()
401 	sg_cpu->last_update = time;  in sugov_update_single_common()
405 	if (!sugov_should_update_freq(sg_cpu->sg_policy, time))  in sugov_update_single_common()
418 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_single_freq()
419 	unsigned int cached_freq = sg_policy->cached_raw_freq;  in sugov_update_single_freq()
423 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_update_single_freq()
428 	next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap);  in sugov_update_single_freq()
430 	if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq &&  in sugov_update_single_freq()
431 	    !sg_policy->need_freq_update) {  in sugov_update_single_freq()
432 		next_f = sg_policy->next_freq;  in sugov_update_single_freq()
434 		/* Restore cached freq as next_freq has changed */  in sugov_update_single_freq()
435 		sg_policy->cached_raw_freq = cached_freq;  in sugov_update_single_freq()
442 	 * This code runs under rq->lock for the target CPU, so it won't run  in sugov_update_single_freq()
446 	if (sg_policy->policy->fast_switch_enabled) {  in sugov_update_single_freq()
447 		cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_single_freq()
449 		raw_spin_lock(&sg_policy->update_lock);  in sugov_update_single_freq()
451 		raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_single_freq()
459 	unsigned long prev_util = sg_cpu->util;  in sugov_update_single_perf()
472 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_update_single_perf()
477 	if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util)  in sugov_update_single_perf()
478 		sg_cpu->util = prev_util;  in sugov_update_single_perf()
480 	cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min,  in sugov_update_single_perf()
481 				   sg_cpu->util, max_cap);  in sugov_update_single_perf()
483 	sg_cpu->sg_policy->last_freq_update_time = time;  in sugov_update_single_perf()
488 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_next_freq_shared()
489 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_next_freq_shared()
493 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_next_freq_shared()
495 	for_each_cpu(j, policy->cpus) {  in sugov_next_freq_shared()
502 		util = max(j_sg_cpu->util, util);  in sugov_next_freq_shared()
512 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_shared()
515 	raw_spin_lock(&sg_policy->update_lock);  in sugov_update_shared()
518 	sg_cpu->last_update = time;  in sugov_update_shared()
528 		if (sg_policy->policy->fast_switch_enabled)  in sugov_update_shared()
529 			cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_shared()
534 	raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_shared()
540 	unsigned int freq;  in sugov_work()  local
544 	 * Hold sg_policy->update_lock shortly to handle the case where:  in sugov_work()
545 	 * in case sg_policy->next_freq is read here, and then updated by  in sugov_work()
553 	raw_spin_lock_irqsave(&sg_policy->update_lock, flags);  in sugov_work()
554 	freq = sg_policy->next_freq;  in sugov_work()
555 	sg_policy->work_in_progress = false;  in sugov_work()
556 	raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);  in sugov_work()
558 	mutex_lock(&sg_policy->work_lock);  in sugov_work()
559 	__cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);  in sugov_work()
560 	mutex_unlock(&sg_policy->work_lock);  in sugov_work()
569 	kthread_queue_work(&sg_policy->worker, &sg_policy->work);  in sugov_irq_work()
586 	return sprintf(buf, "%u\n", tunables->rate_limit_us);  in rate_limit_us_show()
597 		return -EINVAL;  in rate_limit_us_store()
599 	tunables->rate_limit_us = rate_limit_us;  in rate_limit_us_store()
601 	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)  in rate_limit_us_store()
602 		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;  in rate_limit_us_store()
640 	sg_policy->policy = policy;  in sugov_policy_alloc()
641 	raw_spin_lock_init(&sg_policy->update_lock);  in sugov_policy_alloc()
667 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_kthread_create()
671 	if (policy->fast_switch_enabled)  in sugov_kthread_create()
674 	kthread_init_work(&sg_policy->work, sugov_work);  in sugov_kthread_create()
675 	kthread_init_worker(&sg_policy->worker);  in sugov_kthread_create()
676 	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,  in sugov_kthread_create()
678 				cpumask_first(policy->related_cpus));  in sugov_kthread_create()
691 	sg_policy->thread = thread;  in sugov_kthread_create()
692 	if (policy->dvfs_possible_from_any_cpu)  in sugov_kthread_create()
693 		set_cpus_allowed_ptr(thread, policy->related_cpus);  in sugov_kthread_create()
695 		kthread_bind_mask(thread, policy->related_cpus);  in sugov_kthread_create()
697 	init_irq_work(&sg_policy->irq_work, sugov_irq_work);  in sugov_kthread_create()
698 	mutex_init(&sg_policy->work_lock);  in sugov_kthread_create()
708 	if (sg_policy->policy->fast_switch_enabled)  in sugov_kthread_stop()
711 	kthread_flush_worker(&sg_policy->worker);  in sugov_kthread_stop()
712 	kthread_stop(sg_policy->thread);  in sugov_kthread_stop()
713 	mutex_destroy(&sg_policy->work_lock);  in sugov_kthread_stop()
722 		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_tunables_alloc()
742 	if (policy->governor_data)  in sugov_init()
743 		return -EBUSY;  in sugov_init()
749 		ret = -ENOMEM;  in sugov_init()
761 			ret = -EINVAL;  in sugov_init()
764 		policy->governor_data = sg_policy;  in sugov_init()
765 		sg_policy->tunables = global_tunables;  in sugov_init()
767 		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);  in sugov_init()
773 		ret = -ENOMEM;  in sugov_init()
777 	tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);  in sugov_init()
779 	policy->governor_data = sg_policy;  in sugov_init()
780 	sg_policy->tunables = tunables;  in sugov_init()
782 	ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,  in sugov_init()
798 	kobject_put(&tunables->attr_set.kobj);  in sugov_init()
799 	policy->governor_data = NULL;  in sugov_init()
818 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_exit()
819 	struct sugov_tunables *tunables = sg_policy->tunables;  in sugov_exit()
824 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_exit()
825 	policy->governor_data = NULL;  in sugov_exit()
840 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_start()
844 	sg_policy->freq_update_delay_ns	= sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;  in sugov_start()
845 	sg_policy->last_freq_update_time	= 0;  in sugov_start()
846 	sg_policy->next_freq			= 0;  in sugov_start()
847 	sg_policy->work_in_progress		= false;  in sugov_start()
848 	sg_policy->limits_changed		= false;  in sugov_start()
849 	sg_policy->cached_raw_freq		= 0;  in sugov_start()
851 	sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_start()
855 	else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf())  in sugov_start()
860 	for_each_cpu(cpu, policy->cpus) {  in sugov_start()
864 		sg_cpu->cpu = cpu;  in sugov_start()
865 		sg_cpu->sg_policy = sg_policy;  in sugov_start()
866 		cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu);  in sugov_start()
873 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_stop()
876 	for_each_cpu(cpu, policy->cpus)  in sugov_stop()
881 	if (!policy->fast_switch_enabled) {  in sugov_stop()
882 		irq_work_sync(&sg_policy->irq_work);  in sugov_stop()
883 		kthread_cancel_work_sync(&sg_policy->work);  in sugov_stop()
889 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_limits()
891 	if (!policy->fast_switch_enabled) {  in sugov_limits()
892 		mutex_lock(&sg_policy->work_lock);  in sugov_limits()
894 		mutex_unlock(&sg_policy->work_lock);  in sugov_limits()
906 	WRITE_ONCE(sg_policy->limits_changed, true);  in sugov_limits()