319 static int amd_pstate_set_energy_pref_index(struct cpufreq_policy *policy,  in amd_pstate_set_energy_pref_index()  argument
322 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_set_energy_pref_index()
330 	if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {  in amd_pstate_set_energy_pref_index()
331 		pr_debug("EPP cannot be set under performance policy\n");  in amd_pstate_set_energy_pref_index()
340 					  policy->boost_enabled);  in amd_pstate_set_energy_pref_index()
538 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu);  in amd_pstate_update()  local
541 	if (!policy)  in amd_pstate_update()
547 	policy->cur = div_u64(des_perf * max_freq, max_perf);  in amd_pstate_update()
566 	cpufreq_cpu_put(policy);  in amd_pstate_update()
572 	 * Initialize lower frequency limit (i.e.policy->min) with  in amd_pstate_verify()
578 		struct cpufreq_policy *policy = cpufreq_cpu_get(policy_data->cpu);  in amd_pstate_verify()  local
581 		if (!policy)  in amd_pstate_verify()
584 		cpudata = policy->driver_data;  in amd_pstate_verify()
586 		cpufreq_cpu_put(policy);  in amd_pstate_verify()
595 static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy)  in amd_pstate_update_min_max_limit()  argument
599 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_update_min_max_limit()
603 	max_limit_perf = div_u64(policy->max * max_perf, max_freq);  in amd_pstate_update_min_max_limit()
604 	min_limit_perf = div_u64(policy->min * max_perf, max_freq);  in amd_pstate_update_min_max_limit()
606 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)  in amd_pstate_update_min_max_limit()
611 	WRITE_ONCE(cpudata->max_limit_freq, policy->max);  in amd_pstate_update_min_max_limit()
612 	WRITE_ONCE(cpudata->min_limit_freq, policy->min);  in amd_pstate_update_min_max_limit()
617 static int amd_pstate_update_freq(struct cpufreq_policy *policy,  in amd_pstate_update_freq()  argument
621 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_update_freq()
627 	if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)  in amd_pstate_update_freq()
628 		amd_pstate_update_min_max_limit(policy);  in amd_pstate_update_freq()
632 	freqs.old = policy->cur;  in amd_pstate_update_freq()
638 	WARN_ON(fast_switch && !policy->fast_switch_enabled);  in amd_pstate_update_freq()
645 		cpufreq_freq_transition_begin(policy, &freqs);  in amd_pstate_update_freq()
649 			  policy->governor->flags);  in amd_pstate_update_freq()
652 		cpufreq_freq_transition_end(policy, &freqs, false);  in amd_pstate_update_freq()
657 static int amd_pstate_target(struct cpufreq_policy *policy,  in amd_pstate_target()  argument
661 	return amd_pstate_update_freq(policy, target_freq, false);  in amd_pstate_target()
664 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,  in amd_pstate_fast_switch()  argument
667 	if (!amd_pstate_update_freq(policy, target_freq, true))  in amd_pstate_fast_switch()
669 	return policy->cur;  in amd_pstate_fast_switch()
678 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in amd_pstate_adjust_perf()  local
681 	if (!policy)  in amd_pstate_adjust_perf()
684 	cpudata = policy->driver_data;  in amd_pstate_adjust_perf()
686 	if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)  in amd_pstate_adjust_perf()
687 		amd_pstate_update_min_max_limit(policy);  in amd_pstate_adjust_perf()
711 			policy->governor->flags);  in amd_pstate_adjust_perf()
712 	cpufreq_cpu_put(policy);  in amd_pstate_adjust_perf()
715 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on)  in amd_pstate_cpu_boost_update()  argument
717 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_cpu_boost_update()
725 		policy->cpuinfo.max_freq = max_freq;  in amd_pstate_cpu_boost_update()
726 	else if (policy->cpuinfo.max_freq > nominal_freq)  in amd_pstate_cpu_boost_update()
727 		policy->cpuinfo.max_freq = nominal_freq;  in amd_pstate_cpu_boost_update()
729 	policy->max = policy->cpuinfo.max_freq;  in amd_pstate_cpu_boost_update()
732 		ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq);  in amd_pstate_cpu_boost_update()
740 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)  in amd_pstate_set_boost()  argument
742 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_set_boost()
751 	ret = amd_pstate_cpu_boost_update(policy, state);  in amd_pstate_set_boost()
752 	refresh_frequency_limits(policy);  in amd_pstate_set_boost()
826 	struct cpufreq_policy *policy = NULL;  in amd_pstate_update_limits()  local
835 	policy = cpufreq_cpu_get(cpu);  in amd_pstate_update_limits()
836 	if (!policy)  in amd_pstate_update_limits()
839 	cpudata = policy->driver_data;  in amd_pstate_update_limits()
845 		cpufreq_cpu_put(policy);  in amd_pstate_update_limits()
857 	cpufreq_cpu_put(policy);  in amd_pstate_update_limits()
963 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)  in amd_pstate_cpu_init()  argument
973 	amd_perf_ctl_reset(policy->cpu);  in amd_pstate_cpu_init()
974 	dev = get_cpu_device(policy->cpu);  in amd_pstate_cpu_init()
982 	cpudata->cpu = policy->cpu;  in amd_pstate_cpu_init()
1001 	policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu);  in amd_pstate_cpu_init()
1002 	policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu);  in amd_pstate_cpu_init()
1004 	policy->min = min_freq;  in amd_pstate_cpu_init()
1005 	policy->max = max_freq;  in amd_pstate_cpu_init()
1007 	policy->cpuinfo.min_freq = min_freq;  in amd_pstate_cpu_init()
1008 	policy->cpuinfo.max_freq = max_freq;  in amd_pstate_cpu_init()
1010 	policy->boost_enabled = READ_ONCE(cpudata->boost_supported);  in amd_pstate_cpu_init()
1013 	policy->cur = policy->cpuinfo.min_freq;  in amd_pstate_cpu_init()
1016 		policy->fast_switch_possible = true;  in amd_pstate_cpu_init()
1018 	ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],  in amd_pstate_cpu_init()
1025 	ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],  in amd_pstate_cpu_init()
1026 				   FREQ_QOS_MAX, policy->cpuinfo.max_freq);  in amd_pstate_cpu_init()
1035 	policy->driver_data = cpudata;  in amd_pstate_cpu_init()
1049 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy)  in amd_pstate_cpu_exit()  argument
1051 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_cpu_exit()
1055 	policy->fast_switch_possible = false;  in amd_pstate_cpu_exit()
1059 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)  in amd_pstate_cpu_resume()  argument
1070 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)  in amd_pstate_cpu_suspend()  argument
1088 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,  in show_amd_pstate_max_freq()  argument
1092 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_max_freq()
1101 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,  in show_amd_pstate_lowest_nonlinear_freq()  argument
1105 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_lowest_nonlinear_freq()
1118 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,  in show_amd_pstate_highest_perf()  argument
1122 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_highest_perf()
1129 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy,  in show_amd_pstate_prefcore_ranking()  argument
1133 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_prefcore_ranking()
1140 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy,  in show_amd_pstate_hw_prefcore()  argument
1144 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_hw_prefcore()
1152 				struct cpufreq_policy *policy, char *buf)  in show_energy_performance_available_preferences()  argument
1156 	struct amd_cpudata *cpudata = policy->driver_data;  in show_energy_performance_available_preferences()
1158 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)  in show_energy_performance_available_preferences()
1171 		struct cpufreq_policy *policy, const char *buf, size_t count)  in store_energy_performance_preference()  argument
1186 	ret = amd_pstate_set_energy_pref_index(policy, ret);  in store_energy_performance_preference()
1192 				struct cpufreq_policy *policy, char *buf)  in show_energy_performance_preference()  argument
1194 	struct amd_cpudata *cpudata = policy->driver_data;  in show_energy_performance_preference()
1454 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)  in amd_pstate_epp_cpu_init()  argument
1465 	amd_perf_ctl_reset(policy->cpu);  in amd_pstate_epp_cpu_init()
1466 	dev = get_cpu_device(policy->cpu);  in amd_pstate_epp_cpu_init()
1474 	cpudata->cpu = policy->cpu;  in amd_pstate_epp_cpu_init()
1493 	policy->cpuinfo.min_freq = min_freq;  in amd_pstate_epp_cpu_init()
1494 	policy->cpuinfo.max_freq = max_freq;  in amd_pstate_epp_cpu_init()
1496 	policy->cur = policy->cpuinfo.min_freq;  in amd_pstate_epp_cpu_init()
1498 	policy->driver_data = cpudata;  in amd_pstate_epp_cpu_init()
1500 	policy->min = policy->cpuinfo.min_freq;  in amd_pstate_epp_cpu_init()
1501 	policy->max = policy->cpuinfo.max_freq;  in amd_pstate_epp_cpu_init()
1503 	policy->boost_enabled = READ_ONCE(cpudata->boost_supported);  in amd_pstate_epp_cpu_init()
1506 	 * Set the policy to provide a valid fallback value in case  in amd_pstate_epp_cpu_init()
1511 		policy->policy = CPUFREQ_POLICY_PERFORMANCE;  in amd_pstate_epp_cpu_init()
1514 		policy->policy = CPUFREQ_POLICY_POWERSAVE;  in amd_pstate_epp_cpu_init()
1542 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)  in amd_pstate_epp_cpu_exit()  argument
1544 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_cpu_exit()
1548 		policy->driver_data = NULL;  in amd_pstate_epp_cpu_exit()
1551 	pr_debug("CPU %d exiting\n", policy->cpu);  in amd_pstate_epp_cpu_exit()
1554 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy)  in amd_pstate_epp_update_limit()  argument
1556 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_update_limit()
1559 	amd_pstate_update_min_max_limit(policy);  in amd_pstate_epp_update_limit()
1561 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)  in amd_pstate_epp_update_limit()
1570 					  policy->boost_enabled);  in amd_pstate_epp_update_limit()
1577 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)  in amd_pstate_epp_set_policy()  argument
1579 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_set_policy()
1582 	if (!policy->cpuinfo.max_freq)  in amd_pstate_epp_set_policy()
1585 	pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",  in amd_pstate_epp_set_policy()
1586 				policy->cpuinfo.max_freq, policy->max);  in amd_pstate_epp_set_policy()
1588 	cpudata->policy = policy->policy;  in amd_pstate_epp_set_policy()
1590 	ret = amd_pstate_epp_update_limit(policy);  in amd_pstate_epp_set_policy()
1595 	 * policy->cur is never updated with the amd_pstate_epp driver, but it  in amd_pstate_epp_set_policy()
1598 	policy->cur = policy->min;  in amd_pstate_epp_set_policy()
1603 static int amd_pstate_epp_reenable(struct cpufreq_policy *policy)  in amd_pstate_epp_reenable()  argument
1605 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_reenable()
1619 					  max_perf, policy->boost_enabled);  in amd_pstate_epp_reenable()
1622 	return amd_pstate_epp_update_limit(policy);  in amd_pstate_epp_reenable()
1625 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)  in amd_pstate_epp_cpu_online()  argument
1627 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_cpu_online()
1632 	ret = amd_pstate_epp_reenable(policy);  in amd_pstate_epp_cpu_online()
1640 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)  in amd_pstate_epp_cpu_offline()  argument
1642 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_cpu_offline()
1655 					  min_perf, min_perf, policy->boost_enabled);  in amd_pstate_epp_cpu_offline()
1662 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)  in amd_pstate_epp_suspend()  argument
1664 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_suspend()
1685 static int amd_pstate_epp_resume(struct cpufreq_policy *policy)  in amd_pstate_epp_resume()  argument
1687 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_resume()
1693 		amd_pstate_epp_reenable(policy);  in amd_pstate_epp_resume()