1 // SPDX-License-Identifier: GPL-2.0
7 #include <linux/clk-provider.h>
50 	 * Mutex to synchronize between de-init sequence and re-starting LMh
80 	dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_set_bw()
82 		return -ENODEV;  in qcom_cpufreq_set_bw()
116 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_target_index()
118 	unsigned long freq = policy->freq_table[index].frequency;  in qcom_cpufreq_hw_target_index()
121 	writel_relaxed(index, data->base + soc_data->reg_perf_state);  in qcom_cpufreq_hw_target_index()
123 	if (data->per_core_dcvs)  in qcom_cpufreq_hw_target_index()
124 		for (i = 1; i < cpumask_weight(policy->related_cpus); i++)  in qcom_cpufreq_hw_target_index()
125 			writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4);  in qcom_cpufreq_hw_target_index()
137 	if (qcom_cpufreq.soc_data->reg_current_vote)  in qcom_lmh_get_throttle_freq()
138 		lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_current_vote) & 0x3ff;  in qcom_lmh_get_throttle_freq()
140 		lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_domain_state) & 0xff;  in qcom_lmh_get_throttle_freq()
145 /* Get the frequency requested by the cpufreq core for the CPU */
155 	data = policy->driver_data;  in qcom_cpufreq_get_freq()
158 	index = readl_relaxed(data->base + soc_data->reg_perf_state);  in qcom_cpufreq_get_freq()
159 	index = min(index, LUT_MAX_ENTRIES - 1);  in qcom_cpufreq_get_freq()
161 	return policy->freq_table[index].frequency;  in qcom_cpufreq_get_freq()
171 	data = policy->driver_data;  in __qcom_cpufreq_hw_get()
173 	if (data->throttle_irq >= 0)  in __qcom_cpufreq_hw_get()
187 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_fast_switch()
192 	index = policy->cached_resolved_idx;  in qcom_cpufreq_hw_fast_switch()
193 	writel_relaxed(index, data->base + soc_data->reg_perf_state);  in qcom_cpufreq_hw_fast_switch()
195 	if (data->per_core_dcvs)  in qcom_cpufreq_hw_fast_switch()
196 		for (i = 1; i < cpumask_weight(policy->related_cpus); i++)  in qcom_cpufreq_hw_fast_switch()
197 			writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4);  in qcom_cpufreq_hw_fast_switch()
199 	return policy->freq_table[index].frequency;  in qcom_cpufreq_hw_fast_switch()
211 	struct qcom_cpufreq_data *drv_data = policy->driver_data;  in qcom_cpufreq_hw_read_lut()
216 		return -ENOMEM;  in qcom_cpufreq_hw_read_lut()
220 		/* Disable all opps and cross-validate against LUT later */  in qcom_cpufreq_hw_read_lut()
230 	} else if (ret != -ENODEV) {  in qcom_cpufreq_hw_read_lut()
235 		policy->fast_switch_possible = true;  in qcom_cpufreq_hw_read_lut()
240 		data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut +  in qcom_cpufreq_hw_read_lut()
241 				      i * soc_data->lut_row_size);  in qcom_cpufreq_hw_read_lut()
246 		data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut +  in qcom_cpufreq_hw_read_lut()
247 				      i * soc_data->lut_row_size);  in qcom_cpufreq_hw_read_lut()
257 				table[i].frequency = freq;  in qcom_cpufreq_hw_read_lut()
262 				table[i].frequency = CPUFREQ_ENTRY_INVALID;  in qcom_cpufreq_hw_read_lut()
266 			table[i].frequency = CPUFREQ_ENTRY_INVALID;  in qcom_cpufreq_hw_read_lut()
274 			struct cpufreq_frequency_table *prev = &table[i - 1];  in qcom_cpufreq_hw_read_lut()
277 			 * Only treat the last frequency that might be a boost  in qcom_cpufreq_hw_read_lut()
278 			 * as the boost frequency  in qcom_cpufreq_hw_read_lut()
280 			if (prev->frequency == CPUFREQ_ENTRY_INVALID) {  in qcom_cpufreq_hw_read_lut()
282 					prev->frequency = prev_freq;  in qcom_cpufreq_hw_read_lut()
283 					prev->flags = CPUFREQ_BOOST_FREQ;  in qcom_cpufreq_hw_read_lut()
296 	table[i].frequency = CPUFREQ_TABLE_END;  in qcom_cpufreq_hw_read_lut()
297 	policy->freq_table = table;  in qcom_cpufreq_hw_read_lut()
298 	dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);  in qcom_cpufreq_hw_read_lut()
314 		ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain",  in qcom_get_related_cpus()
315 						 "#freq-domain-cells", 0,  in qcom_get_related_cpus()
328 	struct cpufreq_policy *policy = data->policy;  in qcom_lmh_dcvs_notify()
329 	int cpu = cpumask_first(policy->related_cpus);  in qcom_lmh_dcvs_notify()
335 	 * Get the h/w throttled frequency, normalize it using the  in qcom_lmh_dcvs_notify()
341 	if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE)  in qcom_lmh_dcvs_notify()
353 	arch_update_hw_pressure(policy->related_cpus, throttled_freq);  in qcom_lmh_dcvs_notify()
359 	mutex_lock(&data->throttle_lock);  in qcom_lmh_dcvs_notify()
360 	if (data->cancel_throttle)  in qcom_lmh_dcvs_notify()
361 		goto out;  in qcom_lmh_dcvs_notify()
364 	 * If h/w throttled frequency is higher than what cpufreq has requested  in qcom_lmh_dcvs_notify()
368 		enable_irq(data->throttle_irq);  in qcom_lmh_dcvs_notify()
370 		mod_delayed_work(system_highpri_wq, &data->throttle_work,  in qcom_lmh_dcvs_notify()
373 out:  in qcom_lmh_dcvs_notify()
374 	mutex_unlock(&data->throttle_lock);  in qcom_lmh_dcvs_notify()
390 	disable_irq_nosync(c_data->throttle_irq);  in qcom_lmh_dcvs_handle_irq()
391 	schedule_delayed_work(&c_data->throttle_work, 0);  in qcom_lmh_dcvs_handle_irq()
393 	if (qcom_cpufreq.soc_data->reg_intr_clr)  in qcom_lmh_dcvs_handle_irq()
395 			       c_data->base + qcom_cpufreq.soc_data->reg_intr_clr);  in qcom_lmh_dcvs_handle_irq()
422 	{ .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data },
423 	{ .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data },
430 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_lmh_init()
438 	data->throttle_irq = platform_get_irq_optional(pdev, index);  in qcom_cpufreq_hw_lmh_init()
439 	if (data->throttle_irq == -ENXIO)  in qcom_cpufreq_hw_lmh_init()
441 	if (data->throttle_irq < 0)  in qcom_cpufreq_hw_lmh_init()
442 		return data->throttle_irq;  in qcom_cpufreq_hw_lmh_init()
444 	data->cancel_throttle = false;  in qcom_cpufreq_hw_lmh_init()
446 	mutex_init(&data->throttle_lock);  in qcom_cpufreq_hw_lmh_init()
447 	INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll);  in qcom_cpufreq_hw_lmh_init()
449 	snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu);  in qcom_cpufreq_hw_lmh_init()
450 	ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq,  in qcom_cpufreq_hw_lmh_init()
451 				   IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data);  in qcom_cpufreq_hw_lmh_init()
453 		dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret);  in qcom_cpufreq_hw_lmh_init()
457 	ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus);  in qcom_cpufreq_hw_lmh_init()
459 		dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n",  in qcom_cpufreq_hw_lmh_init()
460 			data->irq_name, data->throttle_irq);  in qcom_cpufreq_hw_lmh_init()
467 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_online()
471 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_cpu_online()
474 	mutex_lock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_online()
475 	data->cancel_throttle = false;  in qcom_cpufreq_hw_cpu_online()
476 	mutex_unlock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_online()
478 	ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus);  in qcom_cpufreq_hw_cpu_online()
480 		dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n",  in qcom_cpufreq_hw_cpu_online()
481 			data->irq_name, data->throttle_irq);  in qcom_cpufreq_hw_cpu_online()
488 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_offline()
490 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_cpu_offline()
493 	mutex_lock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_offline()
494 	data->cancel_throttle = true;  in qcom_cpufreq_hw_cpu_offline()
495 	mutex_unlock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_offline()
497 	cancel_delayed_work_sync(&data->throttle_work);  in qcom_cpufreq_hw_cpu_offline()
498 	irq_set_affinity_and_hint(data->throttle_irq, NULL);  in qcom_cpufreq_hw_cpu_offline()
499 	disable_irq_nosync(data->throttle_irq);  in qcom_cpufreq_hw_cpu_offline()
506 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_lmh_exit()
509 	free_irq(data->throttle_irq, data);  in qcom_cpufreq_hw_lmh_exit()
515 	struct device *dev = &pdev->dev;  in qcom_cpufreq_hw_cpu_init()
522 	cpu_dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_hw_cpu_init()
525 		       policy->cpu);  in qcom_cpufreq_hw_cpu_init()
526 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
529 	cpu_np = of_cpu_device_node_get(policy->cpu);  in qcom_cpufreq_hw_cpu_init()
531 		return -EINVAL;  in qcom_cpufreq_hw_cpu_init()
533 	ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain",  in qcom_cpufreq_hw_cpu_init()
534 					 "#freq-domain-cells", 0, &args);  in qcom_cpufreq_hw_cpu_init()
543 	if (!(readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_enable) & 0x1)) {  in qcom_cpufreq_hw_cpu_init()
544 		dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index);  in qcom_cpufreq_hw_cpu_init()
545 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
548 	if (readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_dcvs_ctrl) & 0x1)  in qcom_cpufreq_hw_cpu_init()
549 		data->per_core_dcvs = true;  in qcom_cpufreq_hw_cpu_init()
551 	qcom_get_related_cpus(index, policy->cpus);  in qcom_cpufreq_hw_cpu_init()
553 	policy->driver_data = data;  in qcom_cpufreq_hw_cpu_init()
554 	policy->dvfs_possible_from_any_cpu = true;  in qcom_cpufreq_hw_cpu_init()
555 	data->policy = policy;  in qcom_cpufreq_hw_cpu_init()
559 		dev_err(dev, "Domain-%d failed to read LUT\n", index);  in qcom_cpufreq_hw_cpu_init()
566 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
580 	struct device *cpu_dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_hw_cpu_exit()
581 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_exit()
584 	dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);  in qcom_cpufreq_hw_cpu_exit()
586 	kfree(policy->freq_table);  in qcom_cpufreq_hw_cpu_exit()
592 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_ready()
594 	if (data->throttle_irq >= 0)  in qcom_cpufreq_ready()
595 		enable_irq(data->throttle_irq);  in qcom_cpufreq_ready()
617 	.name		= "qcom-cpufreq-hw",
626 	return __qcom_cpufreq_hw_get(data->policy) * HZ_PER_KHZ;  in qcom_cpufreq_hw_recalc_rate()
636 	req->rate = qcom_cpufreq_hw_recalc_rate(hw, 0);  in qcom_cpufreq_hw_determine_rate()
649 	struct device *dev = &pdev->dev;  in qcom_cpufreq_hw_driver_probe()
651 	struct clk *clk;  in qcom_cpufreq_hw_driver_probe()  local
654 	clk = clk_get(dev, "xo");  in qcom_cpufreq_hw_driver_probe()
655 	if (IS_ERR(clk))  in qcom_cpufreq_hw_driver_probe()
656 		return PTR_ERR(clk);  in qcom_cpufreq_hw_driver_probe()
658 	xo_rate = clk_get_rate(clk);  in qcom_cpufreq_hw_driver_probe()
659 	clk_put(clk);  in qcom_cpufreq_hw_driver_probe()
661 	clk = clk_get(dev, "alternate");  in qcom_cpufreq_hw_driver_probe()
662 	if (IS_ERR(clk))  in qcom_cpufreq_hw_driver_probe()
663 		return PTR_ERR(clk);  in qcom_cpufreq_hw_driver_probe()
665 	cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV;  in qcom_cpufreq_hw_driver_probe()
666 	clk_put(clk);  in qcom_cpufreq_hw_driver_probe()
673 		return -EPROBE_DEFER;  in qcom_cpufreq_hw_driver_probe()
686 		return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
690 		return -ENODEV;  in qcom_cpufreq_hw_driver_probe()
694 		return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
696 	clk_data->num = num_domains;  in qcom_cpufreq_hw_driver_probe()
709 		data->base = base;  in qcom_cpufreq_hw_driver_probe()
711 		/* Register CPU clock for each frequency domain */  in qcom_cpufreq_hw_driver_probe()
714 			return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
718 		data->cpu_clk.init = &clk_init;  in qcom_cpufreq_hw_driver_probe()
720 		ret = devm_clk_hw_register(dev, &data->cpu_clk);  in qcom_cpufreq_hw_driver_probe()
727 		clk_data->hws[i] = &data->cpu_clk;  in qcom_cpufreq_hw_driver_probe()
755 		.name = "qcom-cpufreq-hw",