1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Apple SoC CPU cluster performance state driver
4 *
5 * Copyright The Asahi Linux Contributors
6 *
7 * Based on scpi-cpufreq.c
8 */
9
10 #include <linux/bitfield.h>
11 #include <linux/bitops.h>
12 #include <linux/cpu.h>
13 #include <linux/cpufreq.h>
14 #include <linux/cpumask.h>
15 #include <linux/delay.h>
16 #include <linux/err.h>
17 #include <linux/io.h>
18 #include <linux/iopoll.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_address.h>
22 #include <linux/pm_opp.h>
23 #include <linux/slab.h>
24
25 #define APPLE_DVFS_CMD 0x20
26 #define APPLE_DVFS_CMD_BUSY BIT(31)
27 #define APPLE_DVFS_CMD_SET BIT(25)
28 #define APPLE_DVFS_CMD_PS1_S5L8960X GENMASK(24, 22)
29 #define APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT 22
30 #define APPLE_DVFS_CMD_PS2 GENMASK(15, 12)
31 #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0)
32 #define APPLE_DVFS_CMD_PS1_SHIFT 0
33
34 /* Same timebase as CPU counter (24MHz) */
35 #define APPLE_DVFS_LAST_CHG_TIME 0x38
36
37 /*
38 * Apple ran out of bits and had to shift this in T8112...
39 */
40 #define APPLE_DVFS_STATUS 0x50
41 #define APPLE_DVFS_STATUS_CUR_PS_S5L8960X GENMASK(5, 3)
42 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X 3
43 #define APPLE_DVFS_STATUS_TGT_PS_S5L8960X GENMASK(2, 0)
44 #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4)
45 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4
46 #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0)
47 #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5)
48 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5
49 #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0)
50
51 /*
52 * Div is +1, base clock is 12MHz on existing SoCs.
53 * For documentation purposes. We use the OPP table to
54 * get the frequency.
55 */
56 #define APPLE_DVFS_PLL_STATUS 0xc0
57 #define APPLE_DVFS_PLL_FACTOR 0xc8
58 #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16)
59 #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0)
60
61 #define APPLE_DVFS_TRANSITION_TIMEOUT 400
62
63 struct apple_soc_cpufreq_info {
64 bool has_ps2;
65 u64 max_pstate;
66 u64 cur_pstate_mask;
67 u64 cur_pstate_shift;
68 u64 ps1_mask;
69 u64 ps1_shift;
70 };
71
72 struct apple_cpu_priv {
73 struct device *cpu_dev;
74 void __iomem *reg_base;
75 const struct apple_soc_cpufreq_info *info;
76 };
77
78 static struct cpufreq_driver apple_soc_cpufreq_driver;
79
80 static const struct apple_soc_cpufreq_info soc_s5l8960x_info = {
81 .has_ps2 = false,
82 .max_pstate = 7,
83 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_S5L8960X,
84 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X,
85 .ps1_mask = APPLE_DVFS_CMD_PS1_S5L8960X,
86 .ps1_shift = APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT,
87 };
88
89 static const struct apple_soc_cpufreq_info soc_t8103_info = {
90 .has_ps2 = true,
91 .max_pstate = 15,
92 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,
93 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,
94 .ps1_mask = APPLE_DVFS_CMD_PS1,
95 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT,
96 };
97
98 static const struct apple_soc_cpufreq_info soc_t8112_info = {
99 .has_ps2 = false,
100 .max_pstate = 31,
101 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,
102 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,
103 .ps1_mask = APPLE_DVFS_CMD_PS1,
104 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT,
105 };
106
107 static const struct apple_soc_cpufreq_info soc_default_info = {
108 .has_ps2 = false,
109 .max_pstate = 15,
110 .cur_pstate_mask = 0, /* fallback */
111 .ps1_mask = APPLE_DVFS_CMD_PS1,
112 .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT,
113 };
114
115 static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = {
116 {
117 .compatible = "apple,s5l8960x-cluster-cpufreq",
118 .data = &soc_s5l8960x_info,
119 },
120 {
121 .compatible = "apple,t8103-cluster-cpufreq",
122 .data = &soc_t8103_info,
123 },
124 {
125 .compatible = "apple,t8112-cluster-cpufreq",
126 .data = &soc_t8112_info,
127 },
128 {
129 .compatible = "apple,cluster-cpufreq",
130 .data = &soc_default_info,
131 },
132 {}
133 };
134
apple_soc_cpufreq_get_rate(unsigned int cpu)135 static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)
136 {
137 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
138 struct apple_cpu_priv *priv = policy->driver_data;
139 struct cpufreq_frequency_table *p;
140 unsigned int pstate;
141
142 if (priv->info->cur_pstate_mask) {
143 u32 reg = readl_relaxed(priv->reg_base + APPLE_DVFS_STATUS);
144
145 pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift;
146 } else {
147 /*
148 * For the fallback case we might not know the layout of DVFS_STATUS,
149 * so just use the command register value (which ignores boost limitations).
150 */
151 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);
152
153 pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);
154 }
155
156 cpufreq_for_each_valid_entry(p, policy->freq_table)
157 if (p->driver_data == pstate)
158 return p->frequency;
159
160 dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",
161 pstate);
162 return 0;
163 }
164
apple_soc_cpufreq_set_target(struct cpufreq_policy * policy,unsigned int index)165 static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
166 unsigned int index)
167 {
168 struct apple_cpu_priv *priv = policy->driver_data;
169 unsigned int pstate = policy->freq_table[index].driver_data;
170 u64 reg;
171
172 /* Fallback for newer SoCs */
173 if (index > priv->info->max_pstate)
174 index = priv->info->max_pstate;
175
176 if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,
177 !(reg & APPLE_DVFS_CMD_BUSY), 2,
178 APPLE_DVFS_TRANSITION_TIMEOUT)) {
179 return -EIO;
180 }
181
182 reg &= ~priv->info->ps1_mask;
183 reg |= pstate << priv->info->ps1_shift;
184 if (priv->info->has_ps2) {
185 reg &= ~APPLE_DVFS_CMD_PS2;
186 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
187 }
188 reg |= APPLE_DVFS_CMD_SET;
189
190 writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
191
192 return 0;
193 }
194
apple_soc_cpufreq_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)195 static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,
196 unsigned int target_freq)
197 {
198 if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)
199 return 0;
200
201 return policy->freq_table[policy->cached_resolved_idx].frequency;
202 }
203
apple_soc_cpufreq_find_cluster(struct cpufreq_policy * policy,void __iomem ** reg_base,const struct apple_soc_cpufreq_info ** info)204 static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,
205 void __iomem **reg_base,
206 const struct apple_soc_cpufreq_info **info)
207 {
208 struct of_phandle_args args;
209 const struct of_device_id *match;
210 int ret = 0;
211
212 ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
213 "#performance-domain-cells",
214 policy->cpus, &args);
215 if (ret < 0)
216 return ret;
217
218 match = of_match_node(apple_soc_cpufreq_of_match, args.np);
219 of_node_put(args.np);
220 if (!match)
221 return -ENODEV;
222
223 *info = match->data;
224
225 *reg_base = of_iomap(args.np, 0);
226 if (!*reg_base)
227 return -ENOMEM;
228
229 return 0;
230 }
231
232 static struct freq_attr *apple_soc_cpufreq_hw_attr[] = {
233 &cpufreq_freq_attr_scaling_available_freqs,
234 NULL, /* Filled in below if boost is enabled */
235 NULL,
236 };
237
apple_soc_cpufreq_init(struct cpufreq_policy * policy)238 static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
239 {
240 int ret, i;
241 unsigned int transition_latency;
242 void __iomem *reg_base;
243 struct device *cpu_dev;
244 struct apple_cpu_priv *priv;
245 const struct apple_soc_cpufreq_info *info;
246 struct cpufreq_frequency_table *freq_table;
247
248 cpu_dev = get_cpu_device(policy->cpu);
249 if (!cpu_dev) {
250 pr_err("failed to get cpu%d device\n", policy->cpu);
251 return -ENODEV;
252 }
253
254 ret = dev_pm_opp_of_add_table(cpu_dev);
255 if (ret < 0) {
256 dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
257 return ret;
258 }
259
260 ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info);
261 if (ret) {
262 dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
263 return ret;
264 }
265
266 ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
267 if (ret) {
268 dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
269 goto out_iounmap;
270 }
271
272 ret = dev_pm_opp_get_opp_count(cpu_dev);
273 if (ret <= 0) {
274 dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
275 ret = -EPROBE_DEFER;
276 goto out_free_opp;
277 }
278
279 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
280 if (!priv) {
281 ret = -ENOMEM;
282 goto out_free_opp;
283 }
284
285 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
286 if (ret) {
287 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
288 goto out_free_priv;
289 }
290
291 /* Get OPP levels (p-state indexes) and stash them in driver_data */
292 for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
293 unsigned long rate = freq_table[i].frequency * 1000 + 999;
294 struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);
295
296 if (IS_ERR(opp)) {
297 ret = PTR_ERR(opp);
298 goto out_free_cpufreq_table;
299 }
300 freq_table[i].driver_data = dev_pm_opp_get_level(opp);
301 dev_pm_opp_put(opp);
302 }
303
304 priv->cpu_dev = cpu_dev;
305 priv->reg_base = reg_base;
306 priv->info = info;
307 policy->driver_data = priv;
308 policy->freq_table = freq_table;
309
310 transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
311 if (!transition_latency)
312 transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC;
313
314 policy->cpuinfo.transition_latency = transition_latency;
315 policy->dvfs_possible_from_any_cpu = true;
316 policy->fast_switch_possible = true;
317 policy->suspend_freq = freq_table[0].frequency;
318
319 if (policy_has_boost_freq(policy)) {
320 ret = cpufreq_enable_boost_support();
321 if (ret) {
322 dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);
323 } else {
324 apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;
325 apple_soc_cpufreq_driver.boost_enabled = true;
326 }
327 }
328
329 return 0;
330
331 out_free_cpufreq_table:
332 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
333 out_free_priv:
334 kfree(priv);
335 out_free_opp:
336 dev_pm_opp_remove_all_dynamic(cpu_dev);
337 out_iounmap:
338 iounmap(reg_base);
339 return ret;
340 }
341
apple_soc_cpufreq_exit(struct cpufreq_policy * policy)342 static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
343 {
344 struct apple_cpu_priv *priv = policy->driver_data;
345
346 dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
347 dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
348 iounmap(priv->reg_base);
349 kfree(priv);
350 }
351
352 static struct cpufreq_driver apple_soc_cpufreq_driver = {
353 .name = "apple-cpufreq",
354 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
355 CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,
356 .verify = cpufreq_generic_frequency_table_verify,
357 .get = apple_soc_cpufreq_get_rate,
358 .init = apple_soc_cpufreq_init,
359 .exit = apple_soc_cpufreq_exit,
360 .target_index = apple_soc_cpufreq_set_target,
361 .fast_switch = apple_soc_cpufreq_fast_switch,
362 .register_em = cpufreq_register_em_with_opp,
363 .attr = apple_soc_cpufreq_hw_attr,
364 .suspend = cpufreq_generic_suspend,
365 };
366
apple_soc_cpufreq_module_init(void)367 static int __init apple_soc_cpufreq_module_init(void)
368 {
369 if (!of_machine_is_compatible("apple,arm-platform"))
370 return -ENODEV;
371
372 return cpufreq_register_driver(&apple_soc_cpufreq_driver);
373 }
374 module_init(apple_soc_cpufreq_module_init);
375
apple_soc_cpufreq_module_exit(void)376 static void __exit apple_soc_cpufreq_module_exit(void)
377 {
378 cpufreq_unregister_driver(&apple_soc_cpufreq_driver);
379 }
380 module_exit(apple_soc_cpufreq_module_exit);
381
382 MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);
383 MODULE_AUTHOR("Hector Martin <[email protected]>");
384 MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");
385 MODULE_LICENSE("GPL");
386