1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * itmt.c: Support Intel Turbo Boost Max Technology 3.0
4 *
5 * (C) Copyright 2016 Intel Corporation
6 * Author: Tim Chen <[email protected]>
7 *
8 * On platforms supporting Intel Turbo Boost Max Technology 3.0, (ITMT),
9 * the maximum turbo frequencies of some cores in a CPU package may be
10 * higher than for the other cores in the same package. In that case,
11 * better performance can be achieved by making the scheduler prefer
12 * to run tasks on the CPUs with higher max turbo frequencies.
13 *
14 * This file provides functions and data structures for enabling the
15 * scheduler to favor scheduling on cores can be boosted to a higher
16 * frequency under ITMT.
17 */
18
19 #include <linux/sched.h>
20 #include <linux/cpumask.h>
21 #include <linux/cpuset.h>
22 #include <linux/debugfs.h>
23 #include <linux/mutex.h>
24 #include <linux/sysctl.h>
25 #include <linux/nodemask.h>
26
27 static DEFINE_MUTEX(itmt_update_mutex);
28 DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
29
30 /* Boolean to track if system has ITMT capabilities */
31 static bool __read_mostly sched_itmt_capable;
32
33 /*
34 * Boolean to control whether we want to move processes to cpu capable
35 * of higher turbo frequency for cpus supporting Intel Turbo Boost Max
36 * Technology 3.0.
37 *
38 * It can be set via /sys/kernel/debug/x86/sched_itmt_enabled
39 */
40 bool __read_mostly sysctl_sched_itmt_enabled;
41
sched_itmt_enabled_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)42 static ssize_t sched_itmt_enabled_write(struct file *filp,
43 const char __user *ubuf,
44 size_t cnt, loff_t *ppos)
45 {
46 ssize_t result;
47 bool orig;
48
49 guard(mutex)(&itmt_update_mutex);
50
51 orig = sysctl_sched_itmt_enabled;
52 result = debugfs_write_file_bool(filp, ubuf, cnt, ppos);
53
54 if (sysctl_sched_itmt_enabled != orig) {
55 x86_topology_update = true;
56 rebuild_sched_domains();
57 }
58
59 return result;
60 }
61
62 static const struct file_operations dfs_sched_itmt_fops = {
63 .read = debugfs_read_file_bool,
64 .write = sched_itmt_enabled_write,
65 .open = simple_open,
66 .llseek = default_llseek,
67 };
68
69 static struct dentry *dfs_sched_itmt;
70
71 /**
72 * sched_set_itmt_support() - Indicate platform supports ITMT
73 *
74 * This function is used by the OS to indicate to scheduler that the platform
75 * is capable of supporting the ITMT feature.
76 *
77 * The current scheme has the pstate driver detects if the system
78 * is ITMT capable and call sched_set_itmt_support.
79 *
80 * This must be done only after sched_set_itmt_core_prio
81 * has been called to set the cpus' priorities.
82 * It must not be called with cpu hot plug lock
83 * held as we need to acquire the lock to rebuild sched domains
84 * later.
85 *
86 * Return: 0 on success
87 */
sched_set_itmt_support(void)88 int sched_set_itmt_support(void)
89 {
90 guard(mutex)(&itmt_update_mutex);
91
92 if (sched_itmt_capable)
93 return 0;
94
95 dfs_sched_itmt = debugfs_create_file_unsafe("sched_itmt_enabled",
96 0644,
97 arch_debugfs_dir,
98 &sysctl_sched_itmt_enabled,
99 &dfs_sched_itmt_fops);
100 if (IS_ERR_OR_NULL(dfs_sched_itmt)) {
101 dfs_sched_itmt = NULL;
102 return -ENOMEM;
103 }
104
105 sched_itmt_capable = true;
106
107 sysctl_sched_itmt_enabled = 1;
108
109 x86_topology_update = true;
110 rebuild_sched_domains();
111
112 return 0;
113 }
114
115 /**
116 * sched_clear_itmt_support() - Revoke platform's support of ITMT
117 *
118 * This function is used by the OS to indicate that it has
119 * revoked the platform's support of ITMT feature.
120 *
121 * It must not be called with cpu hot plug lock
122 * held as we need to acquire the lock to rebuild sched domains
123 * later.
124 */
sched_clear_itmt_support(void)125 void sched_clear_itmt_support(void)
126 {
127 guard(mutex)(&itmt_update_mutex);
128
129 if (!sched_itmt_capable)
130 return;
131
132 sched_itmt_capable = false;
133
134 debugfs_remove(dfs_sched_itmt);
135 dfs_sched_itmt = NULL;
136
137 if (sysctl_sched_itmt_enabled) {
138 /* disable sched_itmt if we are no longer ITMT capable */
139 sysctl_sched_itmt_enabled = 0;
140 x86_topology_update = true;
141 rebuild_sched_domains();
142 }
143 }
144
arch_asym_cpu_priority(int cpu)145 int arch_asym_cpu_priority(int cpu)
146 {
147 return per_cpu(sched_core_priority, cpu);
148 }
149
150 /**
151 * sched_set_itmt_core_prio() - Set CPU priority based on ITMT
152 * @prio: Priority of @cpu
153 * @cpu: The CPU number
154 *
155 * The pstate driver will find out the max boost frequency
156 * and call this function to set a priority proportional
157 * to the max boost frequency. CPUs with higher boost
158 * frequency will receive higher priority.
159 *
160 * No need to rebuild sched domain after updating
161 * the CPU priorities. The sched domains have no
162 * dependency on CPU priorities.
163 */
sched_set_itmt_core_prio(int prio,int cpu)164 void sched_set_itmt_core_prio(int prio, int cpu)
165 {
166 per_cpu(sched_core_priority, cpu) = prio;
167 }
168