1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Copyright (c) 2021 SUSE LLC <[email protected]> */
3 /*\
4 * [Description]
5 *
6 * Check that something (e.g. irqbalance daemon) is performing IRQ
7 * load balancing.
8 *
9 * On many systems userland needs to set /proc/irq/$IRQ/smp_affinity
10 * to prevent many IRQs being delivered to the same CPU.
11 *
12 * Note some drivers and IRQ controllers will distribute IRQs
13 * evenly. Some systems will have housekeeping CPUs configured. Some
14 * IRQs can not be masked etc. So this test is not appropriate for all
15 * scenarios.
16 *
17 * Furthermore, exactly how IRQs should be distributed is a
18 * performance and/or security issue. This is only a generic smoke
19 * test. It will hopefully detect misconfigured systems and total
20 * balancing failures which are often silent errors.
21 *
22 * Heuristic: Evidence of Change
23 *
24 * 1. Find IRQs with a non-zero count
25 * 2. Check if they are now disallowed
26 *
27 * There are two sources of information we need to parse:
28 *
29 * 1. /proc/interrupts
30 * 2. /proc/irq/$IRQ/smp_affinity
31 *
32 * We get the active IRQs and CPUs from /proc/interrupts. It also
33 * contains the per-CPU IRQ counts and info we do not care about.
34 *
35 * We get the IRQ masks from each active IRQ's smp_affinity file. This
36 * is a bitmask written out in hexadecimal format. It shows which CPUs
37 * an IRQ may be received by.
38 */
39
40 #include <stdlib.h>
41
42 #include "tst_test.h"
43 #include "tst_safe_stdio.h"
44 #include "tst_safe_file_at.h"
45
46 enum affinity {
47 ALLOW = '+',
48 DENY = '-',
49 };
50
51 static unsigned int *irq_stats;
52 static enum affinity *irq_affinity;
53
54 static unsigned int nr_cpus;
55 static unsigned int nr_irqs;
56 static unsigned int *irq_ids;
57
read_proc_file(const char * const path,size_t * const len_out)58 static char *read_proc_file(const char *const path, size_t *const len_out)
59 {
60 const size_t pg_len = SAFE_SYSCONF(_SC_PAGESIZE);
61 int fd = SAFE_OPEN(path, O_RDONLY);
62 size_t ret = 0, used_len = 0;
63 static size_t total_len;
64 static char *buf;
65
66 do {
67 if (used_len + 1 >= total_len) {
68 total_len += pg_len;
69 buf = SAFE_REALLOC(buf, total_len);
70 }
71
72 ret = SAFE_READ(0, fd,
73 buf + used_len,
74 total_len - used_len - 1);
75 used_len += ret;
76 } while (ret);
77
78 if (!used_len)
79 tst_brk(TBROK, "Empty %s?", path);
80
81 buf[used_len] = '\0';
82
83 SAFE_CLOSE(fd);
84
85 if (len_out)
86 *len_out = used_len;
87 return buf;
88 }
89
collect_irq_info(void)90 static void collect_irq_info(void)
91 {
92 char *buf, *c, *first_row;
93 char path[PATH_MAX];
94 size_t row, col, len;
95 long acc;
96 unsigned int cpu_total, bit, row_parsed;
97
98 nr_cpus = 0;
99 nr_irqs = 0;
100
101 buf = read_proc_file("/proc/interrupts", NULL);
102
103 /* Count CPUs, header columns are like /CPU[0-9]+/ */
104 for (c = buf; *c != '\0' && *c != '\n'; c++) {
105 if (!strncmp(c, "CPU", 3))
106 nr_cpus++;
107 }
108
109 c++;
110 first_row = c;
111 /* Count IRQs, real IRQs start with /[0-9]+:/ */
112 while (*c != '\0') {
113 switch (*c) {
114 case ' ':
115 case '\t':
116 case '\n':
117 case '0' ... '9':
118 c++;
119 break;
120 case ':':
121 nr_irqs++;
122 /* fall-through */
123 default:
124 while (*c != '\n' && *c != '\0')
125 c++;
126 }
127 }
128
129 tst_res(TINFO, "Found %u CPUS, %u IRQs", nr_cpus, nr_irqs);
130
131 irq_ids = SAFE_REALLOC(irq_ids, nr_irqs * sizeof(*irq_ids));
132 irq_stats = SAFE_REALLOC(irq_stats,
133 nr_cpus * (nr_irqs + 1) * sizeof(*irq_stats));
134 irq_affinity = SAFE_REALLOC(irq_affinity,
135 nr_cpus * nr_irqs * sizeof(*irq_affinity));
136
137 c = first_row;
138 acc = -1;
139 row = col = row_parsed = 0;
140 /* Parse columns containing IRQ counts and IRQ IDs into acc. Ignore
141 * everything else.
142 */
143 while (*c != '\0') {
144 switch (*c) {
145 case ' ':
146 case '\t':
147 if (acc >= 0) {
148 irq_stats[row * nr_cpus + col] = acc;
149 acc = -1;
150 col++;
151 }
152 break;
153 case '\n':
154 if (acc != -1)
155 tst_brk(TBROK, "Unexpected EOL");
156 col = 0;
157 if (row_parsed)
158 row++;
159 row_parsed = 0;
160 break;
161 case '0' ... '9':
162 if (acc == -1)
163 acc = 0;
164
165 acc *= 10;
166 acc += *c - '0';
167 break;
168 case ':':
169 if (acc == -1 || col != 0)
170 tst_brk(TBROK, "Unexpected ':'");
171 irq_ids[row] = acc;
172 acc = -1;
173 row_parsed = 1;
174 break;
175 default:
176 acc = -1;
177 while (*c != '\n' && *c != '\0')
178 c++;
179 continue;
180 }
181
182 c++;
183 }
184
185 for (col = 0; col < nr_cpus; col++) {
186 cpu_total = 0;
187
188 for (row = 0; row < nr_irqs; row++)
189 cpu_total += irq_stats[row * nr_cpus + col];
190
191 irq_stats[row * nr_cpus + col] = cpu_total;
192 }
193
194 /* Read the CPU affinity masks for each IRQ. The first CPU is in the
195 * right most (least significant) bit. See bitmap_string() in the kernel
196 * (%*pb)
197 */
198 for (row = 0; row < nr_irqs; row++) {
199 sprintf(path, "/proc/irq/%u/smp_affinity", irq_ids[row]);
200 buf = read_proc_file(path, &len);
201 c = buf + len;
202 col = 0;
203
204 while (--c >= buf) {
205 if (col > nr_cpus) {
206 tst_res(TINFO, "%u/smp_affnity: %s",
207 irq_ids[row], buf);
208 tst_brk(TBROK, "More mask char bits than cpus");
209 }
210
211 switch (*c) {
212 case '\n':
213 case ' ':
214 case ',':
215 continue;
216 case '0' ... '9':
217 acc = *c - '0';
218 break;
219 case 'a' ... 'f':
220 acc = 10 + *c - 'a';
221 break;
222 default:
223 tst_res(TINFO, "%u/smp_affnity: %s",
224 irq_ids[row], buf);
225 tst_brk(TBROK, "Wasn't expecting 0x%02x", *c);
226 }
227
228 for (bit = 0; bit < 4 && col < nr_cpus; bit++) {
229 irq_affinity[row * nr_cpus + col++] =
230 (acc & (1 << bit)) ? ALLOW : DENY;
231 }
232 }
233
234 if (col < nr_cpus) {
235 tst_res(TINFO, "%u/smp_affnity: %s", irq_ids[row], buf);
236 tst_brk(TBROK, "Only found %zu cpus", col);
237 }
238 }
239 }
240
print_irq_info(void)241 static void print_irq_info(void)
242 {
243 size_t row, col;
244 unsigned int count;
245 enum affinity aff;
246
247 tst_printf(" IRQ ");
248 for (col = 0; col < nr_cpus; col++)
249 tst_printf("CPU%-8zu", col);
250
251 tst_printf("\n");
252
253 for (row = 0; row < nr_irqs; row++) {
254 tst_printf("%5u:", irq_ids[row]);
255
256 for (col = 0; col < nr_cpus; col++) {
257 count = irq_stats[row * nr_cpus + col];
258 aff = irq_affinity[row * nr_cpus + col];
259
260 tst_printf("%10u%c", count, aff);
261 }
262
263 tst_printf("\n");
264 }
265
266 tst_printf("Total:");
267
268 for (col = 0; col < nr_cpus; col++)
269 tst_printf("%10u ", irq_stats[row * nr_cpus + col]);
270
271 tst_printf("\n");
272 }
273
evidence_of_change(void)274 static void evidence_of_change(void)
275 {
276 size_t row, col, changed = 0;
277
278 for (row = 0; row < nr_irqs; row++) {
279 for (col = 0; col < nr_cpus; col++) {
280 if (!irq_stats[row * nr_cpus + col])
281 continue;
282
283 if (irq_affinity[row * nr_cpus + col] == ALLOW)
284 continue;
285
286 changed++;
287 }
288 }
289
290 tst_res(changed ? TPASS : TFAIL,
291 "Heuristic: Detected %zu irq-cpu pairs have been dissallowed",
292 changed);
293 }
294
setup(void)295 static void setup(void)
296 {
297 collect_irq_info();
298 print_irq_info();
299
300 if (nr_cpus < 1)
301 tst_brk(TBROK, "No CPUs found in /proc/interrupts?");
302
303 if (nr_irqs < 1)
304 tst_brk(TBROK, "No IRQs found in /proc/interrupts?");
305 }
306
run(void)307 static void run(void)
308 {
309 collect_irq_info();
310
311 evidence_of_change();
312 }
313
314 static struct tst_test test = {
315 .test_all = run,
316 .setup = setup,
317 .min_cpus = 2,
318 };
319