1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * The main purpose of the tests here is to exercise the migration entry code
4 * paths in the kernel.
5 */
6
7 #include "../kselftest_harness.h"
8 #include <strings.h>
9 #include <pthread.h>
10 #include <numa.h>
11 #include <numaif.h>
12 #include <sys/mman.h>
13 #include <sys/prctl.h>
14 #include <sys/types.h>
15 #include <signal.h>
16 #include <time.h>
17
18 #define TWOMEG (2<<20)
19 #define RUNTIME (20)
20 #define MAX_RETRIES 100
21 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
22
FIXTURE(migration)23 FIXTURE(migration)
24 {
25 pthread_t *threads;
26 pid_t *pids;
27 int nthreads;
28 int n1;
29 int n2;
30 };
31
FIXTURE_SETUP(migration)32 FIXTURE_SETUP(migration)
33 {
34 int n;
35
36 ASSERT_EQ(numa_available(), 0);
37 self->nthreads = numa_num_task_cpus() - 1;
38 self->n1 = -1;
39 self->n2 = -1;
40
41 for (n = 0; n < numa_max_possible_node(); n++)
42 if (numa_bitmask_isbitset(numa_all_nodes_ptr, n)) {
43 if (self->n1 == -1) {
44 self->n1 = n;
45 } else {
46 self->n2 = n;
47 break;
48 }
49 }
50
51 self->threads = malloc(self->nthreads * sizeof(*self->threads));
52 ASSERT_NE(self->threads, NULL);
53 self->pids = malloc(self->nthreads * sizeof(*self->pids));
54 ASSERT_NE(self->pids, NULL);
55 };
56
FIXTURE_TEARDOWN(migration)57 FIXTURE_TEARDOWN(migration)
58 {
59 free(self->threads);
60 free(self->pids);
61 }
62
migrate(uint64_t * ptr,int n1,int n2)63 int migrate(uint64_t *ptr, int n1, int n2)
64 {
65 int ret, tmp;
66 int status = 0;
67 struct timespec ts1, ts2;
68 int failures = 0;
69
70 if (clock_gettime(CLOCK_MONOTONIC, &ts1))
71 return -1;
72
73 while (1) {
74 if (clock_gettime(CLOCK_MONOTONIC, &ts2))
75 return -1;
76
77 if (ts2.tv_sec - ts1.tv_sec >= RUNTIME)
78 return 0;
79
80 ret = move_pages(0, 1, (void **) &ptr, &n2, &status,
81 MPOL_MF_MOVE_ALL);
82 if (ret) {
83 if (ret > 0) {
84 /* Migration is best effort; try again */
85 if (++failures < MAX_RETRIES)
86 continue;
87 printf("Didn't migrate %d pages\n", ret);
88 }
89 else
90 perror("Couldn't migrate pages");
91 return -2;
92 }
93 failures = 0;
94 tmp = n2;
95 n2 = n1;
96 n1 = tmp;
97 }
98
99 return 0;
100 }
101
access_mem(void * ptr)102 void *access_mem(void *ptr)
103 {
104 volatile uint64_t y = 0;
105 volatile uint64_t *x = ptr;
106
107 while (1) {
108 pthread_testcancel();
109 y += *x;
110
111 /* Prevent the compiler from optimizing out the writes to y: */
112 asm volatile("" : "+r" (y));
113 }
114
115 return NULL;
116 }
117
118 /*
119 * Basic migration entry testing. One thread will move pages back and forth
120 * between nodes whilst other threads try and access them triggering the
121 * migration entry wait paths in the kernel.
122 */
123 TEST_F_TIMEOUT(migration, private_anon, 2*RUNTIME)
124 {
125 uint64_t *ptr;
126 int i;
127
128 if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0)
129 SKIP(return, "Not enough threads or NUMA nodes available");
130
131 ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE,
132 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
133 ASSERT_NE(ptr, MAP_FAILED);
134
135 memset(ptr, 0xde, TWOMEG);
136 for (i = 0; i < self->nthreads - 1; i++)
137 if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
138 perror("Couldn't create thread");
139
140 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
141 for (i = 0; i < self->nthreads - 1; i++)
142 ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
143 }
144
145 /*
146 * Same as the previous test but with shared memory.
147 */
148 TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME)
149 {
150 pid_t pid;
151 uint64_t *ptr;
152 int i;
153
154 if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0)
155 SKIP(return, "Not enough threads or NUMA nodes available");
156
157 ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE,
158 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
159 ASSERT_NE(ptr, MAP_FAILED);
160
161 memset(ptr, 0xde, TWOMEG);
162 for (i = 0; i < self->nthreads - 1; i++) {
163 pid = fork();
164 if (!pid) {
165 prctl(PR_SET_PDEATHSIG, SIGHUP);
166 /* Parent may have died before prctl so check now. */
167 if (getppid() == 1)
168 kill(getpid(), SIGHUP);
169 access_mem(ptr);
170 } else {
171 self->pids[i] = pid;
172 }
173 }
174
175 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
176 for (i = 0; i < self->nthreads - 1; i++)
177 ASSERT_EQ(kill(self->pids[i], SIGTERM), 0);
178 }
179
180 /*
181 * Tests the pmd migration entry paths.
182 */
183 TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME)
184 {
185 uint64_t *ptr;
186 int i;
187
188 if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0)
189 SKIP(return, "Not enough threads or NUMA nodes available");
190
191 ptr = mmap(NULL, 2*TWOMEG, PROT_READ | PROT_WRITE,
192 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
193 ASSERT_NE(ptr, MAP_FAILED);
194
195 ptr = (uint64_t *) ALIGN((uintptr_t) ptr, TWOMEG);
196 ASSERT_EQ(madvise(ptr, TWOMEG, MADV_HUGEPAGE), 0);
197 memset(ptr, 0xde, TWOMEG);
198 for (i = 0; i < self->nthreads - 1; i++)
199 if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
200 perror("Couldn't create thread");
201
202 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
203 for (i = 0; i < self->nthreads - 1; i++)
204 ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
205 }
206
207 /*
208 * migration test with shared anon THP page
209 */
210
211 TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME)
212 {
213 pid_t pid;
214 uint64_t *ptr;
215 int i;
216
217 if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0)
218 SKIP(return, "Not enough threads or NUMA nodes available");
219
220 ptr = mmap(NULL, 2 * TWOMEG, PROT_READ | PROT_WRITE,
221 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
222 ASSERT_NE(ptr, MAP_FAILED);
223
224 ptr = (uint64_t *) ALIGN((uintptr_t) ptr, TWOMEG);
225 ASSERT_EQ(madvise(ptr, TWOMEG, MADV_HUGEPAGE), 0);
226
227 memset(ptr, 0xde, TWOMEG);
228 for (i = 0; i < self->nthreads - 1; i++) {
229 pid = fork();
230 if (!pid) {
231 prctl(PR_SET_PDEATHSIG, SIGHUP);
232 /* Parent may have died before prctl so check now. */
233 if (getppid() == 1)
234 kill(getpid(), SIGHUP);
235 access_mem(ptr);
236 } else {
237 self->pids[i] = pid;
238 }
239 }
240
241 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
242 for (i = 0; i < self->nthreads - 1; i++)
243 ASSERT_EQ(kill(self->pids[i], SIGTERM), 0);
244 }
245
246 /*
247 * migration test with private anon hugetlb page
248 */
249 TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME)
250 {
251 uint64_t *ptr;
252 int i;
253
254 if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0)
255 SKIP(return, "Not enough threads or NUMA nodes available");
256
257 ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE,
258 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
259 ASSERT_NE(ptr, MAP_FAILED);
260
261 memset(ptr, 0xde, TWOMEG);
262 for (i = 0; i < self->nthreads - 1; i++)
263 if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
264 perror("Couldn't create thread");
265
266 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
267 for (i = 0; i < self->nthreads - 1; i++)
268 ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
269 }
270
271 /*
272 * migration test with shared anon hugetlb page
273 */
274 TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME)
275 {
276 pid_t pid;
277 uint64_t *ptr;
278 int i;
279
280 if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0)
281 SKIP(return, "Not enough threads or NUMA nodes available");
282
283 ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE,
284 MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
285 ASSERT_NE(ptr, MAP_FAILED);
286
287 memset(ptr, 0xde, TWOMEG);
288 for (i = 0; i < self->nthreads - 1; i++) {
289 pid = fork();
290 if (!pid) {
291 prctl(PR_SET_PDEATHSIG, SIGHUP);
292 /* Parent may have died before prctl so check now. */
293 if (getppid() == 1)
294 kill(getpid(), SIGHUP);
295 access_mem(ptr);
296 } else {
297 self->pids[i] = pid;
298 }
299 }
300
301 ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
302 for (i = 0; i < self->nthreads - 1; i++)
303 ASSERT_EQ(kill(self->pids[i], SIGTERM), 0);
304 }
305
306 TEST_HARNESS_MAIN
307