1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * KVM demand paging test
4 * Adapted from dirty_log_test.c
5 *
6 * Copyright (C) 2018, Red Hat, Inc.
7 * Copyright (C) 2019, Google, Inc.
8 */
9
10 #define _GNU_SOURCE /* for pipe2 */
11
12 #include <inttypes.h>
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <time.h>
16 #include <poll.h>
17 #include <pthread.h>
18 #include <linux/userfaultfd.h>
19 #include <sys/syscall.h>
20
21 #include "kvm_util.h"
22 #include "test_util.h"
23 #include "perf_test_util.h"
24 #include "guest_modes.h"
25
26 #ifdef __NR_userfaultfd
27
28 #ifdef PRINT_PER_PAGE_UPDATES
29 #define PER_PAGE_DEBUG(...) printf(__VA_ARGS__)
30 #else
31 #define PER_PAGE_DEBUG(...) _no_printf(__VA_ARGS__)
32 #endif
33
34 #ifdef PRINT_PER_VCPU_UPDATES
35 #define PER_VCPU_DEBUG(...) printf(__VA_ARGS__)
36 #else
37 #define PER_VCPU_DEBUG(...) _no_printf(__VA_ARGS__)
38 #endif
39
40 static int nr_vcpus = 1;
41 static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
42 static size_t demand_paging_size;
43 static char *guest_data_prototype;
44
vcpu_worker(struct perf_test_vcpu_args * vcpu_args)45 static void vcpu_worker(struct perf_test_vcpu_args *vcpu_args)
46 {
47 struct kvm_vcpu *vcpu = vcpu_args->vcpu;
48 int vcpu_idx = vcpu_args->vcpu_idx;
49 struct kvm_run *run = vcpu->run;
50 struct timespec start;
51 struct timespec ts_diff;
52 int ret;
53
54 clock_gettime(CLOCK_MONOTONIC, &start);
55
56 /* Let the guest access its memory */
57 ret = _vcpu_run(vcpu);
58 TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
59 if (get_ucall(vcpu, NULL) != UCALL_SYNC) {
60 TEST_ASSERT(false,
61 "Invalid guest sync status: exit_reason=%s\n",
62 exit_reason_str(run->exit_reason));
63 }
64
65 ts_diff = timespec_elapsed(start);
66 PER_VCPU_DEBUG("vCPU %d execution time: %ld.%.9lds\n", vcpu_idx,
67 ts_diff.tv_sec, ts_diff.tv_nsec);
68 }
69
handle_uffd_page_request(int uffd_mode,int uffd,uint64_t addr)70 static int handle_uffd_page_request(int uffd_mode, int uffd, uint64_t addr)
71 {
72 pid_t tid = syscall(__NR_gettid);
73 struct timespec start;
74 struct timespec ts_diff;
75 int r;
76
77 clock_gettime(CLOCK_MONOTONIC, &start);
78
79 if (uffd_mode == UFFDIO_REGISTER_MODE_MISSING) {
80 struct uffdio_copy copy;
81
82 copy.src = (uint64_t)guest_data_prototype;
83 copy.dst = addr;
84 copy.len = demand_paging_size;
85 copy.mode = 0;
86
87 r = ioctl(uffd, UFFDIO_COPY, ©);
88 if (r == -1) {
89 pr_info("Failed UFFDIO_COPY in 0x%lx from thread %d with errno: %d\n",
90 addr, tid, errno);
91 return r;
92 }
93 } else if (uffd_mode == UFFDIO_REGISTER_MODE_MINOR) {
94 struct uffdio_continue cont = {0};
95
96 cont.range.start = addr;
97 cont.range.len = demand_paging_size;
98
99 r = ioctl(uffd, UFFDIO_CONTINUE, &cont);
100 if (r == -1) {
101 pr_info("Failed UFFDIO_CONTINUE in 0x%lx from thread %d with errno: %d\n",
102 addr, tid, errno);
103 return r;
104 }
105 } else {
106 TEST_FAIL("Invalid uffd mode %d", uffd_mode);
107 }
108
109 ts_diff = timespec_elapsed(start);
110
111 PER_PAGE_DEBUG("UFFD page-in %d \t%ld ns\n", tid,
112 timespec_to_ns(ts_diff));
113 PER_PAGE_DEBUG("Paged in %ld bytes at 0x%lx from thread %d\n",
114 demand_paging_size, addr, tid);
115
116 return 0;
117 }
118
119 bool quit_uffd_thread;
120
121 struct uffd_handler_args {
122 int uffd_mode;
123 int uffd;
124 int pipefd;
125 useconds_t delay;
126 };
127
uffd_handler_thread_fn(void * arg)128 static void *uffd_handler_thread_fn(void *arg)
129 {
130 struct uffd_handler_args *uffd_args = (struct uffd_handler_args *)arg;
131 int uffd = uffd_args->uffd;
132 int pipefd = uffd_args->pipefd;
133 useconds_t delay = uffd_args->delay;
134 int64_t pages = 0;
135 struct timespec start;
136 struct timespec ts_diff;
137
138 clock_gettime(CLOCK_MONOTONIC, &start);
139 while (!quit_uffd_thread) {
140 struct uffd_msg msg;
141 struct pollfd pollfd[2];
142 char tmp_chr;
143 int r;
144 uint64_t addr;
145
146 pollfd[0].fd = uffd;
147 pollfd[0].events = POLLIN;
148 pollfd[1].fd = pipefd;
149 pollfd[1].events = POLLIN;
150
151 r = poll(pollfd, 2, -1);
152 switch (r) {
153 case -1:
154 pr_info("poll err");
155 continue;
156 case 0:
157 continue;
158 case 1:
159 break;
160 default:
161 pr_info("Polling uffd returned %d", r);
162 return NULL;
163 }
164
165 if (pollfd[0].revents & POLLERR) {
166 pr_info("uffd revents has POLLERR");
167 return NULL;
168 }
169
170 if (pollfd[1].revents & POLLIN) {
171 r = read(pollfd[1].fd, &tmp_chr, 1);
172 TEST_ASSERT(r == 1,
173 "Error reading pipefd in UFFD thread\n");
174 return NULL;
175 }
176
177 if (!(pollfd[0].revents & POLLIN))
178 continue;
179
180 r = read(uffd, &msg, sizeof(msg));
181 if (r == -1) {
182 if (errno == EAGAIN)
183 continue;
184 pr_info("Read of uffd got errno %d\n", errno);
185 return NULL;
186 }
187
188 if (r != sizeof(msg)) {
189 pr_info("Read on uffd returned unexpected size: %d bytes", r);
190 return NULL;
191 }
192
193 if (!(msg.event & UFFD_EVENT_PAGEFAULT))
194 continue;
195
196 if (delay)
197 usleep(delay);
198 addr = msg.arg.pagefault.address;
199 r = handle_uffd_page_request(uffd_args->uffd_mode, uffd, addr);
200 if (r < 0)
201 return NULL;
202 pages++;
203 }
204
205 ts_diff = timespec_elapsed(start);
206 PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",
207 pages, ts_diff.tv_sec, ts_diff.tv_nsec,
208 pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
209
210 return NULL;
211 }
212
setup_demand_paging(struct kvm_vm * vm,pthread_t * uffd_handler_thread,int pipefd,int uffd_mode,useconds_t uffd_delay,struct uffd_handler_args * uffd_args,void * hva,void * alias,uint64_t len)213 static void setup_demand_paging(struct kvm_vm *vm,
214 pthread_t *uffd_handler_thread, int pipefd,
215 int uffd_mode, useconds_t uffd_delay,
216 struct uffd_handler_args *uffd_args,
217 void *hva, void *alias, uint64_t len)
218 {
219 bool is_minor = (uffd_mode == UFFDIO_REGISTER_MODE_MINOR);
220 int uffd;
221 struct uffdio_api uffdio_api;
222 struct uffdio_register uffdio_register;
223 uint64_t expected_ioctls = ((uint64_t) 1) << _UFFDIO_COPY;
224 int ret;
225
226 PER_PAGE_DEBUG("Userfaultfd %s mode, faults resolved with %s\n",
227 is_minor ? "MINOR" : "MISSING",
228 is_minor ? "UFFDIO_CONINUE" : "UFFDIO_COPY");
229
230 /* In order to get minor faults, prefault via the alias. */
231 if (is_minor) {
232 size_t p;
233
234 expected_ioctls = ((uint64_t) 1) << _UFFDIO_CONTINUE;
235
236 TEST_ASSERT(alias != NULL, "Alias required for minor faults");
237 for (p = 0; p < (len / demand_paging_size); ++p) {
238 memcpy(alias + (p * demand_paging_size),
239 guest_data_prototype, demand_paging_size);
240 }
241 }
242
243 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
244 TEST_ASSERT(uffd >= 0, __KVM_SYSCALL_ERROR("userfaultfd()", uffd));
245
246 uffdio_api.api = UFFD_API;
247 uffdio_api.features = 0;
248 ret = ioctl(uffd, UFFDIO_API, &uffdio_api);
249 TEST_ASSERT(ret != -1, __KVM_SYSCALL_ERROR("UFFDIO_API", ret));
250
251 uffdio_register.range.start = (uint64_t)hva;
252 uffdio_register.range.len = len;
253 uffdio_register.mode = uffd_mode;
254 ret = ioctl(uffd, UFFDIO_REGISTER, &uffdio_register);
255 TEST_ASSERT(ret != -1, __KVM_SYSCALL_ERROR("UFFDIO_REGISTER", ret));
256 TEST_ASSERT((uffdio_register.ioctls & expected_ioctls) ==
257 expected_ioctls, "missing userfaultfd ioctls");
258
259 uffd_args->uffd_mode = uffd_mode;
260 uffd_args->uffd = uffd;
261 uffd_args->pipefd = pipefd;
262 uffd_args->delay = uffd_delay;
263 pthread_create(uffd_handler_thread, NULL, uffd_handler_thread_fn,
264 uffd_args);
265
266 PER_VCPU_DEBUG("Created uffd thread for HVA range [%p, %p)\n",
267 hva, hva + len);
268 }
269
270 struct test_params {
271 int uffd_mode;
272 useconds_t uffd_delay;
273 enum vm_mem_backing_src_type src_type;
274 bool partition_vcpu_memory_access;
275 };
276
run_test(enum vm_guest_mode mode,void * arg)277 static void run_test(enum vm_guest_mode mode, void *arg)
278 {
279 struct test_params *p = arg;
280 pthread_t *uffd_handler_threads = NULL;
281 struct uffd_handler_args *uffd_args = NULL;
282 struct timespec start;
283 struct timespec ts_diff;
284 int *pipefds = NULL;
285 struct kvm_vm *vm;
286 int r, i;
287
288 vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size, 1,
289 p->src_type, p->partition_vcpu_memory_access);
290
291 demand_paging_size = get_backing_src_pagesz(p->src_type);
292
293 guest_data_prototype = malloc(demand_paging_size);
294 TEST_ASSERT(guest_data_prototype,
295 "Failed to allocate buffer for guest data pattern");
296 memset(guest_data_prototype, 0xAB, demand_paging_size);
297
298 if (p->uffd_mode) {
299 uffd_handler_threads =
300 malloc(nr_vcpus * sizeof(*uffd_handler_threads));
301 TEST_ASSERT(uffd_handler_threads, "Memory allocation failed");
302
303 uffd_args = malloc(nr_vcpus * sizeof(*uffd_args));
304 TEST_ASSERT(uffd_args, "Memory allocation failed");
305
306 pipefds = malloc(sizeof(int) * nr_vcpus * 2);
307 TEST_ASSERT(pipefds, "Unable to allocate memory for pipefd");
308
309 for (i = 0; i < nr_vcpus; i++) {
310 struct perf_test_vcpu_args *vcpu_args;
311 void *vcpu_hva;
312 void *vcpu_alias;
313
314 vcpu_args = &perf_test_args.vcpu_args[i];
315
316 /* Cache the host addresses of the region */
317 vcpu_hva = addr_gpa2hva(vm, vcpu_args->gpa);
318 vcpu_alias = addr_gpa2alias(vm, vcpu_args->gpa);
319
320 /*
321 * Set up user fault fd to handle demand paging
322 * requests.
323 */
324 r = pipe2(&pipefds[i * 2],
325 O_CLOEXEC | O_NONBLOCK);
326 TEST_ASSERT(!r, "Failed to set up pipefd");
327
328 setup_demand_paging(vm, &uffd_handler_threads[i],
329 pipefds[i * 2], p->uffd_mode,
330 p->uffd_delay, &uffd_args[i],
331 vcpu_hva, vcpu_alias,
332 vcpu_args->pages * perf_test_args.guest_page_size);
333 }
334 }
335
336 pr_info("Finished creating vCPUs and starting uffd threads\n");
337
338 clock_gettime(CLOCK_MONOTONIC, &start);
339 perf_test_start_vcpu_threads(nr_vcpus, vcpu_worker);
340 pr_info("Started all vCPUs\n");
341
342 perf_test_join_vcpu_threads(nr_vcpus);
343 ts_diff = timespec_elapsed(start);
344 pr_info("All vCPU threads joined\n");
345
346 if (p->uffd_mode) {
347 char c;
348
349 /* Tell the user fault fd handler threads to quit */
350 for (i = 0; i < nr_vcpus; i++) {
351 r = write(pipefds[i * 2 + 1], &c, 1);
352 TEST_ASSERT(r == 1, "Unable to write to pipefd");
353
354 pthread_join(uffd_handler_threads[i], NULL);
355 }
356 }
357
358 pr_info("Total guest execution time: %ld.%.9lds\n",
359 ts_diff.tv_sec, ts_diff.tv_nsec);
360 pr_info("Overall demand paging rate: %f pgs/sec\n",
361 perf_test_args.vcpu_args[0].pages * nr_vcpus /
362 ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
363
364 perf_test_destroy_vm(vm);
365
366 free(guest_data_prototype);
367 if (p->uffd_mode) {
368 free(uffd_handler_threads);
369 free(uffd_args);
370 free(pipefds);
371 }
372 }
373
help(char * name)374 static void help(char *name)
375 {
376 puts("");
377 printf("usage: %s [-h] [-m vm_mode] [-u uffd_mode] [-d uffd_delay_usec]\n"
378 " [-b memory] [-s type] [-v vcpus] [-o]\n", name);
379 guest_modes_help();
380 printf(" -u: use userfaultfd to handle vCPU page faults. Mode is a\n"
381 " UFFD registration mode: 'MISSING' or 'MINOR'.\n");
382 printf(" -d: add a delay in usec to the User Fault\n"
383 " FD handler to simulate demand paging\n"
384 " overheads. Ignored without -u.\n");
385 printf(" -b: specify the size of the memory region which should be\n"
386 " demand paged by each vCPU. e.g. 10M or 3G.\n"
387 " Default: 1G\n");
388 backing_src_help("-s");
389 printf(" -v: specify the number of vCPUs to run.\n");
390 printf(" -o: Overlap guest memory accesses instead of partitioning\n"
391 " them into a separate region of memory for each vCPU.\n");
392 puts("");
393 exit(0);
394 }
395
main(int argc,char * argv[])396 int main(int argc, char *argv[])
397 {
398 int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
399 struct test_params p = {
400 .src_type = DEFAULT_VM_MEM_SRC,
401 .partition_vcpu_memory_access = true,
402 };
403 int opt;
404
405 guest_modes_append_default();
406
407 while ((opt = getopt(argc, argv, "hm:u:d:b:s:v:o")) != -1) {
408 switch (opt) {
409 case 'm':
410 guest_modes_cmdline(optarg);
411 break;
412 case 'u':
413 if (!strcmp("MISSING", optarg))
414 p.uffd_mode = UFFDIO_REGISTER_MODE_MISSING;
415 else if (!strcmp("MINOR", optarg))
416 p.uffd_mode = UFFDIO_REGISTER_MODE_MINOR;
417 TEST_ASSERT(p.uffd_mode, "UFFD mode must be 'MISSING' or 'MINOR'.");
418 break;
419 case 'd':
420 p.uffd_delay = strtoul(optarg, NULL, 0);
421 TEST_ASSERT(p.uffd_delay >= 0, "A negative UFFD delay is not supported.");
422 break;
423 case 'b':
424 guest_percpu_mem_size = parse_size(optarg);
425 break;
426 case 's':
427 p.src_type = parse_backing_src_type(optarg);
428 break;
429 case 'v':
430 nr_vcpus = atoi(optarg);
431 TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
432 "Invalid number of vcpus, must be between 1 and %d", max_vcpus);
433 break;
434 case 'o':
435 p.partition_vcpu_memory_access = false;
436 break;
437 case 'h':
438 default:
439 help(argv[0]);
440 break;
441 }
442 }
443
444 if (p.uffd_mode == UFFDIO_REGISTER_MODE_MINOR &&
445 !backing_src_is_shared(p.src_type)) {
446 TEST_FAIL("userfaultfd MINOR mode requires shared memory; pick a different -s");
447 }
448
449 for_each_guest_mode(run_test, &p);
450
451 return 0;
452 }
453
454 #else /* __NR_userfaultfd */
455
456 #warning "missing __NR_userfaultfd definition"
457
main(void)458 int main(void)
459 {
460 print_skip("__NR_userfaultfd must be present for userfaultfd test");
461 return KSFT_SKIP;
462 }
463
464 #endif /* __NR_userfaultfd */
465