1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * fs/userfaultfd.c
4 *
5 * Copyright (C) 2007 Davide Libenzi <[email protected]>
6 * Copyright (C) 2008-2009 Red Hat, Inc.
7 * Copyright (C) 2015 Red Hat, Inc.
8 *
9 * Some part derived from fs/eventfd.c (anon inode setup) and
10 * mm/ksm.c (mm hashing).
11 */
12
13 #include <linux/list.h>
14 #include <linux/hashtable.h>
15 #include <linux/sched/signal.h>
16 #include <linux/sched/mm.h>
17 #include <linux/mm.h>
18 #include <linux/mm_inline.h>
19 #include <linux/mmu_notifier.h>
20 #include <linux/poll.h>
21 #include <linux/slab.h>
22 #include <linux/seq_file.h>
23 #include <linux/file.h>
24 #include <linux/bug.h>
25 #include <linux/anon_inodes.h>
26 #include <linux/syscalls.h>
27 #include <linux/userfaultfd_k.h>
28 #include <linux/mempolicy.h>
29 #include <linux/ioctl.h>
30 #include <linux/security.h>
31 #include <linux/hugetlb.h>
32 #include <linux/swapops.h>
33 #include <linux/miscdevice.h>
34 #include <linux/uio.h>
35
36 static int sysctl_unprivileged_userfaultfd __read_mostly;
37
38 #ifdef CONFIG_SYSCTL
39 static const struct ctl_table vm_userfaultfd_table[] = {
40 {
41 .procname = "unprivileged_userfaultfd",
42 .data = &sysctl_unprivileged_userfaultfd,
43 .maxlen = sizeof(sysctl_unprivileged_userfaultfd),
44 .mode = 0644,
45 .proc_handler = proc_dointvec_minmax,
46 .extra1 = SYSCTL_ZERO,
47 .extra2 = SYSCTL_ONE,
48 },
49 };
50 #endif
51
52 static struct kmem_cache *userfaultfd_ctx_cachep __ro_after_init;
53
54 struct userfaultfd_fork_ctx {
55 struct userfaultfd_ctx *orig;
56 struct userfaultfd_ctx *new;
57 struct list_head list;
58 };
59
60 struct userfaultfd_unmap_ctx {
61 struct userfaultfd_ctx *ctx;
62 unsigned long start;
63 unsigned long end;
64 struct list_head list;
65 };
66
67 struct userfaultfd_wait_queue {
68 struct uffd_msg msg;
69 wait_queue_entry_t wq;
70 struct userfaultfd_ctx *ctx;
71 bool waken;
72 };
73
74 struct userfaultfd_wake_range {
75 unsigned long start;
76 unsigned long len;
77 };
78
79 /* internal indication that UFFD_API ioctl was successfully executed */
80 #define UFFD_FEATURE_INITIALIZED (1u << 31)
81
userfaultfd_is_initialized(struct userfaultfd_ctx * ctx)82 static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)
83 {
84 return ctx->features & UFFD_FEATURE_INITIALIZED;
85 }
86
userfaultfd_wp_async_ctx(struct userfaultfd_ctx * ctx)87 static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx)
88 {
89 return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC);
90 }
91
92 /*
93 * Whether WP_UNPOPULATED is enabled on the uffd context. It is only
94 * meaningful when userfaultfd_wp()==true on the vma and when it's
95 * anonymous.
96 */
userfaultfd_wp_unpopulated(struct vm_area_struct * vma)97 bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma)
98 {
99 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
100
101 if (!ctx)
102 return false;
103
104 return ctx->features & UFFD_FEATURE_WP_UNPOPULATED;
105 }
106
userfaultfd_wake_function(wait_queue_entry_t * wq,unsigned mode,int wake_flags,void * key)107 static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
108 int wake_flags, void *key)
109 {
110 struct userfaultfd_wake_range *range = key;
111 int ret;
112 struct userfaultfd_wait_queue *uwq;
113 unsigned long start, len;
114
115 uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
116 ret = 0;
117 /* len == 0 means wake all */
118 start = range->start;
119 len = range->len;
120 if (len && (start > uwq->msg.arg.pagefault.address ||
121 start + len <= uwq->msg.arg.pagefault.address))
122 goto out;
123 WRITE_ONCE(uwq->waken, true);
124 /*
125 * The Program-Order guarantees provided by the scheduler
126 * ensure uwq->waken is visible before the task is woken.
127 */
128 ret = wake_up_state(wq->private, mode);
129 if (ret) {
130 /*
131 * Wake only once, autoremove behavior.
132 *
133 * After the effect of list_del_init is visible to the other
134 * CPUs, the waitqueue may disappear from under us, see the
135 * !list_empty_careful() in handle_userfault().
136 *
137 * try_to_wake_up() has an implicit smp_mb(), and the
138 * wq->private is read before calling the extern function
139 * "wake_up_state" (which in turns calls try_to_wake_up).
140 */
141 list_del_init(&wq->entry);
142 }
143 out:
144 return ret;
145 }
146
147 /**
148 * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
149 * context.
150 * @ctx: [in] Pointer to the userfaultfd context.
151 */
userfaultfd_ctx_get(struct userfaultfd_ctx * ctx)152 static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
153 {
154 refcount_inc(&ctx->refcount);
155 }
156
157 /**
158 * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
159 * context.
160 * @ctx: [in] Pointer to userfaultfd context.
161 *
162 * The userfaultfd context reference must have been previously acquired either
163 * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
164 */
userfaultfd_ctx_put(struct userfaultfd_ctx * ctx)165 static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
166 {
167 if (refcount_dec_and_test(&ctx->refcount)) {
168 VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
169 VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
170 VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
171 VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
172 VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
173 VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
174 VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
175 VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
176 mmdrop(ctx->mm);
177 kmem_cache_free(userfaultfd_ctx_cachep, ctx);
178 }
179 }
180
msg_init(struct uffd_msg * msg)181 static inline void msg_init(struct uffd_msg *msg)
182 {
183 BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
184 /*
185 * Must use memset to zero out the paddings or kernel data is
186 * leaked to userland.
187 */
188 memset(msg, 0, sizeof(struct uffd_msg));
189 }
190
userfault_msg(unsigned long address,unsigned long real_address,unsigned int flags,unsigned long reason,unsigned int features)191 static inline struct uffd_msg userfault_msg(unsigned long address,
192 unsigned long real_address,
193 unsigned int flags,
194 unsigned long reason,
195 unsigned int features)
196 {
197 struct uffd_msg msg;
198
199 msg_init(&msg);
200 msg.event = UFFD_EVENT_PAGEFAULT;
201
202 msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ?
203 real_address : address;
204
205 /*
206 * These flags indicate why the userfault occurred:
207 * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault.
208 * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault.
209 * - Neither of these flags being set indicates a MISSING fault.
210 *
211 * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write
212 * fault. Otherwise, it was a read fault.
213 */
214 if (flags & FAULT_FLAG_WRITE)
215 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
216 if (reason & VM_UFFD_WP)
217 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
218 if (reason & VM_UFFD_MINOR)
219 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
220 if (features & UFFD_FEATURE_THREAD_ID)
221 msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
222 return msg;
223 }
224
225 #ifdef CONFIG_HUGETLB_PAGE
226 /*
227 * Same functionality as userfaultfd_must_wait below with modifications for
228 * hugepmd ranges.
229 */
userfaultfd_huge_must_wait(struct userfaultfd_ctx * ctx,struct vm_fault * vmf,unsigned long reason)230 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
231 struct vm_fault *vmf,
232 unsigned long reason)
233 {
234 struct vm_area_struct *vma = vmf->vma;
235 pte_t *ptep, pte;
236 bool ret = true;
237
238 assert_fault_locked(vmf);
239
240 ptep = hugetlb_walk(vma, vmf->address, vma_mmu_pagesize(vma));
241 if (!ptep)
242 goto out;
243
244 ret = false;
245 pte = huge_ptep_get(vma->vm_mm, vmf->address, ptep);
246
247 /*
248 * Lockless access: we're in a wait_event so it's ok if it
249 * changes under us. PTE markers should be handled the same as none
250 * ptes here.
251 */
252 if (huge_pte_none_mostly(pte))
253 ret = true;
254 if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
255 ret = true;
256 out:
257 return ret;
258 }
259 #else
userfaultfd_huge_must_wait(struct userfaultfd_ctx * ctx,struct vm_fault * vmf,unsigned long reason)260 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
261 struct vm_fault *vmf,
262 unsigned long reason)
263 {
264 return false; /* should never get here */
265 }
266 #endif /* CONFIG_HUGETLB_PAGE */
267
268 /*
269 * Verify the pagetables are still not ok after having reigstered into
270 * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
271 * userfault that has already been resolved, if userfaultfd_read_iter and
272 * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
273 * threads.
274 */
userfaultfd_must_wait(struct userfaultfd_ctx * ctx,struct vm_fault * vmf,unsigned long reason)275 static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
276 struct vm_fault *vmf,
277 unsigned long reason)
278 {
279 struct mm_struct *mm = ctx->mm;
280 unsigned long address = vmf->address;
281 pgd_t *pgd;
282 p4d_t *p4d;
283 pud_t *pud;
284 pmd_t *pmd, _pmd;
285 pte_t *pte;
286 pte_t ptent;
287 bool ret = true;
288
289 assert_fault_locked(vmf);
290
291 pgd = pgd_offset(mm, address);
292 if (!pgd_present(*pgd))
293 goto out;
294 p4d = p4d_offset(pgd, address);
295 if (!p4d_present(*p4d))
296 goto out;
297 pud = pud_offset(p4d, address);
298 if (!pud_present(*pud))
299 goto out;
300 pmd = pmd_offset(pud, address);
301 again:
302 _pmd = pmdp_get_lockless(pmd);
303 if (pmd_none(_pmd))
304 goto out;
305
306 ret = false;
307 if (!pmd_present(_pmd) || pmd_devmap(_pmd))
308 goto out;
309
310 if (pmd_trans_huge(_pmd)) {
311 if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
312 ret = true;
313 goto out;
314 }
315
316 pte = pte_offset_map(pmd, address);
317 if (!pte) {
318 ret = true;
319 goto again;
320 }
321 /*
322 * Lockless access: we're in a wait_event so it's ok if it
323 * changes under us. PTE markers should be handled the same as none
324 * ptes here.
325 */
326 ptent = ptep_get(pte);
327 if (pte_none_mostly(ptent))
328 ret = true;
329 if (!pte_write(ptent) && (reason & VM_UFFD_WP))
330 ret = true;
331 pte_unmap(pte);
332
333 out:
334 return ret;
335 }
336
userfaultfd_get_blocking_state(unsigned int flags)337 static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags)
338 {
339 if (flags & FAULT_FLAG_INTERRUPTIBLE)
340 return TASK_INTERRUPTIBLE;
341
342 if (flags & FAULT_FLAG_KILLABLE)
343 return TASK_KILLABLE;
344
345 return TASK_UNINTERRUPTIBLE;
346 }
347
348 /*
349 * The locking rules involved in returning VM_FAULT_RETRY depending on
350 * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
351 * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
352 * recommendation in __lock_page_or_retry is not an understatement.
353 *
354 * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released
355 * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
356 * not set.
357 *
358 * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
359 * set, VM_FAULT_RETRY can still be returned if and only if there are
360 * fatal_signal_pending()s, and the mmap_lock must be released before
361 * returning it.
362 */
handle_userfault(struct vm_fault * vmf,unsigned long reason)363 vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
364 {
365 struct vm_area_struct *vma = vmf->vma;
366 struct mm_struct *mm = vma->vm_mm;
367 struct userfaultfd_ctx *ctx;
368 struct userfaultfd_wait_queue uwq;
369 vm_fault_t ret = VM_FAULT_SIGBUS;
370 bool must_wait;
371 unsigned int blocking_state;
372
373 /*
374 * We don't do userfault handling for the final child pid update
375 * and when coredumping (faults triggered by get_dump_page()).
376 */
377 if (current->flags & (PF_EXITING|PF_DUMPCORE))
378 goto out;
379
380 assert_fault_locked(vmf);
381
382 ctx = vma->vm_userfaultfd_ctx.ctx;
383 if (!ctx)
384 goto out;
385
386 BUG_ON(ctx->mm != mm);
387
388 /* Any unrecognized flag is a bug. */
389 VM_BUG_ON(reason & ~__VM_UFFD_FLAGS);
390 /* 0 or > 1 flags set is a bug; we expect exactly 1. */
391 VM_BUG_ON(!reason || (reason & (reason - 1)));
392
393 if (ctx->features & UFFD_FEATURE_SIGBUS)
394 goto out;
395 if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY))
396 goto out;
397
398 /*
399 * Check that we can return VM_FAULT_RETRY.
400 *
401 * NOTE: it should become possible to return VM_FAULT_RETRY
402 * even if FAULT_FLAG_TRIED is set without leading to gup()
403 * -EBUSY failures, if the userfaultfd is to be extended for
404 * VM_UFFD_WP tracking and we intend to arm the userfault
405 * without first stopping userland access to the memory. For
406 * VM_UFFD_MISSING userfaults this is enough for now.
407 */
408 if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
409 /*
410 * Validate the invariant that nowait must allow retry
411 * to be sure not to return SIGBUS erroneously on
412 * nowait invocations.
413 */
414 BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
415 #ifdef CONFIG_DEBUG_VM
416 if (printk_ratelimit()) {
417 printk(KERN_WARNING
418 "FAULT_FLAG_ALLOW_RETRY missing %x\n",
419 vmf->flags);
420 dump_stack();
421 }
422 #endif
423 goto out;
424 }
425
426 /*
427 * Handle nowait, not much to do other than tell it to retry
428 * and wait.
429 */
430 ret = VM_FAULT_RETRY;
431 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
432 goto out;
433
434 if (unlikely(READ_ONCE(ctx->released))) {
435 /*
436 * If a concurrent release is detected, do not return
437 * VM_FAULT_SIGBUS or VM_FAULT_NOPAGE, but instead always
438 * return VM_FAULT_RETRY with lock released proactively.
439 *
440 * If we were to return VM_FAULT_SIGBUS here, the non
441 * cooperative manager would be instead forced to
442 * always call UFFDIO_UNREGISTER before it can safely
443 * close the uffd, to avoid involuntary SIGBUS triggered.
444 *
445 * If we were to return VM_FAULT_NOPAGE, it would work for
446 * the fault path, in which the lock will be released
447 * later. However for GUP, faultin_page() does nothing
448 * special on NOPAGE, so GUP would spin retrying without
449 * releasing the mmap read lock, causing possible livelock.
450 *
451 * Here only VM_FAULT_RETRY would make sure the mmap lock
452 * be released immediately, so that the thread concurrently
453 * releasing the userfault would always make progress.
454 */
455 release_fault_lock(vmf);
456 goto out;
457 }
458
459 /* take the reference before dropping the mmap_lock */
460 userfaultfd_ctx_get(ctx);
461
462 init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
463 uwq.wq.private = current;
464 uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags,
465 reason, ctx->features);
466 uwq.ctx = ctx;
467 uwq.waken = false;
468
469 blocking_state = userfaultfd_get_blocking_state(vmf->flags);
470
471 /*
472 * Take the vma lock now, in order to safely call
473 * userfaultfd_huge_must_wait() later. Since acquiring the
474 * (sleepable) vma lock can modify the current task state, that
475 * must be before explicitly calling set_current_state().
476 */
477 if (is_vm_hugetlb_page(vma))
478 hugetlb_vma_lock_read(vma);
479
480 spin_lock_irq(&ctx->fault_pending_wqh.lock);
481 /*
482 * After the __add_wait_queue the uwq is visible to userland
483 * through poll/read().
484 */
485 __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
486 /*
487 * The smp_mb() after __set_current_state prevents the reads
488 * following the spin_unlock to happen before the list_add in
489 * __add_wait_queue.
490 */
491 set_current_state(blocking_state);
492 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
493
494 if (!is_vm_hugetlb_page(vma))
495 must_wait = userfaultfd_must_wait(ctx, vmf, reason);
496 else
497 must_wait = userfaultfd_huge_must_wait(ctx, vmf, reason);
498 if (is_vm_hugetlb_page(vma))
499 hugetlb_vma_unlock_read(vma);
500 release_fault_lock(vmf);
501
502 if (likely(must_wait && !READ_ONCE(ctx->released))) {
503 wake_up_poll(&ctx->fd_wqh, EPOLLIN);
504 schedule();
505 }
506
507 __set_current_state(TASK_RUNNING);
508
509 /*
510 * Here we race with the list_del; list_add in
511 * userfaultfd_ctx_read(), however because we don't ever run
512 * list_del_init() to refile across the two lists, the prev
513 * and next pointers will never point to self. list_add also
514 * would never let any of the two pointers to point to
515 * self. So list_empty_careful won't risk to see both pointers
516 * pointing to self at any time during the list refile. The
517 * only case where list_del_init() is called is the full
518 * removal in the wake function and there we don't re-list_add
519 * and it's fine not to block on the spinlock. The uwq on this
520 * kernel stack can be released after the list_del_init.
521 */
522 if (!list_empty_careful(&uwq.wq.entry)) {
523 spin_lock_irq(&ctx->fault_pending_wqh.lock);
524 /*
525 * No need of list_del_init(), the uwq on the stack
526 * will be freed shortly anyway.
527 */
528 list_del(&uwq.wq.entry);
529 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
530 }
531
532 /*
533 * ctx may go away after this if the userfault pseudo fd is
534 * already released.
535 */
536 userfaultfd_ctx_put(ctx);
537
538 out:
539 return ret;
540 }
541
userfaultfd_event_wait_completion(struct userfaultfd_ctx * ctx,struct userfaultfd_wait_queue * ewq)542 static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
543 struct userfaultfd_wait_queue *ewq)
544 {
545 struct userfaultfd_ctx *release_new_ctx;
546
547 if (WARN_ON_ONCE(current->flags & PF_EXITING))
548 goto out;
549
550 ewq->ctx = ctx;
551 init_waitqueue_entry(&ewq->wq, current);
552 release_new_ctx = NULL;
553
554 spin_lock_irq(&ctx->event_wqh.lock);
555 /*
556 * After the __add_wait_queue the uwq is visible to userland
557 * through poll/read().
558 */
559 __add_wait_queue(&ctx->event_wqh, &ewq->wq);
560 for (;;) {
561 set_current_state(TASK_KILLABLE);
562 if (ewq->msg.event == 0)
563 break;
564 if (READ_ONCE(ctx->released) ||
565 fatal_signal_pending(current)) {
566 /*
567 * &ewq->wq may be queued in fork_event, but
568 * __remove_wait_queue ignores the head
569 * parameter. It would be a problem if it
570 * didn't.
571 */
572 __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
573 if (ewq->msg.event == UFFD_EVENT_FORK) {
574 struct userfaultfd_ctx *new;
575
576 new = (struct userfaultfd_ctx *)
577 (unsigned long)
578 ewq->msg.arg.reserved.reserved1;
579 release_new_ctx = new;
580 }
581 break;
582 }
583
584 spin_unlock_irq(&ctx->event_wqh.lock);
585
586 wake_up_poll(&ctx->fd_wqh, EPOLLIN);
587 schedule();
588
589 spin_lock_irq(&ctx->event_wqh.lock);
590 }
591 __set_current_state(TASK_RUNNING);
592 spin_unlock_irq(&ctx->event_wqh.lock);
593
594 if (release_new_ctx) {
595 userfaultfd_release_new(release_new_ctx);
596 userfaultfd_ctx_put(release_new_ctx);
597 }
598
599 /*
600 * ctx may go away after this if the userfault pseudo fd is
601 * already released.
602 */
603 out:
604 atomic_dec(&ctx->mmap_changing);
605 VM_BUG_ON(atomic_read(&ctx->mmap_changing) < 0);
606 userfaultfd_ctx_put(ctx);
607 }
608
userfaultfd_event_complete(struct userfaultfd_ctx * ctx,struct userfaultfd_wait_queue * ewq)609 static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
610 struct userfaultfd_wait_queue *ewq)
611 {
612 ewq->msg.event = 0;
613 wake_up_locked(&ctx->event_wqh);
614 __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
615 }
616
dup_userfaultfd(struct vm_area_struct * vma,struct list_head * fcs)617 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
618 {
619 struct userfaultfd_ctx *ctx = NULL, *octx;
620 struct userfaultfd_fork_ctx *fctx;
621
622 octx = vma->vm_userfaultfd_ctx.ctx;
623 if (!octx)
624 return 0;
625
626 if (!(octx->features & UFFD_FEATURE_EVENT_FORK)) {
627 userfaultfd_reset_ctx(vma);
628 return 0;
629 }
630
631 list_for_each_entry(fctx, fcs, list)
632 if (fctx->orig == octx) {
633 ctx = fctx->new;
634 break;
635 }
636
637 if (!ctx) {
638 fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
639 if (!fctx)
640 return -ENOMEM;
641
642 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
643 if (!ctx) {
644 kfree(fctx);
645 return -ENOMEM;
646 }
647
648 refcount_set(&ctx->refcount, 1);
649 ctx->flags = octx->flags;
650 ctx->features = octx->features;
651 ctx->released = false;
652 init_rwsem(&ctx->map_changing_lock);
653 atomic_set(&ctx->mmap_changing, 0);
654 ctx->mm = vma->vm_mm;
655 mmgrab(ctx->mm);
656
657 userfaultfd_ctx_get(octx);
658 down_write(&octx->map_changing_lock);
659 atomic_inc(&octx->mmap_changing);
660 up_write(&octx->map_changing_lock);
661 fctx->orig = octx;
662 fctx->new = ctx;
663 list_add_tail(&fctx->list, fcs);
664 }
665
666 vma->vm_userfaultfd_ctx.ctx = ctx;
667 return 0;
668 }
669
dup_fctx(struct userfaultfd_fork_ctx * fctx)670 static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
671 {
672 struct userfaultfd_ctx *ctx = fctx->orig;
673 struct userfaultfd_wait_queue ewq;
674
675 msg_init(&ewq.msg);
676
677 ewq.msg.event = UFFD_EVENT_FORK;
678 ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
679
680 userfaultfd_event_wait_completion(ctx, &ewq);
681 }
682
dup_userfaultfd_complete(struct list_head * fcs)683 void dup_userfaultfd_complete(struct list_head *fcs)
684 {
685 struct userfaultfd_fork_ctx *fctx, *n;
686
687 list_for_each_entry_safe(fctx, n, fcs, list) {
688 dup_fctx(fctx);
689 list_del(&fctx->list);
690 kfree(fctx);
691 }
692 }
693
dup_userfaultfd_fail(struct list_head * fcs)694 void dup_userfaultfd_fail(struct list_head *fcs)
695 {
696 struct userfaultfd_fork_ctx *fctx, *n;
697
698 /*
699 * An error has occurred on fork, we will tear memory down, but have
700 * allocated memory for fctx's and raised reference counts for both the
701 * original and child contexts (and on the mm for each as a result).
702 *
703 * These would ordinarily be taken care of by a user handling the event,
704 * but we are no longer doing so, so manually clean up here.
705 *
706 * mm tear down will take care of cleaning up VMA contexts.
707 */
708 list_for_each_entry_safe(fctx, n, fcs, list) {
709 struct userfaultfd_ctx *octx = fctx->orig;
710 struct userfaultfd_ctx *ctx = fctx->new;
711
712 atomic_dec(&octx->mmap_changing);
713 VM_BUG_ON(atomic_read(&octx->mmap_changing) < 0);
714 userfaultfd_ctx_put(octx);
715 userfaultfd_ctx_put(ctx);
716
717 list_del(&fctx->list);
718 kfree(fctx);
719 }
720 }
721
mremap_userfaultfd_prep(struct vm_area_struct * vma,struct vm_userfaultfd_ctx * vm_ctx)722 void mremap_userfaultfd_prep(struct vm_area_struct *vma,
723 struct vm_userfaultfd_ctx *vm_ctx)
724 {
725 struct userfaultfd_ctx *ctx;
726
727 ctx = vma->vm_userfaultfd_ctx.ctx;
728
729 if (!ctx)
730 return;
731
732 if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
733 vm_ctx->ctx = ctx;
734 userfaultfd_ctx_get(ctx);
735 down_write(&ctx->map_changing_lock);
736 atomic_inc(&ctx->mmap_changing);
737 up_write(&ctx->map_changing_lock);
738 } else {
739 /* Drop uffd context if remap feature not enabled */
740 userfaultfd_reset_ctx(vma);
741 }
742 }
743
mremap_userfaultfd_complete(struct vm_userfaultfd_ctx * vm_ctx,unsigned long from,unsigned long to,unsigned long len)744 void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
745 unsigned long from, unsigned long to,
746 unsigned long len)
747 {
748 struct userfaultfd_ctx *ctx = vm_ctx->ctx;
749 struct userfaultfd_wait_queue ewq;
750
751 if (!ctx)
752 return;
753
754 if (to & ~PAGE_MASK) {
755 userfaultfd_ctx_put(ctx);
756 return;
757 }
758
759 msg_init(&ewq.msg);
760
761 ewq.msg.event = UFFD_EVENT_REMAP;
762 ewq.msg.arg.remap.from = from;
763 ewq.msg.arg.remap.to = to;
764 ewq.msg.arg.remap.len = len;
765
766 userfaultfd_event_wait_completion(ctx, &ewq);
767 }
768
userfaultfd_remove(struct vm_area_struct * vma,unsigned long start,unsigned long end)769 bool userfaultfd_remove(struct vm_area_struct *vma,
770 unsigned long start, unsigned long end)
771 {
772 struct mm_struct *mm = vma->vm_mm;
773 struct userfaultfd_ctx *ctx;
774 struct userfaultfd_wait_queue ewq;
775
776 ctx = vma->vm_userfaultfd_ctx.ctx;
777 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
778 return true;
779
780 userfaultfd_ctx_get(ctx);
781 down_write(&ctx->map_changing_lock);
782 atomic_inc(&ctx->mmap_changing);
783 up_write(&ctx->map_changing_lock);
784 mmap_read_unlock(mm);
785
786 msg_init(&ewq.msg);
787
788 ewq.msg.event = UFFD_EVENT_REMOVE;
789 ewq.msg.arg.remove.start = start;
790 ewq.msg.arg.remove.end = end;
791
792 userfaultfd_event_wait_completion(ctx, &ewq);
793
794 return false;
795 }
796
has_unmap_ctx(struct userfaultfd_ctx * ctx,struct list_head * unmaps,unsigned long start,unsigned long end)797 static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
798 unsigned long start, unsigned long end)
799 {
800 struct userfaultfd_unmap_ctx *unmap_ctx;
801
802 list_for_each_entry(unmap_ctx, unmaps, list)
803 if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
804 unmap_ctx->end == end)
805 return true;
806
807 return false;
808 }
809
userfaultfd_unmap_prep(struct vm_area_struct * vma,unsigned long start,unsigned long end,struct list_head * unmaps)810 int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start,
811 unsigned long end, struct list_head *unmaps)
812 {
813 struct userfaultfd_unmap_ctx *unmap_ctx;
814 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
815
816 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
817 has_unmap_ctx(ctx, unmaps, start, end))
818 return 0;
819
820 unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
821 if (!unmap_ctx)
822 return -ENOMEM;
823
824 userfaultfd_ctx_get(ctx);
825 down_write(&ctx->map_changing_lock);
826 atomic_inc(&ctx->mmap_changing);
827 up_write(&ctx->map_changing_lock);
828 unmap_ctx->ctx = ctx;
829 unmap_ctx->start = start;
830 unmap_ctx->end = end;
831 list_add_tail(&unmap_ctx->list, unmaps);
832
833 return 0;
834 }
835
userfaultfd_unmap_complete(struct mm_struct * mm,struct list_head * uf)836 void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
837 {
838 struct userfaultfd_unmap_ctx *ctx, *n;
839 struct userfaultfd_wait_queue ewq;
840
841 list_for_each_entry_safe(ctx, n, uf, list) {
842 msg_init(&ewq.msg);
843
844 ewq.msg.event = UFFD_EVENT_UNMAP;
845 ewq.msg.arg.remove.start = ctx->start;
846 ewq.msg.arg.remove.end = ctx->end;
847
848 userfaultfd_event_wait_completion(ctx->ctx, &ewq);
849
850 list_del(&ctx->list);
851 kfree(ctx);
852 }
853 }
854
userfaultfd_release(struct inode * inode,struct file * file)855 static int userfaultfd_release(struct inode *inode, struct file *file)
856 {
857 struct userfaultfd_ctx *ctx = file->private_data;
858 struct mm_struct *mm = ctx->mm;
859 /* len == 0 means wake all */
860 struct userfaultfd_wake_range range = { .len = 0, };
861
862 WRITE_ONCE(ctx->released, true);
863
864 userfaultfd_release_all(mm, ctx);
865
866 /*
867 * After no new page faults can wait on this fault_*wqh, flush
868 * the last page faults that may have been already waiting on
869 * the fault_*wqh.
870 */
871 spin_lock_irq(&ctx->fault_pending_wqh.lock);
872 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
873 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
874 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
875
876 /* Flush pending events that may still wait on event_wqh */
877 wake_up_all(&ctx->event_wqh);
878
879 wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
880 userfaultfd_ctx_put(ctx);
881 return 0;
882 }
883
884 /* fault_pending_wqh.lock must be hold by the caller */
find_userfault_in(wait_queue_head_t * wqh)885 static inline struct userfaultfd_wait_queue *find_userfault_in(
886 wait_queue_head_t *wqh)
887 {
888 wait_queue_entry_t *wq;
889 struct userfaultfd_wait_queue *uwq;
890
891 lockdep_assert_held(&wqh->lock);
892
893 uwq = NULL;
894 if (!waitqueue_active(wqh))
895 goto out;
896 /* walk in reverse to provide FIFO behavior to read userfaults */
897 wq = list_last_entry(&wqh->head, typeof(*wq), entry);
898 uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
899 out:
900 return uwq;
901 }
902
find_userfault(struct userfaultfd_ctx * ctx)903 static inline struct userfaultfd_wait_queue *find_userfault(
904 struct userfaultfd_ctx *ctx)
905 {
906 return find_userfault_in(&ctx->fault_pending_wqh);
907 }
908
find_userfault_evt(struct userfaultfd_ctx * ctx)909 static inline struct userfaultfd_wait_queue *find_userfault_evt(
910 struct userfaultfd_ctx *ctx)
911 {
912 return find_userfault_in(&ctx->event_wqh);
913 }
914
userfaultfd_poll(struct file * file,poll_table * wait)915 static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
916 {
917 struct userfaultfd_ctx *ctx = file->private_data;
918 __poll_t ret;
919
920 poll_wait(file, &ctx->fd_wqh, wait);
921
922 if (!userfaultfd_is_initialized(ctx))
923 return EPOLLERR;
924
925 /*
926 * poll() never guarantees that read won't block.
927 * userfaults can be waken before they're read().
928 */
929 if (unlikely(!(file->f_flags & O_NONBLOCK)))
930 return EPOLLERR;
931 /*
932 * lockless access to see if there are pending faults
933 * __pollwait last action is the add_wait_queue but
934 * the spin_unlock would allow the waitqueue_active to
935 * pass above the actual list_add inside
936 * add_wait_queue critical section. So use a full
937 * memory barrier to serialize the list_add write of
938 * add_wait_queue() with the waitqueue_active read
939 * below.
940 */
941 ret = 0;
942 smp_mb();
943 if (waitqueue_active(&ctx->fault_pending_wqh))
944 ret = EPOLLIN;
945 else if (waitqueue_active(&ctx->event_wqh))
946 ret = EPOLLIN;
947
948 return ret;
949 }
950
951 static const struct file_operations userfaultfd_fops;
952
resolve_userfault_fork(struct userfaultfd_ctx * new,struct inode * inode,struct uffd_msg * msg)953 static int resolve_userfault_fork(struct userfaultfd_ctx *new,
954 struct inode *inode,
955 struct uffd_msg *msg)
956 {
957 int fd;
958
959 fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, new,
960 O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
961 if (fd < 0)
962 return fd;
963
964 msg->arg.reserved.reserved1 = 0;
965 msg->arg.fork.ufd = fd;
966 return 0;
967 }
968
userfaultfd_ctx_read(struct userfaultfd_ctx * ctx,int no_wait,struct uffd_msg * msg,struct inode * inode)969 static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
970 struct uffd_msg *msg, struct inode *inode)
971 {
972 ssize_t ret;
973 DECLARE_WAITQUEUE(wait, current);
974 struct userfaultfd_wait_queue *uwq;
975 /*
976 * Handling fork event requires sleeping operations, so
977 * we drop the event_wqh lock, then do these ops, then
978 * lock it back and wake up the waiter. While the lock is
979 * dropped the ewq may go away so we keep track of it
980 * carefully.
981 */
982 LIST_HEAD(fork_event);
983 struct userfaultfd_ctx *fork_nctx = NULL;
984
985 /* always take the fd_wqh lock before the fault_pending_wqh lock */
986 spin_lock_irq(&ctx->fd_wqh.lock);
987 __add_wait_queue(&ctx->fd_wqh, &wait);
988 for (;;) {
989 set_current_state(TASK_INTERRUPTIBLE);
990 spin_lock(&ctx->fault_pending_wqh.lock);
991 uwq = find_userfault(ctx);
992 if (uwq) {
993 /*
994 * Use a seqcount to repeat the lockless check
995 * in wake_userfault() to avoid missing
996 * wakeups because during the refile both
997 * waitqueue could become empty if this is the
998 * only userfault.
999 */
1000 write_seqcount_begin(&ctx->refile_seq);
1001
1002 /*
1003 * The fault_pending_wqh.lock prevents the uwq
1004 * to disappear from under us.
1005 *
1006 * Refile this userfault from
1007 * fault_pending_wqh to fault_wqh, it's not
1008 * pending anymore after we read it.
1009 *
1010 * Use list_del() by hand (as
1011 * userfaultfd_wake_function also uses
1012 * list_del_init() by hand) to be sure nobody
1013 * changes __remove_wait_queue() to use
1014 * list_del_init() in turn breaking the
1015 * !list_empty_careful() check in
1016 * handle_userfault(). The uwq->wq.head list
1017 * must never be empty at any time during the
1018 * refile, or the waitqueue could disappear
1019 * from under us. The "wait_queue_head_t"
1020 * parameter of __remove_wait_queue() is unused
1021 * anyway.
1022 */
1023 list_del(&uwq->wq.entry);
1024 add_wait_queue(&ctx->fault_wqh, &uwq->wq);
1025
1026 write_seqcount_end(&ctx->refile_seq);
1027
1028 /* careful to always initialize msg if ret == 0 */
1029 *msg = uwq->msg;
1030 spin_unlock(&ctx->fault_pending_wqh.lock);
1031 ret = 0;
1032 break;
1033 }
1034 spin_unlock(&ctx->fault_pending_wqh.lock);
1035
1036 spin_lock(&ctx->event_wqh.lock);
1037 uwq = find_userfault_evt(ctx);
1038 if (uwq) {
1039 *msg = uwq->msg;
1040
1041 if (uwq->msg.event == UFFD_EVENT_FORK) {
1042 fork_nctx = (struct userfaultfd_ctx *)
1043 (unsigned long)
1044 uwq->msg.arg.reserved.reserved1;
1045 list_move(&uwq->wq.entry, &fork_event);
1046 /*
1047 * fork_nctx can be freed as soon as
1048 * we drop the lock, unless we take a
1049 * reference on it.
1050 */
1051 userfaultfd_ctx_get(fork_nctx);
1052 spin_unlock(&ctx->event_wqh.lock);
1053 ret = 0;
1054 break;
1055 }
1056
1057 userfaultfd_event_complete(ctx, uwq);
1058 spin_unlock(&ctx->event_wqh.lock);
1059 ret = 0;
1060 break;
1061 }
1062 spin_unlock(&ctx->event_wqh.lock);
1063
1064 if (signal_pending(current)) {
1065 ret = -ERESTARTSYS;
1066 break;
1067 }
1068 if (no_wait) {
1069 ret = -EAGAIN;
1070 break;
1071 }
1072 spin_unlock_irq(&ctx->fd_wqh.lock);
1073 schedule();
1074 spin_lock_irq(&ctx->fd_wqh.lock);
1075 }
1076 __remove_wait_queue(&ctx->fd_wqh, &wait);
1077 __set_current_state(TASK_RUNNING);
1078 spin_unlock_irq(&ctx->fd_wqh.lock);
1079
1080 if (!ret && msg->event == UFFD_EVENT_FORK) {
1081 ret = resolve_userfault_fork(fork_nctx, inode, msg);
1082 spin_lock_irq(&ctx->event_wqh.lock);
1083 if (!list_empty(&fork_event)) {
1084 /*
1085 * The fork thread didn't abort, so we can
1086 * drop the temporary refcount.
1087 */
1088 userfaultfd_ctx_put(fork_nctx);
1089
1090 uwq = list_first_entry(&fork_event,
1091 typeof(*uwq),
1092 wq.entry);
1093 /*
1094 * If fork_event list wasn't empty and in turn
1095 * the event wasn't already released by fork
1096 * (the event is allocated on fork kernel
1097 * stack), put the event back to its place in
1098 * the event_wq. fork_event head will be freed
1099 * as soon as we return so the event cannot
1100 * stay queued there no matter the current
1101 * "ret" value.
1102 */
1103 list_del(&uwq->wq.entry);
1104 __add_wait_queue(&ctx->event_wqh, &uwq->wq);
1105
1106 /*
1107 * Leave the event in the waitqueue and report
1108 * error to userland if we failed to resolve
1109 * the userfault fork.
1110 */
1111 if (likely(!ret))
1112 userfaultfd_event_complete(ctx, uwq);
1113 } else {
1114 /*
1115 * Here the fork thread aborted and the
1116 * refcount from the fork thread on fork_nctx
1117 * has already been released. We still hold
1118 * the reference we took before releasing the
1119 * lock above. If resolve_userfault_fork
1120 * failed we've to drop it because the
1121 * fork_nctx has to be freed in such case. If
1122 * it succeeded we'll hold it because the new
1123 * uffd references it.
1124 */
1125 if (ret)
1126 userfaultfd_ctx_put(fork_nctx);
1127 }
1128 spin_unlock_irq(&ctx->event_wqh.lock);
1129 }
1130
1131 return ret;
1132 }
1133
userfaultfd_read_iter(struct kiocb * iocb,struct iov_iter * to)1134 static ssize_t userfaultfd_read_iter(struct kiocb *iocb, struct iov_iter *to)
1135 {
1136 struct file *file = iocb->ki_filp;
1137 struct userfaultfd_ctx *ctx = file->private_data;
1138 ssize_t _ret, ret = 0;
1139 struct uffd_msg msg;
1140 struct inode *inode = file_inode(file);
1141 bool no_wait;
1142
1143 if (!userfaultfd_is_initialized(ctx))
1144 return -EINVAL;
1145
1146 no_wait = file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT;
1147 for (;;) {
1148 if (iov_iter_count(to) < sizeof(msg))
1149 return ret ? ret : -EINVAL;
1150 _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode);
1151 if (_ret < 0)
1152 return ret ? ret : _ret;
1153 _ret = !copy_to_iter_full(&msg, sizeof(msg), to);
1154 if (_ret)
1155 return ret ? ret : -EFAULT;
1156 ret += sizeof(msg);
1157 /*
1158 * Allow to read more than one fault at time but only
1159 * block if waiting for the very first one.
1160 */
1161 no_wait = true;
1162 }
1163 }
1164
__wake_userfault(struct userfaultfd_ctx * ctx,struct userfaultfd_wake_range * range)1165 static void __wake_userfault(struct userfaultfd_ctx *ctx,
1166 struct userfaultfd_wake_range *range)
1167 {
1168 spin_lock_irq(&ctx->fault_pending_wqh.lock);
1169 /* wake all in the range and autoremove */
1170 if (waitqueue_active(&ctx->fault_pending_wqh))
1171 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
1172 range);
1173 if (waitqueue_active(&ctx->fault_wqh))
1174 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
1175 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
1176 }
1177
wake_userfault(struct userfaultfd_ctx * ctx,struct userfaultfd_wake_range * range)1178 static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1179 struct userfaultfd_wake_range *range)
1180 {
1181 unsigned seq;
1182 bool need_wakeup;
1183
1184 /*
1185 * To be sure waitqueue_active() is not reordered by the CPU
1186 * before the pagetable update, use an explicit SMP memory
1187 * barrier here. PT lock release or mmap_read_unlock(mm) still
1188 * have release semantics that can allow the
1189 * waitqueue_active() to be reordered before the pte update.
1190 */
1191 smp_mb();
1192
1193 /*
1194 * Use waitqueue_active because it's very frequent to
1195 * change the address space atomically even if there are no
1196 * userfaults yet. So we take the spinlock only when we're
1197 * sure we've userfaults to wake.
1198 */
1199 do {
1200 seq = read_seqcount_begin(&ctx->refile_seq);
1201 need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1202 waitqueue_active(&ctx->fault_wqh);
1203 cond_resched();
1204 } while (read_seqcount_retry(&ctx->refile_seq, seq));
1205 if (need_wakeup)
1206 __wake_userfault(ctx, range);
1207 }
1208
validate_unaligned_range(struct mm_struct * mm,__u64 start,__u64 len)1209 static __always_inline int validate_unaligned_range(
1210 struct mm_struct *mm, __u64 start, __u64 len)
1211 {
1212 __u64 task_size = mm->task_size;
1213
1214 if (len & ~PAGE_MASK)
1215 return -EINVAL;
1216 if (!len)
1217 return -EINVAL;
1218 if (start < mmap_min_addr)
1219 return -EINVAL;
1220 if (start >= task_size)
1221 return -EINVAL;
1222 if (len > task_size - start)
1223 return -EINVAL;
1224 if (start + len <= start)
1225 return -EINVAL;
1226 return 0;
1227 }
1228
validate_range(struct mm_struct * mm,__u64 start,__u64 len)1229 static __always_inline int validate_range(struct mm_struct *mm,
1230 __u64 start, __u64 len)
1231 {
1232 if (start & ~PAGE_MASK)
1233 return -EINVAL;
1234
1235 return validate_unaligned_range(mm, start, len);
1236 }
1237
userfaultfd_register(struct userfaultfd_ctx * ctx,unsigned long arg)1238 static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1239 unsigned long arg)
1240 {
1241 struct mm_struct *mm = ctx->mm;
1242 struct vm_area_struct *vma, *cur;
1243 int ret;
1244 struct uffdio_register uffdio_register;
1245 struct uffdio_register __user *user_uffdio_register;
1246 unsigned long vm_flags;
1247 bool found;
1248 bool basic_ioctls;
1249 unsigned long start, end;
1250 struct vma_iterator vmi;
1251 bool wp_async = userfaultfd_wp_async_ctx(ctx);
1252
1253 user_uffdio_register = (struct uffdio_register __user *) arg;
1254
1255 ret = -EFAULT;
1256 if (copy_from_user(&uffdio_register, user_uffdio_register,
1257 sizeof(uffdio_register)-sizeof(__u64)))
1258 goto out;
1259
1260 ret = -EINVAL;
1261 if (!uffdio_register.mode)
1262 goto out;
1263 if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES)
1264 goto out;
1265 vm_flags = 0;
1266 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1267 vm_flags |= VM_UFFD_MISSING;
1268 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1269 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1270 goto out;
1271 #endif
1272 vm_flags |= VM_UFFD_WP;
1273 }
1274 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
1275 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1276 goto out;
1277 #endif
1278 vm_flags |= VM_UFFD_MINOR;
1279 }
1280
1281 ret = validate_range(mm, uffdio_register.range.start,
1282 uffdio_register.range.len);
1283 if (ret)
1284 goto out;
1285
1286 start = uffdio_register.range.start;
1287 end = start + uffdio_register.range.len;
1288
1289 ret = -ENOMEM;
1290 if (!mmget_not_zero(mm))
1291 goto out;
1292
1293 ret = -EINVAL;
1294 mmap_write_lock(mm);
1295 vma_iter_init(&vmi, mm, start);
1296 vma = vma_find(&vmi, end);
1297 if (!vma)
1298 goto out_unlock;
1299
1300 /*
1301 * If the first vma contains huge pages, make sure start address
1302 * is aligned to huge page size.
1303 */
1304 if (is_vm_hugetlb_page(vma)) {
1305 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1306
1307 if (start & (vma_hpagesize - 1))
1308 goto out_unlock;
1309 }
1310
1311 /*
1312 * Search for not compatible vmas.
1313 */
1314 found = false;
1315 basic_ioctls = false;
1316 cur = vma;
1317 do {
1318 cond_resched();
1319
1320 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1321 !!(cur->vm_flags & __VM_UFFD_FLAGS));
1322
1323 /* check not compatible vmas */
1324 ret = -EINVAL;
1325 if (!vma_can_userfault(cur, vm_flags, wp_async))
1326 goto out_unlock;
1327
1328 /*
1329 * UFFDIO_COPY will fill file holes even without
1330 * PROT_WRITE. This check enforces that if this is a
1331 * MAP_SHARED, the process has write permission to the backing
1332 * file. If VM_MAYWRITE is set it also enforces that on a
1333 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1334 * F_WRITE_SEAL can be taken until the vma is destroyed.
1335 */
1336 ret = -EPERM;
1337 if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1338 goto out_unlock;
1339
1340 /*
1341 * If this vma contains ending address, and huge pages
1342 * check alignment.
1343 */
1344 if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1345 end > cur->vm_start) {
1346 unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1347
1348 ret = -EINVAL;
1349
1350 if (end & (vma_hpagesize - 1))
1351 goto out_unlock;
1352 }
1353 if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE))
1354 goto out_unlock;
1355
1356 /*
1357 * Check that this vma isn't already owned by a
1358 * different userfaultfd. We can't allow more than one
1359 * userfaultfd to own a single vma simultaneously or we
1360 * wouldn't know which one to deliver the userfaults to.
1361 */
1362 ret = -EBUSY;
1363 if (cur->vm_userfaultfd_ctx.ctx &&
1364 cur->vm_userfaultfd_ctx.ctx != ctx)
1365 goto out_unlock;
1366
1367 /*
1368 * Note vmas containing huge pages
1369 */
1370 if (is_vm_hugetlb_page(cur))
1371 basic_ioctls = true;
1372
1373 found = true;
1374 } for_each_vma_range(vmi, cur, end);
1375 BUG_ON(!found);
1376
1377 ret = userfaultfd_register_range(ctx, vma, vm_flags, start, end,
1378 wp_async);
1379
1380 out_unlock:
1381 mmap_write_unlock(mm);
1382 mmput(mm);
1383 if (!ret) {
1384 __u64 ioctls_out;
1385
1386 ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
1387 UFFD_API_RANGE_IOCTLS;
1388
1389 /*
1390 * Declare the WP ioctl only if the WP mode is
1391 * specified and all checks passed with the range
1392 */
1393 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP))
1394 ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT);
1395
1396 /* CONTINUE ioctl is only supported for MINOR ranges. */
1397 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
1398 ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
1399
1400 /*
1401 * Now that we scanned all vmas we can already tell
1402 * userland which ioctls methods are guaranteed to
1403 * succeed on this range.
1404 */
1405 if (put_user(ioctls_out, &user_uffdio_register->ioctls))
1406 ret = -EFAULT;
1407 }
1408 out:
1409 return ret;
1410 }
1411
userfaultfd_unregister(struct userfaultfd_ctx * ctx,unsigned long arg)1412 static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1413 unsigned long arg)
1414 {
1415 struct mm_struct *mm = ctx->mm;
1416 struct vm_area_struct *vma, *prev, *cur;
1417 int ret;
1418 struct uffdio_range uffdio_unregister;
1419 bool found;
1420 unsigned long start, end, vma_end;
1421 const void __user *buf = (void __user *)arg;
1422 struct vma_iterator vmi;
1423 bool wp_async = userfaultfd_wp_async_ctx(ctx);
1424
1425 ret = -EFAULT;
1426 if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1427 goto out;
1428
1429 ret = validate_range(mm, uffdio_unregister.start,
1430 uffdio_unregister.len);
1431 if (ret)
1432 goto out;
1433
1434 start = uffdio_unregister.start;
1435 end = start + uffdio_unregister.len;
1436
1437 ret = -ENOMEM;
1438 if (!mmget_not_zero(mm))
1439 goto out;
1440
1441 mmap_write_lock(mm);
1442 ret = -EINVAL;
1443 vma_iter_init(&vmi, mm, start);
1444 vma = vma_find(&vmi, end);
1445 if (!vma)
1446 goto out_unlock;
1447
1448 /*
1449 * If the first vma contains huge pages, make sure start address
1450 * is aligned to huge page size.
1451 */
1452 if (is_vm_hugetlb_page(vma)) {
1453 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1454
1455 if (start & (vma_hpagesize - 1))
1456 goto out_unlock;
1457 }
1458
1459 /*
1460 * Search for not compatible vmas.
1461 */
1462 found = false;
1463 cur = vma;
1464 do {
1465 cond_resched();
1466
1467 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1468 !!(cur->vm_flags & __VM_UFFD_FLAGS));
1469
1470 /*
1471 * Check not compatible vmas, not strictly required
1472 * here as not compatible vmas cannot have an
1473 * userfaultfd_ctx registered on them, but this
1474 * provides for more strict behavior to notice
1475 * unregistration errors.
1476 */
1477 if (!vma_can_userfault(cur, cur->vm_flags, wp_async))
1478 goto out_unlock;
1479
1480 found = true;
1481 } for_each_vma_range(vmi, cur, end);
1482 BUG_ON(!found);
1483
1484 vma_iter_set(&vmi, start);
1485 prev = vma_prev(&vmi);
1486 if (vma->vm_start < start)
1487 prev = vma;
1488
1489 ret = 0;
1490 for_each_vma_range(vmi, vma, end) {
1491 cond_resched();
1492
1493 BUG_ON(!vma_can_userfault(vma, vma->vm_flags, wp_async));
1494
1495 /*
1496 * Nothing to do: this vma is already registered into this
1497 * userfaultfd and with the right tracking mode too.
1498 */
1499 if (!vma->vm_userfaultfd_ctx.ctx)
1500 goto skip;
1501
1502 WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1503
1504 if (vma->vm_start > start)
1505 start = vma->vm_start;
1506 vma_end = min(end, vma->vm_end);
1507
1508 if (userfaultfd_missing(vma)) {
1509 /*
1510 * Wake any concurrent pending userfault while
1511 * we unregister, so they will not hang
1512 * permanently and it avoids userland to call
1513 * UFFDIO_WAKE explicitly.
1514 */
1515 struct userfaultfd_wake_range range;
1516 range.start = start;
1517 range.len = vma_end - start;
1518 wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1519 }
1520
1521 vma = userfaultfd_clear_vma(&vmi, prev, vma,
1522 start, vma_end);
1523 if (IS_ERR(vma)) {
1524 ret = PTR_ERR(vma);
1525 break;
1526 }
1527
1528 skip:
1529 prev = vma;
1530 start = vma->vm_end;
1531 }
1532
1533 out_unlock:
1534 mmap_write_unlock(mm);
1535 mmput(mm);
1536 out:
1537 return ret;
1538 }
1539
1540 /*
1541 * userfaultfd_wake may be used in combination with the
1542 * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
1543 */
userfaultfd_wake(struct userfaultfd_ctx * ctx,unsigned long arg)1544 static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1545 unsigned long arg)
1546 {
1547 int ret;
1548 struct uffdio_range uffdio_wake;
1549 struct userfaultfd_wake_range range;
1550 const void __user *buf = (void __user *)arg;
1551
1552 ret = -EFAULT;
1553 if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1554 goto out;
1555
1556 ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
1557 if (ret)
1558 goto out;
1559
1560 range.start = uffdio_wake.start;
1561 range.len = uffdio_wake.len;
1562
1563 /*
1564 * len == 0 means wake all and we don't want to wake all here,
1565 * so check it again to be sure.
1566 */
1567 VM_BUG_ON(!range.len);
1568
1569 wake_userfault(ctx, &range);
1570 ret = 0;
1571
1572 out:
1573 return ret;
1574 }
1575
userfaultfd_copy(struct userfaultfd_ctx * ctx,unsigned long arg)1576 static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1577 unsigned long arg)
1578 {
1579 __s64 ret;
1580 struct uffdio_copy uffdio_copy;
1581 struct uffdio_copy __user *user_uffdio_copy;
1582 struct userfaultfd_wake_range range;
1583 uffd_flags_t flags = 0;
1584
1585 user_uffdio_copy = (struct uffdio_copy __user *) arg;
1586
1587 ret = -EAGAIN;
1588 if (atomic_read(&ctx->mmap_changing))
1589 goto out;
1590
1591 ret = -EFAULT;
1592 if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1593 /* don't copy "copy" last field */
1594 sizeof(uffdio_copy)-sizeof(__s64)))
1595 goto out;
1596
1597 ret = validate_unaligned_range(ctx->mm, uffdio_copy.src,
1598 uffdio_copy.len);
1599 if (ret)
1600 goto out;
1601 ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
1602 if (ret)
1603 goto out;
1604
1605 ret = -EINVAL;
1606 if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))
1607 goto out;
1608 if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP)
1609 flags |= MFILL_ATOMIC_WP;
1610 if (mmget_not_zero(ctx->mm)) {
1611 ret = mfill_atomic_copy(ctx, uffdio_copy.dst, uffdio_copy.src,
1612 uffdio_copy.len, flags);
1613 mmput(ctx->mm);
1614 } else {
1615 return -ESRCH;
1616 }
1617 if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1618 return -EFAULT;
1619 if (ret < 0)
1620 goto out;
1621 BUG_ON(!ret);
1622 /* len == 0 would wake all */
1623 range.len = ret;
1624 if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1625 range.start = uffdio_copy.dst;
1626 wake_userfault(ctx, &range);
1627 }
1628 ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1629 out:
1630 return ret;
1631 }
1632
userfaultfd_zeropage(struct userfaultfd_ctx * ctx,unsigned long arg)1633 static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1634 unsigned long arg)
1635 {
1636 __s64 ret;
1637 struct uffdio_zeropage uffdio_zeropage;
1638 struct uffdio_zeropage __user *user_uffdio_zeropage;
1639 struct userfaultfd_wake_range range;
1640
1641 user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1642
1643 ret = -EAGAIN;
1644 if (atomic_read(&ctx->mmap_changing))
1645 goto out;
1646
1647 ret = -EFAULT;
1648 if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1649 /* don't copy "zeropage" last field */
1650 sizeof(uffdio_zeropage)-sizeof(__s64)))
1651 goto out;
1652
1653 ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
1654 uffdio_zeropage.range.len);
1655 if (ret)
1656 goto out;
1657 ret = -EINVAL;
1658 if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1659 goto out;
1660
1661 if (mmget_not_zero(ctx->mm)) {
1662 ret = mfill_atomic_zeropage(ctx, uffdio_zeropage.range.start,
1663 uffdio_zeropage.range.len);
1664 mmput(ctx->mm);
1665 } else {
1666 return -ESRCH;
1667 }
1668 if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1669 return -EFAULT;
1670 if (ret < 0)
1671 goto out;
1672 /* len == 0 would wake all */
1673 BUG_ON(!ret);
1674 range.len = ret;
1675 if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1676 range.start = uffdio_zeropage.range.start;
1677 wake_userfault(ctx, &range);
1678 }
1679 ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1680 out:
1681 return ret;
1682 }
1683
userfaultfd_writeprotect(struct userfaultfd_ctx * ctx,unsigned long arg)1684 static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
1685 unsigned long arg)
1686 {
1687 int ret;
1688 struct uffdio_writeprotect uffdio_wp;
1689 struct uffdio_writeprotect __user *user_uffdio_wp;
1690 struct userfaultfd_wake_range range;
1691 bool mode_wp, mode_dontwake;
1692
1693 if (atomic_read(&ctx->mmap_changing))
1694 return -EAGAIN;
1695
1696 user_uffdio_wp = (struct uffdio_writeprotect __user *) arg;
1697
1698 if (copy_from_user(&uffdio_wp, user_uffdio_wp,
1699 sizeof(struct uffdio_writeprotect)))
1700 return -EFAULT;
1701
1702 ret = validate_range(ctx->mm, uffdio_wp.range.start,
1703 uffdio_wp.range.len);
1704 if (ret)
1705 return ret;
1706
1707 if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE |
1708 UFFDIO_WRITEPROTECT_MODE_WP))
1709 return -EINVAL;
1710
1711 mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP;
1712 mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
1713
1714 if (mode_wp && mode_dontwake)
1715 return -EINVAL;
1716
1717 if (mmget_not_zero(ctx->mm)) {
1718 ret = mwriteprotect_range(ctx, uffdio_wp.range.start,
1719 uffdio_wp.range.len, mode_wp);
1720 mmput(ctx->mm);
1721 } else {
1722 return -ESRCH;
1723 }
1724
1725 if (ret)
1726 return ret;
1727
1728 if (!mode_wp && !mode_dontwake) {
1729 range.start = uffdio_wp.range.start;
1730 range.len = uffdio_wp.range.len;
1731 wake_userfault(ctx, &range);
1732 }
1733 return ret;
1734 }
1735
userfaultfd_continue(struct userfaultfd_ctx * ctx,unsigned long arg)1736 static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
1737 {
1738 __s64 ret;
1739 struct uffdio_continue uffdio_continue;
1740 struct uffdio_continue __user *user_uffdio_continue;
1741 struct userfaultfd_wake_range range;
1742 uffd_flags_t flags = 0;
1743
1744 user_uffdio_continue = (struct uffdio_continue __user *)arg;
1745
1746 ret = -EAGAIN;
1747 if (atomic_read(&ctx->mmap_changing))
1748 goto out;
1749
1750 ret = -EFAULT;
1751 if (copy_from_user(&uffdio_continue, user_uffdio_continue,
1752 /* don't copy the output fields */
1753 sizeof(uffdio_continue) - (sizeof(__s64))))
1754 goto out;
1755
1756 ret = validate_range(ctx->mm, uffdio_continue.range.start,
1757 uffdio_continue.range.len);
1758 if (ret)
1759 goto out;
1760
1761 ret = -EINVAL;
1762 if (uffdio_continue.mode & ~(UFFDIO_CONTINUE_MODE_DONTWAKE |
1763 UFFDIO_CONTINUE_MODE_WP))
1764 goto out;
1765 if (uffdio_continue.mode & UFFDIO_CONTINUE_MODE_WP)
1766 flags |= MFILL_ATOMIC_WP;
1767
1768 if (mmget_not_zero(ctx->mm)) {
1769 ret = mfill_atomic_continue(ctx, uffdio_continue.range.start,
1770 uffdio_continue.range.len, flags);
1771 mmput(ctx->mm);
1772 } else {
1773 return -ESRCH;
1774 }
1775
1776 if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))
1777 return -EFAULT;
1778 if (ret < 0)
1779 goto out;
1780
1781 /* len == 0 would wake all */
1782 BUG_ON(!ret);
1783 range.len = ret;
1784 if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) {
1785 range.start = uffdio_continue.range.start;
1786 wake_userfault(ctx, &range);
1787 }
1788 ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN;
1789
1790 out:
1791 return ret;
1792 }
1793
userfaultfd_poison(struct userfaultfd_ctx * ctx,unsigned long arg)1794 static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long arg)
1795 {
1796 __s64 ret;
1797 struct uffdio_poison uffdio_poison;
1798 struct uffdio_poison __user *user_uffdio_poison;
1799 struct userfaultfd_wake_range range;
1800
1801 user_uffdio_poison = (struct uffdio_poison __user *)arg;
1802
1803 ret = -EAGAIN;
1804 if (atomic_read(&ctx->mmap_changing))
1805 goto out;
1806
1807 ret = -EFAULT;
1808 if (copy_from_user(&uffdio_poison, user_uffdio_poison,
1809 /* don't copy the output fields */
1810 sizeof(uffdio_poison) - (sizeof(__s64))))
1811 goto out;
1812
1813 ret = validate_range(ctx->mm, uffdio_poison.range.start,
1814 uffdio_poison.range.len);
1815 if (ret)
1816 goto out;
1817
1818 ret = -EINVAL;
1819 if (uffdio_poison.mode & ~UFFDIO_POISON_MODE_DONTWAKE)
1820 goto out;
1821
1822 if (mmget_not_zero(ctx->mm)) {
1823 ret = mfill_atomic_poison(ctx, uffdio_poison.range.start,
1824 uffdio_poison.range.len, 0);
1825 mmput(ctx->mm);
1826 } else {
1827 return -ESRCH;
1828 }
1829
1830 if (unlikely(put_user(ret, &user_uffdio_poison->updated)))
1831 return -EFAULT;
1832 if (ret < 0)
1833 goto out;
1834
1835 /* len == 0 would wake all */
1836 BUG_ON(!ret);
1837 range.len = ret;
1838 if (!(uffdio_poison.mode & UFFDIO_POISON_MODE_DONTWAKE)) {
1839 range.start = uffdio_poison.range.start;
1840 wake_userfault(ctx, &range);
1841 }
1842 ret = range.len == uffdio_poison.range.len ? 0 : -EAGAIN;
1843
1844 out:
1845 return ret;
1846 }
1847
userfaultfd_wp_async(struct vm_area_struct * vma)1848 bool userfaultfd_wp_async(struct vm_area_struct *vma)
1849 {
1850 return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx);
1851 }
1852
uffd_ctx_features(__u64 user_features)1853 static inline unsigned int uffd_ctx_features(__u64 user_features)
1854 {
1855 /*
1856 * For the current set of features the bits just coincide. Set
1857 * UFFD_FEATURE_INITIALIZED to mark the features as enabled.
1858 */
1859 return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED;
1860 }
1861
userfaultfd_move(struct userfaultfd_ctx * ctx,unsigned long arg)1862 static int userfaultfd_move(struct userfaultfd_ctx *ctx,
1863 unsigned long arg)
1864 {
1865 __s64 ret;
1866 struct uffdio_move uffdio_move;
1867 struct uffdio_move __user *user_uffdio_move;
1868 struct userfaultfd_wake_range range;
1869 struct mm_struct *mm = ctx->mm;
1870
1871 user_uffdio_move = (struct uffdio_move __user *) arg;
1872
1873 if (atomic_read(&ctx->mmap_changing))
1874 return -EAGAIN;
1875
1876 if (copy_from_user(&uffdio_move, user_uffdio_move,
1877 /* don't copy "move" last field */
1878 sizeof(uffdio_move)-sizeof(__s64)))
1879 return -EFAULT;
1880
1881 /* Do not allow cross-mm moves. */
1882 if (mm != current->mm)
1883 return -EINVAL;
1884
1885 ret = validate_range(mm, uffdio_move.dst, uffdio_move.len);
1886 if (ret)
1887 return ret;
1888
1889 ret = validate_range(mm, uffdio_move.src, uffdio_move.len);
1890 if (ret)
1891 return ret;
1892
1893 if (uffdio_move.mode & ~(UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES|
1894 UFFDIO_MOVE_MODE_DONTWAKE))
1895 return -EINVAL;
1896
1897 if (mmget_not_zero(mm)) {
1898 ret = move_pages(ctx, uffdio_move.dst, uffdio_move.src,
1899 uffdio_move.len, uffdio_move.mode);
1900 mmput(mm);
1901 } else {
1902 return -ESRCH;
1903 }
1904
1905 if (unlikely(put_user(ret, &user_uffdio_move->move)))
1906 return -EFAULT;
1907 if (ret < 0)
1908 goto out;
1909
1910 /* len == 0 would wake all */
1911 VM_WARN_ON(!ret);
1912 range.len = ret;
1913 if (!(uffdio_move.mode & UFFDIO_MOVE_MODE_DONTWAKE)) {
1914 range.start = uffdio_move.dst;
1915 wake_userfault(ctx, &range);
1916 }
1917 ret = range.len == uffdio_move.len ? 0 : -EAGAIN;
1918
1919 out:
1920 return ret;
1921 }
1922
1923 /*
1924 * userland asks for a certain API version and we return which bits
1925 * and ioctl commands are implemented in this kernel for such API
1926 * version or -EINVAL if unknown.
1927 */
userfaultfd_api(struct userfaultfd_ctx * ctx,unsigned long arg)1928 static int userfaultfd_api(struct userfaultfd_ctx *ctx,
1929 unsigned long arg)
1930 {
1931 struct uffdio_api uffdio_api;
1932 void __user *buf = (void __user *)arg;
1933 unsigned int ctx_features;
1934 int ret;
1935 __u64 features;
1936
1937 ret = -EFAULT;
1938 if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
1939 goto out;
1940 features = uffdio_api.features;
1941 ret = -EINVAL;
1942 if (uffdio_api.api != UFFD_API)
1943 goto err_out;
1944 ret = -EPERM;
1945 if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
1946 goto err_out;
1947
1948 /* WP_ASYNC relies on WP_UNPOPULATED, choose it unconditionally */
1949 if (features & UFFD_FEATURE_WP_ASYNC)
1950 features |= UFFD_FEATURE_WP_UNPOPULATED;
1951
1952 /* report all available features and ioctls to userland */
1953 uffdio_api.features = UFFD_API_FEATURES;
1954 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1955 uffdio_api.features &=
1956 ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
1957 #endif
1958 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1959 uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
1960 #endif
1961 #ifndef CONFIG_PTE_MARKER_UFFD_WP
1962 uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
1963 uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED;
1964 uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC;
1965 #endif
1966
1967 ret = -EINVAL;
1968 if (features & ~uffdio_api.features)
1969 goto err_out;
1970
1971 uffdio_api.ioctls = UFFD_API_IOCTLS;
1972 ret = -EFAULT;
1973 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1974 goto out;
1975
1976 /* only enable the requested features for this uffd context */
1977 ctx_features = uffd_ctx_features(features);
1978 ret = -EINVAL;
1979 if (cmpxchg(&ctx->features, 0, ctx_features) != 0)
1980 goto err_out;
1981
1982 ret = 0;
1983 out:
1984 return ret;
1985 err_out:
1986 memset(&uffdio_api, 0, sizeof(uffdio_api));
1987 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1988 ret = -EFAULT;
1989 goto out;
1990 }
1991
userfaultfd_ioctl(struct file * file,unsigned cmd,unsigned long arg)1992 static long userfaultfd_ioctl(struct file *file, unsigned cmd,
1993 unsigned long arg)
1994 {
1995 int ret = -EINVAL;
1996 struct userfaultfd_ctx *ctx = file->private_data;
1997
1998 if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx))
1999 return -EINVAL;
2000
2001 switch(cmd) {
2002 case UFFDIO_API:
2003 ret = userfaultfd_api(ctx, arg);
2004 break;
2005 case UFFDIO_REGISTER:
2006 ret = userfaultfd_register(ctx, arg);
2007 break;
2008 case UFFDIO_UNREGISTER:
2009 ret = userfaultfd_unregister(ctx, arg);
2010 break;
2011 case UFFDIO_WAKE:
2012 ret = userfaultfd_wake(ctx, arg);
2013 break;
2014 case UFFDIO_COPY:
2015 ret = userfaultfd_copy(ctx, arg);
2016 break;
2017 case UFFDIO_ZEROPAGE:
2018 ret = userfaultfd_zeropage(ctx, arg);
2019 break;
2020 case UFFDIO_MOVE:
2021 ret = userfaultfd_move(ctx, arg);
2022 break;
2023 case UFFDIO_WRITEPROTECT:
2024 ret = userfaultfd_writeprotect(ctx, arg);
2025 break;
2026 case UFFDIO_CONTINUE:
2027 ret = userfaultfd_continue(ctx, arg);
2028 break;
2029 case UFFDIO_POISON:
2030 ret = userfaultfd_poison(ctx, arg);
2031 break;
2032 }
2033 return ret;
2034 }
2035
2036 #ifdef CONFIG_PROC_FS
userfaultfd_show_fdinfo(struct seq_file * m,struct file * f)2037 static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
2038 {
2039 struct userfaultfd_ctx *ctx = f->private_data;
2040 wait_queue_entry_t *wq;
2041 unsigned long pending = 0, total = 0;
2042
2043 spin_lock_irq(&ctx->fault_pending_wqh.lock);
2044 list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
2045 pending++;
2046 total++;
2047 }
2048 list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
2049 total++;
2050 }
2051 spin_unlock_irq(&ctx->fault_pending_wqh.lock);
2052
2053 /*
2054 * If more protocols will be added, there will be all shown
2055 * separated by a space. Like this:
2056 * protocols: aa:... bb:...
2057 */
2058 seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
2059 pending, total, UFFD_API, ctx->features,
2060 UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
2061 }
2062 #endif
2063
2064 static const struct file_operations userfaultfd_fops = {
2065 #ifdef CONFIG_PROC_FS
2066 .show_fdinfo = userfaultfd_show_fdinfo,
2067 #endif
2068 .release = userfaultfd_release,
2069 .poll = userfaultfd_poll,
2070 .read_iter = userfaultfd_read_iter,
2071 .unlocked_ioctl = userfaultfd_ioctl,
2072 .compat_ioctl = compat_ptr_ioctl,
2073 .llseek = noop_llseek,
2074 };
2075
init_once_userfaultfd_ctx(void * mem)2076 static void init_once_userfaultfd_ctx(void *mem)
2077 {
2078 struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
2079
2080 init_waitqueue_head(&ctx->fault_pending_wqh);
2081 init_waitqueue_head(&ctx->fault_wqh);
2082 init_waitqueue_head(&ctx->event_wqh);
2083 init_waitqueue_head(&ctx->fd_wqh);
2084 seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);
2085 }
2086
new_userfaultfd(int flags)2087 static int new_userfaultfd(int flags)
2088 {
2089 struct userfaultfd_ctx *ctx;
2090 struct file *file;
2091 int fd;
2092
2093 BUG_ON(!current->mm);
2094
2095 /* Check the UFFD_* constants for consistency. */
2096 BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS);
2097 BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
2098 BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
2099
2100 if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY))
2101 return -EINVAL;
2102
2103 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
2104 if (!ctx)
2105 return -ENOMEM;
2106
2107 refcount_set(&ctx->refcount, 1);
2108 ctx->flags = flags;
2109 ctx->features = 0;
2110 ctx->released = false;
2111 init_rwsem(&ctx->map_changing_lock);
2112 atomic_set(&ctx->mmap_changing, 0);
2113 ctx->mm = current->mm;
2114
2115 fd = get_unused_fd_flags(flags & UFFD_SHARED_FCNTL_FLAGS);
2116 if (fd < 0)
2117 goto err_out;
2118
2119 /* Create a new inode so that the LSM can block the creation. */
2120 file = anon_inode_create_getfile("[userfaultfd]", &userfaultfd_fops, ctx,
2121 O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
2122 if (IS_ERR(file)) {
2123 put_unused_fd(fd);
2124 fd = PTR_ERR(file);
2125 goto err_out;
2126 }
2127 /* prevent the mm struct to be freed */
2128 mmgrab(ctx->mm);
2129 file->f_mode |= FMODE_NOWAIT;
2130 fd_install(fd, file);
2131 return fd;
2132 err_out:
2133 kmem_cache_free(userfaultfd_ctx_cachep, ctx);
2134 return fd;
2135 }
2136
userfaultfd_syscall_allowed(int flags)2137 static inline bool userfaultfd_syscall_allowed(int flags)
2138 {
2139 /* Userspace-only page faults are always allowed */
2140 if (flags & UFFD_USER_MODE_ONLY)
2141 return true;
2142
2143 /*
2144 * The user is requesting a userfaultfd which can handle kernel faults.
2145 * Privileged users are always allowed to do this.
2146 */
2147 if (capable(CAP_SYS_PTRACE))
2148 return true;
2149
2150 /* Otherwise, access to kernel fault handling is sysctl controlled. */
2151 return sysctl_unprivileged_userfaultfd;
2152 }
2153
SYSCALL_DEFINE1(userfaultfd,int,flags)2154 SYSCALL_DEFINE1(userfaultfd, int, flags)
2155 {
2156 if (!userfaultfd_syscall_allowed(flags))
2157 return -EPERM;
2158
2159 return new_userfaultfd(flags);
2160 }
2161
userfaultfd_dev_ioctl(struct file * file,unsigned int cmd,unsigned long flags)2162 static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags)
2163 {
2164 if (cmd != USERFAULTFD_IOC_NEW)
2165 return -EINVAL;
2166
2167 return new_userfaultfd(flags);
2168 }
2169
2170 static const struct file_operations userfaultfd_dev_fops = {
2171 .unlocked_ioctl = userfaultfd_dev_ioctl,
2172 .compat_ioctl = userfaultfd_dev_ioctl,
2173 .owner = THIS_MODULE,
2174 .llseek = noop_llseek,
2175 };
2176
2177 static struct miscdevice userfaultfd_misc = {
2178 .minor = MISC_DYNAMIC_MINOR,
2179 .name = "userfaultfd",
2180 .fops = &userfaultfd_dev_fops
2181 };
2182
userfaultfd_init(void)2183 static int __init userfaultfd_init(void)
2184 {
2185 int ret;
2186
2187 ret = misc_register(&userfaultfd_misc);
2188 if (ret)
2189 return ret;
2190
2191 userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
2192 sizeof(struct userfaultfd_ctx),
2193 0,
2194 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2195 init_once_userfaultfd_ctx);
2196 #ifdef CONFIG_SYSCTL
2197 register_sysctl_init("vm", vm_userfaultfd_table);
2198 #endif
2199 return 0;
2200 }
2201 __initcall(userfaultfd_init);
2202