Lines Matching +full:mm +full:- +full:0
1 // SPDX-License-Identifier: GPL-2.0-only
9 * 'fork.c' contains the help-routines for the 'fork' system call
12 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
18 #include <linux/sched/mm.h>
45 #include <linux/mm.h>
84 #include <linux/posix-timers.h>
85 #include <linux/user-return-notifier.h>
149 DEFINE_PER_CPU(unsigned long, process_counts) = 0;
164 int total = 0; in nr_processes()
211 for (i = 0; i < NR_CACHED_STACKS; i++) { in try_release_thread_stack_to_cache()
224 if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) in thread_stack_free_rcu()
232 struct vm_stack *vm_stack = tsk->stack; in thread_stack_delayed_free()
234 vm_stack->stack_vm_area = tsk->stack_vm_area; in thread_stack_delayed_free()
235 call_rcu(&vm_stack->rcu, thread_stack_free_rcu); in thread_stack_delayed_free()
243 for (i = 0; i < NR_CACHED_STACKS; i++) { in free_vm_stack_cache()
249 vfree(vm_stack->addr); in free_vm_stack_cache()
253 return 0; in free_vm_stack_cache()
260 int nr_charged = 0; in memcg_charge_kernel_stack()
262 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); in memcg_charge_kernel_stack()
264 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { in memcg_charge_kernel_stack()
265 ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, 0); in memcg_charge_kernel_stack()
270 return 0; in memcg_charge_kernel_stack()
272 for (i = 0; i < nr_charged; i++) in memcg_charge_kernel_stack()
273 memcg_kmem_uncharge_page(vm->pages[i], 0); in memcg_charge_kernel_stack()
283 for (i = 0; i < NR_CACHED_STACKS; i++) { in alloc_thread_stack_node()
292 kasan_unpoison_range(s->addr, THREAD_SIZE); in alloc_thread_stack_node()
294 stack = kasan_reset_tag(s->addr); in alloc_thread_stack_node()
297 memset(stack, 0, THREAD_SIZE); in alloc_thread_stack_node()
300 vfree(s->addr); in alloc_thread_stack_node()
301 return -ENOMEM; in alloc_thread_stack_node()
304 tsk->stack_vm_area = s; in alloc_thread_stack_node()
305 tsk->stack = stack; in alloc_thread_stack_node()
306 return 0; in alloc_thread_stack_node()
318 0, node, __builtin_return_address(0)); in alloc_thread_stack_node()
320 return -ENOMEM; in alloc_thread_stack_node()
325 return -ENOMEM; in alloc_thread_stack_node()
332 tsk->stack_vm_area = vm; in alloc_thread_stack_node()
334 tsk->stack = stack; in alloc_thread_stack_node()
335 return 0; in alloc_thread_stack_node()
340 if (!try_release_thread_stack_to_cache(tsk->stack_vm_area)) in free_thread_stack()
343 tsk->stack = NULL; in free_thread_stack()
344 tsk->stack_vm_area = NULL; in free_thread_stack()
356 struct rcu_head *rh = tsk->stack; in thread_stack_delayed_free()
367 tsk->stack = kasan_reset_tag(page_address(page)); in alloc_thread_stack_node()
368 return 0; in alloc_thread_stack_node()
370 return -ENOMEM; in alloc_thread_stack_node()
376 tsk->stack = NULL; in free_thread_stack()
391 struct rcu_head *rh = tsk->stack; in thread_stack_delayed_free()
401 tsk->stack = stack; in alloc_thread_stack_node()
402 return stack ? 0 : -ENOMEM; in alloc_thread_stack_node()
408 tsk->stack = NULL; in free_thread_stack()
414 THREAD_SIZE, THREAD_SIZE, 0, 0, in thread_stack_cache_init()
421 /* SLAB cache for signal_struct structures (tsk->signal) */
424 /* SLAB cache for sighand_struct structures (tsk->sighand) */
427 /* SLAB cache for files_struct structures (tsk->files) */
430 /* SLAB cache for fs_struct structures (tsk->fs) */
436 /* SLAB cache for mm_struct structures (tsk->mm) */
446 vma->vm_lock = kmem_cache_alloc(vma_lock_cachep, GFP_KERNEL); in vma_lock_alloc()
447 if (!vma->vm_lock) in vma_lock_alloc()
450 init_rwsem(&vma->vm_lock->lock); in vma_lock_alloc()
451 vma->vm_lock_seq = UINT_MAX; in vma_lock_alloc()
458 kmem_cache_free(vma_lock_cachep, vma->vm_lock); in vma_lock_free()
468 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) in vm_area_alloc() argument
476 vma_init(vma, mm); in vm_area_alloc()
492 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); in vm_area_dup()
493 ASSERT_EXCLUSIVE_WRITER(orig->vm_file); in vm_area_dup()
495 * orig->shared.rb may be modified concurrently, but the clone in vm_area_dup()
503 INIT_LIST_HEAD(&new->anon_vma_chain); in vm_area_dup()
508 if (unlikely(new->vm_flags & VM_PFNMAP)) in vm_area_dup()
529 VM_BUG_ON_VMA(rwsem_is_locked(&vma->vm_lock->lock), vma); in vm_area_free_rcu_cb()
537 call_rcu(&vma->vm_rcu, vm_area_free_rcu_cb); in vm_area_free()
549 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) in account_kernel_stack()
550 mod_lruvec_page_state(vm->pages[i], NR_KERNEL_STACK_KB, in account_kernel_stack()
563 account_kernel_stack(tsk, -1); in exit_task_stack_account()
570 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) in exit_task_stack_account()
571 memcg_kmem_uncharge_page(vm->pages[i], 0); in exit_task_stack_account()
577 if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD)) in release_task_stack()
586 if (refcount_dec_and_test(&tsk->stack_refcount)) in put_task_stack()
594 WARN_ON_ONCE(tsk->seccomp.filter); in free_task()
610 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0); in free_task()
615 if (tsk->flags & PF_KTHREAD) in free_task()
622 static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mm_exe_file() argument
627 RCU_INIT_POINTER(mm->exe_file, exe_file); in dup_mm_exe_file()
637 static __latent_entropy int dup_mmap(struct mm_struct *mm, in dup_mmap() argument
642 unsigned long charge = 0; in dup_mmap()
644 VMA_ITERATOR(vmi, mm, 0); in dup_mmap()
647 return -EINTR; in dup_mmap()
649 uprobe_dup_mmap(oldmm, mm); in dup_mmap()
651 * Not linked in yet - no deadlock potential: in dup_mmap()
653 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING); in dup_mmap()
656 dup_mm_exe_file(mm, oldmm); in dup_mmap()
658 mm->total_vm = oldmm->total_vm; in dup_mmap()
659 mm->data_vm = oldmm->data_vm; in dup_mmap()
660 mm->exec_vm = oldmm->exec_vm; in dup_mmap()
661 mm->stack_vm = oldmm->stack_vm; in dup_mmap()
664 retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL); in dup_mmap()
673 if (mpnt->vm_flags & VM_DONTCOPY) { in dup_mmap()
674 retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start, in dup_mmap()
675 mpnt->vm_end, GFP_KERNEL); in dup_mmap()
679 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); in dup_mmap()
682 charge = 0; in dup_mmap()
684 * Don't duplicate many vmas if we've been oom-killed (for in dup_mmap()
688 retval = -EINTR; in dup_mmap()
691 if (mpnt->vm_flags & VM_ACCOUNT) { in dup_mmap()
704 tmp->vm_mm = mm; in dup_mmap()
708 if (tmp->vm_flags & VM_WIPEONFORK) { in dup_mmap()
714 tmp->anon_vma = NULL; in dup_mmap()
730 mm->map_count++; in dup_mmap()
732 if (tmp->vm_ops && tmp->vm_ops->open) in dup_mmap()
733 tmp->vm_ops->open(tmp); in dup_mmap()
735 file = tmp->vm_file; in dup_mmap()
737 struct address_space *mapping = file->f_mapping; in dup_mmap()
746 &mapping->i_mmap); in dup_mmap()
751 if (!(tmp->vm_flags & VM_WIPEONFORK)) in dup_mmap()
759 /* a new mm has just been created */ in dup_mmap()
760 retval = arch_dup_mmap(oldmm, mm); in dup_mmap()
765 ksm_fork(mm, oldmm); in dup_mmap()
766 khugepaged_fork(mm, oldmm); in dup_mmap()
777 mas_set_range(&vmi.mas, mpnt->vm_start, mpnt->vm_end - 1); in dup_mmap()
780 set_bit(MMF_OOM_SKIP, &mm->flags); in dup_mmap()
787 set_bit(MMF_UNSTABLE, &mm->flags); in dup_mmap()
790 mmap_write_unlock(mm); in dup_mmap()
804 retval = -ENOMEM; in dup_mmap()
809 static inline int mm_alloc_pgd(struct mm_struct *mm) in mm_alloc_pgd() argument
811 mm->pgd = pgd_alloc(mm); in mm_alloc_pgd()
812 if (unlikely(!mm->pgd)) in mm_alloc_pgd()
813 return -ENOMEM; in mm_alloc_pgd()
814 return 0; in mm_alloc_pgd()
817 static inline void mm_free_pgd(struct mm_struct *mm) in mm_free_pgd() argument
819 pgd_free(mm, mm->pgd); in mm_free_pgd()
822 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mmap() argument
825 dup_mm_exe_file(mm, oldmm); in dup_mmap()
827 return 0; in dup_mmap()
829 #define mm_alloc_pgd(mm) (0) argument
830 #define mm_free_pgd(mm) argument
833 static void check_mm(struct mm_struct *mm) in check_mm() argument
840 for (i = 0; i < NR_MM_COUNTERS; i++) { in check_mm()
841 long x = percpu_counter_sum(&mm->rss_stat[i]); in check_mm()
844 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", in check_mm()
845 mm, resident_page_types[i], x); in check_mm()
848 if (mm_pgtables_bytes(mm)) in check_mm()
849 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", in check_mm()
850 mm_pgtables_bytes(mm)); in check_mm()
853 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); in check_mm()
858 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) argument
862 struct mm_struct *mm = arg; in do_check_lazy_tlb() local
864 WARN_ON_ONCE(current->active_mm == mm); in do_check_lazy_tlb()
869 struct mm_struct *mm = arg; in do_shoot_lazy_tlb() local
871 if (current->active_mm == mm) { in do_shoot_lazy_tlb()
872 WARN_ON_ONCE(current->mm); in do_shoot_lazy_tlb()
873 current->active_mm = &init_mm; in do_shoot_lazy_tlb()
874 switch_mm(mm, &init_mm, current); in do_shoot_lazy_tlb()
878 static void cleanup_lazy_tlbs(struct mm_struct *mm) in cleanup_lazy_tlbs() argument
889 * Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it in cleanup_lazy_tlbs()
890 * requires lazy mm users to switch to another mm when the refcount in cleanup_lazy_tlbs()
891 * drops to zero, before the mm is freed. This requires IPIs here to in cleanup_lazy_tlbs()
894 * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm in cleanup_lazy_tlbs()
896 * mm lazy on this CPU but no others, reducing the need for additional in cleanup_lazy_tlbs()
899 * one exiting, or kernel threads using the mm when userspace exits. in cleanup_lazy_tlbs()
904 * - The last lazy reference created by exit_mm() could instead switch in cleanup_lazy_tlbs()
907 * - A batch of mms requiring IPIs could be gathered and freed at once. in cleanup_lazy_tlbs()
908 * - CPUs store active_mm where it can be remotely checked without a in cleanup_lazy_tlbs()
909 * lock, to filter out false-positives in the cpumask. in cleanup_lazy_tlbs()
910 * - After mm_users or mm_count reaches zero, switching away from the in cleanup_lazy_tlbs()
911 * mm could clear mm_cpumask to reduce some IPIs, perhaps together in cleanup_lazy_tlbs()
913 * - A delayed freeing and RCU-like quiescing sequence based on mm in cleanup_lazy_tlbs()
916 on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1); in cleanup_lazy_tlbs()
918 on_each_cpu(do_check_lazy_tlb, (void *)mm, 1); in cleanup_lazy_tlbs()
922 * Called when the last reference to the mm
924 * mmput. Free the page directory and the mm.
926 void __mmdrop(struct mm_struct *mm) in __mmdrop() argument
928 BUG_ON(mm == &init_mm); in __mmdrop()
929 WARN_ON_ONCE(mm == current->mm); in __mmdrop()
931 /* Ensure no CPUs are using this as their lazy tlb mm */ in __mmdrop()
932 cleanup_lazy_tlbs(mm); in __mmdrop()
934 WARN_ON_ONCE(mm == current->active_mm); in __mmdrop()
935 mm_free_pgd(mm); in __mmdrop()
936 destroy_context(mm); in __mmdrop()
937 mmu_notifier_subscriptions_destroy(mm); in __mmdrop()
938 check_mm(mm); in __mmdrop()
939 put_user_ns(mm->user_ns); in __mmdrop()
940 mm_pasid_drop(mm); in __mmdrop()
941 mm_destroy_cid(mm); in __mmdrop()
942 percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS); in __mmdrop()
944 free_mm(mm); in __mmdrop()
950 struct mm_struct *mm; in mmdrop_async_fn() local
952 mm = container_of(work, struct mm_struct, async_put_work); in mmdrop_async_fn()
953 __mmdrop(mm); in mmdrop_async_fn()
956 static void mmdrop_async(struct mm_struct *mm) in mmdrop_async() argument
958 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { in mmdrop_async()
959 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); in mmdrop_async()
960 schedule_work(&mm->async_put_work); in mmdrop_async()
972 if (sig->oom_mm) in free_signal_struct()
973 mmdrop_async(sig->oom_mm); in free_signal_struct()
979 if (refcount_dec_and_test(&sig->sigcnt)) in put_signal_struct()
985 WARN_ON(!tsk->exit_state); in __put_task_struct()
986 WARN_ON(refcount_read(&tsk->usage)); in __put_task_struct()
996 put_signal_struct(tsk->signal); in __put_task_struct()
1047 * Handle zero-sized whitelist or empty thread_struct, otherwise in task_struct_whitelist()
1050 if (unlikely(*size == 0)) in task_struct_whitelist()
1051 *offset = 0; in task_struct_whitelist()
1060 #define ARCH_MIN_TASKALIGN 0 in fork_init()
1077 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; in fork_init()
1078 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; in fork_init()
1079 init_task.signal->rlim[RLIMIT_SIGPENDING] = in fork_init()
1080 init_task.signal->rlim[RLIMIT_NPROC]; in fork_init()
1082 for (i = 0; i < UCOUNT_COUNTS; i++) in fork_init()
1105 return 0; in arch_dup_task_struct()
1136 refcount_set(&tsk->stack_refcount, 1); in dup_task_struct()
1151 tsk->seccomp.filter = NULL; in dup_task_struct()
1161 tsk->stack_canary = get_random_canary(); in dup_task_struct()
1163 if (orig->cpus_ptr == &orig->cpus_mask) in dup_task_struct()
1164 tsk->cpus_ptr = &tsk->cpus_mask; in dup_task_struct()
1171 refcount_set(&tsk->rcu_users, 2); in dup_task_struct()
1173 refcount_set(&tsk->usage, 1); in dup_task_struct()
1175 tsk->btrace_seq = 0; in dup_task_struct()
1177 tsk->splice_pipe = NULL; in dup_task_struct()
1178 tsk->task_frag.page = NULL; in dup_task_struct()
1179 tsk->wake_q.next = NULL; in dup_task_struct()
1180 tsk->worker_private = NULL; in dup_task_struct()
1187 tsk->fail_nth = 0; in dup_task_struct()
1191 tsk->throttle_disk = NULL; in dup_task_struct()
1192 tsk->use_memdelay = 0; in dup_task_struct()
1196 tsk->pasid_activated = 0; in dup_task_struct()
1200 tsk->active_memcg = NULL; in dup_task_struct()
1204 tsk->reported_split_lock = 0; in dup_task_struct()
1208 tsk->mm_cid = -1; in dup_task_struct()
1209 tsk->last_mm_cid = -1; in dup_task_struct()
1210 tsk->mm_cid_active = 0; in dup_task_struct()
1211 tsk->migrate_from_cpu = -1; in dup_task_struct()
1230 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) & in coredump_filter_setup()
1239 static void mm_init_aio(struct mm_struct *mm) in mm_init_aio() argument
1242 spin_lock_init(&mm->ioctx_lock); in mm_init_aio()
1243 mm->ioctx_table = NULL; in mm_init_aio()
1247 static __always_inline void mm_clear_owner(struct mm_struct *mm, in mm_clear_owner() argument
1251 if (mm->owner == p) in mm_clear_owner()
1252 WRITE_ONCE(mm->owner, NULL); in mm_clear_owner()
1256 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) in mm_init_owner() argument
1259 mm->owner = p; in mm_init_owner()
1263 static void mm_init_uprobes_state(struct mm_struct *mm) in mm_init_uprobes_state() argument
1266 mm->uprobes_state.xol_area = NULL; in mm_init_uprobes_state()
1270 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, in mm_init() argument
1273 mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); in mm_init()
1274 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); in mm_init()
1275 atomic_set(&mm->mm_users, 1); in mm_init()
1276 atomic_set(&mm->mm_count, 1); in mm_init()
1277 seqcount_init(&mm->write_protect_seq); in mm_init()
1278 mmap_init_lock(mm); in mm_init()
1279 INIT_LIST_HEAD(&mm->mmlist); in mm_init()
1280 mm_pgtables_bytes_init(mm); in mm_init()
1281 mm->map_count = 0; in mm_init()
1282 mm->locked_vm = 0; in mm_init()
1283 atomic64_set(&mm->pinned_vm, 0); in mm_init()
1284 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); in mm_init()
1285 spin_lock_init(&mm->page_table_lock); in mm_init()
1286 spin_lock_init(&mm->arg_lock); in mm_init()
1287 mm_init_cpumask(mm); in mm_init()
1288 mm_init_aio(mm); in mm_init()
1289 mm_init_owner(mm, p); in mm_init()
1290 mm_pasid_init(mm); in mm_init()
1291 RCU_INIT_POINTER(mm->exe_file, NULL); in mm_init()
1292 mmu_notifier_subscriptions_init(mm); in mm_init()
1293 init_tlb_flush_pending(mm); in mm_init()
1295 mm->pmd_huge_pte = NULL; in mm_init()
1297 mm_init_uprobes_state(mm); in mm_init()
1298 hugetlb_count_init(mm); in mm_init()
1300 if (current->mm) { in mm_init()
1301 mm->flags = mmf_init_flags(current->mm->flags); in mm_init()
1302 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; in mm_init()
1304 mm->flags = default_dump_filter; in mm_init()
1305 mm->def_flags = 0; in mm_init()
1308 if (mm_alloc_pgd(mm)) in mm_init()
1311 if (init_new_context(p, mm)) in mm_init()
1314 if (mm_alloc_cid(mm, p)) in mm_init()
1317 if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT, in mm_init()
1321 mm->user_ns = get_user_ns(user_ns); in mm_init()
1322 lru_gen_init_mm(mm); in mm_init()
1323 return mm; in mm_init()
1326 mm_destroy_cid(mm); in mm_init()
1328 destroy_context(mm); in mm_init()
1330 mm_free_pgd(mm); in mm_init()
1332 free_mm(mm); in mm_init()
1341 struct mm_struct *mm; in mm_alloc() local
1343 mm = allocate_mm(); in mm_alloc()
1344 if (!mm) in mm_alloc()
1347 memset(mm, 0, sizeof(*mm)); in mm_alloc()
1348 return mm_init(mm, current, current_user_ns()); in mm_alloc()
1352 static inline void __mmput(struct mm_struct *mm) in __mmput() argument
1354 VM_BUG_ON(atomic_read(&mm->mm_users)); in __mmput()
1356 uprobe_clear_state(mm); in __mmput()
1357 exit_aio(mm); in __mmput()
1358 ksm_exit(mm); in __mmput()
1359 khugepaged_exit(mm); /* must run before exit_mmap */ in __mmput()
1360 exit_mmap(mm); in __mmput()
1361 mm_put_huge_zero_folio(mm); in __mmput()
1362 set_mm_exe_file(mm, NULL); in __mmput()
1363 if (!list_empty(&mm->mmlist)) { in __mmput()
1365 list_del(&mm->mmlist); in __mmput()
1368 if (mm->binfmt) in __mmput()
1369 module_put(mm->binfmt->module); in __mmput()
1370 lru_gen_del_mm(mm); in __mmput()
1371 mmdrop(mm); in __mmput()
1375 * Decrement the use count and release all resources for an mm.
1377 void mmput(struct mm_struct *mm) in mmput() argument
1381 if (atomic_dec_and_test(&mm->mm_users)) in mmput()
1382 __mmput(mm); in mmput()
1389 struct mm_struct *mm = container_of(work, struct mm_struct, in mmput_async_fn() local
1392 __mmput(mm); in mmput_async_fn()
1395 void mmput_async(struct mm_struct *mm) in mmput_async() argument
1397 if (atomic_dec_and_test(&mm->mm_users)) { in mmput_async()
1398 INIT_WORK(&mm->async_put_work, mmput_async_fn); in mmput_async()
1399 schedule_work(&mm->async_put_work); in mmput_async()
1406 * set_mm_exe_file - change a reference to the mm's executable file
1407 * @mm: The mm to change.
1410 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1414 * the new mm is made visible to anyone.
1418 int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in set_mm_exe_file() argument
1425 * this mm -- see comment above for justification. in set_mm_exe_file()
1427 old_exe_file = rcu_dereference_raw(mm->exe_file); in set_mm_exe_file()
1435 return -EACCES; in set_mm_exe_file()
1438 rcu_assign_pointer(mm->exe_file, new_exe_file); in set_mm_exe_file()
1443 return 0; in set_mm_exe_file()
1447 * replace_mm_exe_file - replace a reference to the mm's executable file
1448 * @mm: The mm to change.
1451 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1455 int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in replace_mm_exe_file() argument
1459 int ret = 0; in replace_mm_exe_file()
1461 /* Forbid mm->exe_file change if old file still mapped. */ in replace_mm_exe_file()
1462 old_exe_file = get_mm_exe_file(mm); in replace_mm_exe_file()
1464 VMA_ITERATOR(vmi, mm, 0); in replace_mm_exe_file()
1465 mmap_read_lock(mm); in replace_mm_exe_file()
1467 if (!vma->vm_file) in replace_mm_exe_file()
1469 if (path_equal(&vma->vm_file->f_path, in replace_mm_exe_file()
1470 &old_exe_file->f_path)) { in replace_mm_exe_file()
1471 ret = -EBUSY; in replace_mm_exe_file()
1475 mmap_read_unlock(mm); in replace_mm_exe_file()
1483 return -EACCES; in replace_mm_exe_file()
1487 mmap_write_lock(mm); in replace_mm_exe_file()
1488 old_exe_file = rcu_dereference_raw(mm->exe_file); in replace_mm_exe_file()
1489 rcu_assign_pointer(mm->exe_file, new_exe_file); in replace_mm_exe_file()
1490 mmap_write_unlock(mm); in replace_mm_exe_file()
1496 return 0; in replace_mm_exe_file()
1500 * get_mm_exe_file - acquire a reference to the mm's executable file
1501 * @mm: The mm of interest.
1503 * Returns %NULL if mm has no associated executable file.
1506 struct file *get_mm_exe_file(struct mm_struct *mm) in get_mm_exe_file() argument
1511 exe_file = get_file_rcu(&mm->exe_file); in get_mm_exe_file()
1517 * get_task_exe_file - acquire a reference to the task's executable file
1520 * Returns %NULL if task's mm (if any) has no associated executable file or
1521 * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1527 struct mm_struct *mm; in get_task_exe_file() local
1529 if (task->flags & PF_KTHREAD) in get_task_exe_file()
1533 mm = task->mm; in get_task_exe_file()
1534 if (mm) in get_task_exe_file()
1535 exe_file = get_mm_exe_file(mm); in get_task_exe_file()
1541 * get_task_mm - acquire a reference to the task's mm
1544 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
1545 * this kernel workthread has transiently adopted a user mm with use_mm,
1547 * bumping up the use count. User must release the mm via mmput()
1552 struct mm_struct *mm; in get_task_mm() local
1554 if (task->flags & PF_KTHREAD) in get_task_mm()
1558 mm = task->mm; in get_task_mm()
1559 if (mm) in get_task_mm()
1560 mmget(mm); in get_task_mm()
1562 return mm; in get_task_mm()
1568 struct mm_struct *mm; in mm_access() local
1571 err = down_read_killable(&task->signal->exec_update_lock); in mm_access()
1575 mm = get_task_mm(task); in mm_access()
1576 if (!mm) { in mm_access()
1577 mm = ERR_PTR(-ESRCH); in mm_access()
1578 } else if (mm != current->mm && !ptrace_may_access(task, mode)) { in mm_access()
1579 mmput(mm); in mm_access()
1580 mm = ERR_PTR(-EACCES); in mm_access()
1582 up_read(&task->signal->exec_update_lock); in mm_access()
1584 return mm; in mm_access()
1592 vfork = tsk->vfork_done; in complete_vfork_done()
1594 tsk->vfork_done = NULL; in complete_vfork_done()
1612 child->vfork_done = NULL; in wait_for_vfork_done()
1633 static void mm_release(struct task_struct *tsk, struct mm_struct *mm) in mm_release() argument
1638 deactivate_mm(tsk, mm); in mm_release()
1645 if (tsk->clear_child_tid) { in mm_release()
1646 if (atomic_read(&mm->mm_users) > 1) { in mm_release()
1648 * We don't check the error code - if userspace has in mm_release()
1651 put_user(0, tsk->clear_child_tid); in mm_release()
1652 do_futex(tsk->clear_child_tid, FUTEX_WAKE, in mm_release()
1653 1, NULL, NULL, 0, 0); in mm_release()
1655 tsk->clear_child_tid = NULL; in mm_release()
1659 * All done, finally we can wake up parent and return this mm to him. in mm_release()
1662 if (tsk->vfork_done) in mm_release()
1666 void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm) in exit_mm_release() argument
1669 mm_release(tsk, mm); in exit_mm_release()
1672 void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm) in exec_mm_release() argument
1675 mm_release(tsk, mm); in exec_mm_release()
1679 * dup_mm() - duplicates an existing mm structure
1680 * @tsk: the task_struct with which the new mm will be associated.
1681 * @oldmm: the mm to duplicate.
1683 * Allocates a new mm structure and duplicates the provided @oldmm structure
1686 * Return: the duplicated mm or NULL on failure.
1691 struct mm_struct *mm; in dup_mm() local
1694 mm = allocate_mm(); in dup_mm()
1695 if (!mm) in dup_mm()
1698 memcpy(mm, oldmm, sizeof(*mm)); in dup_mm()
1700 if (!mm_init(mm, tsk, mm->user_ns)) in dup_mm()
1704 err = dup_mmap(mm, oldmm); in dup_mm()
1709 mm->hiwater_rss = get_mm_rss(mm); in dup_mm()
1710 mm->hiwater_vm = mm->total_vm; in dup_mm()
1712 if (mm->binfmt && !try_module_get(mm->binfmt->module)) in dup_mm()
1715 return mm; in dup_mm()
1719 mm->binfmt = NULL; in dup_mm()
1720 mm_init_owner(mm, NULL); in dup_mm()
1721 mmput(mm); in dup_mm()
1731 struct mm_struct *mm, *oldmm; in copy_mm() local
1733 tsk->min_flt = tsk->maj_flt = 0; in copy_mm()
1734 tsk->nvcsw = tsk->nivcsw = 0; in copy_mm()
1736 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; in copy_mm()
1737 tsk->last_switch_time = 0; in copy_mm()
1740 tsk->mm = NULL; in copy_mm()
1741 tsk->active_mm = NULL; in copy_mm()
1748 oldmm = current->mm; in copy_mm()
1750 return 0; in copy_mm()
1754 mm = oldmm; in copy_mm()
1756 mm = dup_mm(tsk, current->mm); in copy_mm()
1757 if (!mm) in copy_mm()
1758 return -ENOMEM; in copy_mm()
1761 tsk->mm = mm; in copy_mm()
1762 tsk->active_mm = mm; in copy_mm()
1764 return 0; in copy_mm()
1769 struct fs_struct *fs = current->fs; in copy_fs()
1771 /* tsk->fs is already what we want */ in copy_fs()
1772 spin_lock(&fs->lock); in copy_fs()
1774 if (fs->in_exec) { in copy_fs()
1775 spin_unlock(&fs->lock); in copy_fs()
1776 return -EAGAIN; in copy_fs()
1778 fs->users++; in copy_fs()
1779 spin_unlock(&fs->lock); in copy_fs()
1780 return 0; in copy_fs()
1782 tsk->fs = copy_fs_struct(fs); in copy_fs()
1783 if (!tsk->fs) in copy_fs()
1784 return -ENOMEM; in copy_fs()
1785 return 0; in copy_fs()
1796 oldf = current->files; in copy_files()
1798 return 0; in copy_files()
1801 tsk->files = NULL; in copy_files()
1802 return 0; in copy_files()
1806 atomic_inc(&oldf->count); in copy_files()
1807 return 0; in copy_files()
1814 tsk->files = newf; in copy_files()
1815 return 0; in copy_files()
1823 refcount_inc(¤t->sighand->count); in copy_sighand()
1824 return 0; in copy_sighand()
1827 RCU_INIT_POINTER(tsk->sighand, sig); in copy_sighand()
1829 return -ENOMEM; in copy_sighand()
1831 refcount_set(&sig->count, 1); in copy_sighand()
1832 spin_lock_irq(¤t->sighand->siglock); in copy_sighand()
1833 memcpy(sig->action, current->sighand->action, sizeof(sig->action)); in copy_sighand()
1834 spin_unlock_irq(¤t->sighand->siglock); in copy_sighand()
1838 flush_signal_handlers(tsk, 0); in copy_sighand()
1840 return 0; in copy_sighand()
1845 if (refcount_dec_and_test(&sighand->count)) { in __cleanup_sighand()
1860 struct posix_cputimers *pct = &sig->posix_cputimers; in posix_cpu_timers_init_group()
1863 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); in posix_cpu_timers_init_group()
1872 return 0; in copy_signal()
1875 tsk->signal = sig; in copy_signal()
1877 return -ENOMEM; in copy_signal()
1879 sig->nr_threads = 1; in copy_signal()
1880 sig->quick_threads = 1; in copy_signal()
1881 atomic_set(&sig->live, 1); in copy_signal()
1882 refcount_set(&sig->sigcnt, 1); in copy_signal()
1885 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); in copy_signal()
1886 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); in copy_signal()
1888 init_waitqueue_head(&sig->wait_chldexit); in copy_signal()
1889 sig->curr_target = tsk; in copy_signal()
1890 init_sigpending(&sig->shared_pending); in copy_signal()
1891 INIT_HLIST_HEAD(&sig->multiprocess); in copy_signal()
1892 seqlock_init(&sig->stats_lock); in copy_signal()
1893 prev_cputime_init(&sig->prev_cputime); in copy_signal()
1896 INIT_HLIST_HEAD(&sig->posix_timers); in copy_signal()
1897 INIT_HLIST_HEAD(&sig->ignored_posix_timers); in copy_signal()
1898 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in copy_signal()
1899 sig->real_timer.function = it_real_fn; in copy_signal()
1902 task_lock(current->group_leader); in copy_signal()
1903 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); in copy_signal()
1904 task_unlock(current->group_leader); in copy_signal()
1911 sig->oom_score_adj = current->signal->oom_score_adj; in copy_signal()
1912 sig->oom_score_adj_min = current->signal->oom_score_adj_min; in copy_signal()
1914 mutex_init(&sig->cred_guard_mutex); in copy_signal()
1915 init_rwsem(&sig->exec_update_lock); in copy_signal()
1917 return 0; in copy_signal()
1924 * Must be called with sighand->lock held, which is common to in copy_seccomp()
1929 assert_spin_locked(¤t->sighand->siglock); in copy_seccomp()
1931 /* Ref-count the new filter user, and assign it. */ in copy_seccomp()
1933 p->seccomp = current->seccomp; in copy_seccomp()
1948 if (p->seccomp.mode != SECCOMP_MODE_DISABLED) in copy_seccomp()
1955 current->clear_child_tid = tidptr; in SYSCALL_DEFINE1()
1962 raw_spin_lock_init(&p->pi_lock); in rt_mutex_init_task()
1964 p->pi_waiters = RB_ROOT_CACHED; in rt_mutex_init_task()
1965 p->pi_top_task = NULL; in rt_mutex_init_task()
1966 p->pi_blocked_on = NULL; in rt_mutex_init_task()
1975 INIT_HLIST_NODE(&task->pid_links[type]); in init_task_pid_links()
1982 task->thread_pid = pid; in init_task_pid()
1984 task->signal->pids[type] = pid; in init_task_pid()
1990 p->rcu_read_lock_nesting = 0; in rcu_copy_process()
1991 p->rcu_read_unlock_special.s = 0; in rcu_copy_process()
1992 p->rcu_blocked_node = NULL; in rcu_copy_process()
1993 INIT_LIST_HEAD(&p->rcu_node_entry); in rcu_copy_process()
1996 p->rcu_tasks_holdout = false; in rcu_copy_process()
1997 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); in rcu_copy_process()
1998 p->rcu_tasks_idle_cpu = -1; in rcu_copy_process()
1999 INIT_LIST_HEAD(&p->rcu_tasks_exit_list); in rcu_copy_process()
2002 p->trc_reader_nesting = 0; in rcu_copy_process()
2003 p->trc_reader_special.s = 0; in rcu_copy_process()
2004 INIT_LIST_HEAD(&p->trc_holdout_list); in rcu_copy_process()
2005 INIT_LIST_HEAD(&p->trc_blkd_node); in rcu_copy_process()
2010 * __pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
2043 if (pidfd < 0) in __pidfd_prepare()
2055 pidfd_file->f_flags |= (flags & PIDFD_THREAD); in __pidfd_prepare()
2061 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
2070 * task identified by @pid must be a thread-group leader.
2092 return -EINVAL; in pidfd_prepare()
2107 call_rcu(&tsk->rcu, __delayed_free_task); in delayed_free_task()
2115 if (!tsk->mm) in copy_oom_score_adj()
2124 set_bit(MMF_MULTIPROCESS, &tsk->mm->flags); in copy_oom_score_adj()
2126 tsk->signal->oom_score_adj = current->signal->oom_score_adj; in copy_oom_score_adj()
2127 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min; in copy_oom_score_adj()
2136 for (i = 0; i < RV_PER_TASK_MONITORS; i++) in rv_task_fork()
2137 p->rv[i].da_mon.monitoring = false; in rv_task_fork()
2140 #define rv_task_fork(p) do {} while (0)
2149 * flags). The actual kick-off is left to the caller.
2157 int pidfd = -1, retval; in copy_process()
2161 const u64 clone_flags = args->flags; in copy_process()
2162 struct nsproxy *nsp = current->nsproxy; in copy_process()
2169 return ERR_PTR(-EINVAL); in copy_process()
2172 return ERR_PTR(-EINVAL); in copy_process()
2179 return ERR_PTR(-EINVAL); in copy_process()
2187 return ERR_PTR(-EINVAL); in copy_process()
2192 * multi-rooted process trees, prevent global and container-inits in copy_process()
2196 current->signal->flags & SIGNAL_UNKILLABLE) in copy_process()
2197 return ERR_PTR(-EINVAL); in copy_process()
2205 (task_active_pid_ns(current) != nsp->pid_ns_for_children)) in copy_process()
2206 return ERR_PTR(-EINVAL); in copy_process()
2211 * - CLONE_DETACHED is blocked so that we can potentially in copy_process()
2215 return ERR_PTR(-EINVAL); in copy_process()
2227 spin_lock_irq(¤t->sighand->siglock); in copy_process()
2229 hlist_add_head(&delayed.node, ¤t->signal->multiprocess); in copy_process()
2231 spin_unlock_irq(¤t->sighand->siglock); in copy_process()
2232 retval = -ERESTARTNOINTR; in copy_process()
2236 retval = -ENOMEM; in copy_process()
2240 p->flags &= ~PF_KTHREAD; in copy_process()
2241 if (args->kthread) in copy_process()
2242 p->flags |= PF_KTHREAD; in copy_process()
2243 if (args->user_worker) { in copy_process()
2248 p->flags |= PF_USER_WORKER; in copy_process()
2249 siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); in copy_process()
2251 if (args->io_thread) in copy_process()
2252 p->flags |= PF_IO_WORKER; in copy_process()
2254 if (args->name) in copy_process()
2255 strscpy_pad(p->comm, args->name, sizeof(p->comm)); in copy_process()
2257 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL; in copy_process()
2261 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; in copy_process()
2269 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); in copy_process()
2272 if (retval < 0) in copy_process()
2275 retval = -EAGAIN; in copy_process()
2277 if (p->real_cred->user != INIT_USER && in copy_process()
2281 current->flags &= ~PF_NPROC_EXCEEDED; in copy_process()
2288 retval = -EAGAIN; in copy_process()
2293 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE | PF_NO_SETAFFINITY); in copy_process()
2294 p->flags |= PF_FORKNOEXEC; in copy_process()
2295 INIT_LIST_HEAD(&p->children); in copy_process()
2296 INIT_LIST_HEAD(&p->sibling); in copy_process()
2298 p->vfork_done = NULL; in copy_process()
2299 spin_lock_init(&p->alloc_lock); in copy_process()
2301 init_sigpending(&p->pending); in copy_process()
2303 p->utime = p->stime = p->gtime = 0; in copy_process()
2305 p->utimescaled = p->stimescaled = 0; in copy_process()
2307 prev_cputime_init(&p->prev_cputime); in copy_process()
2310 seqcount_init(&p->vtime.seqcount); in copy_process()
2311 p->vtime.starttime = 0; in copy_process()
2312 p->vtime.state = VTIME_INACTIVE; in copy_process()
2316 p->io_uring = NULL; in copy_process()
2319 p->default_timer_slack_ns = current->timer_slack_ns; in copy_process()
2322 p->psi_flags = 0; in copy_process()
2325 task_io_accounting_init(&p->ioac); in copy_process()
2328 posix_cputimers_init(&p->posix_cputimers); in copy_process()
2331 p->io_context = NULL; in copy_process()
2334 if (args->kthread) { in copy_process()
2339 p->mempolicy = mpol_dup(p->mempolicy); in copy_process()
2340 if (IS_ERR(p->mempolicy)) { in copy_process()
2341 retval = PTR_ERR(p->mempolicy); in copy_process()
2342 p->mempolicy = NULL; in copy_process()
2347 p->cpuset_mem_spread_rotor = NUMA_NO_NODE; in copy_process()
2348 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock); in copy_process()
2351 memset(&p->irqtrace, 0, sizeof(p->irqtrace)); in copy_process()
2352 p->irqtrace.hardirq_disable_ip = _THIS_IP_; in copy_process()
2353 p->irqtrace.softirq_enable_ip = _THIS_IP_; in copy_process()
2354 p->softirqs_enabled = 1; in copy_process()
2355 p->softirq_context = 0; in copy_process()
2358 p->pagefault_disabled = 0; in copy_process()
2365 p->blocked_on = NULL; /* not blocked yet */ in copy_process()
2368 p->sequential_io = 0; in copy_process()
2369 p->sequential_io_avg = 0; in copy_process()
2372 RCU_INIT_POINTER(p->bpf_storage, NULL); in copy_process()
2373 p->bpf_ctx = NULL; in copy_process()
2395 retval = copy_files(clone_flags, p, args->no_files); in copy_process()
2423 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid, in copy_process()
2424 args->set_tid_size); in copy_process()
2437 int flags = (clone_flags & CLONE_THREAD) ? PIDFD_THREAD : 0; in copy_process()
2441 if (retval < 0) in copy_process()
2445 retval = put_user(pidfd, args->pidfd); in copy_process()
2451 p->plug = NULL; in copy_process()
2473 p->pid = pid_nr(pid); in copy_process()
2475 p->group_leader = current->group_leader; in copy_process()
2476 p->tgid = current->tgid; in copy_process()
2478 p->group_leader = p; in copy_process()
2479 p->tgid = p->pid; in copy_process()
2482 p->nr_dirtied = 0; in copy_process()
2483 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); in copy_process()
2484 p->dirty_paused_when = 0; in copy_process()
2486 p->pdeath_signal = 0; in copy_process()
2487 p->task_works = NULL; in copy_process()
2491 p->kretprobe_instances.first = NULL; in copy_process()
2494 p->rethooks.first = NULL; in copy_process()
2508 * Now that the cgroups are pinned, re-clone the parent cgroup and put in copy_process()
2512 * This isn't part of ->can_fork() because while the re-cloning is in copy_process()
2521 * From this point on we must avoid any synchronous user-space in copy_process()
2522 * communication until we take the tasklist-lock. In particular, we do in copy_process()
2523 * not want user-space to be able to predict the process start-time by in copy_process()
2528 p->start_time = ktime_get_ns(); in copy_process()
2529 p->start_boottime = ktime_get_boottime_ns(); in copy_process()
2537 /* CLONE_PARENT re-uses the old parent */ in copy_process()
2539 p->real_parent = current->real_parent; in copy_process()
2540 p->parent_exec_id = current->parent_exec_id; in copy_process()
2542 p->exit_signal = -1; in copy_process()
2544 p->exit_signal = current->group_leader->exit_signal; in copy_process()
2546 p->real_parent = current; in copy_process()
2547 p->parent_exec_id = current->self_exec_id; in copy_process()
2548 p->exit_signal = args->exit_signal; in copy_process()
2555 spin_lock(¤t->sighand->siglock); in copy_process()
2562 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { in copy_process()
2563 retval = -ENOMEM; in copy_process()
2569 retval = -EINTR; in copy_process()
2582 if (likely(p->pid)) { in copy_process()
2592 ns_of_pid(pid)->child_reaper = p; in copy_process()
2593 p->signal->flags |= SIGNAL_UNKILLABLE; in copy_process()
2595 p->signal->shared_pending.signal = delayed.signal; in copy_process()
2596 p->signal->tty = tty_kref_get(current->signal->tty); in copy_process()
2602 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || in copy_process()
2603 p->real_parent->signal->is_child_subreaper; in copy_process()
2604 list_add_tail(&p->sibling, &p->real_parent->children); in copy_process()
2605 list_add_tail_rcu(&p->tasks, &init_task.tasks); in copy_process()
2611 current->signal->nr_threads++; in copy_process()
2612 current->signal->quick_threads++; in copy_process()
2613 atomic_inc(¤t->signal->live); in copy_process()
2614 refcount_inc(¤t->signal->sigcnt); in copy_process()
2616 list_add_tail_rcu(&p->thread_node, in copy_process()
2617 &p->signal->thread_head); in copy_process()
2624 spin_unlock(¤t->sighand->siglock); in copy_process()
2646 spin_unlock(¤t->sighand->siglock); in copy_process()
2661 if (p->io_context) in copy_process()
2666 if (p->mm) { in copy_process()
2667 mm_clear_owner(p->mm, p); in copy_process()
2668 mmput(p->mm); in copy_process()
2672 free_signal_struct(p->signal); in copy_process()
2674 __cleanup_sighand(p->sighand); in copy_process()
2692 mpol_put(p->mempolicy); in copy_process()
2700 WRITE_ONCE(p->__state, TASK_DEAD); in copy_process()
2705 spin_lock_irq(¤t->sighand->siglock); in copy_process()
2707 spin_unlock_irq(¤t->sighand->siglock); in copy_process()
2716 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */ in init_idle_pids()
2724 return 0; in idle_dummy()
2738 task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args); in fork_idle()
2767 return copy_process(NULL, 0, node, &args); in create_io_thread()
2771 * Ok, this is the main fork-routine.
2773 * It copies the process, and if successful kick-starts
2776 * args->exit_signal is expected to be checked for sanity by the caller.
2780 u64 clone_flags = args->flags; in kernel_clone()
2784 int trace = 0; in kernel_clone()
2798 (args->pidfd == args->parent_tid)) in kernel_clone()
2799 return -EINVAL; in kernel_clone()
2810 else if (args->exit_signal != SIGCHLD) in kernel_clone()
2816 trace = 0; in kernel_clone()
2826 * Do this prior waking up the new thread - the thread pointer in kernel_clone()
2835 put_user(nr, args->parent_tid); in kernel_clone()
2838 p->vfork_done = &vfork; in kernel_clone()
2846 lru_gen_add_mm(p->mm); in kernel_clone()
2911 return -EINVAL; in SYSCALL_DEFINE0()
2972 pid_t *kset_tid = kargs->set_tid; in copy_clone_args_from_user()
2983 return -E2BIG; in copy_clone_args_from_user()
2985 return -EINVAL; in copy_clone_args_from_user()
2992 return -EINVAL; in copy_clone_args_from_user()
2994 if (unlikely(!args.set_tid && args.set_tid_size > 0)) in copy_clone_args_from_user()
2995 return -EINVAL; in copy_clone_args_from_user()
2997 if (unlikely(args.set_tid && args.set_tid_size == 0)) in copy_clone_args_from_user()
2998 return -EINVAL; in copy_clone_args_from_user()
3006 return -EINVAL; in copy_clone_args_from_user()
3010 return -EINVAL; in copy_clone_args_from_user()
3027 (kargs->set_tid_size * sizeof(pid_t)))) in copy_clone_args_from_user()
3028 return -EFAULT; in copy_clone_args_from_user()
3030 kargs->set_tid = kset_tid; in copy_clone_args_from_user()
3032 return 0; in copy_clone_args_from_user()
3036 * clone3_stack_valid - check and prepare stack
3045 if (kargs->stack == 0) { in clone3_stack_valid()
3046 if (kargs->stack_size > 0) in clone3_stack_valid()
3049 if (kargs->stack_size == 0) in clone3_stack_valid()
3052 if (!access_ok((void __user *)kargs->stack, kargs->stack_size)) in clone3_stack_valid()
3056 kargs->stack += kargs->stack_size; in clone3_stack_valid()
3066 if (kargs->flags & in clone3_args_valid()
3071 * - make the CLONE_DETACHED bit reusable for clone3 in clone3_args_valid()
3072 * - make the CSIGNAL bits reusable for clone3 in clone3_args_valid()
3074 if (kargs->flags & (CLONE_DETACHED | (CSIGNAL & (~CLONE_NEWTIME)))) in clone3_args_valid()
3077 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) == in clone3_args_valid()
3081 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) && in clone3_args_valid()
3082 kargs->exit_signal) in clone3_args_valid()
3092 * sys_clone3 - create a new process with specific properties
3111 return -ENOSYS; in SYSCALL_DEFINE2()
3121 return -EINVAL; in SYSCALL_DEFINE2()
3132 leader = top = top->group_leader; in walk_process_tree()
3135 list_for_each_entry(child, &parent->children, sibling) { in walk_process_tree()
3138 if (res < 0) in walk_process_tree()
3150 parent = child->real_parent; in walk_process_tree()
3151 leader = parent->group_leader; in walk_process_tree()
3159 #define ARCH_MIN_MMSTRUCT_ALIGN 0
3166 spin_lock_init(&sighand->siglock); in sighand_ctor()
3167 init_waitqueue_head(&sighand->signalfd_wqh); in sighand_ctor()
3192 sizeof(struct sighand_struct), 0, in proc_caches_init()
3196 sizeof(struct signal_struct), 0, in proc_caches_init()
3200 sizeof(struct files_struct), 0, in proc_caches_init()
3204 sizeof(struct fs_struct), 0, in proc_caches_init()
3226 return -EINVAL; in check_unshare_flags()
3235 return -EINVAL; in check_unshare_flags()
3238 if (refcount_read(¤t->sighand->count) > 1) in check_unshare_flags()
3239 return -EINVAL; in check_unshare_flags()
3243 return -EINVAL; in check_unshare_flags()
3246 return 0; in check_unshare_flags()
3254 struct fs_struct *fs = current->fs; in unshare_fs()
3257 return 0; in unshare_fs()
3260 if (fs->users == 1) in unshare_fs()
3261 return 0; in unshare_fs()
3265 return -ENOMEM; in unshare_fs()
3267 return 0; in unshare_fs()
3275 struct files_struct *fd = current->files; in unshare_fd()
3278 (fd && atomic_read(&fd->count) > 1)) { in unshare_fd()
3285 return 0; in unshare_fd()
3302 int do_sysvsem = 0; in ksys_unshare()
3376 fs = current->fs; in ksys_unshare()
3377 spin_lock(&fs->lock); in ksys_unshare()
3378 current->fs = new_fs; in ksys_unshare()
3379 if (--fs->users) in ksys_unshare()
3383 spin_unlock(&fs->lock); in ksys_unshare()
3387 swap(current->files, new_fd); in ksys_unshare()
3436 old = task->files; in unshare_files()
3438 task->files = copy; in unshare_files()
3441 return 0; in unshare_files()
3464 return 0; in sysctl_max_threads()