fork.c 47.8 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 *  linux/kernel/fork.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 *  'fork.c' contains the help-routines for the 'fork' system call
 * (see also entry.S and others).
 * Fork is rather simple, once you get the hang of it, but the memory
 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
 */

#include <linux/slab.h>
#include <linux/init.h>
#include <linux/unistd.h>
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/completion.h>
#include <linux/personality.h>
#include <linux/mempolicy.h>
#include <linux/sem.h>
#include <linux/file.h>
Al Viro's avatar
Al Viro committed
24
#include <linux/fdtable.h>
25
#include <linux/iocontext.h>
Linus Torvalds's avatar
Linus Torvalds committed
26 27 28
#include <linux/key.h>
#include <linux/binfmts.h>
#include <linux/mman.h>
Andrea Arcangeli's avatar
Andrea Arcangeli committed
29
#include <linux/mmu_notifier.h>
Linus Torvalds's avatar
Linus Torvalds committed
30
#include <linux/fs.h>
31
#include <linux/nsproxy.h>
32
#include <linux/capability.h>
Linus Torvalds's avatar
Linus Torvalds committed
33
#include <linux/cpu.h>
34
#include <linux/cgroup.h>
Linus Torvalds's avatar
Linus Torvalds committed
35
#include <linux/security.h>
36
#include <linux/hugetlb.h>
37
#include <linux/seccomp.h>
Linus Torvalds's avatar
Linus Torvalds committed
38 39 40 41
#include <linux/swap.h>
#include <linux/syscalls.h>
#include <linux/jiffies.h>
#include <linux/futex.h>
42
#include <linux/compat.h>
43
#include <linux/kthread.h>
44
#include <linux/task_io_accounting_ops.h>
45
#include <linux/rcupdate.h>
Linus Torvalds's avatar
Linus Torvalds committed
46 47 48
#include <linux/ptrace.h>
#include <linux/mount.h>
#include <linux/audit.h>
49
#include <linux/memcontrol.h>
50
#include <linux/ftrace.h>
51
#include <linux/proc_fs.h>
Linus Torvalds's avatar
Linus Torvalds committed
52 53
#include <linux/profile.h>
#include <linux/rmap.h>
54
#include <linux/ksm.h>
Linus Torvalds's avatar
Linus Torvalds committed
55
#include <linux/acct.h>
56
#include <linux/tsacct_kern.h>
57
#include <linux/cn_proc.h>
58
#include <linux/freezer.h>
59
#include <linux/delayacct.h>
60
#include <linux/taskstats_kern.h>
61
#include <linux/random.h>
62
#include <linux/tty.h>
63
#include <linux/blkdev.h>
64
#include <linux/fs_struct.h>
65
#include <linux/magic.h>
66
#include <linux/perf_event.h>
67
#include <linux/posix-timers.h>
68
#include <linux/user-return-notifier.h>
69
#include <linux/oom.h>
Andrea Arcangeli's avatar
Andrea Arcangeli committed
70
#include <linux/khugepaged.h>
71
#include <linux/signalfd.h>
72
#include <linux/uprobes.h>
73
#include <linux/aio.h>
Linus Torvalds's avatar
Linus Torvalds committed
74 75 76 77 78 79 80 81

#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/uaccess.h>
#include <asm/mmu_context.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>

82 83
#include <trace/events/sched.h>

84 85 86
#define CREATE_TRACE_POINTS
#include <trace/events/task.h>

Linus Torvalds's avatar
Linus Torvalds committed
87 88 89 90
/*
 * Protected counters by write_lock_irq(&tasklist_lock)
 */
unsigned long total_forks;	/* Handle normal Linux uptimes. */
91
int nr_threads;			/* The idle threads do not count.. */
Linus Torvalds's avatar
Linus Torvalds committed
92 93 94 95 96

int max_threads;		/* tunable limit on nr_threads */

DEFINE_PER_CPU(unsigned long, process_counts) = 0;

Rohit Gupta's avatar
Rohit Gupta committed
97
DEFINE_RWLOCK(tasklist_lock);  /* outer */
98 99 100 101 102 103 104 105

#ifdef CONFIG_PROVE_RCU
int lockdep_tasklist_lock_is_held(void)
{
	return lockdep_is_held(&tasklist_lock);
}
EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
#endif /* #ifdef CONFIG_PROVE_RCU */
Linus Torvalds's avatar
Linus Torvalds committed
106 107 108 109 110 111

int nr_processes(void)
{
	int cpu;
	int total = 0;

112
	for_each_possible_cpu(cpu)
Linus Torvalds's avatar
Linus Torvalds committed
113 114 115 116 117
		total += per_cpu(process_counts, cpu);

	return total;
}

118 119 120 121
void __weak arch_release_task_struct(struct task_struct *tsk)
{
}

122
#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
123
static struct kmem_cache *task_struct_cachep;
124 125 126 127 128 129 130 131 132 133

static inline struct task_struct *alloc_task_struct_node(int node)
{
	return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
}

static inline void free_task_struct(struct task_struct *tsk)
{
	kmem_cache_free(task_struct_cachep, tsk);
}
Linus Torvalds's avatar
Linus Torvalds committed
134 135
#endif

136 137 138 139
void __weak arch_release_thread_info(struct thread_info *ti)
{
}

140
#ifndef CONFIG_ARCH_THREAD_INFO_ALLOCATOR
141

142 143 144 145 146
/*
 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
 * kmemcache based allocator.
 */
# if THREAD_SIZE >= PAGE_SIZE
147 148
static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
						  int node)
149
{
150
	struct page *page = alloc_pages_node(node, THREADINFO_GFP_ACCOUNTED,
151
					     THREAD_SIZE_ORDER);
152 153

	return page ? page_address(page) : NULL;
154 155 156 157
}

static inline void free_thread_info(struct thread_info *ti)
{
158
	free_memcg_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER);
159
}
160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
# else
static struct kmem_cache *thread_info_cache;

static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
						  int node)
{
	return kmem_cache_alloc_node(thread_info_cache, THREADINFO_GFP, node);
}

static void free_thread_info(struct thread_info *ti)
{
	kmem_cache_free(thread_info_cache, ti);
}

void thread_info_cache_init(void)
{
	thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE,
					      THREAD_SIZE, 0, NULL);
	BUG_ON(thread_info_cache == NULL);
}
# endif
181 182
#endif

Linus Torvalds's avatar
Linus Torvalds committed
183
/* SLAB cache for signal_struct structures (tsk->signal) */
184
static struct kmem_cache *signal_cachep;
Linus Torvalds's avatar
Linus Torvalds committed
185 186

/* SLAB cache for sighand_struct structures (tsk->sighand) */
187
struct kmem_cache *sighand_cachep;
Linus Torvalds's avatar
Linus Torvalds committed
188 189

/* SLAB cache for files_struct structures (tsk->files) */
190
struct kmem_cache *files_cachep;
Linus Torvalds's avatar
Linus Torvalds committed
191 192

/* SLAB cache for fs_struct structures (tsk->fs) */
193
struct kmem_cache *fs_cachep;
Linus Torvalds's avatar
Linus Torvalds committed
194 195

/* SLAB cache for vm_area_struct structures */
196
struct kmem_cache *vm_area_cachep;
Linus Torvalds's avatar
Linus Torvalds committed
197 198

/* SLAB cache for mm_struct structures (tsk->mm) */
199
static struct kmem_cache *mm_cachep;
Linus Torvalds's avatar
Linus Torvalds committed
200

Mauro Ribeiro's avatar
Mauro Ribeiro committed
201 202 203
/* Notifier list called when a task struct is freed */
static ATOMIC_NOTIFIER_HEAD(task_free_notifier);

204 205 206 207 208 209 210
static void account_kernel_stack(struct thread_info *ti, int account)
{
	struct zone *zone = page_zone(virt_to_page(ti));

	mod_zone_page_state(zone, NR_KERNEL_STACK, account);
}

Linus Torvalds's avatar
Linus Torvalds committed
211 212
void free_task(struct task_struct *tsk)
{
213
	account_kernel_stack(tsk->stack, -1);
214
	arch_release_thread_info(tsk->stack);
215
	free_thread_info(tsk->stack);
216
	rt_mutex_debug_task_free(tsk);
217
	ftrace_graph_exit_task(tsk);
218
	put_seccomp_filter(tsk);
219
	arch_release_task_struct(tsk);
Linus Torvalds's avatar
Linus Torvalds committed
220 221 222 223
	free_task_struct(tsk);
}
EXPORT_SYMBOL(free_task);

224 225
static inline void free_signal_struct(struct signal_struct *sig)
{
226
	taskstats_tgid_free(sig);
227
	sched_autogroup_exit(sig);
228 229 230 231 232
	kmem_cache_free(signal_cachep, sig);
}

static inline void put_signal_struct(struct signal_struct *sig)
{
233
	if (atomic_dec_and_test(&sig->sigcnt))
234 235 236
		free_signal_struct(sig);
}

Mauro Ribeiro's avatar
Mauro Ribeiro committed
237 238 239 240 241 242 243 244 245 246 247 248
int task_free_register(struct notifier_block *n)
{
	return atomic_notifier_chain_register(&task_free_notifier, n);
}
EXPORT_SYMBOL(task_free_register);

int task_free_unregister(struct notifier_block *n)
{
	return atomic_notifier_chain_unregister(&task_free_notifier, n);
}
EXPORT_SYMBOL(task_free_unregister);

249 250 251
#ifdef CONFIG_PREEMPT_RT_BASE
static
#endif
252
void __put_task_struct(struct task_struct *tsk)
Linus Torvalds's avatar
Linus Torvalds committed
253
{
Eugene Teo's avatar
Eugene Teo committed
254
	WARN_ON(!tsk->exit_state);
Linus Torvalds's avatar
Linus Torvalds committed
255 256 257
	WARN_ON(atomic_read(&tsk->usage));
	WARN_ON(tsk == current);

258
	security_task_free(tsk);
259
	exit_creds(tsk);
260
	delayacct_tsk_free(tsk);
261
	put_signal_struct(tsk->signal);
Linus Torvalds's avatar
Linus Torvalds committed
262

Mauro Ribeiro's avatar
Mauro Ribeiro committed
263
	atomic_notifier_call_chain(&task_free_notifier, 0, tsk);
Linus Torvalds's avatar
Linus Torvalds committed
264 265 266
	if (!profile_handoff_task(tsk))
		free_task(tsk);
}
Rohit Gupta's avatar
Rohit Gupta committed
267
#ifndef CONFIG_PREEMPT_RT_BASE
268
EXPORT_SYMBOL_GPL(__put_task_struct);
Rohit Gupta's avatar
Rohit Gupta committed
269 270 271 272 273 274 275 276 277 278
#else
void __put_task_struct_cb(struct rcu_head *rhp)
{
	struct task_struct *tsk = container_of(rhp, struct task_struct, put_rcu);

	__put_task_struct(tsk);

}
EXPORT_SYMBOL_GPL(__put_task_struct_cb);
#endif
Linus Torvalds's avatar
Linus Torvalds committed
279

280
void __init __weak arch_task_cache_init(void) { }
281

Linus Torvalds's avatar
Linus Torvalds committed
282 283
void __init fork_init(unsigned long mempages)
{
284
#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
Linus Torvalds's avatar
Linus Torvalds committed
285 286 287 288 289 290
#ifndef ARCH_MIN_TASKALIGN
#define ARCH_MIN_TASKALIGN	L1_CACHE_BYTES
#endif
	/* create a slab on which task_structs can be allocated */
	task_struct_cachep =
		kmem_cache_create("task_struct", sizeof(struct task_struct),
291
			ARCH_MIN_TASKALIGN, SLAB_PANIC | SLAB_NOTRACK, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
292 293
#endif

294 295 296
	/* do the arch specific task caches init */
	arch_task_cache_init();

Linus Torvalds's avatar
Linus Torvalds committed
297 298 299 300 301 302 303 304 305 306
	/*
	 * The default maximum number of threads is set to a safe
	 * value: the thread structures can take up at most half
	 * of memory.
	 */
	max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE);

	/*
	 * we need to allow at least 20 threads to boot a system
	 */
307
	if (max_threads < 20)
Linus Torvalds's avatar
Linus Torvalds committed
308 309 310 311 312 313 314 315
		max_threads = 20;

	init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
	init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
	init_task.signal->rlim[RLIMIT_SIGPENDING] =
		init_task.signal->rlim[RLIMIT_NPROC];
}

316 317 318 319 320 321 322
int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst,
					       struct task_struct *src)
{
	*dst = *src;
	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
323 324 325 326
static struct task_struct *dup_task_struct(struct task_struct *orig)
{
	struct task_struct *tsk;
	struct thread_info *ti;
327
	unsigned long *stackend;
328
	int node = tsk_fork_get_node(orig);
329
	int err;
Linus Torvalds's avatar
Linus Torvalds committed
330

331
	tsk = alloc_task_struct_node(node);
Linus Torvalds's avatar
Linus Torvalds committed
332 333 334
	if (!tsk)
		return NULL;

335
	ti = alloc_thread_info_node(tsk, node);
336 337
	if (!ti)
		goto free_tsk;
Linus Torvalds's avatar
Linus Torvalds committed
338

339
	err = arch_dup_task_struct(tsk, orig);
340
	if (err)
341
		goto free_ti;
342

343 344 345
	tsk->stack = ti;

	setup_thread_stack(tsk, orig);
346
	clear_user_return_notifier(tsk);
347
	clear_tsk_need_resched(tsk);
348 349
	stackend = end_of_stack(tsk);
	*stackend = STACK_END_MAGIC;	/* for overflow detection */
Linus Torvalds's avatar
Linus Torvalds committed
350

351 352 353 354
#ifdef CONFIG_CC_STACKPROTECTOR
	tsk->stack_canary = get_random_int();
#endif

355 356 357 358 359
	/*
	 * One for us, one for whoever does the "release_task()" (usually
	 * parent)
	 */
	atomic_set(&tsk->usage, 2);
360
#ifdef CONFIG_BLK_DEV_IO_TRACE
361
	tsk->btrace_seq = 0;
362
#endif
363
	tsk->splice_pipe = NULL;
364
	tsk->task_frag.page = NULL;
365 366 367

	account_kernel_stack(ti, 1);

Linus Torvalds's avatar
Linus Torvalds committed
368
	return tsk;
369

370
free_ti:
371
	free_thread_info(ti);
372
free_tsk:
373 374
	free_task_struct(tsk);
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
375 376 377
}

#ifdef CONFIG_MMU
Alexey Dobriyan's avatar
Alexey Dobriyan committed
378
static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
Linus Torvalds's avatar
Linus Torvalds committed
379
{
380
	struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
Linus Torvalds's avatar
Linus Torvalds committed
381 382 383 384 385
	struct rb_node **rb_link, *rb_parent;
	int retval;
	unsigned long charge;
	struct mempolicy *pol;

386
	uprobe_start_dup_mmap();
Linus Torvalds's avatar
Linus Torvalds committed
387
	down_write(&oldmm->mmap_sem);
388
	flush_cache_dup_mm(oldmm);
389
	uprobe_dup_mmap(oldmm, mm);
390 391 392 393
	/*
	 * Not linked in yet - no deadlock potential:
	 */
	down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
394

Linus Torvalds's avatar
Linus Torvalds committed
395 396 397 398
	mm->locked_vm = 0;
	mm->mmap = NULL;
	mm->mmap_cache = NULL;
	mm->free_area_cache = oldmm->mmap_base;
399
	mm->cached_hole_size = ~0UL;
Linus Torvalds's avatar
Linus Torvalds committed
400
	mm->map_count = 0;
401
	cpumask_clear(mm_cpumask(mm));
Linus Torvalds's avatar
Linus Torvalds committed
402 403 404 405
	mm->mm_rb = RB_ROOT;
	rb_link = &mm->mm_rb.rb_node;
	rb_parent = NULL;
	pprev = &mm->mmap;
406
	retval = ksm_fork(mm, oldmm);
Andrea Arcangeli's avatar
Andrea Arcangeli committed
407 408 409
	if (retval)
		goto out;
	retval = khugepaged_fork(mm, oldmm);
410 411
	if (retval)
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
412

413
	prev = NULL;
414
	for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
Linus Torvalds's avatar
Linus Torvalds committed
415 416 417
		struct file *file;

		if (mpnt->vm_flags & VM_DONTCOPY) {
418
			vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
419
							-vma_pages(mpnt));
Linus Torvalds's avatar
Linus Torvalds committed
420 421 422 423
			continue;
		}
		charge = 0;
		if (mpnt->vm_flags & VM_ACCOUNT) {
424 425
			unsigned long len = vma_pages(mpnt);

426
			if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
Linus Torvalds's avatar
Linus Torvalds committed
427 428 429
				goto fail_nomem;
			charge = len;
		}
430
		tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
Linus Torvalds's avatar
Linus Torvalds committed
431 432 433
		if (!tmp)
			goto fail_nomem;
		*tmp = *mpnt;
434
		INIT_LIST_HEAD(&tmp->anon_vma_chain);
435
		pol = mpol_dup(vma_policy(mpnt));
Linus Torvalds's avatar
Linus Torvalds committed
436 437 438 439
		retval = PTR_ERR(pol);
		if (IS_ERR(pol))
			goto fail_nomem_policy;
		vma_set_policy(tmp, pol);
440
		tmp->vm_mm = mm;
441 442
		if (anon_vma_fork(tmp, mpnt))
			goto fail_nomem_anon_vma_fork;
Linus Torvalds's avatar
Linus Torvalds committed
443
		tmp->vm_flags &= ~VM_LOCKED;
444
		tmp->vm_next = tmp->vm_prev = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
445 446
		file = tmp->vm_file;
		if (file) {
Al Viro's avatar
Al Viro committed
447
			struct inode *inode = file_inode(file);
448 449
			struct address_space *mapping = file->f_mapping;

450
			vma_get_file(tmp);
Linus Torvalds's avatar
Linus Torvalds committed
451 452
			if (tmp->vm_flags & VM_DENYWRITE)
				atomic_dec(&inode->i_writecount);
453
			mutex_lock(&mapping->i_mmap_mutex);
454 455 456 457
			if (tmp->vm_flags & VM_SHARED)
				mapping->i_mmap_writable++;
			flush_dcache_mmap_lock(mapping);
			/* insert tmp into the share list, just after mpnt */
458 459 460 461 462 463
			if (unlikely(tmp->vm_flags & VM_NONLINEAR))
				vma_nonlinear_insert(tmp,
						&mapping->i_mmap_nonlinear);
			else
				vma_interval_tree_insert_after(tmp, mpnt,
							&mapping->i_mmap);
464
			flush_dcache_mmap_unlock(mapping);
465
			mutex_unlock(&mapping->i_mmap_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
466 467
		}

468 469 470 471 472 473 474 475
		/*
		 * Clear hugetlb-related page reserves for children. This only
		 * affects MAP_PRIVATE mappings. Faults generated by the child
		 * are not guaranteed to succeed, even if read-only
		 */
		if (is_vm_hugetlb_page(tmp))
			reset_vma_resv_huge_pages(tmp);

Linus Torvalds's avatar
Linus Torvalds committed
476
		/*
477
		 * Link in the new vma and copy the page table entries.
Linus Torvalds's avatar
Linus Torvalds committed
478 479 480
		 */
		*pprev = tmp;
		pprev = &tmp->vm_next;
481 482
		tmp->vm_prev = prev;
		prev = tmp;
Linus Torvalds's avatar
Linus Torvalds committed
483 484 485 486 487 488

		__vma_link_rb(mm, tmp, rb_link, rb_parent);
		rb_link = &tmp->vm_rb.rb_right;
		rb_parent = &tmp->vm_rb;

		mm->map_count++;
489
		retval = copy_page_range(mm, oldmm, mpnt);
Linus Torvalds's avatar
Linus Torvalds committed
490 491 492 493 494 495 496

		if (tmp->vm_ops && tmp->vm_ops->open)
			tmp->vm_ops->open(tmp);

		if (retval)
			goto out;
	}
497 498
	/* a new mm has just been created */
	arch_dup_mmap(oldmm, mm);
Linus Torvalds's avatar
Linus Torvalds committed
499 500
	retval = 0;
out:
501
	up_write(&mm->mmap_sem);
502
	flush_tlb_mm(oldmm);
Linus Torvalds's avatar
Linus Torvalds committed
503
	up_write(&oldmm->mmap_sem);
504
	uprobe_end_dup_mmap();
Linus Torvalds's avatar
Linus Torvalds committed
505
	return retval;
506 507
fail_nomem_anon_vma_fork:
	mpol_put(pol);
Linus Torvalds's avatar
Linus Torvalds committed
508 509 510 511 512 513 514 515
fail_nomem_policy:
	kmem_cache_free(vm_area_cachep, tmp);
fail_nomem:
	retval = -ENOMEM;
	vm_unacct_memory(charge);
	goto out;
}

516
static inline int mm_alloc_pgd(struct mm_struct *mm)
Linus Torvalds's avatar
Linus Torvalds committed
517 518 519 520 521 522 523
{
	mm->pgd = pgd_alloc(mm);
	if (unlikely(!mm->pgd))
		return -ENOMEM;
	return 0;
}

524
static inline void mm_free_pgd(struct mm_struct *mm)
Linus Torvalds's avatar
Linus Torvalds committed
525
{
526
	pgd_free(mm, mm->pgd);
Linus Torvalds's avatar
Linus Torvalds committed
527 528 529 530 531 532 533
}
#else
#define dup_mmap(mm, oldmm)	(0)
#define mm_alloc_pgd(mm)	(0)
#define mm_free_pgd(mm)
#endif /* CONFIG_MMU */

Daniel Walker's avatar
Daniel Walker committed
534
__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
Linus Torvalds's avatar
Linus Torvalds committed
535

536
#define allocate_mm()	(kmem_cache_alloc(mm_cachep, GFP_KERNEL))
Linus Torvalds's avatar
Linus Torvalds committed
537 538
#define free_mm(mm)	(kmem_cache_free(mm_cachep, (mm)))

539 540 541 542 543 544 545 546 547 548 549 550
static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;

static int __init coredump_filter_setup(char *s)
{
	default_dump_filter =
		(simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
		MMF_DUMP_FILTER_MASK;
	return 1;
}

__setup("coredump_filter=", coredump_filter_setup);

Linus Torvalds's avatar
Linus Torvalds committed
551 552
#include <linux/init_task.h>

553 554 555 556 557 558 559 560
static void mm_init_aio(struct mm_struct *mm)
{
#ifdef CONFIG_AIO
	spin_lock_init(&mm->ioctx_lock);
	INIT_HLIST_HEAD(&mm->ioctx_list);
#endif
}

561
static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
Linus Torvalds's avatar
Linus Torvalds committed
562 563 564 565 566
{
	atomic_set(&mm->mm_users, 1);
	atomic_set(&mm->mm_count, 1);
	init_rwsem(&mm->mmap_sem);
	INIT_LIST_HEAD(&mm->mmlist);
567 568
	mm->flags = (current->mm) ?
		(current->mm->flags & MMF_INIT_MASK) : default_dump_filter;
569
	mm->core_state = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
570
	mm->nr_ptes = 0;
571
	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
Linus Torvalds's avatar
Linus Torvalds committed
572 573
	spin_lock_init(&mm->page_table_lock);
	mm->free_area_cache = TASK_UNMAPPED_BASE;
574
	mm->cached_hole_size = ~0UL;
575
	mm_init_aio(mm);
576
	mm_init_owner(mm, p);
577
	clear_tlb_flush_pending(mm);
Linus Torvalds's avatar
Linus Torvalds committed
578 579 580

	if (likely(!mm_alloc_pgd(mm))) {
		mm->def_flags = 0;
Andrea Arcangeli's avatar
Andrea Arcangeli committed
581
		mmu_notifier_mm_init(mm);
Linus Torvalds's avatar
Linus Torvalds committed
582 583
		return mm;
	}
584

Linus Torvalds's avatar
Linus Torvalds committed
585 586 587 588
	free_mm(mm);
	return NULL;
}

589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
static void check_mm(struct mm_struct *mm)
{
	int i;

	for (i = 0; i < NR_MM_COUNTERS; i++) {
		long x = atomic_long_read(&mm->rss_stat.count[i]);

		if (unlikely(x))
			printk(KERN_ALERT "BUG: Bad rss-counter state "
					  "mm:%p idx:%d val:%ld\n", mm, i, x);
	}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	VM_BUG_ON(mm->pmd_huge_pte);
#endif
}

Linus Torvalds's avatar
Linus Torvalds committed
606 607 608
/*
 * Allocate and initialize an mm_struct.
 */
609
struct mm_struct *mm_alloc(void)
Linus Torvalds's avatar
Linus Torvalds committed
610
{
611
	struct mm_struct *mm;
Linus Torvalds's avatar
Linus Torvalds committed
612 613

	mm = allocate_mm();
614 615 616 617
	if (!mm)
		return NULL;

	memset(mm, 0, sizeof(*mm));
618 619
	mm_init_cpumask(mm);
	return mm_init(mm, current);
Linus Torvalds's avatar
Linus Torvalds committed
620 621 622 623 624 625 626
}

/*
 * Called when the last reference to the mm
 * is dropped: either by a lazy thread or by
 * mmput. Free the page directory and the mm.
 */
627
void __mmdrop(struct mm_struct *mm)
Linus Torvalds's avatar
Linus Torvalds committed
628 629 630 631
{
	BUG_ON(mm == &init_mm);
	mm_free_pgd(mm);
	destroy_context(mm);
Andrea Arcangeli's avatar
Andrea Arcangeli committed
632
	mmu_notifier_mm_destroy(mm);
633
	check_mm(mm);
Linus Torvalds's avatar
Linus Torvalds committed
634 635
	free_mm(mm);
}
636
EXPORT_SYMBOL_GPL(__mmdrop);
Linus Torvalds's avatar
Linus Torvalds committed
637

Rohit Gupta's avatar
Rohit Gupta committed
638 639 640 641 642 643 644 645 646 647 648 649 650
#ifdef CONFIG_PREEMPT_RT_BASE
/*
 * RCU callback for delayed mm drop. Not strictly rcu, but we don't
 * want another facility to make this work.
 */
void __mmdrop_delayed(struct rcu_head *rhp)
{
	struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop);

	__mmdrop(mm);
}
#endif

Linus Torvalds's avatar
Linus Torvalds committed
651 652 653 654 655
/*
 * Decrement the use count and release all resources for an mm.
 */
void mmput(struct mm_struct *mm)
{
656 657
	might_sleep();

Linus Torvalds's avatar
Linus Torvalds committed
658
	if (atomic_dec_and_test(&mm->mm_users)) {
659
		uprobe_clear_state(mm);
Linus Torvalds's avatar
Linus Torvalds committed
660
		exit_aio(mm);
661
		ksm_exit(mm);
Andrea Arcangeli's avatar
Andrea Arcangeli committed
662
		khugepaged_exit(mm); /* must run before exit_mmap */
Linus Torvalds's avatar
Linus Torvalds committed
663
		exit_mmap(mm);
Matt Helsley's avatar
Matt Helsley committed
664
		set_mm_exe_file(mm, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
665 666 667 668 669
		if (!list_empty(&mm->mmlist)) {
			spin_lock(&mmlist_lock);
			list_del(&mm->mmlist);
			spin_unlock(&mmlist_lock);
		}
670 671
		if (mm->binfmt)
			module_put(mm->binfmt->module);
Linus Torvalds's avatar
Linus Torvalds committed
672 673 674 675 676
		mmdrop(mm);
	}
}
EXPORT_SYMBOL_GPL(mmput);

677 678 679 680 681 682 683 684 685 686 687 688 689
void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
{
	if (new_exe_file)
		get_file(new_exe_file);
	if (mm->exe_file)
		fput(mm->exe_file);
	mm->exe_file = new_exe_file;
}

struct file *get_mm_exe_file(struct mm_struct *mm)
{
	struct file *exe_file;

690
	/* We need mmap_sem to protect against races with removal of exe_file */
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705
	down_read(&mm->mmap_sem);
	exe_file = mm->exe_file;
	if (exe_file)
		get_file(exe_file);
	up_read(&mm->mmap_sem);
	return exe_file;
}

static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm)
{
	/* It's safe to write the exe_file pointer without exe_file_lock because
	 * this is called during fork when the task is not yet in /proc */
	newmm->exe_file = get_mm_exe_file(oldmm);
}

Linus Torvalds's avatar
Linus Torvalds committed
706 707 708
/**
 * get_task_mm - acquire a reference to the task's mm
 *
709
 * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
Linus Torvalds's avatar
Linus Torvalds committed
710 711 712 713 714 715 716 717 718 719 720 721
 * this kernel workthread has transiently adopted a user mm with use_mm,
 * to do its AIO) is not set and if so returns a reference to it, after
 * bumping up the use count.  User must release the mm via mmput()
 * after use.  Typically used by /proc and ptrace.
 */
struct mm_struct *get_task_mm(struct task_struct *task)
{
	struct mm_struct *mm;

	task_lock(task);
	mm = task->mm;
	if (mm) {
722
		if (task->flags & PF_KTHREAD)
Linus Torvalds's avatar
Linus Torvalds committed
723 724 725 726 727 728 729 730 731
			mm = NULL;
		else
			atomic_inc(&mm->mm_users);
	}
	task_unlock(task);
	return mm;
}
EXPORT_SYMBOL_GPL(get_task_mm);

732 733 734 735 736 737 738 739 740 741 742
struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
{
	struct mm_struct *mm;
	int err;

	err =  mutex_lock_killable(&task->signal->cred_guard_mutex);
	if (err)
		return ERR_PTR(err);

	mm = get_task_mm(task);
	if (mm && mm != current->mm &&
Mauro Ribeiro's avatar
Mauro Ribeiro committed
743 744
			!ptrace_may_access(task, mode) &&
			!capable(CAP_SYS_RESOURCE)) {
745 746 747 748 749 750 751 752
		mmput(mm);
		mm = ERR_PTR(-EACCES);
	}
	mutex_unlock(&task->signal->cred_guard_mutex);

	return mm;
}

753
static void complete_vfork_done(struct task_struct *tsk)
754
{
Oleg Nesterov's avatar
Oleg Nesterov committed
755
	struct completion *vfork;
756

Oleg Nesterov's avatar
Oleg Nesterov committed
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782
	task_lock(tsk);
	vfork = tsk->vfork_done;
	if (likely(vfork)) {
		tsk->vfork_done = NULL;
		complete(vfork);
	}
	task_unlock(tsk);
}

static int wait_for_vfork_done(struct task_struct *child,
				struct completion *vfork)
{
	int killed;

	freezer_do_not_count();
	killed = wait_for_completion_killable(vfork);
	freezer_count();

	if (killed) {
		task_lock(child);
		child->vfork_done = NULL;
		task_unlock(child);
	}

	put_task_struct(child);
	return killed;
783 784
}

Linus Torvalds's avatar
Linus Torvalds committed
785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
/* Please note the differences between mmput and mm_release.
 * mmput is called whenever we stop holding onto a mm_struct,
 * error success whatever.
 *
 * mm_release is called after a mm_struct has been removed
 * from the current process.
 *
 * This difference is important for error handling, when we
 * only half set up a mm_struct for a new process and need to restore
 * the old one.  Because we mmput the new mm_struct before
 * restoring the old one. . .
 * Eric Biederman 10 January 1998
 */
void mm_release(struct task_struct *tsk, struct mm_struct *mm)
{
800 801
	/* Get rid of any futexes when releasing the mm */
#ifdef CONFIG_FUTEX
802
	if (unlikely(tsk->robust_list)) {
803
		exit_robust_list(tsk);
804 805
		tsk->robust_list = NULL;
	}
806
#ifdef CONFIG_COMPAT
807
	if (unlikely(tsk->compat_robust_list)) {
808
		compat_exit_robust_list(tsk);
809 810
		tsk->compat_robust_list = NULL;
	}
811
#endif
812 813
	if (unlikely(!list_empty(&tsk->pi_state_list)))
		exit_pi_state_list(tsk);
814 815
#endif

816 817
	uprobe_free_utask(tsk);

Linus Torvalds's avatar
Linus Torvalds committed
818 819 820
	/* Get rid of any cached register state */
	deactivate_mm(tsk, mm);

821 822 823 824
	/*
	 * If we're exiting normally, clear a user-space tid field if
	 * requested.  We leave this alone when dying by signal, to leave
	 * the value intact in a core dump, and to save the unnecessary
Oleg Nesterov's avatar
Oleg Nesterov committed
825 826
	 * trouble, say, a killed vfork parent shouldn't touch this mm.
	 * Userland only wants this done for a sys_exit.
827
	 */
828 829 830 831 832 833 834 835 836 837 838
	if (tsk->clear_child_tid) {
		if (!(tsk->flags & PF_SIGNALED) &&
		    atomic_read(&mm->mm_users) > 1) {
			/*
			 * We don't check the error code - if userspace has
			 * not set up a proper pointer then tough luck.
			 */
			put_user(0, tsk->clear_child_tid);
			sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
					1, NULL, NULL, 0);
		}
Linus Torvalds's avatar
Linus Torvalds committed
839 840
		tsk->clear_child_tid = NULL;
	}
841 842 843 844 845 846 847

	/*
	 * All done, finally we can wake up parent and return this mm to him.
	 * Also kthread_stop() uses this completion for synchronization.
	 */
	if (tsk->vfork_done)
		complete_vfork_done(tsk);
Linus Torvalds's avatar
Linus Torvalds committed
848 849
}

850 851 852 853
/*
 * Allocate a new mm structure and copy contents from the
 * mm structure of the passed in task structure.
 */
854
struct mm_struct *dup_mm(struct task_struct *tsk)
855 856 857 858 859 860 861 862 863 864 865 866
{
	struct mm_struct *mm, *oldmm = current->mm;
	int err;

	if (!oldmm)
		return NULL;

	mm = allocate_mm();
	if (!mm)
		goto fail_nomem;

	memcpy(mm, oldmm, sizeof(*mm));
867
	mm_init_cpumask(mm);
868

869 870
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	mm->pmd_huge_pte = NULL;
871 872 873
#endif
#ifdef CONFIG_NUMA_BALANCING
	mm->first_nid = NUMA_PTE_SCAN_INIT;
874
#endif
875
	if (!mm_init(mm, tsk))
876 877 878 879 880
		goto fail_nomem;

	if (init_new_context(tsk, mm))
		goto fail_nocontext;

Matt Helsley's avatar
Matt Helsley committed
881 882
	dup_mm_exe_file(oldmm, mm);

883 884 885 886 887 888 889
	err = dup_mmap(mm, oldmm);
	if (err)
		goto free_pt;

	mm->hiwater_rss = get_mm_rss(mm);
	mm->hiwater_vm = mm->total_vm;

890 891 892
	if (mm->binfmt && !try_module_get(mm->binfmt->module))
		goto free_pt;

893 894 895
	return mm;

free_pt:
896 897
	/* don't put binfmt in mmput, we haven't got module yet */
	mm->binfmt = NULL;
898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
	mmput(mm);

fail_nomem:
	return NULL;

fail_nocontext:
	/*
	 * If init_new_context() failed, we cannot use mmput() to free the mm
	 * because it calls destroy_context()
	 */
	mm_free_pgd(mm);
	free_mm(mm);
	return NULL;
}

913
static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds's avatar
Linus Torvalds committed
914
{
915
	struct mm_struct *mm, *oldmm;
Linus Torvalds's avatar
Linus Torvalds committed
916 917 918 919
	int retval;

	tsk->min_flt = tsk->maj_flt = 0;
	tsk->nvcsw = tsk->nivcsw = 0;
920 921 922
#ifdef CONFIG_DETECT_HUNG_TASK
	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942

	tsk->mm = NULL;
	tsk->active_mm = NULL;

	/*
	 * Are we cloning a kernel thread?
	 *
	 * We need to steal a active VM for that..
	 */
	oldmm = current->mm;
	if (!oldmm)
		return 0;

	if (clone_flags & CLONE_VM) {
		atomic_inc(&oldmm->mm_users);
		mm = oldmm;
		goto good_mm;
	}

	retval = -ENOMEM;
943
	mm = dup_mm(tsk);
Linus Torvalds's avatar
Linus Torvalds committed
944 945 946 947 948 949 950 951 952 953 954 955
	if (!mm)
		goto fail_nomem;

good_mm:
	tsk->mm = mm;
	tsk->active_mm = mm;
	return 0;

fail_nomem:
	return retval;
}

Alexey Dobriyan's avatar
Alexey Dobriyan committed
956
static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds's avatar
Linus Torvalds committed
957
{
958
	struct fs_struct *fs = current->fs;
Linus Torvalds's avatar
Linus Torvalds committed
959
	if (clone_flags & CLONE_FS) {
960
		/* tsk->fs is already what we want */
961
		spin_lock(&fs->lock);
962
		if (fs->in_exec) {
963
			spin_unlock(&fs->lock);
964 965 966
			return -EAGAIN;
		}
		fs->users++;
967
		spin_unlock(&fs->lock);
Linus Torvalds's avatar
Linus Torvalds committed
968 969
		return 0;
	}
970
	tsk->fs = copy_fs_struct(fs);
Linus Torvalds's avatar
Linus Torvalds committed
971 972 973 974 975
	if (!tsk->fs)
		return -ENOMEM;
	return 0;
}

976
static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
{
	struct files_struct *oldf, *newf;
	int error = 0;

	/*
	 * A background process may not have any files ...
	 */
	oldf = current->files;
	if (!oldf)
		goto out;

	if (clone_flags & CLONE_FILES) {
		atomic_inc(&oldf->count);
		goto out;
	}

	newf = dup_fd(oldf, &error);
	if (!newf)
		goto out;

	tsk->files = newf;
	error = 0;
out:
	return error;
}

1003
static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
1004 1005 1006
{
#ifdef CONFIG_BLOCK
	struct io_context *ioc = current->io_context;
1007
	struct io_context *new_ioc;
1008 1009 1010

	if (!ioc)
		return 0;
1011 1012 1013 1014
	/*
	 * Share io context with parent, if CLONE_IO is set
	 */
	if (clone_flags & CLONE_IO) {
1015 1016
		ioc_task_link(ioc);
		tsk->io_context = ioc;
1017
	} else if (ioprio_valid(ioc->ioprio)) {
1018 1019
		new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
		if (unlikely(!new_ioc))
1020 1021
			return -ENOMEM;

1022
		new_ioc->ioprio = ioc->ioprio;
1023
		put_io_context(new_ioc);
1024 1025 1026 1027 1028
	}
#endif
	return 0;
}

Alexey Dobriyan's avatar
Alexey Dobriyan committed
1029
static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds's avatar
Linus Torvalds committed
1030 1031 1032
{
	struct sighand_struct *sig;

1033
	if (clone_flags & CLONE_SIGHAND) {
Linus Torvalds's avatar
Linus Torvalds committed
1034 1035 1036 1037
		atomic_inc(&current->sighand->count);
		return 0;
	}
	sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1038
	rcu_assign_pointer(tsk->sighand, sig);
Linus Torvalds's avatar
Linus Torvalds committed
1039 1040 1041 1042 1043 1044 1045
	if (!sig)
		return -ENOMEM;
	atomic_set(&sig->count, 1);
	memcpy(sig->action, current->sighand->action, sizeof(sig->action));
	return 0;
}

1046
void __cleanup_sighand(struct sighand_struct *sighand)
1047
{
1048 1049
	if (atomic_dec_and_test(&sighand->count)) {
		signalfd_cleanup(sighand);
1050
		kmem_cache_free(sighand_cachep, sighand);
1051
	}
1052 1053
}

1054 1055 1056 1057 1058 1059

/*
 * Initialize POSIX timer handling for a thread group.
 */
static void posix_cpu_timers_init_group(struct signal_struct *sig)
{
1060 1061
	unsigned long cpu_limit;

1062 1063 1064
	/* Thread group counters. */
	thread_group_cputime_init(sig);

1065 1066 1067
	cpu_limit = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
	if (cpu_limit != RLIM_INFINITY) {
		sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit);
1068 1069 1070
		sig->cputimer.running = 1;
	}

1071 1072 1073 1074 1075 1076
	/* The timer lists. */
	INIT_LIST_HEAD(&sig->cpu_timers[0]);
	INIT_LIST_HEAD(&sig->cpu_timers[1]);
	INIT_LIST_HEAD(&sig->cpu_timers[2]);
}

Alexey Dobriyan's avatar
Alexey Dobriyan committed
1077
static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds's avatar
Linus Torvalds committed
1078 1079 1080
{
	struct signal_struct *sig;

1081
	if (clone_flags & CLONE_THREAD)
1082 1083
		return 0;

1084
	sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
Linus Torvalds's avatar
Linus Torvalds committed
1085 1086 1087 1088
	tsk->signal = sig;
	if (!sig)
		return -ENOMEM;

1089
	sig->nr_threads = 1;
Linus Torvalds's avatar
Linus Torvalds committed
1090
	atomic_set(&sig->live, 1);
1091
	atomic_set(&sig->sigcnt, 1);
1092 1093 1094 1095 1096

	/* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
	sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
	tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);

Linus Torvalds's avatar
Linus Torvalds committed
1097
	init_waitqueue_head(&sig->wait_chldexit);
1098
	sig->curr_target = tsk;
Linus Torvalds's avatar
Linus Torvalds committed
1099 1100 1101
	init_sigpending(&sig->shared_pending);
	INIT_LIST_HEAD(&sig->posix_timers);

1102
	hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
Linus Torvalds's avatar
Linus Torvalds committed
1103 1104 1105 1106 1107 1108
	sig->real_timer.function = it_real_fn;

	task_lock(current->group_leader);
	memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
	task_unlock(current->group_leader);

1109 1110
	posix_cpu_timers_init_group(sig);

1111
	tty_audit_fork(sig);
1112
	sched_autogroup_fork(sig);
1113

1114
#ifdef CONFIG_CGROUPS
1115
	init_rwsem(&sig->group_rwsem);
1116 1117
#endif

1118
	sig->oom_score_adj = current->signal->oom_score_adj;
1119
	sig->oom_score_adj_min = current->signal->oom_score_adj_min;
1120

1121 1122 1123
	sig->has_child_subreaper = current->signal->has_child_subreaper ||
				   current->signal->is_child_subreaper;

1124 1125
	mutex_init(&sig->cred_guard_mutex);

Linus Torvalds's avatar
Linus Torvalds committed
1126 1127 1128
	return 0;
}

Alexey Dobriyan's avatar
Alexey Dobriyan committed
1129
static void copy_flags(unsigned long clone_flags, struct task_struct *p)
Linus Torvalds's avatar
Linus Torvalds committed
1130 1131 1132
{
	unsigned long new_flags = p->flags;

1133
	new_flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
Linus Torvalds's avatar
Linus Torvalds committed
1134 1135 1136 1137
	new_flags |= PF_FORKNOEXEC;
	p->flags = new_flags;
}

1138
SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
Linus Torvalds's avatar
Linus Torvalds committed
1139 1140 1141
{
	current->clear_child_tid = tidptr;

1142
	return task_pid_vnr(current);
Linus Torvalds's avatar
Linus Torvalds committed
1143 1144
}

Alexey Dobriyan's avatar
Alexey Dobriyan committed
1145
static void rt_mutex_init_task(struct task_struct *p)
1146
{
1147
	raw_spin_lock_init(&p->pi_lock);
1148
#ifdef CONFIG_RT_MUTEXES
1149
	plist_head_init(&p->pi_waiters);
1150 1151 1152 1153
	p->pi_blocked_on = NULL;
#endif
}

1154 1155 1156 1157 1158 1159 1160
#ifdef CONFIG_MM_OWNER
void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
{
	mm->owner = p;
}
#endif /* CONFIG_MM_OWNER */

1161 1162 1163 1164 1165
/*
 * Initialize POSIX timer handling for a single task.
 */
static void posix_cpu_timers_init(struct task_struct *tsk)
{
Rohit Gupta's avatar
Rohit Gupta committed
1166 1167 1168
#ifdef CONFIG_PREEMPT_RT_BASE
	tsk->posix_timer_list = NULL;
#endif
1169 1170
	tsk->cputime_expires.prof_exp = 0;
	tsk->cputime_expires.virt_exp = 0;
1171 1172 1173 1174 1175 1176
	tsk->cputime_expires.sched_exp = 0;
	INIT_LIST_HEAD(&tsk->cpu_timers[0]);
	INIT_LIST_HEAD(&tsk->cpu_timers[1]);
	INIT_LIST_HEAD(&tsk->cpu_timers[2]);
}

Linus Torvalds's avatar
Linus Torvalds committed
1177 1178 1179 1180 1181 1182 1183 1184
/*
 * This creates a new process as a copy of the old one,
 * but does not actually start it yet.
 *
 * It copies the registers, and all the appropriate
 * parts of the process environment (as per the clone
 * flags). The actual kick-off is left to the caller.
 */
1185 1186 1187 1188
static struct task_struct *copy_process(unsigned long clone_flags,
					unsigned long stack_start,
					unsigned long stack_size,
					int __user *child_tidptr,
Roland McGrath's avatar
Roland McGrath committed
1189 1190
					struct pid *pid,
					int trace)
Linus Torvalds's avatar
Linus Torvalds committed
1191 1192
{
	int retval;
1193
	struct task_struct *p;
Linus Torvalds's avatar
Linus Torvalds committed
1194 1195 1196 1197

	if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
		return ERR_PTR(-EINVAL);

1198 1199 1200
	if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
		return ERR_PTR(-EINVAL);

Linus Torvalds's avatar
Linus Torvalds committed
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215
	/*
	 * Thread groups must share signals as well, and detached threads
	 * can only be started up within the thread group.
	 */
	if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
		return ERR_PTR(-EINVAL);

	/*
	 * Shared signal handlers imply shared VM. By way of the above,
	 * thread groups also imply shared VM. Blocking this case allows
	 * for various simplifications in other code.
	 */
	if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
		return ERR_PTR(-EINVAL);

1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
	/*
	 * Siblings of global init remain as zombies on exit since they are
	 * not reaped by their parent (swapper). To solve this and to avoid
	 * multi-rooted process trees, prevent global and container-inits
	 * from creating siblings.
	 */
	if ((clone_flags & CLONE_PARENT) &&
				current->signal->flags & SIGNAL_UNKILLABLE)
		return ERR_PTR(-EINVAL);

1226
	/*
1227 1228 1229
	 * If the new process will be in a different pid namespace don't
	 * allow it to share a thread group or signal handlers with the
	 * forking task.
1230
	 */
1231
	if ((clone_flags & (CLONE_SIGHAND | CLONE_NEWPID)) &&
1232 1233 1234
	    (task_active_pid_ns(current) != current->nsproxy->pid_ns))
		return ERR_PTR(-EINVAL);

Linus Torvalds's avatar
Linus Torvalds committed
1235 1236 1237 1238 1239 1240 1241 1242 1243
	retval = security_task_create(clone_flags);
	if (retval)
		goto fork_out;

	retval = -ENOMEM;
	p = dup_task_struct(current);
	if (!p)
		goto fork_out;

1244
	ftrace_graph_init_task(p);
1245
	get_seccomp_filter(p);
1246

1247 1248
	rt_mutex_init_task(p);

1249
#ifdef CONFIG_PROVE_LOCKING
1250 1251 1252
	DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
	DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1253
	retval = -EAGAIN;
1254
	if (atomic_read(&p->real_cred->user->processes) >=
1255
			task_rlimit(p, RLIMIT_NPROC)) {
Linus Torvalds's avatar
Linus Torvalds committed
1256
		if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) &&
1257
		    p->real_cred->user != INIT_USER)
Linus Torvalds's avatar
Linus Torvalds committed
1258 1259
			goto bad_fork_free;
	}
1260
	current->flags &= ~PF_NPROC_EXCEEDED;
Linus Torvalds's avatar
Linus Torvalds committed
1261

1262 1263 1264
	retval = copy_creds(p, clone_flags);
	if (retval < 0)
		goto bad_fork_free;
Linus Torvalds's avatar
Linus Torvalds committed
1265 1266 1267 1268 1269 1270

	/*
	 * If multiple threads are within copy_process(), then this check
	 * triggers too late. This doesn't hurt, the check is only there
	 * to stop root fork bombs.
	 */
1271
	retval = -EAGAIN;
Linus Torvalds's avatar
Linus Torvalds committed
1272 1273 1274
	if (nr_threads >= max_threads)
		goto bad_fork_cleanup_count;

1275
	if (!try_module_get(task_thread_info(p)->exec_domain->module))
Linus Torvalds's avatar
Linus Torvalds committed
1276 1277 1278
		goto bad_fork_cleanup_count;

	p->did_exec = 0;
1279
	delayacct_tsk_init(p);	/* Must remain after dup_task_struct() */
Linus Torvalds's avatar
Linus Torvalds committed
1280 1281 1282
	copy_flags(clone_flags, p);
	INIT_LIST_HEAD(&p->children);
	INIT_LIST_HEAD(&p->sibling);
1283
	rcu_copy_process(p);
Linus Torvalds's avatar
Linus Torvalds committed
1284 1285 1286 1287
	p->vfork_done = NULL;
	spin_lock_init(&p->alloc_lock);

	init_sigpending(&p->pending);
Rohit Gupta's avatar
Rohit Gupta committed
1288
	p->sigqueue_cache = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1289

1290 1291
	p->utime = p->stime = p->gtime = 0;
	p->utimescaled = p->stimescaled = 0;
1292
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
1293
	p->prev_cputime.utime = p->prev_cputime.stime = 0;
1294
#endif
1295
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
Rohit Gupta's avatar
Rohit Gupta committed
1296 1297
	raw_spin_lock_init(&p->vtime_lock);
	seqcount_init(&p->vtime_seq);
1298 1299 1300 1301
	p->vtime_snap = 0;
	p->vtime_snap_whence = VTIME_SLEEPING;
#endif

1302 1303 1304
#if defined(SPLIT_RSS_COUNTING)
	memset(&p->rss_stat, 0, sizeof(p->rss_stat));
#endif
1305

1306 1307
	p->default_timer_slack_ns = current->timer_slack_ns;

1308
	task_io_accounting_init(&p->ioac);
Linus Torvalds's avatar
Linus Torvalds committed
1309 1310
	acct_clear_integrals(p);

1311
	posix_cpu_timers_init(p);
Linus Torvalds's avatar
Linus Torvalds committed
1312 1313

	do_posix_clock_monotonic_gettime(&p->start_time);
1314 1315
	p->real_start_time = p->start_time;
	monotonic_to_bootbased(&p->real_start_time);
Linus Torvalds's avatar
Linus Torvalds committed
1316 1317
	p->io_context = NULL;
	p->audit_context = NULL;
1318
	if (clone_flags & CLONE_THREAD)
1319
		threadgroup_change_begin(current);
1320
	cgroup_fork(p);
Linus Torvalds's avatar
Linus Torvalds committed
1321
#ifdef CONFIG_NUMA
1322
	p->mempolicy = mpol_dup(p->mempolicy);
1323 1324 1325 1326 1327
	if (IS_ERR(p->mempolicy)) {
		retval = PTR_ERR(p->mempolicy);
		p->mempolicy = NULL;
		goto bad_fork_cleanup_cgroup;
	}
1328
	mpol_fix_fork_child_flag(p);
Linus Torvalds's avatar
Linus Torvalds committed
1329
#endif
1330 1331 1332
#ifdef CONFIG_CPUSETS
	p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
	p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
1333
	seqcount_init(&p->mems_allowed_seq);
1334
#endif
1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
#ifdef CONFIG_TRACE_IRQFLAGS
	p->irq_events = 0;
	p->hardirqs_enabled = 0;
	p->hardirq_enable_ip = 0;
	p->hardirq_enable_event = 0;
	p->hardirq_disable_ip = _THIS_IP_;
	p->hardirq_disable_event = 0;
	p->softirqs_enabled = 1;
	p->softirq_enable_ip = _THIS_IP_;
	p->softirq_enable_event = 0;
	p->softirq_disable_ip = 0;
	p->softirq_disable_event = 0;
	p->hardirq_context = 0;
	p->softirq_context = 0;
#endif
Rohit Gupta's avatar
Rohit Gupta committed
1350 1351 1352
#ifdef CONFIG_PREEMPT_RT_FULL
	p->pagefault_disabled = 0;
#endif
Ingo Molnar's avatar
Ingo Molnar committed
1353 1354 1355 1356 1357
#ifdef CONFIG_LOCKDEP
	p->lockdep_depth = 0; /* no locks held yet */
	p->curr_chain_key = 0;
	p->lockdep_recursion = 0;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1358

1359 1360 1361
#ifdef CONFIG_DEBUG_MUTEXES
	p->blocked_on = NULL; /* not blocked yet */
#endif
1362
#ifdef CONFIG_MEMCG