From 35f71bc0a09a45924bed268d8ccd0d3407bc476f Mon Sep 17 00:00:00 2001 From: Michal Hocko Date: Thu, 16 Apr 2015 12:47:38 -0700 Subject: fork: report pid reservation failure properly copy_process will report any failure in alloc_pid as ENOMEM currently which is misleading because the pid allocation might fail not only when the memory is short but also when the pid space is consumed already. The current man page even mentions this case: : EAGAIN : : A system-imposed limit on the number of threads was encountered. : There are a number of limits that may trigger this error: the : RLIMIT_NPROC soft resource limit (set via setrlimit(2)), which : limits the number of processes and threads for a real user ID, was : reached; the kernel's system-wide limit on the number of processes : and threads, /proc/sys/kernel/threads-max, was reached (see : proc(5)); or the maximum number of PIDs, /proc/sys/kernel/pid_max, : was reached (see proc(5)). so the current behavior is also incorrect wrt. documentation. POSIX man page also suggest returing EAGAIN when the process count limit is reached. This patch simply propagates error code from alloc_pid and makes sure we return -EAGAIN due to reservation failure. This will make behavior of fork closer to both our documentation and POSIX. alloc_pid might alsoo fail when the reaper in the pid namespace is dead (the namespace basically disallows all new processes) and there is no good error code which would match documented ones. We have traditionally returned ENOMEM for this case which is misleading as well but as per Eric W. Biederman this behavior is documented in man pid_namespaces(7) : If the "init" process of a PID namespace terminates, the kernel : terminates all of the processes in the namespace via a SIGKILL signal. : This behavior reflects the fact that the "init" process is essential for : the correct operation of a PID namespace. In this case, a subsequent : fork(2) into this PID namespace will fail with the error ENOMEM; it is : not possible to create a new processes in a PID namespace whose "init" : process has terminated. and introducing a new error code would be too risky so let's stick to ENOMEM for this case. Signed-off-by: Michal Hocko Cc: Oleg Nesterov Cc: "Eric W. Biederman" Cc: Michael Kerrisk Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/fork.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel/fork.c') diff --git a/kernel/fork.c b/kernel/fork.c index f2c1e7352298..d778016ac1e3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1403,10 +1403,11 @@ static struct task_struct *copy_process(unsigned long clone_flags, goto bad_fork_cleanup_io; if (pid != &init_struct_pid) { - retval = -ENOMEM; pid = alloc_pid(p->nsproxy->pid_ns_for_children); - if (!pid) + if (IS_ERR(pid)) { + retval = PTR_ERR(pid); goto bad_fork_cleanup_io; + } } p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; -- cgit v1.2.3 From 3ea7f5e25ec271909451b7dc17be37581b888de6 Mon Sep 17 00:00:00 2001 From: Jean Delvare Date: Thu, 16 Apr 2015 12:47:41 -0700 Subject: fork_init: update max_threads comment The comment explaining what value max_threads is set to is outdated. The maximum memory consumption ratio for thread structures was 1/2 until February 2002, then it was briefly changed to 1/16 before being set to 1/8 which we still use today. The comment was never updated to reflect that change, it's about time. Signed-off-by: Jean Delvare Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/fork.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/fork.c') diff --git a/kernel/fork.c b/kernel/fork.c index d778016ac1e3..c507e29bcb01 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -270,8 +270,8 @@ void __init fork_init(unsigned long mempages) /* * The default maximum number of threads is set to a safe - * value: the thread structures can take up at most half - * of memory. + * value: the thread structures can take up at most one + * eighth of the memory. */ max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE); -- cgit v1.2.3 From ff691f6e03815dc8f99461ea509df863a879fc3a Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Thu, 16 Apr 2015 12:47:44 -0700 Subject: kernel/fork.c: new function for max_threads PAGE_SIZE is not guaranteed to be equal to or less than 8 times the THREAD_SIZE. E.g. architecture hexagon may have page size 1M and thread size 4096. This would lead to a division by zero in the calculation of max_threads. With this patch the buggy code is moved to a separate function set_max_threads. The error is not fixed. After fixing the problem in a separate patch the new function can be reused to adjust max_threads after adding or removing memory. Argument mempages of function fork_init() is removed as totalram_pages is an exported symbol. The creation of separate patches for refactoring to a new function and for fixing the logic was suggested by Ingo Molnar. Signed-off-by: Heinrich Schuchardt Cc: Oleg Nesterov Cc: Ingo Molnar Cc: Guenter Roeck Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- init/main.c | 4 ++-- kernel/fork.c | 34 +++++++++++++++++++++------------- 2 files changed, 23 insertions(+), 15 deletions(-) (limited to 'kernel/fork.c') diff --git a/init/main.c b/init/main.c index a7e969d12f51..2115055faeac 100644 --- a/init/main.c +++ b/init/main.c @@ -91,7 +91,7 @@ static int kernel_init(void *); extern void init_IRQ(void); -extern void fork_init(unsigned long); +extern void fork_init(void); extern void radix_tree_init(void); #ifndef CONFIG_DEBUG_RODATA static inline void mark_rodata_ro(void) { } @@ -645,7 +645,7 @@ asmlinkage __visible void __init start_kernel(void) #endif thread_info_cache_init(); cred_init(); - fork_init(totalram_pages); + fork_init(); proc_caches_init(); buffer_init(); key_init(); diff --git a/kernel/fork.c b/kernel/fork.c index c507e29bcb01..01038e6f51a8 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -253,7 +253,26 @@ EXPORT_SYMBOL_GPL(__put_task_struct); void __init __weak arch_task_cache_init(void) { } -void __init fork_init(unsigned long mempages) +/* + * set_max_threads + */ +static void set_max_threads(void) +{ + /* + * The default maximum number of threads is set to a safe + * value: the thread structures can take up at most one + * eighth of the memory. + */ + max_threads = totalram_pages / (8 * THREAD_SIZE / PAGE_SIZE); + + /* + * we need to allow at least 20 threads to boot a system + */ + if (max_threads < 20) + max_threads = 20; +} + +void __init fork_init(void) { #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR #ifndef ARCH_MIN_TASKALIGN @@ -268,18 +287,7 @@ void __init fork_init(unsigned long mempages) /* do the arch specific task caches init */ arch_task_cache_init(); - /* - * The default maximum number of threads is set to a safe - * value: the thread structures can take up at most one - * eighth of the memory. - */ - max_threads = mempages / (8 * THREAD_SIZE / PAGE_SIZE); - - /* - * we need to allow at least 20 threads to boot a system - */ - if (max_threads < 20) - max_threads = 20; + set_max_threads(); init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; -- cgit v1.2.3 From ac1b398de1ef94aeee8ba87b0120763526572a6e Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Thu, 16 Apr 2015 12:47:47 -0700 Subject: kernel/fork.c: avoid division by zero PAGE_SIZE is not guaranteed to be equal to or less than 8 times the THREAD_SIZE. E.g. architecture hexagon may have page size 1M and thread size 4096. This would lead to a division by zero in the calculation of max_threads. With 32-bit calculation there is no solution which delivers valid results for all possible combinations of the parameters. The code is only called once. Hence a 64-bit calculation can be used as solution. [akpm@linux-foundation.org: use clamp_t(), per Oleg] Signed-off-by: Heinrich Schuchardt Cc: Oleg Nesterov Cc: Ingo Molnar Cc: Guenter Roeck Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/fork.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) (limited to 'kernel/fork.c') diff --git a/kernel/fork.c b/kernel/fork.c index 01038e6f51a8..c7f2e1a4187a 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -87,6 +87,16 @@ #define CREATE_TRACE_POINTS #include +/* + * Minimum number of threads to boot the kernel + */ +#define MIN_THREADS 20 + +/* + * Maximum number of threads + */ +#define MAX_THREADS FUTEX_TID_MASK + /* * Protected counters by write_lock_irq(&tasklist_lock) */ @@ -258,18 +268,19 @@ void __init __weak arch_task_cache_init(void) { } */ static void set_max_threads(void) { - /* - * The default maximum number of threads is set to a safe - * value: the thread structures can take up at most one - * eighth of the memory. - */ - max_threads = totalram_pages / (8 * THREAD_SIZE / PAGE_SIZE); + u64 threads; /* - * we need to allow at least 20 threads to boot a system + * The number of threads shall be limited such that the thread + * structures may only consume a small part of the available memory. */ - if (max_threads < 20) - max_threads = 20; + if (fls64(totalram_pages) + fls64(PAGE_SIZE) > 64) + threads = MAX_THREADS; + else + threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE, + (u64) THREAD_SIZE * 8UL); + + max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS); } void __init fork_init(void) -- cgit v1.2.3 From 16db3d3f1170fb0efca652c9378ce7c5f5cb4232 Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Thu, 16 Apr 2015 12:47:50 -0700 Subject: kernel/sysctl.c: threads-max observe limits Users can change the maximum number of threads by writing to /proc/sys/kernel/threads-max. With the patch the value entered is checked against the same limits that apply when fork_init is called. Signed-off-by: Heinrich Schuchardt Cc: Oleg Nesterov Cc: Ingo Molnar Cc: Guenter Roeck Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/sysctl.h | 3 +++ kernel/fork.c | 31 +++++++++++++++++++++++++++++-- kernel/sysctl.c | 6 ++---- 3 files changed, 34 insertions(+), 6 deletions(-) (limited to 'kernel/fork.c') diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index b7361f831226..795d5fea5697 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -212,4 +212,7 @@ static inline void setup_sysctl_set(struct ctl_table_set *p, #endif /* CONFIG_SYSCTL */ +int sysctl_max_threads(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + #endif /* _LINUX_SYSCTL_H */ diff --git a/kernel/fork.c b/kernel/fork.c index c7f2e1a4187a..8807a129711b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -74,6 +74,7 @@ #include #include #include +#include #include #include @@ -266,7 +267,7 @@ void __init __weak arch_task_cache_init(void) { } /* * set_max_threads */ -static void set_max_threads(void) +static void set_max_threads(unsigned int max_threads_suggested) { u64 threads; @@ -280,6 +281,9 @@ static void set_max_threads(void) threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE, (u64) THREAD_SIZE * 8UL); + if (threads > max_threads_suggested) + threads = max_threads_suggested; + max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS); } @@ -298,7 +302,7 @@ void __init fork_init(void) /* do the arch specific task caches init */ arch_task_cache_init(); - set_max_threads(); + set_max_threads(MAX_THREADS); init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; @@ -2020,3 +2024,26 @@ int unshare_files(struct files_struct **displaced) task_unlock(task); return 0; } + +int sysctl_max_threads(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table t; + int ret; + int threads = max_threads; + int min = MIN_THREADS; + int max = MAX_THREADS; + + t = *table; + t.data = &threads; + t.extra1 = &min; + t.extra2 = &max; + + ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (ret || !write) + return ret; + + set_max_threads(threads); + + return 0; +} diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 42b7fc2860c1..3c0998426b57 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -93,11 +93,9 @@ #include #endif - #if defined(CONFIG_SYSCTL) /* External variables not in a header file. */ -extern int max_threads; extern int suid_dumpable; #ifdef CONFIG_COREDUMP extern int core_uses_pid; @@ -710,10 +708,10 @@ static struct ctl_table kern_table[] = { #endif { .procname = "threads-max", - .data = &max_threads, + .data = NULL, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = sysctl_max_threads, }, { .procname = "random", -- cgit v1.2.3 From 90f31d0ea88880f780574f3d0bb1a227c4c66ca3 Mon Sep 17 00:00:00 2001 From: Konstantin Khlebnikov Date: Thu, 16 Apr 2015 12:47:56 -0700 Subject: mm: rcu-protected get_mm_exe_file() This patch removes mm->mmap_sem from mm->exe_file read side. Also it kills dup_mm_exe_file() and moves exe_file duplication into dup_mmap() where both mmap_sems are locked. [akpm@linux-foundation.org: fix comment typo] Signed-off-by: Konstantin Khlebnikov Cc: Davidlohr Bueso Cc: Al Viro Cc: Oleg Nesterov Cc: "Paul E. McKenney" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/file.c | 3 +-- include/linux/fs.h | 1 + include/linux/mm_types.h | 2 +- kernel/fork.c | 56 ++++++++++++++++++++++++++++++++---------------- 4 files changed, 40 insertions(+), 22 deletions(-) (limited to 'kernel/fork.c') diff --git a/fs/file.c b/fs/file.c index ee738ea028fa..93c5f89c248b 100644 --- a/fs/file.c +++ b/fs/file.c @@ -638,8 +638,7 @@ static struct file *__fget(unsigned int fd, fmode_t mask) file = fcheck_files(files, fd); if (file) { /* File object ref couldn't be taken */ - if ((file->f_mode & mask) || - !atomic_long_inc_not_zero(&file->f_count)) + if ((file->f_mode & mask) || !get_file_rcu(file)) file = NULL; } rcu_read_unlock(); diff --git a/include/linux/fs.h b/include/linux/fs.h index f4fc60727b8d..6bf7ab7c1573 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -870,6 +870,7 @@ static inline struct file *get_file(struct file *f) atomic_long_inc(&f->f_count); return f; } +#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count) #define fput_atomic(x) atomic_long_add_unless(&(x)->f_count, -1, 1) #define file_count(x) atomic_long_read(&(x)->f_count) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 590630eb59ba..8d37e26a1007 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -429,7 +429,7 @@ struct mm_struct { #endif /* store ref to file /proc//exe symlink points to */ - struct file *exe_file; + struct file __rcu *exe_file; #ifdef CONFIG_MMU_NOTIFIER struct mmu_notifier_mm *mmu_notifier_mm; #endif diff --git a/kernel/fork.c b/kernel/fork.c index 8807a129711b..259202637531 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -403,6 +403,9 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) */ down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); + /* No ordering required: file already has been exposed. */ + RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); + mm->total_vm = oldmm->total_vm; mm->shared_vm = oldmm->shared_vm; mm->exec_vm = oldmm->exec_vm; @@ -528,7 +531,13 @@ static inline void mm_free_pgd(struct mm_struct *mm) pgd_free(mm, mm->pgd); } #else -#define dup_mmap(mm, oldmm) (0) +static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) +{ + down_write(&oldmm->mmap_sem); + RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm)); + up_write(&oldmm->mmap_sem); + return 0; +} #define mm_alloc_pgd(mm) (0) #define mm_free_pgd(mm) #endif /* CONFIG_MMU */ @@ -697,35 +706,46 @@ void mmput(struct mm_struct *mm) } EXPORT_SYMBOL_GPL(mmput); +/** + * set_mm_exe_file - change a reference to the mm's executable file + * + * This changes mm's executable file (shown as symlink /proc/[pid]/exe). + * + * Main users are mmput(), sys_execve() and sys_prctl(PR_SET_MM_MAP/EXE_FILE). + * Callers prevent concurrent invocations: in mmput() nobody alive left, + * in execve task is single-threaded, prctl holds mmap_sem exclusively. + */ void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) { + struct file *old_exe_file = rcu_dereference_protected(mm->exe_file, + !atomic_read(&mm->mm_users) || current->in_execve || + lockdep_is_held(&mm->mmap_sem)); + if (new_exe_file) get_file(new_exe_file); - if (mm->exe_file) - fput(mm->exe_file); - mm->exe_file = new_exe_file; + rcu_assign_pointer(mm->exe_file, new_exe_file); + if (old_exe_file) + fput(old_exe_file); } +/** + * get_mm_exe_file - acquire a reference to the mm's executable file + * + * Returns %NULL if mm has no associated executable file. + * User must release file via fput(). + */ struct file *get_mm_exe_file(struct mm_struct *mm) { struct file *exe_file; - /* We need mmap_sem to protect against races with removal of exe_file */ - down_read(&mm->mmap_sem); - exe_file = mm->exe_file; - if (exe_file) - get_file(exe_file); - up_read(&mm->mmap_sem); + rcu_read_lock(); + exe_file = rcu_dereference(mm->exe_file); + if (exe_file && !get_file_rcu(exe_file)) + exe_file = NULL; + rcu_read_unlock(); return exe_file; } -static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm) -{ - /* It's safe to write the exe_file pointer without exe_file_lock because - * this is called during fork when the task is not yet in /proc */ - newmm->exe_file = get_mm_exe_file(oldmm); -} - /** * get_task_mm - acquire a reference to the task's mm * @@ -887,8 +907,6 @@ static struct mm_struct *dup_mm(struct task_struct *tsk) if (!mm_init(mm, tsk)) goto fail_nomem; - dup_mm_exe_file(oldmm, mm); - err = dup_mmap(mm, oldmm); if (err) goto free_pt; -- cgit v1.2.3 From 6e399cd144d8500ffb5d40fa6848890e2580a80a Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 16 Apr 2015 12:47:59 -0700 Subject: prctl: avoid using mmap_sem for exe_file serialization Oleg cleverly suggested using xchg() to set the new mm->exe_file instead of calling set_mm_exe_file() which requires some form of serialization -- mmap_sem in this case. For archs that do not have atomic rmw instructions we still fallback to a spinlock alternative, so this should always be safe. As such, we only need the mmap_sem for looking up the backing vm_file, which can be done sharing the lock. Naturally, this means we need to manually deal with both the new and old file reference counting, and we need not worry about the MMF_EXE_FILE_CHANGED bits, which can probably be deleted in the future anyway. Signed-off-by: Davidlohr Bueso Suggested-by: Oleg Nesterov Acked-by: Oleg Nesterov Reviewed-by: Konstantin Khlebnikov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/exec.c | 6 ++++++ kernel/fork.c | 19 +++++++++++++------ kernel/sys.c | 47 ++++++++++++++++++++++++++++------------------- 3 files changed, 47 insertions(+), 25 deletions(-) (limited to 'kernel/fork.c') diff --git a/fs/exec.c b/fs/exec.c index c7f9b733406d..a5fef835ebc5 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1078,7 +1078,13 @@ int flush_old_exec(struct linux_binprm * bprm) if (retval) goto out; + /* + * Must be called _before_ exec_mmap() as bprm->mm is + * not visibile until then. This also enables the update + * to be lockless. + */ set_mm_exe_file(bprm->mm, bprm->file); + /* * Release all of the old mmap stuff */ diff --git a/kernel/fork.c b/kernel/fork.c index 259202637531..0d23e76a0c61 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -711,15 +711,22 @@ EXPORT_SYMBOL_GPL(mmput); * * This changes mm's executable file (shown as symlink /proc/[pid]/exe). * - * Main users are mmput(), sys_execve() and sys_prctl(PR_SET_MM_MAP/EXE_FILE). - * Callers prevent concurrent invocations: in mmput() nobody alive left, - * in execve task is single-threaded, prctl holds mmap_sem exclusively. + * Main users are mmput() and sys_execve(). Callers prevent concurrent + * invocations: in mmput() nobody alive left, in execve task is single + * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the + * mm->exe_file, but does so without using set_mm_exe_file() in order + * to do avoid the need for any locks. */ void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) { - struct file *old_exe_file = rcu_dereference_protected(mm->exe_file, - !atomic_read(&mm->mm_users) || current->in_execve || - lockdep_is_held(&mm->mmap_sem)); + struct file *old_exe_file; + + /* + * It is safe to dereference the exe_file without RCU as + * this function is only called if nobody else can access + * this mm -- see comment above for justification. + */ + old_exe_file = rcu_dereference_raw(mm->exe_file); if (new_exe_file) get_file(new_exe_file); diff --git a/kernel/sys.c b/kernel/sys.c index 3be344902316..a4e372b798a5 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1649,14 +1649,13 @@ SYSCALL_DEFINE1(umask, int, mask) return mask; } -static int prctl_set_mm_exe_file_locked(struct mm_struct *mm, unsigned int fd) +static int prctl_set_mm_exe_file(struct mm_struct *mm, unsigned int fd) { struct fd exe; + struct file *old_exe, *exe_file; struct inode *inode; int err; - VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm); - exe = fdget(fd); if (!exe.file) return -EBADF; @@ -1680,15 +1679,22 @@ static int prctl_set_mm_exe_file_locked(struct mm_struct *mm, unsigned int fd) /* * Forbid mm->exe_file change if old file still mapped. */ + exe_file = get_mm_exe_file(mm); err = -EBUSY; - if (mm->exe_file) { + if (exe_file) { struct vm_area_struct *vma; - for (vma = mm->mmap; vma; vma = vma->vm_next) - if (vma->vm_file && - path_equal(&vma->vm_file->f_path, - &mm->exe_file->f_path)) - goto exit; + down_read(&mm->mmap_sem); + for (vma = mm->mmap; vma; vma = vma->vm_next) { + if (!vma->vm_file) + continue; + if (path_equal(&vma->vm_file->f_path, + &exe_file->f_path)) + goto exit_err; + } + + up_read(&mm->mmap_sem); + fput(exe_file); } /* @@ -1702,10 +1708,18 @@ static int prctl_set_mm_exe_file_locked(struct mm_struct *mm, unsigned int fd) goto exit; err = 0; - set_mm_exe_file(mm, exe.file); /* this grabs a reference to exe.file */ + /* set the new file, lockless */ + get_file(exe.file); + old_exe = xchg(&mm->exe_file, exe.file); + if (old_exe) + fput(old_exe); exit: fdput(exe); return err; +exit_err: + up_read(&mm->mmap_sem); + fput(exe_file); + goto exit; } #ifdef CONFIG_CHECKPOINT_RESTORE @@ -1840,10 +1854,9 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data user_auxv[AT_VECTOR_SIZE - 1] = AT_NULL; } - down_write(&mm->mmap_sem); if (prctl_map.exe_fd != (u32)-1) - error = prctl_set_mm_exe_file_locked(mm, prctl_map.exe_fd); - downgrade_write(&mm->mmap_sem); + error = prctl_set_mm_exe_file(mm, prctl_map.exe_fd); + down_read(&mm->mmap_sem); if (error) goto out; @@ -1909,12 +1922,8 @@ static int prctl_set_mm(int opt, unsigned long addr, if (!capable(CAP_SYS_RESOURCE)) return -EPERM; - if (opt == PR_SET_MM_EXE_FILE) { - down_write(&mm->mmap_sem); - error = prctl_set_mm_exe_file_locked(mm, (unsigned int)addr); - up_write(&mm->mmap_sem); - return error; - } + if (opt == PR_SET_MM_EXE_FILE) + return prctl_set_mm_exe_file(mm, (unsigned int)addr); if (addr >= TASK_SIZE || addr < mmap_min_addr) return -EINVAL; -- cgit v1.2.3 From 11163348a23cdbcdca5fb42485418e75f8566a5c Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 16 Apr 2015 12:49:12 -0700 Subject: oprofile: reduce mmap_sem hold for mm->exe_file sync_buffer() needs the mmap_sem for two distinct operations, both only occurring upon user context switch handling: 1) Dealing with the exe_file. 2) Adding the dcookie data as we need to lookup the vma that backs it. This is done via add_sample() and add_data(). This patch isolates 1), for it will no longer need the mmap_sem for serialization. However, for now, make of the more standard get_mm_exe_file(), requiring only holding the mmap_sem to read the value, and relying on reference counting to make sure that the exe file won't dissappear underneath us while doing the get dcookie. As a consequence, for 2) we move the mmap_sem locking into where we really need it, in lookup_dcookie(). The benefits are twofold: reduce mmap_sem hold times, and cleaner code. [akpm@linux-foundation.org: export get_mm_exe_file for arch/x86/oprofile/oprofile.ko] Signed-off-by: Davidlohr Bueso Cc: Robert Richter Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/oprofile/buffer_sync.c | 30 ++++++++++++++++-------------- kernel/fork.c | 1 + 2 files changed, 17 insertions(+), 14 deletions(-) (limited to 'kernel/fork.c') diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c index d93b2b6b1f7a..82f7000a285d 100644 --- a/drivers/oprofile/buffer_sync.c +++ b/drivers/oprofile/buffer_sync.c @@ -21,6 +21,7 @@ * objects. */ +#include #include #include #include @@ -224,10 +225,18 @@ static inline unsigned long fast_get_dcookie(struct path *path) static unsigned long get_exec_dcookie(struct mm_struct *mm) { unsigned long cookie = NO_COOKIE; + struct file *exe_file; - if (mm && mm->exe_file) - cookie = fast_get_dcookie(&mm->exe_file->f_path); + if (!mm) + goto done; + + exe_file = get_mm_exe_file(mm); + if (!exe_file) + goto done; + cookie = fast_get_dcookie(&exe_file->f_path); + fput(exe_file); +done: return cookie; } @@ -236,6 +245,8 @@ static unsigned long get_exec_dcookie(struct mm_struct *mm) * pair that can then be added to the global event buffer. We make * sure to do this lookup before a mm->mmap modification happens so * we don't lose track. + * + * The caller must ensure the mm is not nil (ie: not a kernel thread). */ static unsigned long lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) @@ -243,6 +254,7 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) unsigned long cookie = NO_COOKIE; struct vm_area_struct *vma; + down_read(&mm->mmap_sem); for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { if (addr < vma->vm_start || addr >= vma->vm_end) @@ -262,6 +274,7 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) if (!vma) cookie = INVALID_COOKIE; + up_read(&mm->mmap_sem); return cookie; } @@ -402,20 +415,9 @@ static void release_mm(struct mm_struct *mm) { if (!mm) return; - up_read(&mm->mmap_sem); mmput(mm); } - -static struct mm_struct *take_tasks_mm(struct task_struct *task) -{ - struct mm_struct *mm = get_task_mm(task); - if (mm) - down_read(&mm->mmap_sem); - return mm; -} - - static inline int is_code(unsigned long val) { return val == ESCAPE_CODE; @@ -532,7 +534,7 @@ void sync_buffer(int cpu) new = (struct task_struct *)val; oldmm = mm; release_mm(oldmm); - mm = take_tasks_mm(new); + mm = get_task_mm(new); if (mm != oldmm) cookie = get_exec_dcookie(mm); add_user_ctx_switch(new, cookie); diff --git a/kernel/fork.c b/kernel/fork.c index 0d23e76a0c61..03c1eaaa6ef5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -752,6 +752,7 @@ struct file *get_mm_exe_file(struct mm_struct *mm) rcu_read_unlock(); return exe_file; } +EXPORT_SYMBOL(get_mm_exe_file); /** * get_task_mm - acquire a reference to the task's mm -- cgit v1.2.3