From 1a23395672658969a4035dcc518ea6cab835c579 Mon Sep 17 00:00:00 2001 From: Manfred Spraul Date: Wed, 12 Jul 2017 14:34:38 -0700 Subject: ipc/sem.c: remove sem_base, embed struct sem sma->sem_base is initialized with sma->sem_base = (struct sem *) &sma[1]; The current code has four problems: - There is an unnecessary pointer dereference - sem_base is not needed. - Alignment for struct sem only works by chance. - The current code causes false positive for static code analysis. - This is a cast between different non-void types, which the future randstruct GCC plugin warns on. And, as bonus, the code size gets smaller: Before: 0 .text 00003770 After: 0 .text 0000374e [manfred@colorfullife.com: s/[0]/[]/, per hch] Link: http://lkml.kernel.org/r/20170525185107.12869-2-manfred@colorfullife.com Link: http://lkml.kernel.org/r/20170515171912.6298-2-manfred@colorfullife.com Signed-off-by: Manfred Spraul Acked-by: Kees Cook Cc: Kees Cook Cc: <1vier1@web.de> Cc: Davidlohr Bueso Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Fabian Frederick Cc: Christoph Hellwig Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- ipc/sem.c | 88 ++++++++++++++++++++++++--------------------------------------- 1 file changed, 34 insertions(+), 54 deletions(-) (limited to 'ipc/sem.c') diff --git a/ipc/sem.c b/ipc/sem.c index 947dc2348271..fff8337ebab3 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -87,24 +87,6 @@ #include #include "util.h" -/* One semaphore structure for each semaphore in the system. */ -struct sem { - int semval; /* current value */ - /* - * PID of the process that last modified the semaphore. For - * Linux, specifically these are: - * - semop - * - semctl, via SETVAL and SETALL. - * - at task exit when performing undo adjustments (see exit_sem). - */ - int sempid; - spinlock_t lock; /* spinlock for fine-grained semtimedop */ - struct list_head pending_alter; /* pending single-sop operations */ - /* that alter the semaphore */ - struct list_head pending_const; /* pending single-sop operations */ - /* that do not alter the semaphore*/ - time_t sem_otime; /* candidate for sem_otime */ -} ____cacheline_aligned_in_smp; /* One queue for each sleeping process in the system. */ struct sem_queue { @@ -175,7 +157,7 @@ static int sysvipc_sem_proc_show(struct seq_file *s, void *it); * sem_array.sem_undo * * b) global or semaphore sem_lock() for read/write: - * sem_array.sem_base[i].pending_{const,alter}: + * sem_array.sems[i].pending_{const,alter}: * * c) special: * sem_undo_list.list_proc: @@ -250,7 +232,7 @@ static void unmerge_queues(struct sem_array *sma) */ list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { struct sem *curr; - curr = &sma->sem_base[q->sops[0].sem_num]; + curr = &sma->sems[q->sops[0].sem_num]; list_add_tail(&q->list, &curr->pending_alter); } @@ -270,7 +252,7 @@ static void merge_queues(struct sem_array *sma) { int i; for (i = 0; i < sma->sem_nsems; i++) { - struct sem *sem = sma->sem_base + i; + struct sem *sem = &sma->sems[i]; list_splice_init(&sem->pending_alter, &sma->pending_alter); } @@ -306,7 +288,7 @@ static void complexmode_enter(struct sem_array *sma) sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; for (i = 0; i < sma->sem_nsems; i++) { - sem = sma->sem_base + i; + sem = &sma->sems[i]; spin_lock(&sem->lock); spin_unlock(&sem->lock); } @@ -366,7 +348,7 @@ static inline int sem_lock(struct sem_array *sma, struct sembuf *sops, * * Both facts are tracked by use_global_mode. */ - sem = sma->sem_base + sops->sem_num; + sem = &sma->sems[sops->sem_num]; /* * Initial check for use_global_lock. Just an optimization, @@ -421,7 +403,7 @@ static inline void sem_unlock(struct sem_array *sma, int locknum) complexmode_tryleave(sma); ipc_unlock_object(&sma->sem_perm); } else { - struct sem *sem = sma->sem_base + locknum; + struct sem *sem = &sma->sems[locknum]; spin_unlock(&sem->lock); } } @@ -487,7 +469,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) if (ns->used_sems + nsems > ns->sc_semmns) return -ENOSPC; - size = sizeof(*sma) + nsems * sizeof(struct sem); + size = sizeof(*sma) + nsems * sizeof(sma->sems[0]); sma = ipc_rcu_alloc(size); if (!sma) return -ENOMEM; @@ -504,12 +486,10 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) return retval; } - sma->sem_base = (struct sem *) &sma[1]; - for (i = 0; i < nsems; i++) { - INIT_LIST_HEAD(&sma->sem_base[i].pending_alter); - INIT_LIST_HEAD(&sma->sem_base[i].pending_const); - spin_lock_init(&sma->sem_base[i].lock); + INIT_LIST_HEAD(&sma->sems[i].pending_alter); + INIT_LIST_HEAD(&sma->sems[i].pending_const); + spin_lock_init(&sma->sems[i].lock); } sma->complex_count = 0; @@ -612,7 +592,7 @@ static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q) un = q->undo; for (sop = sops; sop < sops + nsops; sop++) { - curr = sma->sem_base + sop->sem_num; + curr = &sma->sems[sop->sem_num]; sem_op = sop->sem_op; result = curr->semval; @@ -639,7 +619,7 @@ static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q) sop--; pid = q->pid; while (sop >= sops) { - sma->sem_base[sop->sem_num].sempid = pid; + sma->sems[sop->sem_num].sempid = pid; sop--; } @@ -661,7 +641,7 @@ undo: sop--; while (sop >= sops) { sem_op = sop->sem_op; - sma->sem_base[sop->sem_num].semval -= sem_op; + sma->sems[sop->sem_num].semval -= sem_op; if (sop->sem_flg & SEM_UNDO) un->semadj[sop->sem_num] += sem_op; sop--; @@ -692,7 +672,7 @@ static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q) * until the operations can go through. */ for (sop = sops; sop < sops + nsops; sop++) { - curr = sma->sem_base + sop->sem_num; + curr = &sma->sems[sop->sem_num]; sem_op = sop->sem_op; result = curr->semval; @@ -716,7 +696,7 @@ static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q) } for (sop = sops; sop < sops + nsops; sop++) { - curr = sma->sem_base + sop->sem_num; + curr = &sma->sems[sop->sem_num]; sem_op = sop->sem_op; result = curr->semval; @@ -815,7 +795,7 @@ static int wake_const_ops(struct sem_array *sma, int semnum, if (semnum == -1) pending_list = &sma->pending_const; else - pending_list = &sma->sem_base[semnum].pending_const; + pending_list = &sma->sems[semnum].pending_const; list_for_each_entry_safe(q, tmp, pending_list, list) { int error = perform_atomic_semop(sma, q); @@ -856,7 +836,7 @@ static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops, for (i = 0; i < nsops; i++) { int num = sops[i].sem_num; - if (sma->sem_base[num].semval == 0) { + if (sma->sems[num].semval == 0) { got_zero = 1; semop_completed |= wake_const_ops(sma, num, wake_q); } @@ -867,7 +847,7 @@ static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops, * Assume all were changed. */ for (i = 0; i < sma->sem_nsems; i++) { - if (sma->sem_base[i].semval == 0) { + if (sma->sems[i].semval == 0) { got_zero = 1; semop_completed |= wake_const_ops(sma, i, wake_q); } @@ -909,7 +889,7 @@ static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *w if (semnum == -1) pending_list = &sma->pending_alter; else - pending_list = &sma->sem_base[semnum].pending_alter; + pending_list = &sma->sems[semnum].pending_alter; again: list_for_each_entry_safe(q, tmp, pending_list, list) { @@ -922,7 +902,7 @@ again: * be in the per semaphore pending queue, and decrements * cannot be successful if the value is already 0. */ - if (semnum != -1 && sma->sem_base[semnum].semval == 0) + if (semnum != -1 && sma->sems[semnum].semval == 0) break; error = perform_atomic_semop(sma, q); @@ -959,9 +939,9 @@ again: static void set_semotime(struct sem_array *sma, struct sembuf *sops) { if (sops == NULL) { - sma->sem_base[0].sem_otime = get_seconds(); + sma->sems[0].sem_otime = get_seconds(); } else { - sma->sem_base[sops[0].sem_num].sem_otime = + sma->sems[sops[0].sem_num].sem_otime = get_seconds(); } } @@ -1067,9 +1047,9 @@ static int count_semcnt(struct sem_array *sma, ushort semnum, semcnt = 0; /* First: check the simple operations. They are easy to evaluate */ if (count_zero) - l = &sma->sem_base[semnum].pending_const; + l = &sma->sems[semnum].pending_const; else - l = &sma->sem_base[semnum].pending_alter; + l = &sma->sems[semnum].pending_alter; list_for_each_entry(q, l, list) { /* all task on a per-semaphore list sleep on exactly @@ -1124,7 +1104,7 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp) wake_up_sem_queue_prepare(q, -EIDRM, &wake_q); } for (i = 0; i < sma->sem_nsems; i++) { - struct sem *sem = sma->sem_base + i; + struct sem *sem = &sma->sems[i]; list_for_each_entry_safe(q, tq, &sem->pending_const, list) { unlink_queue(sma, q); wake_up_sem_queue_prepare(q, -EIDRM, &wake_q); @@ -1174,9 +1154,9 @@ static time_t get_semotime(struct sem_array *sma) int i; time_t res; - res = sma->sem_base[0].sem_otime; + res = sma->sems[0].sem_otime; for (i = 1; i < sma->sem_nsems; i++) { - time_t to = sma->sem_base[i].sem_otime; + time_t to = sma->sems[i].sem_otime; if (to > res) res = to; @@ -1325,7 +1305,7 @@ static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum, return -EIDRM; } - curr = &sma->sem_base[semnum]; + curr = &sma->sems[semnum]; ipc_assert_locked_object(&sma->sem_perm); list_for_each_entry(un, &sma->list_id, list_id) @@ -1402,7 +1382,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, } } for (i = 0; i < sma->sem_nsems; i++) - sem_io[i] = sma->sem_base[i].semval; + sem_io[i] = sma->sems[i].semval; sem_unlock(sma, -1); rcu_read_unlock(); err = 0; @@ -1450,8 +1430,8 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, } for (i = 0; i < nsems; i++) { - sma->sem_base[i].semval = sem_io[i]; - sma->sem_base[i].sempid = task_tgid_vnr(current); + sma->sems[i].semval = sem_io[i]; + sma->sems[i].sempid = task_tgid_vnr(current); } ipc_assert_locked_object(&sma->sem_perm); @@ -1476,7 +1456,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, err = -EIDRM; goto out_unlock; } - curr = &sma->sem_base[semnum]; + curr = &sma->sems[semnum]; switch (cmd) { case GETVAL: @@ -1932,7 +1912,7 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops, */ if (nsops == 1) { struct sem *curr; - curr = &sma->sem_base[sops->sem_num]; + curr = &sma->sems[sops->sem_num]; if (alter) { if (sma->complex_count) { @@ -2146,7 +2126,7 @@ void exit_sem(struct task_struct *tsk) /* perform adjustments registered in un */ for (i = 0; i < sma->sem_nsems; i++) { - struct sem *semaphore = &sma->sem_base[i]; + struct sem *semaphore = &sma->sems[i]; if (un->semadj[i]) { semaphore->semval += un->semadj[i]; /* -- cgit v1.2.3 From dba4cdd39e698d8dcdad0656825423052ac90ccd Mon Sep 17 00:00:00 2001 From: Manfred Spraul Date: Wed, 12 Jul 2017 14:34:41 -0700 Subject: ipc: merge ipc_rcu and kern_ipc_perm ipc has two management structures that exist for every id: - struct kern_ipc_perm, it contains e.g. the permissions. - struct ipc_rcu, it contains the rcu head for rcu handling and the refcount. The patch merges both structures. As a bonus, we may save one cacheline, because both structures are cacheline aligned. In addition, it reduces the number of casts, instead most codepaths can use container_of. To simplify code, the ipc_rcu_alloc initializes the allocation to 0. [manfred@colorfullife.com: really include the memset() into ipc_alloc_rcu()] Link: http://lkml.kernel.org/r/564f8612-0601-b267-514f-a9f650ec9b32@colorfullife.com Link: http://lkml.kernel.org/r/20170525185107.12869-3-manfred@colorfullife.com Signed-off-by: Manfred Spraul Cc: Davidlohr Bueso Cc: Kees Cook Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/ipc.h | 3 +++ ipc/msg.c | 19 +++++++++++-------- ipc/sem.c | 34 +++++++++++++++++----------------- ipc/shm.c | 18 +++++++++++------- ipc/util.c | 35 +++++++++++++++++------------------ ipc/util.h | 18 +++++++----------- 6 files changed, 66 insertions(+), 61 deletions(-) (limited to 'ipc/sem.c') diff --git a/include/linux/ipc.h b/include/linux/ipc.h index 71fd92d81b26..5591f055e13f 100644 --- a/include/linux/ipc.h +++ b/include/linux/ipc.h @@ -20,6 +20,9 @@ struct kern_ipc_perm { umode_t mode; unsigned long seq; void *security; + + struct rcu_head rcu; + atomic_t refcount; } ____cacheline_aligned_in_smp; #endif /* _LINUX_IPC_H */ diff --git a/ipc/msg.c b/ipc/msg.c index 104926dc72be..0ed7dae7d4e8 100644 --- a/ipc/msg.c +++ b/ipc/msg.c @@ -97,8 +97,8 @@ static inline void msg_rmid(struct ipc_namespace *ns, struct msg_queue *s) static void msg_rcu_free(struct rcu_head *head) { - struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu); - struct msg_queue *msq = ipc_rcu_to_struct(p); + struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu); + struct msg_queue *msq = container_of(p, struct msg_queue, q_perm); security_msg_queue_free(msq); ipc_rcu_free(head); @@ -118,7 +118,10 @@ static int newque(struct ipc_namespace *ns, struct ipc_params *params) key_t key = params->key; int msgflg = params->flg; - msq = ipc_rcu_alloc(sizeof(*msq)); + BUILD_BUG_ON(offsetof(struct msg_queue, q_perm) != 0); + + msq = container_of(ipc_rcu_alloc(sizeof(*msq)), struct msg_queue, + q_perm); if (!msq) return -ENOMEM; @@ -128,7 +131,7 @@ static int newque(struct ipc_namespace *ns, struct ipc_params *params) msq->q_perm.security = NULL; retval = security_msg_queue_alloc(msq); if (retval) { - ipc_rcu_putref(msq, ipc_rcu_free); + ipc_rcu_putref(&msq->q_perm, ipc_rcu_free); return retval; } @@ -144,7 +147,7 @@ static int newque(struct ipc_namespace *ns, struct ipc_params *params) /* ipc_addid() locks msq upon success. */ id = ipc_addid(&msg_ids(ns), &msq->q_perm, ns->msg_ctlmni); if (id < 0) { - ipc_rcu_putref(msq, msg_rcu_free); + ipc_rcu_putref(&msq->q_perm, msg_rcu_free); return id; } @@ -249,7 +252,7 @@ static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp) free_msg(msg); } atomic_sub(msq->q_cbytes, &ns->msg_bytes); - ipc_rcu_putref(msq, msg_rcu_free); + ipc_rcu_putref(&msq->q_perm, msg_rcu_free); } /* @@ -688,7 +691,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext, /* enqueue the sender and prepare to block */ ss_add(msq, &s, msgsz); - if (!ipc_rcu_getref(msq)) { + if (!ipc_rcu_getref(&msq->q_perm)) { err = -EIDRM; goto out_unlock0; } @@ -700,7 +703,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext, rcu_read_lock(); ipc_lock_object(&msq->q_perm); - ipc_rcu_putref(msq, msg_rcu_free); + ipc_rcu_putref(&msq->q_perm, msg_rcu_free); /* raced with RMID? */ if (!ipc_valid_object(&msq->q_perm)) { err = -EIDRM; diff --git a/ipc/sem.c b/ipc/sem.c index fff8337ebab3..bdff6d93d2c7 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -260,8 +260,8 @@ static void merge_queues(struct sem_array *sma) static void sem_rcu_free(struct rcu_head *head) { - struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu); - struct sem_array *sma = ipc_rcu_to_struct(p); + struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu); + struct sem_array *sma = container_of(p, struct sem_array, sem_perm); security_sem_free(sma); ipc_rcu_free(head); @@ -438,7 +438,7 @@ static inline struct sem_array *sem_obtain_object_check(struct ipc_namespace *ns static inline void sem_lock_and_putref(struct sem_array *sma) { sem_lock(sma, NULL, -1); - ipc_rcu_putref(sma, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); } static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s) @@ -469,20 +469,20 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) if (ns->used_sems + nsems > ns->sc_semmns) return -ENOSPC; + BUILD_BUG_ON(offsetof(struct sem_array, sem_perm) != 0); + size = sizeof(*sma) + nsems * sizeof(sma->sems[0]); - sma = ipc_rcu_alloc(size); + sma = container_of(ipc_rcu_alloc(size), struct sem_array, sem_perm); if (!sma) return -ENOMEM; - memset(sma, 0, size); - sma->sem_perm.mode = (semflg & S_IRWXUGO); sma->sem_perm.key = key; sma->sem_perm.security = NULL; retval = security_sem_alloc(sma); if (retval) { - ipc_rcu_putref(sma, ipc_rcu_free); + ipc_rcu_putref(&sma->sem_perm, ipc_rcu_free); return retval; } @@ -502,7 +502,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) id = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); if (id < 0) { - ipc_rcu_putref(sma, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); return id; } ns->used_sems += nsems; @@ -1122,7 +1122,7 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp) wake_up_q(&wake_q); ns->used_sems -= sma->sem_nsems; - ipc_rcu_putref(sma, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); } static unsigned long copy_semid_to_user(void __user *buf, struct semid64_ds *in, int version) @@ -1362,7 +1362,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, goto out_unlock; } if (nsems > SEMMSL_FAST) { - if (!ipc_rcu_getref(sma)) { + if (!ipc_rcu_getref(&sma->sem_perm)) { err = -EIDRM; goto out_unlock; } @@ -1370,7 +1370,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, rcu_read_unlock(); sem_io = ipc_alloc(sizeof(ushort)*nsems); if (sem_io == NULL) { - ipc_rcu_putref(sma, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); return -ENOMEM; } @@ -1395,7 +1395,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, int i; struct sem_undo *un; - if (!ipc_rcu_getref(sma)) { + if (!ipc_rcu_getref(&sma->sem_perm)) { err = -EIDRM; goto out_rcu_wakeup; } @@ -1404,20 +1404,20 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, if (nsems > SEMMSL_FAST) { sem_io = ipc_alloc(sizeof(ushort)*nsems); if (sem_io == NULL) { - ipc_rcu_putref(sma, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); return -ENOMEM; } } if (copy_from_user(sem_io, p, nsems*sizeof(ushort))) { - ipc_rcu_putref(sma, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); err = -EFAULT; goto out_free; } for (i = 0; i < nsems; i++) { if (sem_io[i] > SEMVMX) { - ipc_rcu_putref(sma, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); err = -ERANGE; goto out_free; } @@ -1699,7 +1699,7 @@ static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid) } nsems = sma->sem_nsems; - if (!ipc_rcu_getref(sma)) { + if (!ipc_rcu_getref(&sma->sem_perm)) { rcu_read_unlock(); un = ERR_PTR(-EIDRM); goto out; @@ -1709,7 +1709,7 @@ static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid) /* step 2: allocate new undo structure */ new = kzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems, GFP_KERNEL); if (!new) { - ipc_rcu_putref(sma, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); return ERR_PTR(-ENOMEM); } diff --git a/ipc/shm.c b/ipc/shm.c index f45c7959b264..5ef6d31a52c5 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -174,9 +174,10 @@ static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp) static void shm_rcu_free(struct rcu_head *head) { - struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu); - struct shmid_kernel *shp = ipc_rcu_to_struct(p); - + struct kern_ipc_perm *ptr = container_of(head, struct kern_ipc_perm, + rcu); + struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel, + shm_perm); security_shm_free(shp); ipc_rcu_free(head); } @@ -241,7 +242,7 @@ static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp) user_shm_unlock(i_size_read(file_inode(shm_file)), shp->mlock_user); fput(shm_file); - ipc_rcu_putref(shp, shm_rcu_free); + ipc_rcu_putref(&shp->shm_perm, shm_rcu_free); } /* @@ -542,7 +543,10 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params) ns->shm_tot + numpages > ns->shm_ctlall) return -ENOSPC; - shp = ipc_rcu_alloc(sizeof(*shp)); + BUILD_BUG_ON(offsetof(struct shmid_kernel, shm_perm) != 0); + + shp = container_of(ipc_rcu_alloc(sizeof(*shp)), struct shmid_kernel, + shm_perm); if (!shp) return -ENOMEM; @@ -553,7 +557,7 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params) shp->shm_perm.security = NULL; error = security_shm_alloc(shp); if (error) { - ipc_rcu_putref(shp, ipc_rcu_free); + ipc_rcu_putref(&shp->shm_perm, ipc_rcu_free); return error; } @@ -624,7 +628,7 @@ no_id: user_shm_unlock(size, shp->mlock_user); fput(file); no_file: - ipc_rcu_putref(shp, shm_rcu_free); + ipc_rcu_putref(&shp->shm_perm, shm_rcu_free); return error; } diff --git a/ipc/util.c b/ipc/util.c index caec7b1bfaa3..5d1ff1035efe 100644 --- a/ipc/util.c +++ b/ipc/util.c @@ -418,46 +418,45 @@ void ipc_free(void *ptr) } /** - * ipc_rcu_alloc - allocate ipc and rcu space + * ipc_rcu_alloc - allocate ipc space * @size: size desired * - * Allocate memory for the rcu header structure + the object. - * Returns the pointer to the object or NULL upon failure. + * Allocate memory for an ipc object. + * The first member must be struct kern_ipc_perm. */ -void *ipc_rcu_alloc(int size) +struct kern_ipc_perm *ipc_rcu_alloc(int size) { /* * We prepend the allocation with the rcu struct */ - struct ipc_rcu *out = ipc_alloc(sizeof(struct ipc_rcu) + size); + struct kern_ipc_perm *out = ipc_alloc(size); if (unlikely(!out)) return NULL; + + memset(out, 0, size); atomic_set(&out->refcount, 1); - return out + 1; + return out; } -int ipc_rcu_getref(void *ptr) +int ipc_rcu_getref(struct kern_ipc_perm *ptr) { - struct ipc_rcu *p = ((struct ipc_rcu *)ptr) - 1; - - return atomic_inc_not_zero(&p->refcount); + return atomic_inc_not_zero(&ptr->refcount); } -void ipc_rcu_putref(void *ptr, void (*func)(struct rcu_head *head)) +void ipc_rcu_putref(struct kern_ipc_perm *ptr, + void (*func)(struct rcu_head *head)) { - struct ipc_rcu *p = ((struct ipc_rcu *)ptr) - 1; - - if (!atomic_dec_and_test(&p->refcount)) + if (!atomic_dec_and_test(&ptr->refcount)) return; - call_rcu(&p->rcu, func); + call_rcu(&ptr->rcu, func); } -void ipc_rcu_free(struct rcu_head *head) +void ipc_rcu_free(struct rcu_head *h) { - struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu); + struct kern_ipc_perm *ptr = container_of(h, struct kern_ipc_perm, rcu); - kvfree(p); + kvfree(ptr); } /** diff --git a/ipc/util.h b/ipc/util.h index 60ddccca464d..09d0f918c3e2 100644 --- a/ipc/util.h +++ b/ipc/util.h @@ -47,13 +47,6 @@ static inline void msg_exit_ns(struct ipc_namespace *ns) { } static inline void shm_exit_ns(struct ipc_namespace *ns) { } #endif -struct ipc_rcu { - struct rcu_head rcu; - atomic_t refcount; -} ____cacheline_aligned_in_smp; - -#define ipc_rcu_to_struct(p) ((void *)(p+1)) - /* * Structure that holds the parameters needed by the ipc operations * (see after) @@ -125,11 +118,14 @@ void ipc_free(void *ptr); * Objects are reference counted, they start with reference count 1. * getref increases the refcount, the putref call that reduces the recount * to 0 schedules the rcu destruction. Caller must guarantee locking. + * + * struct kern_ipc_perm must be the first member in the allocated structure. */ -void *ipc_rcu_alloc(int size); -int ipc_rcu_getref(void *ptr); -void ipc_rcu_putref(void *ptr, void (*func)(struct rcu_head *head)); -void ipc_rcu_free(struct rcu_head *head); +struct kern_ipc_perm *ipc_rcu_alloc(int size); +int ipc_rcu_getref(struct kern_ipc_perm *ptr); +void ipc_rcu_putref(struct kern_ipc_perm *ptr, + void (*func)(struct rcu_head *head)); +void ipc_rcu_free(struct rcu_head *h); struct kern_ipc_perm *ipc_lock(struct ipc_ids *, int); struct kern_ipc_perm *ipc_obtain_object_idr(struct ipc_ids *ids, int id); -- cgit v1.2.3 From f8dbe8d290637ac3f68600e30d092393fe9b40a5 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 12 Jul 2017 14:34:47 -0700 Subject: ipc: drop non-RCU allocation The only users of ipc_alloc() were ipc_rcu_alloc() and the on-heap sem_io fall-back memory. Better to just open-code these to make things easier to read. [manfred@colorfullife.com: Rediff due to inclusion of memset() into ipc_rcu_alloc()] Link: http://lkml.kernel.org/r/20170525185107.12869-5-manfred@colorfullife.com Signed-off-by: Kees Cook Signed-off-by: Manfred Spraul Cc: Davidlohr Bueso Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- ipc/sem.c | 8 +++++--- ipc/util.c | 25 +------------------------ ipc/util.h | 6 ------ 3 files changed, 6 insertions(+), 33 deletions(-) (limited to 'ipc/sem.c') diff --git a/ipc/sem.c b/ipc/sem.c index bdff6d93d2c7..484ccf83cf85 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -1368,7 +1368,8 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, } sem_unlock(sma, -1); rcu_read_unlock(); - sem_io = ipc_alloc(sizeof(ushort)*nsems); + sem_io = kvmalloc_array(nsems, sizeof(ushort), + GFP_KERNEL); if (sem_io == NULL) { ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); return -ENOMEM; @@ -1402,7 +1403,8 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, rcu_read_unlock(); if (nsems > SEMMSL_FAST) { - sem_io = ipc_alloc(sizeof(ushort)*nsems); + sem_io = kvmalloc_array(nsems, sizeof(ushort), + GFP_KERNEL); if (sem_io == NULL) { ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); return -ENOMEM; @@ -1480,7 +1482,7 @@ out_rcu_wakeup: wake_up_q(&wake_q); out_free: if (sem_io != fast_sem_io) - ipc_free(sem_io); + kvfree(sem_io); return err; } diff --git a/ipc/util.c b/ipc/util.c index 5d1ff1035efe..dd73feb1569a 100644 --- a/ipc/util.c +++ b/ipc/util.c @@ -394,29 +394,6 @@ void ipc_rmid(struct ipc_ids *ids, struct kern_ipc_perm *ipcp) ipcp->deleted = true; } -/** - * ipc_alloc - allocate ipc space - * @size: size desired - * - * Allocate memory from the appropriate pools and return a pointer to it. - * NULL is returned if the allocation fails - */ -void *ipc_alloc(int size) -{ - return kvmalloc(size, GFP_KERNEL); -} - -/** - * ipc_free - free ipc space - * @ptr: pointer returned by ipc_alloc - * - * Free a block created with ipc_alloc(). - */ -void ipc_free(void *ptr) -{ - kvfree(ptr); -} - /** * ipc_rcu_alloc - allocate ipc space * @size: size desired @@ -429,7 +406,7 @@ struct kern_ipc_perm *ipc_rcu_alloc(int size) /* * We prepend the allocation with the rcu struct */ - struct kern_ipc_perm *out = ipc_alloc(size); + struct kern_ipc_perm *out = kvmalloc(size, GFP_KERNEL); if (unlikely(!out)) return NULL; diff --git a/ipc/util.h b/ipc/util.h index 09d0f918c3e2..2578fd9be835 100644 --- a/ipc/util.h +++ b/ipc/util.h @@ -107,12 +107,6 @@ void ipc_rmid(struct ipc_ids *, struct kern_ipc_perm *); /* must be called with ipcp locked */ int ipcperms(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp, short flg); -/* for rare, potentially huge allocations. - * both function can sleep - */ -void *ipc_alloc(int size); -void ipc_free(void *ptr); - /* * For allocation that need to be freed by RCU. * Objects are reference counted, they start with reference count 1. -- cgit v1.2.3 From 1b4654ef72f61c84704b3c79b50fdeed8747fc56 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 12 Jul 2017 14:34:50 -0700 Subject: ipc/sem: do not use ipc_rcu_free() Avoid using ipc_rcu_free, since it just re-finds the original structure pointer. For the pre-list-init failure path, there is no RCU needed, since it was just allocated. It can be directly freed. Link: http://lkml.kernel.org/r/20170525185107.12869-6-manfred@colorfullife.com Signed-off-by: Kees Cook Signed-off-by: Manfred Spraul Cc: Davidlohr Bueso Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- ipc/sem.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'ipc/sem.c') diff --git a/ipc/sem.c b/ipc/sem.c index 484ccf83cf85..a04c4d6d120c 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -258,13 +258,18 @@ static void merge_queues(struct sem_array *sma) } } +static void __sem_free(struct sem_array *sma) +{ + kvfree(sma); +} + static void sem_rcu_free(struct rcu_head *head) { struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu); struct sem_array *sma = container_of(p, struct sem_array, sem_perm); security_sem_free(sma); - ipc_rcu_free(head); + __sem_free(sma); } /* @@ -482,7 +487,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) sma->sem_perm.security = NULL; retval = security_sem_alloc(sma); if (retval) { - ipc_rcu_putref(&sma->sem_perm, ipc_rcu_free); + __sem_free(sma); return retval; } -- cgit v1.2.3 From 101ede01dfd5072651965e974bc6e30c8d0748e2 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 12 Jul 2017 14:35:02 -0700 Subject: ipc/sem: avoid ipc_rcu_alloc() Instead of using ipc_rcu_alloc() which only performs the refcount bump, open code it to perform better sem-specific checks. This also allows for sem_array structure layout to be randomized in the future. [manfred@colorfullife.com: Rediff, because the memset was temporarily inside ipc_rcu_alloc()] Link: http://lkml.kernel.org/r/20170525185107.12869-10-manfred@colorfullife.com Signed-off-by: Kees Cook Signed-off-by: Manfred Spraul Cc: Davidlohr Bueso Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- ipc/sem.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'ipc/sem.c') diff --git a/ipc/sem.c b/ipc/sem.c index a04c4d6d120c..445a5b5eb88f 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -451,6 +451,25 @@ static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s) ipc_rmid(&sem_ids(ns), &s->sem_perm); } +static struct sem_array *sem_alloc(size_t nsems) +{ + struct sem_array *sma; + size_t size; + + if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0])) + return NULL; + + size = sizeof(*sma) + nsems * sizeof(sma->sems[0]); + sma = kvmalloc(size, GFP_KERNEL); + if (unlikely(!sma)) + return NULL; + + memset(sma, 0, size); + atomic_set(&sma->sem_perm.refcount, 1); + + return sma; +} + /** * newary - Create a new semaphore set * @ns: namespace @@ -463,7 +482,6 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) int id; int retval; struct sem_array *sma; - int size; key_t key = params->key; int nsems = params->u.nsems; int semflg = params->flg; @@ -474,10 +492,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) if (ns->used_sems + nsems > ns->sc_semmns) return -ENOSPC; - BUILD_BUG_ON(offsetof(struct sem_array, sem_perm) != 0); - - size = sizeof(*sma) + nsems * sizeof(sma->sems[0]); - sma = container_of(ipc_rcu_alloc(size), struct sem_array, sem_perm); + sma = sem_alloc(nsems); if (!sma) return -ENOMEM; -- cgit v1.2.3 From 2ec55f8024db859d70f14c26e91ca044328dd50d Mon Sep 17 00:00:00 2001 From: Manfred Spraul Date: Wed, 12 Jul 2017 14:35:13 -0700 Subject: ipc/sem.c: avoid ipc_rcu_putref for failed ipc_addid() Loosely based on a patch from Kees Cook : - id and retval can be merged - if ipc_addid() fails, then use call_rcu() directly. The difference is that call_rcu is used for failed ipc_addid() calls, to continue to guaranteed an rcu delay for security_sem_free(). Link: http://lkml.kernel.org/r/20170525185107.12869-14-manfred@colorfullife.com Signed-off-by: Manfred Spraul Cc: Kees Cook Cc: Davidlohr Bueso Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- ipc/sem.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'ipc/sem.c') diff --git a/ipc/sem.c b/ipc/sem.c index 445a5b5eb88f..2b2ed56e0fde 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -479,7 +479,6 @@ static struct sem_array *sem_alloc(size_t nsems) */ static int newary(struct ipc_namespace *ns, struct ipc_params *params) { - int id; int retval; struct sem_array *sma; key_t key = params->key; @@ -520,10 +519,10 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) sma->sem_nsems = nsems; sma->sem_ctime = get_seconds(); - id = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); - if (id < 0) { - ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); - return id; + retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); + if (retval < 0) { + call_rcu(&sma->sem_perm.rcu, sem_rcu_free); + return retval; } ns->used_sems += nsems; -- cgit v1.2.3 From 3d3653f9732c73feb8c4addfc1cbdaa292a399fa Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 12 Jul 2017 14:35:22 -0700 Subject: ipc: move atomic_set() to where it is needed Only after ipc_addid() has succeeded will refcounting be used, so move initialization into ipc_addid() and remove from open-coded *_alloc() routines. Link: http://lkml.kernel.org/r/20170525185107.12869-17-manfred@colorfullife.com Signed-off-by: Kees Cook Signed-off-by: Manfred Spraul Cc: Davidlohr Bueso Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- ipc/msg.c | 2 -- ipc/sem.c | 1 - ipc/shm.c | 2 -- ipc/util.c | 1 + 4 files changed, 1 insertion(+), 5 deletions(-) (limited to 'ipc/sem.c') diff --git a/ipc/msg.c b/ipc/msg.c index cd90bfde89a4..770342e1d327 100644 --- a/ipc/msg.c +++ b/ipc/msg.c @@ -117,8 +117,6 @@ static struct msg_queue *msg_alloc(void) if (unlikely(!msq)) return NULL; - atomic_set(&msq->q_perm.refcount, 1); - return msq; } diff --git a/ipc/sem.c b/ipc/sem.c index 2b2ed56e0fde..5f137738819d 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -465,7 +465,6 @@ static struct sem_array *sem_alloc(size_t nsems) return NULL; memset(sma, 0, size); - atomic_set(&sma->sem_perm.refcount, 1); return sma; } diff --git a/ipc/shm.c b/ipc/shm.c index c5976d318ed1..d1988ef821a1 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -526,8 +526,6 @@ static struct shmid_kernel *shm_alloc(void) if (unlikely(!shp)) return NULL; - atomic_set(&shp->shm_perm.refcount, 1); - return shp; } diff --git a/ipc/util.c b/ipc/util.c index 2428dd44ca97..1a2cb02467ab 100644 --- a/ipc/util.c +++ b/ipc/util.c @@ -232,6 +232,7 @@ int ipc_addid(struct ipc_ids *ids, struct kern_ipc_perm *new, int size) idr_preload(GFP_KERNEL); + atomic_set(&new->refcount, 1); spin_lock_init(&new->lock); new->deleted = false; rcu_read_lock(); -- cgit v1.2.3 From e2029dfeef7b09f08ac8572e8be3d4c624d1f79a Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 12 Jul 2017 14:35:31 -0700 Subject: ipc/sem: drop __sem_free() The remaining users of __sem_free() can simply call kvfree() instead for better readability. [manfred@colorfullife.com: Rediff to keep rcu protection for security_sem_alloc()] Link: http://lkml.kernel.org/r/20170525185107.12869-20-manfred@colorfullife.com Signed-off-by: Kees Cook Signed-off-by: Manfred Spraul Cc: Davidlohr Bueso Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- ipc/sem.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'ipc/sem.c') diff --git a/ipc/sem.c b/ipc/sem.c index 5f137738819d..9e70cd7a17da 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -258,18 +258,13 @@ static void merge_queues(struct sem_array *sma) } } -static void __sem_free(struct sem_array *sma) -{ - kvfree(sma); -} - static void sem_rcu_free(struct rcu_head *head) { struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu); struct sem_array *sma = container_of(p, struct sem_array, sem_perm); security_sem_free(sma); - __sem_free(sma); + kvfree(sma); } /* @@ -500,7 +495,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) sma->sem_perm.security = NULL; retval = security_sem_alloc(sma); if (retval) { - __sem_free(sma); + kvfree(sma); return retval; } -- cgit v1.2.3