From 387ad9674b0013c8756ad20d854ff005b0c313ad Mon Sep 17 00:00:00 2001 From: Elena Reshetova Date: Mon, 20 Feb 2017 12:19:00 +0200 Subject: kernel: convert cgroup_namespace.count from atomic_t to refcount_t refcount_t type and corresponding API should be used instead of atomic_t when the variable is used as a reference counter. This allows to avoid accidental refcounter overflows that might lead to use-after-free situations. Signed-off-by: Elena Reshetova Signed-off-by: Hans Liljestrand Signed-off-by: Kees Cook Signed-off-by: David Windsor Signed-off-by: Tejun Heo --- include/linux/cgroup.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f6b43fbb141c..44129793c7b8 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -640,7 +641,7 @@ static inline void cgroup_sk_free(struct sock_cgroup_data *skcd) {} #endif /* CONFIG_CGROUP_DATA */ struct cgroup_namespace { - atomic_t count; + refcount_t count; struct ns_common ns; struct user_namespace *user_ns; struct ucounts *ucounts; @@ -675,12 +676,12 @@ copy_cgroup_ns(unsigned long flags, struct user_namespace *user_ns, static inline void get_cgroup_ns(struct cgroup_namespace *ns) { if (ns) - atomic_inc(&ns->count); + refcount_inc(&ns->count); } static inline void put_cgroup_ns(struct cgroup_namespace *ns) { - if (ns && atomic_dec_and_test(&ns->count)) + if (ns && refcount_dec_and_test(&ns->count)) free_cgroup_ns(ns); } -- cgit v1.2.3 From 4b9502e63b5e2b1b5ef491919d3219b9440fe0b3 Mon Sep 17 00:00:00 2001 From: Elena Reshetova Date: Wed, 8 Mar 2017 10:00:40 +0200 Subject: kernel: convert css_set.refcount from atomic_t to refcount_t refcount_t type and corresponding API should be used instead of atomic_t when the variable is used as a reference counter. This allows to avoid accidental refcounter overflows that might lead to use-after-free situations. Signed-off-by: Elena Reshetova Signed-off-by: Hans Liljestrand Signed-off-by: Kees Cook Signed-off-by: David Windsor Signed-off-by: Tejun Heo --- include/linux/cgroup-defs.h | 3 ++- kernel/cgroup/cgroup-internal.h | 5 +++-- kernel/cgroup/cgroup-v1.c | 4 ++-- kernel/cgroup/cgroup.c | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) (limited to 'include') diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 6a3f850cabab..c74b78ecd583 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -156,7 +157,7 @@ struct css_set { struct cgroup_subsys_state *subsys[CGROUP_SUBSYS_COUNT]; /* reference count */ - atomic_t refcount; + refcount_t refcount; /* the default cgroup associated with this css_set */ struct cgroup *dfl_cgrp; diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 9203bfb05603..4567f12b02e9 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -5,6 +5,7 @@ #include #include #include +#include /* * A cgroup can be associated with multiple css_sets as different tasks may @@ -134,7 +135,7 @@ static inline void put_css_set(struct css_set *cset) * can see it. Similar to atomic_dec_and_lock(), but for an * rwlock */ - if (atomic_add_unless(&cset->refcount, -1, 1)) + if (refcount_dec_not_one(&cset->refcount)) return; spin_lock_irqsave(&css_set_lock, flags); @@ -147,7 +148,7 @@ static inline void put_css_set(struct css_set *cset) */ static inline void get_css_set(struct css_set *cset) { - atomic_inc(&cset->refcount); + refcount_inc(&cset->refcount); } bool cgroup_ssid_enabled(int ssid); diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index 56eba9caa632..c4a68c438fde 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -346,7 +346,7 @@ static int cgroup_task_count(const struct cgroup *cgrp) spin_lock_irq(&css_set_lock); list_for_each_entry(link, &cgrp->cset_links, cset_link) - count += atomic_read(&link->cset->refcount); + count += refcount_read(&link->cset->refcount); spin_unlock_irq(&css_set_lock); return count; } @@ -1286,7 +1286,7 @@ static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css, u64 count; rcu_read_lock(); - count = atomic_read(&task_css_set(current)->refcount); + count = refcount_read(&task_css_set(current)->refcount); rcu_read_unlock(); return count; } diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 8ee78688e36d..b1cc1c306668 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -554,7 +554,7 @@ EXPORT_SYMBOL_GPL(of_css); * haven't been created. */ struct css_set init_css_set = { - .refcount = ATOMIC_INIT(1), + .refcount = REFCOUNT_INIT(1), .tasks = LIST_HEAD_INIT(init_css_set.tasks), .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks), .task_iters = LIST_HEAD_INIT(init_css_set.task_iters), @@ -724,7 +724,7 @@ void put_css_set_locked(struct css_set *cset) lockdep_assert_held(&css_set_lock); - if (!atomic_dec_and_test(&cset->refcount)) + if (!refcount_dec_and_test(&cset->refcount)) return; /* This css_set is dead. unlink it and release cgroup and css refs */ @@ -977,7 +977,7 @@ static struct css_set *find_css_set(struct css_set *old_cset, return NULL; } - atomic_set(&cset->refcount, 1); + refcount_set(&cset->refcount, 1); INIT_LIST_HEAD(&cset->tasks); INIT_LIST_HEAD(&cset->mg_tasks); INIT_LIST_HEAD(&cset->task_iters); -- cgit v1.2.3 From 8f48cfabac57977338f5c828ed3e12fc34373c7d Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Fri, 24 Mar 2017 22:13:35 +0800 Subject: cgroup: drop duplicate header nsproxy.h Drop duplicate header nsproxy.h from linux/cgroup.h. Signed-off-by: Geliang Tang Signed-off-by: Tejun Heo --- include/linux/cgroup.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include') diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 44129793c7b8..34b4a298e52e 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include -- cgit v1.2.3 From 30e03acda5fd9c77ec9bf8b3c5def9540c6b0486 Mon Sep 17 00:00:00 2001 From: Rakib Mullick Date: Sun, 9 Apr 2017 07:36:14 +0600 Subject: cpuset: Remove cpuset_update_active_cpus()'s parameter. In cpuset_update_active_cpus(), cpu_online isn't used anymore. Remove it. Signed-off-by: Rakib Mullick Acked-by: Zefan Li Signed-off-by: Tejun Heo --- include/linux/cpuset.h | 4 ++-- kernel/cgroup/cpuset.c | 2 +- kernel/sched/core.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 611fce58d670..119a3f9604b0 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -42,7 +42,7 @@ static inline void cpuset_dec(void) extern int cpuset_init(void); extern void cpuset_init_smp(void); -extern void cpuset_update_active_cpus(bool cpu_online); +extern void cpuset_update_active_cpus(void); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern void cpuset_cpus_allowed_fallback(struct task_struct *p); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); @@ -155,7 +155,7 @@ static inline bool cpusets_enabled(void) { return false; } static inline int cpuset_init(void) { return 0; } static inline void cpuset_init_smp(void) {} -static inline void cpuset_update_active_cpus(bool cpu_online) +static inline void cpuset_update_active_cpus(void) { partition_sched_domains(1, NULL, NULL); } diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 8b84db2d2d17..f6501f4f6040 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -2351,7 +2351,7 @@ static void cpuset_hotplug_workfn(struct work_struct *work) rebuild_sched_domains(); } -void cpuset_update_active_cpus(bool cpu_online) +void cpuset_update_active_cpus(void) { /* * We're inside cpu hotplug critical region which usually nests diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 956383844116..e62802c044e1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5727,7 +5727,7 @@ static void cpuset_cpu_active(void) * cpuset configurations. */ } - cpuset_update_active_cpus(true); + cpuset_update_active_cpus(); } static int cpuset_cpu_inactive(unsigned int cpu) @@ -5750,7 +5750,7 @@ static int cpuset_cpu_inactive(unsigned int cpu) if (overflow) return -EBUSY; - cpuset_update_active_cpus(false); + cpuset_update_active_cpus(); } else { num_cpus_frozen++; partition_sched_domains(1, NULL, NULL); -- cgit v1.2.3 From b8b1a2e5eca6bbf20e3a29c5f9db8331ec52af2d Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 6 Apr 2017 18:47:57 -0700 Subject: cgroup: move cgroup_subsys_state parent field for cache locality Various structures embed a struct cgroup_subsys_state, typically at the top of the containing structure. It is common for code that accesses the structures to perform operations that iterate over the chain of parent css pointers, also accessing data in each containing structure. In particular, struct cpuacct is used by fairly hot code paths in the scheduler such as cpuacct_charge(). Move the parent css pointer field to the end of the structure to increase the chances of residing in the same cache line as the data from the containing structure. Signed-off-by: Todd Poynor Signed-off-by: Tejun Heo --- include/linux/cgroup-defs.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index c74b78ecd583..21745946cae1 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -107,9 +107,6 @@ struct cgroup_subsys_state { /* reference count - access via css_[try]get() and css_put() */ struct percpu_ref refcnt; - /* PI: the parent css */ - struct cgroup_subsys_state *parent; - /* siblings list anchored at the parent's ->children */ struct list_head sibling; struct list_head children; @@ -139,6 +136,12 @@ struct cgroup_subsys_state { /* percpu_ref killing and RCU release */ struct rcu_head rcu_head; struct work_struct destroy_work; + + /* + * PI: the parent css. Placed here for cache proximity to following + * fields of the containing structure. + */ + struct cgroup_subsys_state *parent; }; /* -- cgit v1.2.3