From 12907fbb1a691807bb0420a27126e15934cb7954 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 15 Dec 2016 11:44:28 +0100 Subject: sched/clock, clocksource: Add optional cs::mark_unstable() method PeterZ reported that we'd fail to mark the TSC unstable when the clocksource watchdog finds it unsuitable. Allow a clocksource to run a custom action when its being marked unstable and hook up the TSC unstable code. Reported-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Cc: Linus Torvalds Cc: Mike Galbraith Cc: Peter Zijlstra Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar --- kernel/time/clocksource.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel/time') diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 665985b0a89a..93621ae718d3 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -141,6 +141,10 @@ static void __clocksource_unstable(struct clocksource *cs) { cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG); cs->flags |= CLOCK_SOURCE_UNSTABLE; + + if (cs->mark_unstable) + cs->mark_unstable(cs); + if (finished_booting) schedule_work(&watchdog_work); } -- cgit v1.2.3 From 93825f2ec736f30e034ab7c9d56b42849c5b00da Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 31 Jan 2017 04:09:16 +0100 Subject: jiffies: Reuse TICK_NSEC instead of NSEC_PER_JIFFY NSEC_PER_JIFFY is an ad-hoc redefinition of TICK_NSEC. Let's rather use a unique and well maintained version. Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Heiko Carstens Cc: Martin Schwidefsky Cc: Tony Luck Cc: Fenghua Yu Cc: Peter Zijlstra Cc: Rik van Riel Cc: Stanislaw Gruszka Cc: Wanpeng Li Link: http://lkml.kernel.org/r/1485832191-26889-1-git-send-email-fweisbec@gmail.com Signed-off-by: Ingo Molnar --- kernel/time/jiffies.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) (limited to 'kernel/time') diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index a4a0e478e44d..7906b3f0c41a 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -27,19 +27,8 @@ #include "timekeeping.h" -/* The Jiffies based clocksource is the lowest common - * denominator clock source which should function on - * all systems. It has the same coarse resolution as - * the timer interrupt frequency HZ and it suffers - * inaccuracies caused by missed or lost timer - * interrupts and the inability for the timer - * interrupt hardware to accuratly tick at the - * requested HZ value. It is also not recommended - * for "tick-less" systems. - */ -#define NSEC_PER_JIFFY ((NSEC_PER_SEC+HZ/2)/HZ) -/* Since jiffies uses a simple NSEC_PER_JIFFY multiplier +/* Since jiffies uses a simple TICK_NSEC multiplier * conversion, the .shift value could be zero. However * this would make NTP adjustments impossible as they are * in units of 1/2^.shift. Thus we use JIFFIES_SHIFT to @@ -47,8 +36,8 @@ * amount, and give ntp adjustments in units of 1/2^8 * * The value 8 is somewhat carefully chosen, as anything - * larger can result in overflows. NSEC_PER_JIFFY grows as - * HZ shrinks, so values greater than 8 overflow 32bits when + * larger can result in overflows. TICK_NSEC grows as HZ + * shrinks, so values greater than 8 overflow 32bits when * HZ=100. */ #if HZ < 34 @@ -64,12 +53,23 @@ static u64 jiffies_read(struct clocksource *cs) return (u64) jiffies; } +/* + * The Jiffies based clocksource is the lowest common + * denominator clock source which should function on + * all systems. It has the same coarse resolution as + * the timer interrupt frequency HZ and it suffers + * inaccuracies caused by missed or lost timer + * interrupts and the inability for the timer + * interrupt hardware to accuratly tick at the + * requested HZ value. It is also not recommended + * for "tick-less" systems. + */ static struct clocksource clocksource_jiffies = { .name = "jiffies", .rating = 1, /* lowest valid rating*/ .read = jiffies_read, .mask = CLOCKSOURCE_MASK(32), - .mult = NSEC_PER_JIFFY << JIFFIES_SHIFT, /* details above */ + .mult = TICK_NSEC << JIFFIES_SHIFT, /* details above */ .shift = JIFFIES_SHIFT, .max_cycles = 10, }; @@ -125,7 +125,7 @@ int register_refined_jiffies(long cycles_per_second) shift_hz += cycles_per_tick/2; do_div(shift_hz, cycles_per_tick); /* Calculate nsec_per_tick using shift_hz */ - nsec_per_tick = (u64)NSEC_PER_SEC << 8; + nsec_per_tick = (u64)TICK_NSEC << 8; nsec_per_tick += (u32)shift_hz/2; do_div(nsec_per_tick, (u32)shift_hz); -- cgit v1.2.3 From 07e5f5e353aaa61696c8353d87050994a0c4648a Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 31 Jan 2017 04:09:17 +0100 Subject: time: Introduce jiffies64_to_nsecs() This will be needed for the cputime_t to nsec conversion. Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Heiko Carstens Cc: Martin Schwidefsky Cc: Tony Luck Cc: Fenghua Yu Cc: Peter Zijlstra Cc: Rik van Riel Cc: Stanislaw Gruszka Cc: Wanpeng Li Link: http://lkml.kernel.org/r/1485832191-26889-2-git-send-email-fweisbec@gmail.com Signed-off-by: Ingo Molnar --- include/linux/jiffies.h | 2 ++ kernel/time/time.c | 10 ++++++++++ kernel/time/timeconst.bc | 6 ++++++ 3 files changed, 18 insertions(+) (limited to 'kernel/time') diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index 589d14e970ad..624215cebee5 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -293,6 +293,8 @@ static inline u64 jiffies_to_nsecs(const unsigned long j) return (u64)jiffies_to_usecs(j) * NSEC_PER_USEC; } +extern u64 jiffies64_to_nsecs(u64 j); + extern unsigned long __msecs_to_jiffies(const unsigned int m); #if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ) /* diff --git a/kernel/time/time.c b/kernel/time/time.c index a3a9a8a029dc..25bdd2504571 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -702,6 +702,16 @@ u64 nsec_to_clock_t(u64 x) #endif } +u64 jiffies64_to_nsecs(u64 j) +{ +#if !(NSEC_PER_SEC % HZ) + return (NSEC_PER_SEC / HZ) * j; +# else + return div_u64(j * HZ_TO_NSEC_NUM, HZ_TO_NSEC_DEN); +#endif +} +EXPORT_SYMBOL(jiffies64_to_nsecs); + /** * nsecs_to_jiffies64 - Convert nsecs in u64 to jiffies64 * diff --git a/kernel/time/timeconst.bc b/kernel/time/timeconst.bc index c48688904f9f..f83bbb81600b 100644 --- a/kernel/time/timeconst.bc +++ b/kernel/time/timeconst.bc @@ -98,6 +98,12 @@ define timeconst(hz) { print "#define HZ_TO_USEC_DEN\t\t", hz/cd, "\n" print "#define USEC_TO_HZ_NUM\t\t", hz/cd, "\n" print "#define USEC_TO_HZ_DEN\t\t", 1000000/cd, "\n" + + cd=gcd(hz,1000000000) + print "#define HZ_TO_NSEC_NUM\t\t", 1000000000/cd, "\n" + print "#define HZ_TO_NSEC_DEN\t\t", hz/cd, "\n" + print "#define NSEC_TO_HZ_NUM\t\t", hz/cd, "\n" + print "#define NSEC_TO_HZ_DEN\t\t", 1000000000/cd, "\n" print "\n" print "#endif /* KERNEL_TIMECONST_H */\n" -- cgit v1.2.3 From a1cecf2ba78e0a6de00ff99df34b662728535aa5 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 31 Jan 2017 04:09:22 +0100 Subject: sched/cputime: Introduce special task_cputime_t() API to return old-typed cputime This API returns a task's cputime in cputime_t in order to ease the conversion of cputime internals to use nsecs units instead. Blindly converting all cputime readers to use this API now will later let us convert more smoothly and step by step all these places to use the new nsec based cputime. Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Fenghua Yu Cc: Heiko Carstens Cc: Linus Torvalds Cc: Martin Schwidefsky Cc: Michael Ellerman Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Rik van Riel Cc: Stanislaw Gruszka Cc: Thomas Gleixner Cc: Tony Luck Cc: Wanpeng Li Link: http://lkml.kernel.org/r/1485832191-26889-7-git-send-email-fweisbec@gmail.com Signed-off-by: Ingo Molnar --- arch/alpha/kernel/osf_sys.c | 2 +- arch/x86/kernel/apm_32.c | 2 +- drivers/isdn/mISDN/stack.c | 2 +- fs/binfmt_elf.c | 6 +++--- fs/binfmt_elf_fdpic.c | 6 +++--- include/linux/sched.h | 32 ++++++++++++++++++++++++++--- kernel/acct.c | 2 +- kernel/delayacct.c | 4 ++-- kernel/signal.c | 4 ++-- kernel/time/itimer.c | 2 +- kernel/time/posix-cpu-timers.c | 46 +++++++++++++++++++++--------------------- kernel/tsacct.c | 6 +++--- 12 files changed, 70 insertions(+), 44 deletions(-) (limited to 'kernel/time') diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index 54d8616644e2..0f92438d736b 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -1154,7 +1154,7 @@ SYSCALL_DEFINE2(osf_getrusage, int, who, struct rusage32 __user *, ru) memset(&r, 0, sizeof(r)); switch (who) { case RUSAGE_SELF: - task_cputime(current, &utime, &stime); + task_cputime_t(current, &utime, &stime); utime_jiffies = cputime_to_jiffies(utime); stime_jiffies = cputime_to_jiffies(stime); jiffies_to_timeval32(utime_jiffies, &r.ru_utime); diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index 45d44c173cf9..89c84fcdd3c0 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c @@ -913,7 +913,7 @@ static int apm_cpu_idle(struct cpuidle_device *dev, unsigned int bucket; recalc: - task_cputime(current, &utime, &stime); + task_cputime_t(current, &utime, &stime); if (jiffies_since_last_check > IDLE_CALC_LIMIT) { use_apm_idle = 0; } else if (jiffies_since_last_check > idle_period) { diff --git a/drivers/isdn/mISDN/stack.c b/drivers/isdn/mISDN/stack.c index 9cb4b621fbc3..0a3661767531 100644 --- a/drivers/isdn/mISDN/stack.c +++ b/drivers/isdn/mISDN/stack.c @@ -306,7 +306,7 @@ mISDNStackd(void *data) "msg %d sleep %d stopped\n", dev_name(&st->dev->dev), st->msg_cnt, st->sleep_cnt, st->stopped_cnt); - task_cputime(st->thread, &utime, &stime); + task_cputime_t(st->thread, &utime, &stime); printk(KERN_DEBUG "mISDNStackd daemon for %s utime(%ld) stime(%ld)\n", dev_name(&st->dev->dev), utime, stime); diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 422370293cfd..68b915650cae 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1421,19 +1421,19 @@ static void fill_prstatus(struct elf_prstatus *prstatus, prstatus->pr_pgrp = task_pgrp_vnr(p); prstatus->pr_sid = task_session_vnr(p); if (thread_group_leader(p)) { - struct task_cputime cputime; + struct task_cputime_t cputime; /* * This is the record for the group leader. It shows the * group-wide total, not its individual thread total. */ - thread_group_cputime(p, &cputime); + thread_group_cputime_t(p, &cputime); cputime_to_timeval(cputime.utime, &prstatus->pr_utime); cputime_to_timeval(cputime.stime, &prstatus->pr_stime); } else { cputime_t utime, stime; - task_cputime(p, &utime, &stime); + task_cputime_t(p, &utime, &stime); cputime_to_timeval(utime, &prstatus->pr_utime); cputime_to_timeval(stime, &prstatus->pr_stime); } diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index d2e36f82c35d..6ccd9df7247a 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -1342,19 +1342,19 @@ static void fill_prstatus(struct elf_prstatus *prstatus, prstatus->pr_pgrp = task_pgrp_vnr(p); prstatus->pr_sid = task_session_vnr(p); if (thread_group_leader(p)) { - struct task_cputime cputime; + struct task_cputime_t cputime; /* * This is the record for the group leader. It shows the * group-wide total, not its individual thread total. */ - thread_group_cputime(p, &cputime); + thread_group_cputime_t(p, &cputime); cputime_to_timeval(cputime.utime, &prstatus->pr_utime); cputime_to_timeval(cputime.stime, &prstatus->pr_stime); } else { cputime_t utime, stime; - task_cputime(p, &utime, &stime); + task_cputime_t(p, &utime, &stime); cputime_to_timeval(utime, &prstatus->pr_utime); cputime_to_timeval(stime, &prstatus->pr_stime); } diff --git a/include/linux/sched.h b/include/linux/sched.h index 252ff25983c8..9cc722f77799 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -615,6 +615,13 @@ struct task_cputime { unsigned long long sum_exec_runtime; }; +/* Temporary type to ease cputime_t to nsecs conversion */ +struct task_cputime_t { + cputime_t utime; + cputime_t stime; + unsigned long long sum_exec_runtime; +}; + /* Alternate field names when used to cache expirations. */ #define virt_exp utime #define prof_exp stime @@ -748,7 +755,7 @@ struct signal_struct { struct thread_group_cputimer cputimer; /* Earliest-expiration cache. */ - struct task_cputime cputime_expires; + struct task_cputime_t cputime_expires; #ifdef CONFIG_NO_HZ_FULL atomic_t tick_dep_mask; @@ -1682,7 +1689,7 @@ struct task_struct { /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */ unsigned long min_flt, maj_flt; - struct task_cputime cputime_expires; + struct task_cputime_t cputime_expires; struct list_head cpu_timers[3]; /* process credentials */ @@ -2286,6 +2293,19 @@ static inline void task_cputime_scaled(struct task_struct *t, } #endif +static inline void task_cputime_t(struct task_struct *t, + cputime_t *utime, cputime_t *stime) +{ + task_cputime(t, utime, stime); +} + +static inline void task_cputime_t_scaled(struct task_struct *t, + cputime_t *utimescaled, + cputime_t *stimescaled) +{ + task_cputime_scaled(t, utimescaled, stimescaled); +} + extern void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st); extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st); @@ -3499,7 +3519,13 @@ static __always_inline bool need_resched(void) * Thread group CPU time accounting. */ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); -void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); +void thread_group_cputimer(struct task_struct *tsk, struct task_cputime_t *times); + +static inline void thread_group_cputime_t(struct task_struct *tsk, + struct task_cputime_t *times) +{ + thread_group_cputime(tsk, (struct task_cputime *)times); +} /* * Reevaluate whether the task has signals pending delivery. diff --git a/kernel/acct.c b/kernel/acct.c index 74963d192c5d..b9b190a8eecf 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -559,7 +559,7 @@ void acct_collect(long exitcode, int group_dead) pacct->ac_flag |= ACORE; if (current->flags & PF_SIGNALED) pacct->ac_flag |= AXSIG; - task_cputime(current, &utime, &stime); + task_cputime_t(current, &utime, &stime); pacct->ac_utime += utime; pacct->ac_stime += stime; pacct->ac_minflt += current->min_flt; diff --git a/kernel/delayacct.c b/kernel/delayacct.c index 435c14a45118..228640f2b3d2 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -87,12 +87,12 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) unsigned long flags, t1; s64 tmp; - task_cputime(tsk, &utime, &stime); + task_cputime_t(tsk, &utime, &stime); tmp = (s64)d->cpu_run_real_total; tmp += cputime_to_nsecs(utime + stime); d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp; - task_cputime_scaled(tsk, &utimescaled, &stimescaled); + task_cputime_t_scaled(tsk, &utimescaled, &stimescaled); tmp = (s64)d->cpu_scaled_run_real_total; tmp += cputime_to_nsecs(utimescaled + stimescaled); d->cpu_scaled_run_real_total = diff --git a/kernel/signal.c b/kernel/signal.c index 3603d93a1968..218048a837ea 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1619,7 +1619,7 @@ bool do_notify_parent(struct task_struct *tsk, int sig) task_uid(tsk)); rcu_read_unlock(); - task_cputime(tsk, &utime, &stime); + task_cputime_t(tsk, &utime, &stime); info.si_utime = cputime_to_clock_t(utime + tsk->signal->utime); info.si_stime = cputime_to_clock_t(stime + tsk->signal->stime); @@ -1704,7 +1704,7 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, info.si_uid = from_kuid_munged(task_cred_xxx(parent, user_ns), task_uid(tsk)); rcu_read_unlock(); - task_cputime(tsk, &utime, &stime); + task_cputime_t(tsk, &utime, &stime); info.si_utime = cputime_to_clock_t(utime); info.si_stime = cputime_to_clock_t(stime); diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c index 8c89143f9ebf..f2d5097bcb6d 100644 --- a/kernel/time/itimer.c +++ b/kernel/time/itimer.c @@ -53,7 +53,7 @@ static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, cval = it->expires; cinterval = it->incr; if (cval) { - struct task_cputime cputime; + struct task_cputime_t cputime; cputime_t t; thread_group_cputimer(tsk, &cputime); diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index e9e8c10f0d9a..d53ff711a2a8 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -115,7 +115,7 @@ static void bump_cpu_timer(struct k_itimer *timer, * Checks @cputime to see if all fields are zero. Returns true if all fields * are zero, false if any field is nonzero. */ -static inline int task_cputime_zero(const struct task_cputime *cputime) +static inline int task_cputime_zero(const struct task_cputime_t *cputime) { if (!cputime->utime && !cputime->stime && !cputime->sum_exec_runtime) return 1; @@ -126,7 +126,7 @@ static inline unsigned long long prof_ticks(struct task_struct *p) { cputime_t utime, stime; - task_cputime(p, &utime, &stime); + task_cputime_t(p, &utime, &stime); return cputime_to_expires(utime + stime); } @@ -134,7 +134,7 @@ static inline unsigned long long virt_ticks(struct task_struct *p) { cputime_t utime, stime; - task_cputime(p, &utime, &stime); + task_cputime_t(p, &utime, &stime); return cputime_to_expires(utime); } @@ -210,7 +210,7 @@ retry: } } -static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct task_cputime *sum) +static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct task_cputime_t *sum) { __update_gt_cputime(&cputime_atomic->utime, sum->utime); __update_gt_cputime(&cputime_atomic->stime, sum->stime); @@ -218,7 +218,7 @@ static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct } /* Sample task_cputime_atomic values in "atomic_timers", store results in "times". */ -static inline void sample_cputime_atomic(struct task_cputime *times, +static inline void sample_cputime_atomic(struct task_cputime_t *times, struct task_cputime_atomic *atomic_times) { times->utime = atomic64_read(&atomic_times->utime); @@ -226,10 +226,10 @@ static inline void sample_cputime_atomic(struct task_cputime *times, times->sum_exec_runtime = atomic64_read(&atomic_times->sum_exec_runtime); } -void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) +void thread_group_cputimer(struct task_struct *tsk, struct task_cputime_t *times) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - struct task_cputime sum; + struct task_cputime_t sum; /* Check if cputimer isn't running. This is accessed without locking. */ if (!READ_ONCE(cputimer->running)) { @@ -238,7 +238,7 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) * values through the TIMER_ABSTIME flag, therefore we have * to synchronize the timer to the clock every time we start it. */ - thread_group_cputime(tsk, &sum); + thread_group_cputime_t(tsk, &sum); update_gt_cputime(&cputimer->cputime_atomic, &sum); /* @@ -262,21 +262,21 @@ static int cpu_clock_sample_group(const clockid_t which_clock, struct task_struct *p, unsigned long long *sample) { - struct task_cputime cputime; + struct task_cputime_t cputime; switch (CPUCLOCK_WHICH(which_clock)) { default: return -EINVAL; case CPUCLOCK_PROF: - thread_group_cputime(p, &cputime); + thread_group_cputime_t(p, &cputime); *sample = cputime_to_expires(cputime.utime + cputime.stime); break; case CPUCLOCK_VIRT: - thread_group_cputime(p, &cputime); + thread_group_cputime_t(p, &cputime); *sample = cputime_to_expires(cputime.utime); break; case CPUCLOCK_SCHED: - thread_group_cputime(p, &cputime); + thread_group_cputime_t(p, &cputime); *sample = cputime.sum_exec_runtime; break; } @@ -466,7 +466,7 @@ static void arm_timer(struct k_itimer *timer) { struct task_struct *p = timer->it.cpu.task; struct list_head *head, *listpos; - struct task_cputime *cputime_expires; + struct task_cputime_t *cputime_expires; struct cpu_timer_list *const nt = &timer->it.cpu; struct cpu_timer_list *next; @@ -562,7 +562,7 @@ static int cpu_timer_sample_group(const clockid_t which_clock, struct task_struct *p, unsigned long long *sample) { - struct task_cputime cputime; + struct task_cputime_t cputime; thread_group_cputimer(p, &cputime); switch (CPUCLOCK_WHICH(which_clock)) { @@ -761,7 +761,7 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) /* * Protect against sighand release/switch in exit/exec and * also make timer sampling safe if it ends up calling - * thread_group_cputime(). + * thread_group_cputime_t(). */ sighand = lock_task_sighand(p, &flags); if (unlikely(sighand == NULL)) { @@ -826,7 +826,7 @@ static void check_thread_timers(struct task_struct *tsk, { struct list_head *timers = tsk->cpu_timers; struct signal_struct *const sig = tsk->signal; - struct task_cputime *tsk_expires = &tsk->cputime_expires; + struct task_cputime_t *tsk_expires = &tsk->cputime_expires; unsigned long long expires; unsigned long soft; @@ -934,7 +934,7 @@ static void check_process_timers(struct task_struct *tsk, unsigned long long utime, ptime, virt_expires, prof_expires; unsigned long long sum_sched_runtime, sched_expires; struct list_head *timers = sig->cpu_timers; - struct task_cputime cputime; + struct task_cputime_t cputime; unsigned long soft; /* @@ -1037,7 +1037,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) } else { /* * Protect arm_timer() and timer sampling in case of call to - * thread_group_cputime(). + * thread_group_cputime_t(). */ sighand = lock_task_sighand(p, &flags); if (unlikely(sighand == NULL)) { @@ -1080,8 +1080,8 @@ out: * Returns true if any field of the former is greater than the corresponding * field of the latter if the latter field is set. Otherwise returns false. */ -static inline int task_cputime_expired(const struct task_cputime *sample, - const struct task_cputime *expires) +static inline int task_cputime_expired(const struct task_cputime_t *sample, + const struct task_cputime_t *expires) { if (expires->utime && sample->utime >= expires->utime) return 1; @@ -1108,9 +1108,9 @@ static inline int fastpath_timer_check(struct task_struct *tsk) struct signal_struct *sig; if (!task_cputime_zero(&tsk->cputime_expires)) { - struct task_cputime task_sample; + struct task_cputime_t task_sample; - task_cputime(tsk, &task_sample.utime, &task_sample.stime); + task_cputime_t(tsk, &task_sample.utime, &task_sample.stime); task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) return 1; @@ -1133,7 +1133,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk) */ if (READ_ONCE(sig->cputimer.running) && !READ_ONCE(sig->cputimer.checking_timer)) { - struct task_cputime group_sample; + struct task_cputime_t group_sample; sample_cputime_atomic(&group_sample, &sig->cputimer.cputime_atomic); diff --git a/kernel/tsacct.c b/kernel/tsacct.c index f8e26ab963ed..040d0a64d0d1 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -66,11 +66,11 @@ void bacct_add_tsk(struct user_namespace *user_ns, task_tgid_nr_ns(rcu_dereference(tsk->real_parent), pid_ns) : 0; rcu_read_unlock(); - task_cputime(tsk, &utime, &stime); + task_cputime_t(tsk, &utime, &stime); stats->ac_utime = cputime_to_usecs(utime); stats->ac_stime = cputime_to_usecs(stime); - task_cputime_scaled(tsk, &utimescaled, &stimescaled); + task_cputime_t_scaled(tsk, &utimescaled, &stimescaled); stats->ac_utimescaled = cputime_to_usecs(utimescaled); stats->ac_stimescaled = cputime_to_usecs(stimescaled); @@ -159,7 +159,7 @@ void acct_update_integrals(struct task_struct *tsk) unsigned long flags; local_irq_save(flags); - task_cputime(tsk, &utime, &stime); + task_cputime_t(tsk, &utime, &stime); __acct_update_integrals(tsk, utime, stime); local_irq_restore(flags); } -- cgit v1.2.3 From 715eb7a9243a058a0722aa2f6ba703ede9113e76 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 31 Jan 2017 04:09:33 +0100 Subject: timers/posix-timers: Use TICK_NSEC instead of a dynamically ad-hoc calculated version Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Fenghua Yu Cc: Heiko Carstens Cc: Linus Torvalds Cc: Martin Schwidefsky Cc: Michael Ellerman Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Rik van Riel Cc: Stanislaw Gruszka Cc: Thomas Gleixner Cc: Tony Luck Cc: Wanpeng Li Link: http://lkml.kernel.org/r/1485832191-26889-18-git-send-email-fweisbec@gmail.com Signed-off-by: Ingo Molnar --- kernel/time/posix-cpu-timers.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'kernel/time') diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index d53ff711a2a8..8349e02b1c0c 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -890,8 +890,6 @@ static inline void stop_process_timers(struct signal_struct *sig) tick_dep_clear_signal(sig, TICK_DEP_BIT_POSIX_TIMER); } -static u32 onecputick; - static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, unsigned long long *expires, unsigned long long cur_time, int signo) @@ -903,9 +901,9 @@ static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, if (it->incr) { it->expires += it->incr; it->error += it->incr_error; - if (it->error >= onecputick) { + if (it->error >= TICK_NSEC) { it->expires -= cputime_one_jiffy; - it->error -= onecputick; + it->error -= TICK_NSEC; } } else { it->expires = 0; @@ -1476,15 +1474,10 @@ static __init int init_posix_cpu_timers(void) .clock_get = thread_cpu_clock_get, .timer_create = thread_cpu_timer_create, }; - struct timespec ts; posix_timers_register_clock(CLOCK_PROCESS_CPUTIME_ID, &process); posix_timers_register_clock(CLOCK_THREAD_CPUTIME_ID, &thread); - cputime_to_timespec(cputime_one_jiffy, &ts); - onecputick = ts.tv_nsec; - WARN_ON(ts.tv_sec != 0); - return 0; } __initcall(init_posix_cpu_timers); -- cgit v1.2.3 From ebd7e7fc4bc63be5eaf9da903b8060b02dd711ea Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 31 Jan 2017 04:09:34 +0100 Subject: timers/posix-timers: Convert internals to use nsecs Use the new nsec based cputime accessors as part of the whole cputime conversion from cputime_t to nsecs. Also convert posix-cpu-timers to use nsec based internal counters to simplify it. Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Fenghua Yu Cc: Heiko Carstens Cc: Linus Torvalds Cc: Martin Schwidefsky Cc: Michael Ellerman Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Rik van Riel Cc: Stanislaw Gruszka Cc: Thomas Gleixner Cc: Tony Luck Cc: Wanpeng Li Link: http://lkml.kernel.org/r/1485832191-26889-19-git-send-email-fweisbec@gmail.com Signed-off-by: Ingo Molnar --- include/linux/posix-timers.h | 12 +-- include/linux/sched.h | 6 +- kernel/fork.c | 2 +- kernel/sched/cputime.c | 6 +- kernel/sched/stats.h | 4 +- kernel/time/itimer.c | 6 +- kernel/time/posix-cpu-timers.c | 210 +++++++++++++++++------------------------ 7 files changed, 100 insertions(+), 146 deletions(-) (limited to 'kernel/time') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 62d44c176071..890de52362d0 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -8,19 +8,9 @@ #include -static inline unsigned long long cputime_to_expires(cputime_t expires) -{ - return (__force unsigned long long)expires; -} - -static inline cputime_t expires_to_cputime(unsigned long long expires) -{ - return (__force cputime_t)expires; -} - struct cpu_timer_list { struct list_head entry; - unsigned long long expires, incr; + u64 expires, incr; struct task_struct *task; int firing; }; diff --git a/include/linux/sched.h b/include/linux/sched.h index dc44366128d8..baa6a2834e0f 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -755,7 +755,7 @@ struct signal_struct { struct thread_group_cputimer cputimer; /* Earliest-expiration cache. */ - struct task_cputime_t cputime_expires; + struct task_cputime cputime_expires; #ifdef CONFIG_NO_HZ_FULL atomic_t tick_dep_mask; @@ -1689,7 +1689,7 @@ struct task_struct { /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */ unsigned long min_flt, maj_flt; - struct task_cputime_t cputime_expires; + struct task_cputime cputime_expires; struct list_head cpu_timers[3]; /* process credentials */ @@ -3527,7 +3527,7 @@ static __always_inline bool need_resched(void) * Thread group CPU time accounting. */ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); -void thread_group_cputimer(struct task_struct *tsk, struct task_cputime_t *times); +void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); static inline void thread_group_cputime_t(struct task_struct *tsk, struct task_cputime_t *cputime) diff --git a/kernel/fork.c b/kernel/fork.c index 11c5c8ab827c..09992ff2f8fa 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1313,7 +1313,7 @@ static void posix_cpu_timers_init_group(struct signal_struct *sig) cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); if (cpu_limit != RLIM_INFINITY) { - sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit); + sig->cputime_expires.prof_exp = cpu_limit * NSEC_PER_SEC; sig->cputimer.running = true; } diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index bee6c97b1e83..f7b9624c7df0 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -122,7 +122,7 @@ void account_user_time(struct task_struct *p, cputime_t cputime) /* Add user time to process. */ p->utime += cputime_to_nsecs(cputime); - account_group_user_time(p, cputime); + account_group_user_time(p, cputime_to_nsecs(cputime)); index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER; @@ -144,7 +144,7 @@ void account_guest_time(struct task_struct *p, cputime_t cputime) /* Add guest time to process. */ p->utime += cputime_to_nsecs(cputime); - account_group_user_time(p, cputime); + account_group_user_time(p, cputime_to_nsecs(cputime)); p->gtime += cputime_to_nsecs(cputime); /* Add guest time to cpustat. */ @@ -168,7 +168,7 @@ void account_system_index_time(struct task_struct *p, { /* Add system time to process. */ p->stime += cputime_to_nsecs(cputime); - account_group_system_time(p, cputime); + account_group_system_time(p, cputime_to_nsecs(cputime)); /* Add system time to cpustat. */ task_group_account_field(p, index, cputime_to_nsecs(cputime)); diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index 34659a853505..9788478a66d4 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -216,7 +216,7 @@ static inline bool cputimer_running(struct task_struct *tsk) * running CPU and update the utime field there. */ static inline void account_group_user_time(struct task_struct *tsk, - cputime_t cputime) + u64 cputime) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; @@ -237,7 +237,7 @@ static inline void account_group_user_time(struct task_struct *tsk, * running CPU and update the stime field there. */ static inline void account_group_system_time(struct task_struct *tsk, - cputime_t cputime) + u64 cputime) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c index f2d5097bcb6d..bb01ff445ce9 100644 --- a/kernel/time/itimer.c +++ b/kernel/time/itimer.c @@ -53,15 +53,15 @@ static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, cval = it->expires; cinterval = it->incr; if (cval) { - struct task_cputime_t cputime; + struct task_cputime cputime; cputime_t t; thread_group_cputimer(tsk, &cputime); if (clock_id == CPUCLOCK_PROF) - t = cputime.utime + cputime.stime; + t = nsecs_to_cputime(cputime.utime + cputime.stime); else /* CPUCLOCK_VIRT */ - t = cputime.utime; + t = nsecs_to_cputime(cputime.utime); if (cval < t) /* about to fire */ diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 8349e02b1c0c..45be3cec0dc2 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -50,39 +50,14 @@ static int check_clock(const clockid_t which_clock) return error; } -static inline unsigned long long -timespec_to_sample(const clockid_t which_clock, const struct timespec *tp) -{ - unsigned long long ret; - - ret = 0; /* high half always zero when .cpu used */ - if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) { - ret = (unsigned long long)tp->tv_sec * NSEC_PER_SEC + tp->tv_nsec; - } else { - ret = cputime_to_expires(timespec_to_cputime(tp)); - } - return ret; -} - -static void sample_to_timespec(const clockid_t which_clock, - unsigned long long expires, - struct timespec *tp) -{ - if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) - *tp = ns_to_timespec(expires); - else - cputime_to_timespec((__force cputime_t)expires, tp); -} - /* * Update expiry time from increment, and increase overrun count, * given the current clock sample. */ -static void bump_cpu_timer(struct k_itimer *timer, - unsigned long long now) +static void bump_cpu_timer(struct k_itimer *timer, u64 now) { int i; - unsigned long long delta, incr; + u64 delta, incr; if (timer->it.cpu.incr == 0) return; @@ -115,28 +90,28 @@ static void bump_cpu_timer(struct k_itimer *timer, * Checks @cputime to see if all fields are zero. Returns true if all fields * are zero, false if any field is nonzero. */ -static inline int task_cputime_zero(const struct task_cputime_t *cputime) +static inline int task_cputime_zero(const struct task_cputime *cputime) { if (!cputime->utime && !cputime->stime && !cputime->sum_exec_runtime) return 1; return 0; } -static inline unsigned long long prof_ticks(struct task_struct *p) +static inline u64 prof_ticks(struct task_struct *p) { - cputime_t utime, stime; + u64 utime, stime; - task_cputime_t(p, &utime, &stime); + task_cputime(p, &utime, &stime); - return cputime_to_expires(utime + stime); + return utime + stime; } -static inline unsigned long long virt_ticks(struct task_struct *p) +static inline u64 virt_ticks(struct task_struct *p) { - cputime_t utime, stime; + u64 utime, stime; - task_cputime_t(p, &utime, &stime); + task_cputime(p, &utime, &stime); - return cputime_to_expires(utime); + return utime; } static int @@ -176,8 +151,8 @@ posix_cpu_clock_set(const clockid_t which_clock, const struct timespec *tp) /* * Sample a per-thread clock for the given task. */ -static int cpu_clock_sample(const clockid_t which_clock, struct task_struct *p, - unsigned long long *sample) +static int cpu_clock_sample(const clockid_t which_clock, + struct task_struct *p, u64 *sample) { switch (CPUCLOCK_WHICH(which_clock)) { default: @@ -210,7 +185,7 @@ retry: } } -static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct task_cputime_t *sum) +static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct task_cputime *sum) { __update_gt_cputime(&cputime_atomic->utime, sum->utime); __update_gt_cputime(&cputime_atomic->stime, sum->stime); @@ -218,7 +193,7 @@ static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct } /* Sample task_cputime_atomic values in "atomic_timers", store results in "times". */ -static inline void sample_cputime_atomic(struct task_cputime_t *times, +static inline void sample_cputime_atomic(struct task_cputime *times, struct task_cputime_atomic *atomic_times) { times->utime = atomic64_read(&atomic_times->utime); @@ -226,10 +201,10 @@ static inline void sample_cputime_atomic(struct task_cputime_t *times, times->sum_exec_runtime = atomic64_read(&atomic_times->sum_exec_runtime); } -void thread_group_cputimer(struct task_struct *tsk, struct task_cputime_t *times) +void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - struct task_cputime_t sum; + struct task_cputime sum; /* Check if cputimer isn't running. This is accessed without locking. */ if (!READ_ONCE(cputimer->running)) { @@ -238,7 +213,7 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime_t *times * values through the TIMER_ABSTIME flag, therefore we have * to synchronize the timer to the clock every time we start it. */ - thread_group_cputime_t(tsk, &sum); + thread_group_cputime(tsk, &sum); update_gt_cputime(&cputimer->cputime_atomic, &sum); /* @@ -260,23 +235,23 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime_t *times */ static int cpu_clock_sample_group(const clockid_t which_clock, struct task_struct *p, - unsigned long long *sample) + u64 *sample) { - struct task_cputime_t cputime; + struct task_cputime cputime; switch (CPUCLOCK_WHICH(which_clock)) { default: return -EINVAL; case CPUCLOCK_PROF: - thread_group_cputime_t(p, &cputime); - *sample = cputime_to_expires(cputime.utime + cputime.stime); + thread_group_cputime(p, &cputime); + *sample = cputime.utime + cputime.stime; break; case CPUCLOCK_VIRT: - thread_group_cputime_t(p, &cputime); - *sample = cputime_to_expires(cputime.utime); + thread_group_cputime(p, &cputime); + *sample = cputime.utime; break; case CPUCLOCK_SCHED: - thread_group_cputime_t(p, &cputime); + thread_group_cputime(p, &cputime); *sample = cputime.sum_exec_runtime; break; } @@ -288,7 +263,7 @@ static int posix_cpu_clock_get_task(struct task_struct *tsk, struct timespec *tp) { int err = -EINVAL; - unsigned long long rtn; + u64 rtn; if (CPUCLOCK_PERTHREAD(which_clock)) { if (same_thread_group(tsk, current)) @@ -299,7 +274,7 @@ static int posix_cpu_clock_get_task(struct task_struct *tsk, } if (!err) - sample_to_timespec(which_clock, rtn, tp); + *tp = ns_to_timespec(rtn); return err; } @@ -453,7 +428,7 @@ void posix_cpu_timers_exit_group(struct task_struct *tsk) cleanup_timers(tsk->signal->cpu_timers); } -static inline int expires_gt(cputime_t expires, cputime_t new_exp) +static inline int expires_gt(u64 expires, u64 new_exp) { return expires == 0 || expires > new_exp; } @@ -466,7 +441,7 @@ static void arm_timer(struct k_itimer *timer) { struct task_struct *p = timer->it.cpu.task; struct list_head *head, *listpos; - struct task_cputime_t *cputime_expires; + struct task_cputime *cputime_expires; struct cpu_timer_list *const nt = &timer->it.cpu; struct cpu_timer_list *next; @@ -488,7 +463,7 @@ static void arm_timer(struct k_itimer *timer) list_add(&nt->entry, listpos); if (listpos == head) { - unsigned long long exp = nt->expires; + u64 exp = nt->expires; /* * We are the new earliest-expiring POSIX 1.b timer, hence @@ -499,16 +474,15 @@ static void arm_timer(struct k_itimer *timer) switch (CPUCLOCK_WHICH(timer->it_clock)) { case CPUCLOCK_PROF: - if (expires_gt(cputime_expires->prof_exp, expires_to_cputime(exp))) - cputime_expires->prof_exp = expires_to_cputime(exp); + if (expires_gt(cputime_expires->prof_exp, exp)) + cputime_expires->prof_exp = exp; break; case CPUCLOCK_VIRT: - if (expires_gt(cputime_expires->virt_exp, expires_to_cputime(exp))) - cputime_expires->virt_exp = expires_to_cputime(exp); + if (expires_gt(cputime_expires->virt_exp, exp)) + cputime_expires->virt_exp = exp; break; case CPUCLOCK_SCHED: - if (cputime_expires->sched_exp == 0 || - cputime_expires->sched_exp > exp) + if (expires_gt(cputime_expires->sched_exp, exp)) cputime_expires->sched_exp = exp; break; } @@ -559,20 +533,19 @@ static void cpu_timer_fire(struct k_itimer *timer) * traversal. */ static int cpu_timer_sample_group(const clockid_t which_clock, - struct task_struct *p, - unsigned long long *sample) + struct task_struct *p, u64 *sample) { - struct task_cputime_t cputime; + struct task_cputime cputime; thread_group_cputimer(p, &cputime); switch (CPUCLOCK_WHICH(which_clock)) { default: return -EINVAL; case CPUCLOCK_PROF: - *sample = cputime_to_expires(cputime.utime + cputime.stime); + *sample = cputime.utime + cputime.stime; break; case CPUCLOCK_VIRT: - *sample = cputime_to_expires(cputime.utime); + *sample = cputime.utime; break; case CPUCLOCK_SCHED: *sample = cputime.sum_exec_runtime; @@ -593,12 +566,12 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, unsigned long flags; struct sighand_struct *sighand; struct task_struct *p = timer->it.cpu.task; - unsigned long long old_expires, new_expires, old_incr, val; + u64 old_expires, new_expires, old_incr, val; int ret; WARN_ON_ONCE(p == NULL); - new_expires = timespec_to_sample(timer->it_clock, &new->it_value); + new_expires = timespec_to_ns(&new->it_value); /* * Protect against sighand release/switch in exit/exec and p->cpu_timers @@ -659,9 +632,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, bump_cpu_timer(timer, val); if (val < timer->it.cpu.expires) { old_expires = timer->it.cpu.expires - val; - sample_to_timespec(timer->it_clock, - old_expires, - &old->it_value); + old->it_value = ns_to_timespec(old_expires); } else { old->it_value.tv_nsec = 1; old->it_value.tv_sec = 0; @@ -699,8 +670,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, * Install the new reload setting, and * set up the signal and overrun bookkeeping. */ - timer->it.cpu.incr = timespec_to_sample(timer->it_clock, - &new->it_interval); + timer->it.cpu.incr = timespec_to_ns(&new->it_interval); /* * This acts as a modification timestamp for the timer, @@ -723,17 +693,15 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, ret = 0; out: - if (old) { - sample_to_timespec(timer->it_clock, - old_incr, &old->it_interval); - } + if (old) + old->it_interval = ns_to_timespec(old_incr); return ret; } static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) { - unsigned long long now; + u64 now; struct task_struct *p = timer->it.cpu.task; WARN_ON_ONCE(p == NULL); @@ -741,8 +709,7 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) /* * Easy part: convert the reload time. */ - sample_to_timespec(timer->it_clock, - timer->it.cpu.incr, &itp->it_interval); + itp->it_interval = ns_to_timespec(timer->it.cpu.incr); if (timer->it.cpu.expires == 0) { /* Timer not armed at all. */ itp->it_value.tv_sec = itp->it_value.tv_nsec = 0; @@ -761,7 +728,7 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) /* * Protect against sighand release/switch in exit/exec and * also make timer sampling safe if it ends up calling - * thread_group_cputime_t(). + * thread_group_cputime(). */ sighand = lock_task_sighand(p, &flags); if (unlikely(sighand == NULL)) { @@ -771,8 +738,7 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) * Call the timer disarmed, nothing else to do. */ timer->it.cpu.expires = 0; - sample_to_timespec(timer->it_clock, timer->it.cpu.expires, - &itp->it_value); + itp->it_value = ns_to_timespec(timer->it.cpu.expires); return; } else { cpu_timer_sample_group(timer->it_clock, p, &now); @@ -781,9 +747,7 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) } if (now < timer->it.cpu.expires) { - sample_to_timespec(timer->it_clock, - timer->it.cpu.expires - now, - &itp->it_value); + itp->it_value = ns_to_timespec(timer->it.cpu.expires - now); } else { /* * The timer should have expired already, but the firing @@ -826,8 +790,8 @@ static void check_thread_timers(struct task_struct *tsk, { struct list_head *timers = tsk->cpu_timers; struct signal_struct *const sig = tsk->signal; - struct task_cputime_t *tsk_expires = &tsk->cputime_expires; - unsigned long long expires; + struct task_cputime *tsk_expires = &tsk->cputime_expires; + u64 expires; unsigned long soft; /* @@ -838,10 +802,10 @@ static void check_thread_timers(struct task_struct *tsk, return; expires = check_timers_list(timers, firing, prof_ticks(tsk)); - tsk_expires->prof_exp = expires_to_cputime(expires); + tsk_expires->prof_exp = expires; expires = check_timers_list(++timers, firing, virt_ticks(tsk)); - tsk_expires->virt_exp = expires_to_cputime(expires); + tsk_expires->virt_exp = expires; tsk_expires->sched_exp = check_timers_list(++timers, firing, tsk->se.sum_exec_runtime); @@ -891,13 +855,12 @@ static inline void stop_process_timers(struct signal_struct *sig) } static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, - unsigned long long *expires, - unsigned long long cur_time, int signo) + u64 *expires, u64 cur_time, int signo) { if (!it->expires) return; - if (cur_time >= it->expires) { + if (cur_time >= cputime_to_nsecs(it->expires)) { if (it->incr) { it->expires += it->incr; it->error += it->incr_error; @@ -915,8 +878,8 @@ static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, __group_send_sig_info(signo, SEND_SIG_PRIV, tsk); } - if (it->expires && (!*expires || it->expires < *expires)) { - *expires = it->expires; + if (it->expires && (!*expires || cputime_to_nsecs(it->expires) < *expires)) { + *expires = cputime_to_nsecs(it->expires); } } @@ -929,10 +892,10 @@ static void check_process_timers(struct task_struct *tsk, struct list_head *firing) { struct signal_struct *const sig = tsk->signal; - unsigned long long utime, ptime, virt_expires, prof_expires; - unsigned long long sum_sched_runtime, sched_expires; + u64 utime, ptime, virt_expires, prof_expires; + u64 sum_sched_runtime, sched_expires; struct list_head *timers = sig->cpu_timers; - struct task_cputime_t cputime; + struct task_cputime cputime; unsigned long soft; /* @@ -952,8 +915,8 @@ static void check_process_timers(struct task_struct *tsk, * Collect the current process totals. */ thread_group_cputimer(tsk, &cputime); - utime = cputime_to_expires(cputime.utime); - ptime = utime + cputime_to_expires(cputime.stime); + utime = cputime.utime; + ptime = utime + cputime.stime; sum_sched_runtime = cputime.sum_exec_runtime; prof_expires = check_timers_list(timers, firing, ptime); @@ -969,10 +932,10 @@ static void check_process_timers(struct task_struct *tsk, SIGVTALRM); soft = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); if (soft != RLIM_INFINITY) { - unsigned long psecs = cputime_to_secs(ptime); + unsigned long psecs = div_u64(ptime, NSEC_PER_SEC); unsigned long hard = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_max); - cputime_t x; + u64 x; if (psecs >= hard) { /* * At the hard limit, we just die. @@ -991,14 +954,13 @@ static void check_process_timers(struct task_struct *tsk, sig->rlim[RLIMIT_CPU].rlim_cur = soft; } } - x = secs_to_cputime(soft); - if (!prof_expires || x < prof_expires) { + x = soft * NSEC_PER_SEC; + if (!prof_expires || x < prof_expires) prof_expires = x; - } } - sig->cputime_expires.prof_exp = expires_to_cputime(prof_expires); - sig->cputime_expires.virt_exp = expires_to_cputime(virt_expires); + sig->cputime_expires.prof_exp = prof_expires; + sig->cputime_expires.virt_exp = virt_expires; sig->cputime_expires.sched_exp = sched_expires; if (task_cputime_zero(&sig->cputime_expires)) stop_process_timers(sig); @@ -1015,7 +977,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) struct sighand_struct *sighand; unsigned long flags; struct task_struct *p = timer->it.cpu.task; - unsigned long long now; + u64 now; WARN_ON_ONCE(p == NULL); @@ -1035,7 +997,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) } else { /* * Protect arm_timer() and timer sampling in case of call to - * thread_group_cputime_t(). + * thread_group_cputime(). */ sighand = lock_task_sighand(p, &flags); if (unlikely(sighand == NULL)) { @@ -1078,8 +1040,8 @@ out: * Returns true if any field of the former is greater than the corresponding * field of the latter if the latter field is set. Otherwise returns false. */ -static inline int task_cputime_expired(const struct task_cputime_t *sample, - const struct task_cputime_t *expires) +static inline int task_cputime_expired(const struct task_cputime *sample, + const struct task_cputime *expires) { if (expires->utime && sample->utime >= expires->utime) return 1; @@ -1106,9 +1068,9 @@ static inline int fastpath_timer_check(struct task_struct *tsk) struct signal_struct *sig; if (!task_cputime_zero(&tsk->cputime_expires)) { - struct task_cputime_t task_sample; + struct task_cputime task_sample; - task_cputime_t(tsk, &task_sample.utime, &task_sample.stime); + task_cputime(tsk, &task_sample.utime, &task_sample.stime); task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) return 1; @@ -1131,7 +1093,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk) */ if (READ_ONCE(sig->cputimer.running) && !READ_ONCE(sig->cputimer.checking_timer)) { - struct task_cputime_t group_sample; + struct task_cputime group_sample; sample_cputime_atomic(&group_sample, &sig->cputimer.cputime_atomic); @@ -1214,7 +1176,7 @@ void run_posix_cpu_timers(struct task_struct *tsk) void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, cputime_t *newval, cputime_t *oldval) { - unsigned long long now; + u64 now, new; WARN_ON_ONCE(clock_idx == CPUCLOCK_SCHED); cpu_timer_sample_group(clock_idx, tsk, &now); @@ -1226,31 +1188,33 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, * it to be absolute. */ if (*oldval) { - if (*oldval <= now) { + if (cputime_to_nsecs(*oldval) <= now) { /* Just about to fire. */ *oldval = cputime_one_jiffy; } else { - *oldval -= now; + *oldval -= nsecs_to_cputime(now); } } if (!*newval) return; - *newval += now; + *newval += nsecs_to_cputime(now); } + new = cputime_to_nsecs(*newval); + /* * Update expiration cache if we are the earliest timer, or eventually * RLIMIT_CPU limit is earlier than prof_exp cpu timer expire. */ switch (clock_idx) { case CPUCLOCK_PROF: - if (expires_gt(tsk->signal->cputime_expires.prof_exp, *newval)) - tsk->signal->cputime_expires.prof_exp = *newval; + if (expires_gt(tsk->signal->cputime_expires.prof_exp, new)) + tsk->signal->cputime_expires.prof_exp = new; break; case CPUCLOCK_VIRT: - if (expires_gt(tsk->signal->cputime_expires.virt_exp, *newval)) - tsk->signal->cputime_expires.virt_exp = *newval; + if (expires_gt(tsk->signal->cputime_expires.virt_exp, new)) + tsk->signal->cputime_expires.virt_exp = new; break; } @@ -1308,7 +1272,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, /* * We were interrupted by a signal. */ - sample_to_timespec(which_clock, timer.it.cpu.expires, rqtp); + *rqtp = ns_to_timespec(timer.it.cpu.expires); error = posix_cpu_timer_set(&timer, 0, &zero_it, it); if (!error) { /* -- cgit v1.2.3 From 858cf3a8c59968e7c5f7c1a1192459a0d52d1ab4 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 31 Jan 2017 04:09:35 +0100 Subject: timers/itimer: Convert internal cputime_t units to nsec Use the new nsec based cputime accessors as part of the whole cputime conversion from cputime_t to nsecs. Also convert itimers to use nsec based internal counters. This simplifies it and removes the whole game with error/inc_error which served to deal with cputime_t random granularity. Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Fenghua Yu Cc: Heiko Carstens Cc: Linus Torvalds Cc: Martin Schwidefsky Cc: Michael Ellerman Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Rik van Riel Cc: Stanislaw Gruszka Cc: Thomas Gleixner Cc: Tony Luck Cc: Wanpeng Li Link: http://lkml.kernel.org/r/1485832191-26889-20-git-send-email-fweisbec@gmail.com Signed-off-by: Ingo Molnar --- include/linux/posix-timers.h | 2 +- include/linux/sched.h | 6 ++-- include/trace/events/timer.h | 26 ++++++++--------- kernel/time/itimer.c | 64 +++++++++++++++--------------------------- kernel/time/posix-cpu-timers.c | 43 +++++++++++----------------- 5 files changed, 55 insertions(+), 86 deletions(-) (limited to 'kernel/time') diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 890de52362d0..64aa189efe21 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -119,7 +119,7 @@ void run_posix_cpu_timers(struct task_struct *task); void posix_cpu_timers_exit(struct task_struct *task); void posix_cpu_timers_exit_group(struct task_struct *task); void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx, - cputime_t *newval, cputime_t *oldval); + u64 *newval, u64 *oldval); long clock_nanosleep_restart(struct restart_block *restart_block); diff --git a/include/linux/sched.h b/include/linux/sched.h index baa6a2834e0f..268fdd713089 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -568,10 +568,8 @@ struct pacct_struct { }; struct cpu_itimer { - cputime_t expires; - cputime_t incr; - u32 error; - u32 incr_error; + u64 expires; + u64 incr; }; /** diff --git a/include/trace/events/timer.h b/include/trace/events/timer.h index 1448637616d6..1bca99dbb98f 100644 --- a/include/trace/events/timer.h +++ b/include/trace/events/timer.h @@ -269,17 +269,17 @@ DEFINE_EVENT(hrtimer_class, hrtimer_cancel, TRACE_EVENT(itimer_state, TP_PROTO(int which, const struct itimerval *const value, - cputime_t expires), + unsigned long long expires), TP_ARGS(which, value, expires), TP_STRUCT__entry( - __field( int, which ) - __field( cputime_t, expires ) - __field( long, value_sec ) - __field( long, value_usec ) - __field( long, interval_sec ) - __field( long, interval_usec ) + __field( int, which ) + __field( unsigned long long, expires ) + __field( long, value_sec ) + __field( long, value_usec ) + __field( long, interval_sec ) + __field( long, interval_usec ) ), TP_fast_assign( @@ -292,7 +292,7 @@ TRACE_EVENT(itimer_state, ), TP_printk("which=%d expires=%llu it_value=%ld.%ld it_interval=%ld.%ld", - __entry->which, (unsigned long long)__entry->expires, + __entry->which, __entry->expires, __entry->value_sec, __entry->value_usec, __entry->interval_sec, __entry->interval_usec) ); @@ -305,14 +305,14 @@ TRACE_EVENT(itimer_state, */ TRACE_EVENT(itimer_expire, - TP_PROTO(int which, struct pid *pid, cputime_t now), + TP_PROTO(int which, struct pid *pid, unsigned long long now), TP_ARGS(which, pid, now), TP_STRUCT__entry( - __field( int , which ) - __field( pid_t, pid ) - __field( cputime_t, now ) + __field( int , which ) + __field( pid_t, pid ) + __field( unsigned long long, now ) ), TP_fast_assign( @@ -322,7 +322,7 @@ TRACE_EVENT(itimer_expire, ), TP_printk("which=%d pid=%d now=%llu", __entry->which, - (int) __entry->pid, (unsigned long long)__entry->now) + (int) __entry->pid, __entry->now) ); #ifdef CONFIG_NO_HZ_COMMON diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c index bb01ff445ce9..a95f13c31464 100644 --- a/kernel/time/itimer.c +++ b/kernel/time/itimer.c @@ -45,35 +45,35 @@ static struct timeval itimer_get_remtime(struct hrtimer *timer) static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, struct itimerval *const value) { - cputime_t cval, cinterval; + u64 val, interval; struct cpu_itimer *it = &tsk->signal->it[clock_id]; spin_lock_irq(&tsk->sighand->siglock); - cval = it->expires; - cinterval = it->incr; - if (cval) { + val = it->expires; + interval = it->incr; + if (val) { struct task_cputime cputime; - cputime_t t; + u64 t; thread_group_cputimer(tsk, &cputime); if (clock_id == CPUCLOCK_PROF) - t = nsecs_to_cputime(cputime.utime + cputime.stime); + t = cputime.utime + cputime.stime; else /* CPUCLOCK_VIRT */ - t = nsecs_to_cputime(cputime.utime); + t = cputime.utime; - if (cval < t) + if (val < t) /* about to fire */ - cval = cputime_one_jiffy; + val = TICK_NSEC; else - cval = cval - t; + val -= t; } spin_unlock_irq(&tsk->sighand->siglock); - cputime_to_timeval(cval, &value->it_value); - cputime_to_timeval(cinterval, &value->it_interval); + value->it_value = ns_to_timeval(val); + value->it_interval = ns_to_timeval(interval); } int do_getitimer(int which, struct itimerval *value) @@ -129,55 +129,35 @@ enum hrtimer_restart it_real_fn(struct hrtimer *timer) return HRTIMER_NORESTART; } -static inline u32 cputime_sub_ns(cputime_t ct, s64 real_ns) -{ - struct timespec ts; - s64 cpu_ns; - - cputime_to_timespec(ct, &ts); - cpu_ns = timespec_to_ns(&ts); - - return (cpu_ns <= real_ns) ? 0 : cpu_ns - real_ns; -} - static void set_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, const struct itimerval *const value, struct itimerval *const ovalue) { - cputime_t cval, nval, cinterval, ninterval; - s64 ns_ninterval, ns_nval; - u32 error, incr_error; + u64 oval, nval, ointerval, ninterval; struct cpu_itimer *it = &tsk->signal->it[clock_id]; - nval = timeval_to_cputime(&value->it_value); - ns_nval = timeval_to_ns(&value->it_value); - ninterval = timeval_to_cputime(&value->it_interval); - ns_ninterval = timeval_to_ns(&value->it_interval); - - error = cputime_sub_ns(nval, ns_nval); - incr_error = cputime_sub_ns(ninterval, ns_ninterval); + nval = timeval_to_ns(&value->it_value); + ninterval = timeval_to_ns(&value->it_interval); spin_lock_irq(&tsk->sighand->siglock); - cval = it->expires; - cinterval = it->incr; - if (cval || nval) { + oval = it->expires; + ointerval = it->incr; + if (oval || nval) { if (nval > 0) - nval += cputime_one_jiffy; - set_process_cpu_timer(tsk, clock_id, &nval, &cval); + nval += TICK_NSEC; + set_process_cpu_timer(tsk, clock_id, &nval, &oval); } it->expires = nval; it->incr = ninterval; - it->error = error; - it->incr_error = incr_error; trace_itimer_state(clock_id == CPUCLOCK_VIRT ? ITIMER_VIRTUAL : ITIMER_PROF, value, nval); spin_unlock_irq(&tsk->sighand->siglock); if (ovalue) { - cputime_to_timeval(cval, &ovalue->it_value); - cputime_to_timeval(cinterval, &ovalue->it_interval); + ovalue->it_value = ns_to_timeval(oval); + ovalue->it_interval = ns_to_timeval(ointerval); } } diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 45be3cec0dc2..b4377a5e4269 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -20,10 +20,10 @@ */ void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) { - cputime_t cputime = secs_to_cputime(rlim_new); + u64 nsecs = rlim_new * NSEC_PER_SEC; spin_lock_irq(&task->sighand->siglock); - set_process_cpu_timer(task, CPUCLOCK_PROF, &cputime, NULL); + set_process_cpu_timer(task, CPUCLOCK_PROF, &nsecs, NULL); spin_unlock_irq(&task->sighand->siglock); } @@ -860,17 +860,11 @@ static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, if (!it->expires) return; - if (cur_time >= cputime_to_nsecs(it->expires)) { - if (it->incr) { + if (cur_time >= it->expires) { + if (it->incr) it->expires += it->incr; - it->error += it->incr_error; - if (it->error >= TICK_NSEC) { - it->expires -= cputime_one_jiffy; - it->error -= TICK_NSEC; - } - } else { + else it->expires = 0; - } trace_itimer_expire(signo == SIGPROF ? ITIMER_PROF : ITIMER_VIRTUAL, @@ -878,9 +872,8 @@ static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, __group_send_sig_info(signo, SEND_SIG_PRIV, tsk); } - if (it->expires && (!*expires || cputime_to_nsecs(it->expires) < *expires)) { - *expires = cputime_to_nsecs(it->expires); - } + if (it->expires && (!*expires || it->expires < *expires)) + *expires = it->expires; } /* @@ -1174,9 +1167,9 @@ void run_posix_cpu_timers(struct task_struct *tsk) * The tsk->sighand->siglock must be held by the caller. */ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, - cputime_t *newval, cputime_t *oldval) + u64 *newval, u64 *oldval) { - u64 now, new; + u64 now; WARN_ON_ONCE(clock_idx == CPUCLOCK_SCHED); cpu_timer_sample_group(clock_idx, tsk, &now); @@ -1188,33 +1181,31 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, * it to be absolute. */ if (*oldval) { - if (cputime_to_nsecs(*oldval) <= now) { + if (*oldval <= now) { /* Just about to fire. */ - *oldval = cputime_one_jiffy; + *oldval = TICK_NSEC; } else { - *oldval -= nsecs_to_cputime(now); + *oldval -= now; } } if (!*newval) return; - *newval += nsecs_to_cputime(now); + *newval += now; } - new = cputime_to_nsecs(*newval); - /* * Update expiration cache if we are the earliest timer, or eventually * RLIMIT_CPU limit is earlier than prof_exp cpu timer expire. */ switch (clock_idx) { case CPUCLOCK_PROF: - if (expires_gt(tsk->signal->cputime_expires.prof_exp, new)) - tsk->signal->cputime_expires.prof_exp = new; + if (expires_gt(tsk->signal->cputime_expires.prof_exp, *newval)) + tsk->signal->cputime_expires.prof_exp = *newval; break; case CPUCLOCK_VIRT: - if (expires_gt(tsk->signal->cputime_expires.virt_exp, new)) - tsk->signal->cputime_expires.virt_exp = new; + if (expires_gt(tsk->signal->cputime_expires.virt_exp, *newval)) + tsk->signal->cputime_expires.virt_exp = *newval; break; } -- cgit v1.2.3