1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
// SPDX-License-Identifier: GPL-2.0-only
#include "sched.h"
/*
* A simple wrapper around refcount. An allocated sched_core_cookie's
* address is used to compute the cookie of the task.
*/
struct sched_core_cookie {
refcount_t refcnt;
};
unsigned long sched_core_alloc_cookie(void)
{
struct sched_core_cookie *ck = kmalloc(sizeof(*ck), GFP_KERNEL);
if (!ck)
return 0;
refcount_set(&ck->refcnt, 1);
sched_core_get();
return (unsigned long)ck;
}
void sched_core_put_cookie(unsigned long cookie)
{
struct sched_core_cookie *ptr = (void *)cookie;
if (ptr && refcount_dec_and_test(&ptr->refcnt)) {
kfree(ptr);
sched_core_put();
}
}
unsigned long sched_core_get_cookie(unsigned long cookie)
{
struct sched_core_cookie *ptr = (void *)cookie;
if (ptr)
refcount_inc(&ptr->refcnt);
return cookie;
}
/*
* sched_core_update_cookie - replace the cookie on a task
* @p: the task to update
* @cookie: the new cookie
*
* Effectively exchange the task cookie; caller is responsible for lifetimes on
* both ends.
*
* Returns: the old cookie
*/
unsigned long sched_core_update_cookie(struct task_struct *p, unsigned long cookie)
{
unsigned long old_cookie;
struct rq_flags rf;
struct rq *rq;
bool enqueued;
rq = task_rq_lock(p, &rf);
/*
* Since creating a cookie implies sched_core_get(), and we cannot set
* a cookie until after we've created it, similarly, we cannot destroy
* a cookie until after we've removed it, we must have core scheduling
* enabled here.
*/
SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq));
enqueued = sched_core_enqueued(p);
if (enqueued)
sched_core_dequeue(rq, p);
old_cookie = p->core_cookie;
p->core_cookie = cookie;
if (enqueued)
sched_core_enqueue(rq, p);
/*
* If task is currently running, it may not be compatible anymore after
* the cookie change, so enter the scheduler on its CPU to schedule it
* away.
*/
if (task_running(rq, p))
resched_curr(rq);
task_rq_unlock(rq, p, &rf);
return old_cookie;
}
static unsigned long sched_core_clone_cookie(struct task_struct *p)
{
unsigned long cookie, flags;
raw_spin_lock_irqsave(&p->pi_lock, flags);
cookie = sched_core_get_cookie(p->core_cookie);
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
return cookie;
}
void sched_core_fork(struct task_struct *p)
{
RB_CLEAR_NODE(&p->core_node);
p->core_cookie = sched_core_clone_cookie(current);
}
void sched_core_free(struct task_struct *p)
{
sched_core_put_cookie(p->core_cookie);
}
|
