1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
// SPDX-License-Identifier: GPL-2.0
/*
* Common Primitives for Data Access Monitoring
*
* Author: SeongJae Park <sj@kernel.org>
*/
#include <linux/mmu_notifier.h>
#include <linux/page_idle.h>
#include <linux/pagemap.h>
#include <linux/rmap.h>
#include "ops-common.h"
/*
* Get an online page for a pfn if it's in the LRU list. Otherwise, returns
* NULL.
*
* The body of this function is stolen from the 'page_idle_get_page()'. We
* steal rather than reuse it because the code is quite simple.
*/
struct page *damon_get_page(unsigned long pfn)
{
struct page *page = pfn_to_online_page(pfn);
if (!page || !PageLRU(page) || !get_page_unless_zero(page))
return NULL;
if (unlikely(!PageLRU(page))) {
put_page(page);
page = NULL;
}
return page;
}
void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
{
bool referenced = false;
struct page *page = damon_get_page(pte_pfn(*pte));
if (!page)
return;
if (pte_young(*pte)) {
referenced = true;
*pte = pte_mkold(*pte);
}
#ifdef CONFIG_MMU_NOTIFIER
if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE))
referenced = true;
#endif /* CONFIG_MMU_NOTIFIER */
if (referenced)
set_page_young(page);
set_page_idle(page);
put_page(page);
}
void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
bool referenced = false;
struct page *page = damon_get_page(pmd_pfn(*pmd));
if (!page)
return;
if (pmd_young(*pmd)) {
referenced = true;
*pmd = pmd_mkold(*pmd);
}
#ifdef CONFIG_MMU_NOTIFIER
if (mmu_notifier_clear_young(mm, addr, addr + HPAGE_PMD_SIZE))
referenced = true;
#endif /* CONFIG_MMU_NOTIFIER */
if (referenced)
set_page_young(page);
set_page_idle(page);
put_page(page);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
}
#define DAMON_MAX_SUBSCORE (100)
#define DAMON_MAX_AGE_IN_LOG (32)
int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
struct damos *s)
{
unsigned int max_nr_accesses;
int freq_subscore;
unsigned int age_in_sec;
int age_in_log, age_subscore;
unsigned int freq_weight = s->quota.weight_nr_accesses;
unsigned int age_weight = s->quota.weight_age;
int hotness;
max_nr_accesses = c->aggr_interval / c->sample_interval;
freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
age_in_log++, age_in_sec >>= 1)
;
/* If frequency is 0, higher age means it's colder */
if (freq_subscore == 0)
age_in_log *= -1;
/*
* Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
* Scale it to be in [0, 100] and set it as age subscore.
*/
age_in_log += DAMON_MAX_AGE_IN_LOG;
age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
DAMON_MAX_AGE_IN_LOG / 2;
hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
if (freq_weight + age_weight)
hotness /= freq_weight + age_weight;
/*
* Transform it to fit in [0, DAMOS_MAX_SCORE]
*/
hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
/* Return coldness of the region */
return DAMOS_MAX_SCORE - hotness;
}
int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
struct damos *s)
{
unsigned int max_nr_accesses;
int freq_subscore;
unsigned int age_in_sec;
int age_in_log, age_subscore;
unsigned int freq_weight = s->quota.weight_nr_accesses;
unsigned int age_weight = s->quota.weight_age;
int hotness;
max_nr_accesses = c->aggr_interval / c->sample_interval;
freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
age_in_log++, age_in_sec >>= 1)
;
/* If frequency is 0, higher age means it's colder */
if (freq_subscore == 0)
age_in_log *= -1;
/*
* Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
* Scale it to be in [0, 100] and set it as age subscore.
*/
age_in_log += DAMON_MAX_AGE_IN_LOG;
age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
DAMON_MAX_AGE_IN_LOG / 2;
hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
if (freq_weight + age_weight)
hotness /= freq_weight + age_weight;
/*
* Transform it to fit in [0, DAMOS_MAX_SCORE]
*/
hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
return hotness;
}
|
