1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
|
// SPDX-License-Identifier: GPL-2.0
#include "blk-rq-qos.h"
/*
* Increment 'v', if 'v' is below 'below'. Returns true if we succeeded,
* false if 'v' + 1 would be bigger than 'below'.
*/
static bool atomic_inc_below(atomic_t *v, unsigned int below)
{
unsigned int cur = atomic_read(v);
for (;;) {
unsigned int old;
if (cur >= below)
return false;
old = atomic_cmpxchg(v, cur, cur + 1);
if (old == cur)
break;
cur = old;
}
return true;
}
bool rq_wait_inc_below(struct rq_wait *rq_wait, unsigned int limit)
{
return atomic_inc_below(&rq_wait->inflight, limit);
}
void __rq_qos_cleanup(struct rq_qos *rqos, struct bio *bio)
{
do {
if (rqos->ops->cleanup)
rqos->ops->cleanup(rqos, bio);
rqos = rqos->next;
} while (rqos);
}
void __rq_qos_done(struct rq_qos *rqos, struct request *rq)
{
do {
if (rqos->ops->done)
rqos->ops->done(rqos, rq);
rqos = rqos->next;
} while (rqos);
}
void __rq_qos_issue(struct rq_qos *rqos, struct request *rq)
{
do {
if (rqos->ops->issue)
rqos->ops->issue(rqos, rq);
rqos = rqos->next;
} while (rqos);
}
void __rq_qos_requeue(struct rq_qos *rqos, struct request *rq)
{
do {
if (rqos->ops->requeue)
rqos->ops->requeue(rqos, rq);
rqos = rqos->next;
} while (rqos);
}
void __rq_qos_throttle(struct rq_qos *rqos, struct bio *bio)
{
do {
if (rqos->ops->throttle)
rqos->ops->throttle(rqos, bio);
rqos = rqos->next;
} while (rqos);
}
void __rq_qos_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)
{
do {
if (rqos->ops->track)
rqos->ops->track(rqos, rq, bio);
rqos = rqos->next;
} while (rqos);
}
void __rq_qos_merge(struct rq_qos *rqos, struct request *rq, struct bio *bio)
{
do {
if (rqos->ops->merge)
rqos->ops->merge(rqos, rq, bio);
rqos = rqos->next;
} while (rqos);
}
void __rq_qos_done_bio(struct rq_qos *rqos, struct bio *bio)
{
do {
if (rqos->ops->done_bio)
rqos->ops->done_bio(rqos, bio);
rqos = rqos->next;
} while (rqos);
}
void __rq_qos_queue_depth_changed(struct rq_qos *rqos)
{
do {
if (rqos->ops->queue_depth_changed)
rqos->ops->queue_depth_changed(rqos);
rqos = rqos->next;
} while (rqos);
}
/*
* Return true, if we can't increase the depth further by scaling
*/
bool rq_depth_calc_max_depth(struct rq_depth *rqd)
{
unsigned int depth;
bool ret = false;
/*
* For QD=1 devices, this is a special case. It's important for those
* to have one request ready when one completes, so force a depth of
* 2 for those devices. On the backend, it'll be a depth of 1 anyway,
* since the device can't have more than that in flight. If we're
* scaling down, then keep a setting of 1/1/1.
*/
if (rqd->queue_depth == 1) {
if (rqd->scale_step > 0)
rqd->max_depth = 1;
else {
rqd->max_depth = 2;
ret = true;
}
} else {
/*
* scale_step == 0 is our default state. If we have suffered
* latency spikes, step will be > 0, and we shrink the
* allowed write depths. If step is < 0, we're only doing
* writes, and we allow a temporarily higher depth to
* increase performance.
*/
depth = min_t(unsigned int, rqd->default_depth,
rqd->queue_depth);
if (rqd->scale_step > 0)
depth = 1 + ((depth - 1) >> min(31, rqd->scale_step));
else if (rqd->scale_step < 0) {
unsigned int maxd = 3 * rqd->queue_depth / 4;
depth = 1 + ((depth - 1) << -rqd->scale_step);
if (depth > maxd) {
depth = maxd;
ret = true;
}
}
rqd->max_depth = depth;
}
return ret;
}
/* Returns true on success and false if scaling up wasn't possible */
bool rq_depth_scale_up(struct rq_depth *rqd)
{
/*
* Hit max in previous round, stop here
*/
if (rqd->scaled_max)
return false;
rqd->scale_step--;
rqd->scaled_max = rq_depth_calc_max_depth(rqd);
return true;
}
/*
* Scale rwb down. If 'hard_throttle' is set, do it quicker, since we
* had a latency violation. Returns true on success and returns false if
* scaling down wasn't possible.
*/
bool rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle)
{
/*
* Stop scaling down when we've hit the limit. This also prevents
* ->scale_step from going to crazy values, if the device can't
* keep up.
*/
if (rqd->max_depth == 1)
return false;
if (rqd->scale_step < 0 && hard_throttle)
rqd->scale_step = 0;
else
rqd->scale_step++;
rqd->scaled_max = false;
rq_depth_calc_max_depth(rqd);
return true;
}
struct rq_qos_wait_data {
struct wait_queue_entry wq;
struct task_struct *task;
struct rq_wait *rqw;
acquire_inflight_cb_t *cb;
void *private_data;
bool got_token;
};
static int rq_qos_wake_function(struct wait_queue_entry *curr,
unsigned int mode, int wake_flags, void *key)
{
struct rq_qos_wait_data *data = container_of(curr,
struct rq_qos_wait_data,
wq);
/*
* If we fail to get a budget, return -1 to interrupt the wake up loop
* in __wake_up_common.
*/
if (!data->cb(data->rqw, data->private_data))
return -1;
data->got_token = true;
smp_wmb();
list_del_init(&curr->entry);
wake_up_process(data->task);
return 1;
}
/**
* rq_qos_wait - throttle on a rqw if we need to
* @rqw: rqw to throttle on
* @private_data: caller provided specific data
* @acquire_inflight_cb: inc the rqw->inflight counter if we can
* @cleanup_cb: the callback to cleanup in case we race with a waker
*
* This provides a uniform place for the rq_qos users to do their throttling.
* Since you can end up with a lot of things sleeping at once, this manages the
* waking up based on the resources available. The acquire_inflight_cb should
* inc the rqw->inflight if we have the ability to do so, or return false if not
* and then we will sleep until the room becomes available.
*
* cleanup_cb is in case that we race with a waker and need to cleanup the
* inflight count accordingly.
*/
void rq_qos_wait(struct rq_wait *rqw, void *private_data,
acquire_inflight_cb_t *acquire_inflight_cb,
cleanup_cb_t *cleanup_cb)
{
struct rq_qos_wait_data data = {
.wq = {
.func = rq_qos_wake_function,
.entry = LIST_HEAD_INIT(data.wq.entry),
},
.task = current,
.rqw = rqw,
.cb = acquire_inflight_cb,
.private_data = private_data,
};
bool has_sleeper;
has_sleeper = wq_has_sleeper(&rqw->wait);
if (!has_sleeper && acquire_inflight_cb(rqw, private_data))
return;
has_sleeper = !prepare_to_wait_exclusive(&rqw->wait, &data.wq,
TASK_UNINTERRUPTIBLE);
do {
/* The memory barrier in set_task_state saves us here. */
if (data.got_token)
break;
if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) {
finish_wait(&rqw->wait, &data.wq);
/*
* We raced with wbt_wake_function() getting a token,
* which means we now have two. Put our local token
* and wake anyone else potentially waiting for one.
*/
smp_rmb();
if (data.got_token)
cleanup_cb(rqw, private_data);
break;
}
io_schedule();
has_sleeper = true;
set_current_state(TASK_UNINTERRUPTIBLE);
} while (1);
finish_wait(&rqw->wait, &data.wq);
}
void rq_qos_exit(struct request_queue *q)
{
while (q->rq_qos) {
struct rq_qos *rqos = q->rq_qos;
q->rq_qos = rqos->next;
rqos->ops->exit(rqos);
}
}
|