summaryrefslogtreecommitdiffstats
path: root/crypto/async_tx/async_xor.c
blob: a057ecb1288d2d8f55dedf513c7310c47d0303ed (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
// SPDX-License-Identifier: GPL-2.0-only
/*
 * xor offload engine api
 *
 * Copyright © 2006, Intel Corporation.
 *
 *      Dan Williams <dan.j.williams@intel.com>
 *
 *      with architecture considerations by:
 *      Neil Brown <neilb@suse.de>
 *      Jeff Garzik <jeff@garzik.org>
 */
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/dma-mapping.h>
#include <linux/raid/xor.h>
#include <linux/async_tx.h>

/* do_async_xor - dma map the pages and perform the xor with an engine */
static __async_inline struct dma_async_tx_descriptor *
do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap,
	     struct async_submit_ctl *submit)
{
	struct dma_device *dma = chan->device;
	struct dma_async_tx_descriptor *tx = NULL;
	dma_async_tx_callback cb_fn_orig = submit->cb_fn;
	void *cb_param_orig = submit->cb_param;
	enum async_tx_flags flags_orig = submit->flags;
	enum dma_ctrl_flags dma_flags = 0;
	int src_cnt = unmap->to_cnt;
	int xor_src_cnt;
	dma_addr_t dma_dest = unmap->addr[unmap->to_cnt];
	dma_addr_t *src_list = unmap->addr;

	while (src_cnt) {
		dma_addr_t tmp;

		submit->flags = flags_orig;
		xor_src_cnt = min(src_cnt, (int)dma->max_xor);
		/* if we are submitting additional xors, leave the chain open
		 * and clear the callback parameters
		 */
		if (src_cnt > xor_src_cnt) {
			submit->flags &= ~ASYNC_TX_ACK;
			submit->flags |= ASYNC_TX_FENCE;
			submit->cb_fn = NULL;
			submit->cb_param = NULL;
		} else {
			submit->cb_fn = cb_fn_orig;
			submit->cb_param = cb_param_orig;
		}
		if (submit->cb_fn)
			dma_flags |= DMA_PREP_INTERRUPT;
		if (submit->flags & ASYNC_TX_FENCE)
			dma_flags |= DMA_PREP_FENCE;

		/* Drivers force forward progress in case they can not provide a
		 * descriptor
		 */
		tmp = src_list[0];
		if (src_list > unmap->addr)
			src_list[0] = dma_dest;
		tx = dma->device_prep_dma_xor(chan, dma_dest, src_list,
					      xor_src_cnt, unmap->len,
					      dma_flags);

		if (unlikely(!tx))
			async_tx_quiesce(&submit->depend_tx);

		/* spin wait for the preceding transactions to complete */
		while (unlikely(!tx)) {
			dma_async_issue_pending(chan);
			tx = dma->device_prep_dma_xor(chan, dma_dest,
						      src_list,
						      xor_src_cnt, unmap->len,
						      dma_flags);
		}
		src_list[0] = tmp;

		dma_set_unmap(tx, unmap);
		async_tx_submit(chan, tx, submit);
		submit->depend_tx = tx;

		if (src_cnt > xor_src_cnt) {
			/* drop completed sources */
			src_cnt -= xor_src_cnt;
			/* use the intermediate result a source */
			src_cnt++;
			src_list += xor_src_cnt - 1;
		} else
			break;
	}

	return tx;
}

static void
do_sync_xor_offs(struct page *dest, unsigned int offset,
		struct page **src_list, unsigned int *src_offs,
	    int src_cnt, size_t len, struct async_submit_ctl *submit)
{
	int i;
	int xor_src_cnt = 0;
	int src_off = 0;
	void *dest_buf;
	void **srcs;

	if (submit->scribble)
		srcs = submit->scribble;
	else
		srcs = (void **) src_list;

	/* convert to buffer pointers */
	for (i = 0; i < src_cnt; i++)
		if (src_list[i])
			srcs[xor_src_cnt++] = page_address(src_list[i]) +
				(src_offs ? src_offs[i] : offset);
	src_cnt = xor_src_cnt;
	/* set destination address */
	dest_buf = page_address(dest) + offset;

	if (submit->flags & ASYNC_TX_XOR_ZERO_DST)
		memset(dest_buf, 0, len);

	while (src_cnt > 0) {
		/* process up to 'MAX_XOR_BLOCKS' sources */
		xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
		xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]);

		/* drop completed sources */
		src_cnt -= xor_src_cnt;
		src_off += xor_src_cnt;
	}

	async_tx_sync_epilog(submit);
}

static inline bool
dma_xor_aligned_offsets(struct dma_device *device, unsigned int offset,
		unsigned int *src_offs, int src_cnt, int len)
{
	int i;

	if (!is_dma_xor_aligned(device, offset, 0, len))
		return false;

	if (!src_offs)
		return true;

	for (i = 0; i < src_cnt; i++) {
		if (!is_dma_xor_aligned(device, src_offs[i], 0, len))
			return false;
	}
	return true;
}

/**
 * async_xor_offs - attempt to xor a set of blocks with a dma engine.
 * @dest: destination page
 * @offset: dst offset to start transaction
 * @src_list: array of source pages
 * @src_offs: array of source pages offset, NULL means common src/dst offset
 * @src_cnt: number of source pages
 * @len: length in bytes
 * @submit: submission / completion modifiers
 *
 * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
 *
 * xor_blocks always uses the dest as a source so the
 * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
 * the calculation.  The assumption with dma eninges is that they only
 * use the destination buffer as a source when it is explicity specified
 * in the source list.
 *
 * src_list note: if the dest is also a source it must be at index zero.
 * The contents of this array will be overwritten if a scribble region
 * is not specified.
 */
struct dma_async_tx_descriptor *
async_xor_offs(struct page *dest, unsigned int offset,
		struct page **src_list, unsigned int *src_offs,
		int src_cnt, size_t len, struct async_submit_ctl *submit)
{
	struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR,
						      &dest, 1, src_list,
						      src_cnt, len);
	struct dma_device *device = chan ? chan->device : NULL;
	struct dmaengine_unmap_data *unmap = NULL;

	BUG_ON(src_cnt <= 1);

	if (device)
		unmap = dmaengine_get_unmap_data(device->dev, src_cnt+1, GFP_NOWAIT);

	if (unmap && dma_xor_aligned_offsets(device, offset,
				src_offs, src_cnt, len)) {
		struct dma_async_tx_descriptor *tx;
		int i, j;

		/* run the xor asynchronously */
		pr_debug("%s (async): len: %zu\n", __func__, len);

		unmap->len = len;
		for (i = 0, j = 0; i < src_cnt; i++) {
			if (!src_list[i])
				continue;
			unmap->to_cnt++;
			unmap->addr[j++] = dma_map_page(device->dev, src_list[i],
					src_offs ? src_offs[i] : offset,
					len, DMA_TO_DEVICE);
		}

		/* map it bidirectional as it may be re-used as a source */
		unmap->addr[j] = dma_map_page(device->dev, dest, offset, len,
					      DMA_BIDIRECTIONAL);
		unmap->bidi_cnt = 1;

		tx = do_async_xor(chan, unmap, submit);
		dmaengine_unmap_put(unmap);
		return tx;
	} else {
		dmaengine_unmap_put(unmap);
		/* run the xor synchronously */
		pr_debug("%s (sync): len: %zu\n", __func__, len);
		WARN_ONCE(chan, "%s: no space for dma address conversion\n",
			  __func__);

		/* in the sync case the dest is an implied source
		 * (assumes the dest is the first source)
		 */
		if (submit->flags & ASYNC_TX_XOR_DROP_DST) {
			src_cnt--;
			src_list++;
		}

		/* wait for any prerequisite operations */
		async_tx_quiesce(&submit->depend_tx);

		do_sync_xor_offs(dest, offset, src_list, src_offs,
				src_cnt, len, submit);

		return NULL;
	}
}
EXPORT_SYMBOL_GPL(async_xor_offs);

/**
 * async_xor - attempt to xor a set of blocks with a dma engine.
 * @dest: destination page
 * @src_list: array of source pages
 * @offset: common src/dst offset to start transaction
 * @src_cnt: number of source pages
 * @len: length in bytes
 * @submit: submission / completion modifiers
 *
 * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
 *
 * xor_blocks always uses the dest as a source so the
 * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
 * the calculation.  The assumption with dma eninges is that they only
 * use the destination buffer as a source when it is explicity specified
 * in the source list.
 *
 * src_list note: if the dest is also a source it must be at index zero.
 * The contents of this array will be overwritten if a scribble region
 * is not specified.
 */
struct dma_async_tx_descriptor *
async_xor(struct page *dest, struct page **src_list, unsigned int offset,
	  int src_cnt, size_t len, struct async_submit_ctl *submit)
{
	return async_xor_offs(dest, offset, src_list, NULL,
			src_cnt, len, submit);
}
EXPORT_SYMBOL_GPL(async_xor);

static int page_is_zero(struct page *p, unsigned int offset, size_t len)
{
	return !memchr_inv(page_address(p) + offset, 0, len);
}

static inline struct dma_chan *
xor_val_chan(struct async_submit_ctl *submit, struct page *dest,
		 struct page **src_list, int src_cnt, size_t len)
{
	#ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
	return NULL;
	#endif
	return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list,
				     src_cnt, len);
}

/**
 * async_xor_val_offs - attempt a xor parity check with a dma engine.
 * @dest: destination page used if the xor is performed synchronously
 * @offset: des offset in pages to start transaction
 * @src_list: array of source pages
 * @src_offs: array of source pages offset, NULL means common src/det offset
 * @src_cnt: number of source pages
 * @len: length in bytes
 * @result: 0 if sum == 0 else non-zero
 * @submit: submission / completion modifiers
 *
 * honored flags: ASYNC_TX_ACK
 *
 * src_list note: if the dest is also a source it must be at index zero.
 * The contents of this array will be overwritten if a scribble region
 * is not specified.
 */
struct dma_async_tx_descriptor *
async_xor_val_offs(struct page *dest, unsigned int offset,
		struct page **src_list, unsigned int *src_offs,
		int src_cnt, size_t len, enum sum_check_flags *result,
		struct async_submit_ctl *submit)
{
	struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len);
	struct dma_device *device = chan ? chan->device : NULL;
	struct dma_async_tx_descriptor *tx = NULL;
	struct dmaengine_unmap_data *unmap = NULL;

	BUG_ON(src_cnt <= 1);

	if (device)
		unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOWAIT);

	if (unmap && src_cnt <= device->max_xor &&
	    dma_xor_aligned_offsets(device, offset, src_offs, src_cnt, len)) {
		unsigned long dma_prep_flags = 0;
		int i;

		pr_debug("%s: (async) len: %zu\n", __func__, len);

		if (submit->cb_fn)
			dma_prep_flags |= DMA_PREP_INTERRUPT;
		if (submit->flags & ASYNC_TX_FENCE)
			dma_prep_flags |= DMA_PREP_FENCE;

		for (i = 0; i < src_cnt; i++) {
			unmap->addr[i] = dma_map_page(device->dev, src_list[i],
					src_offs ? src_offs[i] : offset,
					len, DMA_TO_DEVICE);
			unmap->to_cnt++;
		}
		unmap->len = len;

		tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt,
						     len, result,
						     dma_prep_flags);
		if (unlikely(!tx)) {
			async_tx_quiesce(&submit->depend_tx);

			while (!tx) {
				dma_async_issue_pending(chan);
				tx = device->device_prep_dma_xor_val(chan,
					unmap->addr, src_cnt, len, result,
					dma_prep_flags);
			}
		}
		dma_set_unmap(tx, unmap);
		async_tx_submit(chan, tx, submit);
	} else {
		enum async_tx_flags flags_orig = submit->flags;

		pr_debug("%s: (sync) len: %zu\n", __func__, len);
		WARN_ONCE(device && src_cnt <= device->max_xor,
			  "%s: no space for dma address conversion\n",
			  __func__);

		submit->flags |= ASYNC_TX_XOR_DROP_DST;
		submit->flags &= ~ASYNC_TX_ACK;

		tx = async_xor_offs(dest, offset, src_list, src_offs,
				src_cnt, len, submit);

		async_tx_quiesce(&tx);

		*result = !page_is_zero(dest, offset, len) << SUM_CHECK_P;

		async_tx_sync_epilog(submit);
		submit->flags = flags_orig;
	}
	dmaengine_unmap_put(unmap);

	return tx;
}
EXPORT_SYMBOL_GPL(async_xor_val_offs);

/**
 * async_xor_val - attempt a xor parity check with a dma engine.
 * @dest: destination page used if the xor is performed synchronously
 * @src_list: array of source pages
 * @offset: offset in pages to start transaction
 * @src_cnt: number of source pages
 * @len: length in bytes
 * @result: 0 if sum == 0 else non-zero
 * @submit: submission / completion modifiers
 *
 * honored flags: ASYNC_TX_ACK
 *
 * src_list note: if the dest is also a source it must be at index zero.
 * The contents of this array will be overwritten if a scribble region
 * is not specified.
 */
struct dma_async_tx_descriptor *
async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
	      int src_cnt, size_t len, enum sum_check_flags *result,
	      struct async_submit_ctl *submit)
{
	return async_xor_val_offs(dest, offset, src_list, NULL, src_cnt,
			len, result, submit);
}
EXPORT_SYMBOL_GPL(async_xor_val);

MODULE_AUTHOR("Intel Corporation");
MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
MODULE_LICENSE("GPL");