2 * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
3 * Copyright(c) 2009 Intel Corporation
5 * based on raid6recov.c:
6 * Copyright 2002 H. Peter Anvin
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 51
20 * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
23 #include <linux/kernel.h>
24 #include <linux/interrupt.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/raid/pq.h>
27 #include <linux/async_tx.h>
29 static struct dma_async_tx_descriptor *
30 async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
31 size_t len, struct async_submit_ctl *submit)
33 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
34 &dest, 1, srcs, 2, len);
35 struct dma_device *dma = chan ? chan->device : NULL;
36 const u8 *amul, *bmul;
41 dma_addr_t dma_dest[2];
42 dma_addr_t dma_src[2];
43 struct device *dev = dma->dev;
44 struct dma_async_tx_descriptor *tx;
45 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
47 if (submit->flags & ASYNC_TX_FENCE)
48 dma_flags |= DMA_PREP_FENCE;
49 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
50 dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
51 dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
52 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef,
55 async_tx_submit(chan, tx, submit);
60 /* run the operation synchronously */
61 async_tx_quiesce(&submit->depend_tx);
62 amul = raid6_gfmul[coef[0]];
63 bmul = raid6_gfmul[coef[1]];
64 a = page_address(srcs[0]);
65 b = page_address(srcs[1]);
66 c = page_address(dest);
77 static struct dma_async_tx_descriptor *
78 async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
79 struct async_submit_ctl *submit)
81 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
82 &dest, 1, &src, 1, len);
83 struct dma_device *dma = chan ? chan->device : NULL;
84 const u8 *qmul; /* Q multiplier table */
88 dma_addr_t dma_dest[2];
89 dma_addr_t dma_src[1];
90 struct device *dev = dma->dev;
91 struct dma_async_tx_descriptor *tx;
92 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
94 if (submit->flags & ASYNC_TX_FENCE)
95 dma_flags |= DMA_PREP_FENCE;
96 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
97 dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
98 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef,
101 async_tx_submit(chan, tx, submit);
106 /* no channel available, or failed to allocate a descriptor, so
107 * perform the operation synchronously
109 async_tx_quiesce(&submit->depend_tx);
110 qmul = raid6_gfmul[coef];
111 d = page_address(dest);
112 s = page_address(src);
120 static struct dma_async_tx_descriptor *
121 __2data_recov_4(size_t bytes, int faila, int failb, struct page **blocks,
122 struct async_submit_ctl *submit)
124 struct dma_async_tx_descriptor *tx = NULL;
125 struct page *p, *q, *a, *b;
126 struct page *srcs[2];
127 unsigned char coef[2];
128 enum async_tx_flags flags = submit->flags;
129 dma_async_tx_callback cb_fn = submit->cb_fn;
130 void *cb_param = submit->cb_param;
131 void *scribble = submit->scribble;
139 /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
140 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
143 coef[0] = raid6_gfexi[failb-faila];
144 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
145 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
146 tx = async_sum_product(b, srcs, coef, bytes, submit);
151 init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
153 tx = async_xor(a, srcs, 0, 2, bytes, submit);
159 static struct dma_async_tx_descriptor *
160 __2data_recov_5(size_t bytes, int faila, int failb, struct page **blocks,
161 struct async_submit_ctl *submit)
163 struct dma_async_tx_descriptor *tx = NULL;
164 struct page *p, *q, *g, *dp, *dq;
165 struct page *srcs[2];
166 unsigned char coef[2];
167 enum async_tx_flags flags = submit->flags;
168 dma_async_tx_callback cb_fn = submit->cb_fn;
169 void *cb_param = submit->cb_param;
170 void *scribble = submit->scribble;
171 int uninitialized_var(good);
174 for (i = 0; i < 3; i++) {
175 if (i == faila || i == failb)
188 /* Compute syndrome with zero for the missing data pages
189 * Use the dead data pages as temporary storage for delta p and
195 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
196 tx = async_memcpy(dp, g, 0, 0, bytes, submit);
197 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
198 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
200 /* compute P + Pxy */
203 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
204 NULL, NULL, scribble);
205 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
207 /* compute Q + Qxy */
210 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
211 NULL, NULL, scribble);
212 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
214 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
217 coef[0] = raid6_gfexi[failb-faila];
218 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
219 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
220 tx = async_sum_product(dq, srcs, coef, bytes, submit);
225 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
227 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
232 static struct dma_async_tx_descriptor *
233 __2data_recov_n(int disks, size_t bytes, int faila, int failb,
234 struct page **blocks, struct async_submit_ctl *submit)
236 struct dma_async_tx_descriptor *tx = NULL;
237 struct page *p, *q, *dp, *dq;
238 struct page *srcs[2];
239 unsigned char coef[2];
240 enum async_tx_flags flags = submit->flags;
241 dma_async_tx_callback cb_fn = submit->cb_fn;
242 void *cb_param = submit->cb_param;
243 void *scribble = submit->scribble;
248 /* Compute syndrome with zero for the missing data pages
249 * Use the dead data pages as temporary storage for
250 * delta p and delta q
253 blocks[faila] = (void *)raid6_empty_zero_page;
254 blocks[disks-2] = dp;
256 blocks[failb] = (void *)raid6_empty_zero_page;
257 blocks[disks-1] = dq;
259 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
260 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
262 /* Restore pointer table */
268 /* compute P + Pxy */
271 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
272 NULL, NULL, scribble);
273 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
275 /* compute Q + Qxy */
278 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
279 NULL, NULL, scribble);
280 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
282 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
285 coef[0] = raid6_gfexi[failb-faila];
286 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
287 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
288 tx = async_sum_product(dq, srcs, coef, bytes, submit);
293 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
295 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
301 * async_raid6_2data_recov - asynchronously calculate two missing data blocks
302 * @disks: number of disks in the RAID-6 array
304 * @faila: first failed drive index
305 * @failb: second failed drive index
306 * @blocks: array of source pointers where the last two entries are p and q
307 * @submit: submission/completion modifiers
309 struct dma_async_tx_descriptor *
310 async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
311 struct page **blocks, struct async_submit_ctl *submit)
313 BUG_ON(faila == failb);
317 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
319 /* we need to preserve the contents of 'blocks' for the async
320 * case, so punt to synchronous if a scribble buffer is not available
322 if (!submit->scribble) {
323 void **ptrs = (void **) blocks;
326 async_tx_quiesce(&submit->depend_tx);
327 for (i = 0; i < disks; i++)
328 ptrs[i] = page_address(blocks[i]);
330 raid6_2data_recov(disks, bytes, faila, failb, ptrs);
332 async_tx_sync_epilog(submit);
339 /* dma devices do not uniformly understand a zero source pq
340 * operation (in contrast to the synchronous case), so
341 * explicitly handle the 4 disk special case
343 return __2data_recov_4(bytes, faila, failb, blocks, submit);
345 /* dma devices do not uniformly understand a single
346 * source pq operation (in contrast to the synchronous
347 * case), so explicitly handle the 5 disk special case
349 return __2data_recov_5(bytes, faila, failb, blocks, submit);
351 return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
354 EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
357 * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
358 * @disks: number of disks in the RAID-6 array
360 * @faila: failed drive index
361 * @blocks: array of source pointers where the last two entries are p and q
362 * @submit: submission/completion modifiers
364 struct dma_async_tx_descriptor *
365 async_raid6_datap_recov(int disks, size_t bytes, int faila,
366 struct page **blocks, struct async_submit_ctl *submit)
368 struct dma_async_tx_descriptor *tx = NULL;
369 struct page *p, *q, *dq;
371 enum async_tx_flags flags = submit->flags;
372 dma_async_tx_callback cb_fn = submit->cb_fn;
373 void *cb_param = submit->cb_param;
374 void *scribble = submit->scribble;
375 struct page *srcs[2];
377 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
379 /* we need to preserve the contents of 'blocks' for the async
380 * case, so punt to synchronous if a scribble buffer is not available
383 void **ptrs = (void **) blocks;
386 async_tx_quiesce(&submit->depend_tx);
387 for (i = 0; i < disks; i++)
388 ptrs[i] = page_address(blocks[i]);
390 raid6_datap_recov(disks, bytes, faila, ptrs);
392 async_tx_sync_epilog(submit);
400 /* Compute syndrome with zero for the missing data page
401 * Use the dead data page as temporary storage for delta q
404 blocks[faila] = (void *)raid6_empty_zero_page;
405 blocks[disks-1] = dq;
407 /* in the 4 disk case we only need to perform a single source
411 int good = faila == 0 ? 1 : 0;
412 struct page *g = blocks[good];
414 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
416 tx = async_memcpy(p, g, 0, 0, bytes, submit);
418 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
420 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
422 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
424 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
427 /* Restore pointer table */
431 /* calculate g^{-faila} */
432 coef = raid6_gfinv[raid6_gfexp[faila]];
436 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
437 NULL, NULL, scribble);
438 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
440 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
441 tx = async_mult(dq, dq, coef, bytes, submit);
445 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
447 tx = async_xor(p, srcs, 0, 2, bytes, submit);
451 EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
453 MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
454 MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
455 MODULE_LICENSE("GPL");