2 * include/asm-i386/xor.h
4 * Optimized RAID-5 checksumming functions for MMX and SSE.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2, or (at your option)
11 * You should have received a copy of the GNU General Public License
12 * (for example /usr/src/linux/COPYING); if not, write to the Free
13 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17 * High-speed RAID5 checksumming functions utilizing MMX instructions.
18 * Copyright (C) 1998 Ingo Molnar.
21 #define LD(x,y) " movq 8*("#x")(%1), %%mm"#y" ;\n"
22 #define ST(x,y) " movq %%mm"#y", 8*("#x")(%1) ;\n"
23 #define XO1(x,y) " pxor 8*("#x")(%2), %%mm"#y" ;\n"
24 #define XO2(x,y) " pxor 8*("#x")(%3), %%mm"#y" ;\n"
25 #define XO3(x,y) " pxor 8*("#x")(%4), %%mm"#y" ;\n"
26 #define XO4(x,y) " pxor 8*("#x")(%5), %%mm"#y" ;\n"
31 xor_pII_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
33 unsigned long lines = bytes >> 7;
37 __asm__ __volatile__ (
74 xor_pII_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
77 unsigned long lines = bytes >> 7;
81 __asm__ __volatile__ (
115 "+r" (p1), "+r" (p2), "+r" (p3)
123 xor_pII_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
124 unsigned long *p3, unsigned long *p4)
126 unsigned long lines = bytes >> 7;
130 __asm__ __volatile__ (
169 "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
178 xor_pII_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
179 unsigned long *p3, unsigned long *p4, unsigned long *p5)
181 unsigned long lines = bytes >> 7;
185 /* Make sure GCC forgets anything it knows about p4 or p5,
186 such that it won't pass to the asm volatile below a
187 register that is shared with any other variable. That's
188 because we modify p4 and p5 there, but we can't mark them
189 as read/write, otherwise we'd overflow the 10-asm-operands
190 limit of GCC < 3.1. */
191 __asm__ ("" : "+r" (p4), "+r" (p5));
193 __asm__ __volatile__ (
237 "+r" (p1), "+r" (p2), "+r" (p3)
241 /* p4 and p5 were modified, and now the variables are dead.
242 Clobber them just to be sure nobody does something stupid
243 like assuming they have some legal value. */
244 __asm__ ("" : "=r" (p4), "=r" (p5));
258 xor_p5_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
260 unsigned long lines = bytes >> 6;
264 __asm__ __volatile__ (
267 " movq (%1), %%mm0 ;\n"
268 " movq 8(%1), %%mm1 ;\n"
269 " pxor (%2), %%mm0 ;\n"
270 " movq 16(%1), %%mm2 ;\n"
271 " movq %%mm0, (%1) ;\n"
272 " pxor 8(%2), %%mm1 ;\n"
273 " movq 24(%1), %%mm3 ;\n"
274 " movq %%mm1, 8(%1) ;\n"
275 " pxor 16(%2), %%mm2 ;\n"
276 " movq 32(%1), %%mm4 ;\n"
277 " movq %%mm2, 16(%1) ;\n"
278 " pxor 24(%2), %%mm3 ;\n"
279 " movq 40(%1), %%mm5 ;\n"
280 " movq %%mm3, 24(%1) ;\n"
281 " pxor 32(%2), %%mm4 ;\n"
282 " movq 48(%1), %%mm6 ;\n"
283 " movq %%mm4, 32(%1) ;\n"
284 " pxor 40(%2), %%mm5 ;\n"
285 " movq 56(%1), %%mm7 ;\n"
286 " movq %%mm5, 40(%1) ;\n"
287 " pxor 48(%2), %%mm6 ;\n"
288 " pxor 56(%2), %%mm7 ;\n"
289 " movq %%mm6, 48(%1) ;\n"
290 " movq %%mm7, 56(%1) ;\n"
305 xor_p5_mmx_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
308 unsigned long lines = bytes >> 6;
312 __asm__ __volatile__ (
313 " .align 32,0x90 ;\n"
315 " movq (%1), %%mm0 ;\n"
316 " movq 8(%1), %%mm1 ;\n"
317 " pxor (%2), %%mm0 ;\n"
318 " movq 16(%1), %%mm2 ;\n"
319 " pxor 8(%2), %%mm1 ;\n"
320 " pxor (%3), %%mm0 ;\n"
321 " pxor 16(%2), %%mm2 ;\n"
322 " movq %%mm0, (%1) ;\n"
323 " pxor 8(%3), %%mm1 ;\n"
324 " pxor 16(%3), %%mm2 ;\n"
325 " movq 24(%1), %%mm3 ;\n"
326 " movq %%mm1, 8(%1) ;\n"
327 " movq 32(%1), %%mm4 ;\n"
328 " movq 40(%1), %%mm5 ;\n"
329 " pxor 24(%2), %%mm3 ;\n"
330 " movq %%mm2, 16(%1) ;\n"
331 " pxor 32(%2), %%mm4 ;\n"
332 " pxor 24(%3), %%mm3 ;\n"
333 " pxor 40(%2), %%mm5 ;\n"
334 " movq %%mm3, 24(%1) ;\n"
335 " pxor 32(%3), %%mm4 ;\n"
336 " pxor 40(%3), %%mm5 ;\n"
337 " movq 48(%1), %%mm6 ;\n"
338 " movq %%mm4, 32(%1) ;\n"
339 " movq 56(%1), %%mm7 ;\n"
340 " pxor 48(%2), %%mm6 ;\n"
341 " movq %%mm5, 40(%1) ;\n"
342 " pxor 56(%2), %%mm7 ;\n"
343 " pxor 48(%3), %%mm6 ;\n"
344 " pxor 56(%3), %%mm7 ;\n"
345 " movq %%mm6, 48(%1) ;\n"
346 " movq %%mm7, 56(%1) ;\n"
354 "+r" (p1), "+r" (p2), "+r" (p3)
362 xor_p5_mmx_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
363 unsigned long *p3, unsigned long *p4)
365 unsigned long lines = bytes >> 6;
369 __asm__ __volatile__ (
370 " .align 32,0x90 ;\n"
372 " movq (%1), %%mm0 ;\n"
373 " movq 8(%1), %%mm1 ;\n"
374 " pxor (%2), %%mm0 ;\n"
375 " movq 16(%1), %%mm2 ;\n"
376 " pxor 8(%2), %%mm1 ;\n"
377 " pxor (%3), %%mm0 ;\n"
378 " pxor 16(%2), %%mm2 ;\n"
379 " pxor 8(%3), %%mm1 ;\n"
380 " pxor (%4), %%mm0 ;\n"
381 " movq 24(%1), %%mm3 ;\n"
382 " pxor 16(%3), %%mm2 ;\n"
383 " pxor 8(%4), %%mm1 ;\n"
384 " movq %%mm0, (%1) ;\n"
385 " movq 32(%1), %%mm4 ;\n"
386 " pxor 24(%2), %%mm3 ;\n"
387 " pxor 16(%4), %%mm2 ;\n"
388 " movq %%mm1, 8(%1) ;\n"
389 " movq 40(%1), %%mm5 ;\n"
390 " pxor 32(%2), %%mm4 ;\n"
391 " pxor 24(%3), %%mm3 ;\n"
392 " movq %%mm2, 16(%1) ;\n"
393 " pxor 40(%2), %%mm5 ;\n"
394 " pxor 32(%3), %%mm4 ;\n"
395 " pxor 24(%4), %%mm3 ;\n"
396 " movq %%mm3, 24(%1) ;\n"
397 " movq 56(%1), %%mm7 ;\n"
398 " movq 48(%1), %%mm6 ;\n"
399 " pxor 40(%3), %%mm5 ;\n"
400 " pxor 32(%4), %%mm4 ;\n"
401 " pxor 48(%2), %%mm6 ;\n"
402 " movq %%mm4, 32(%1) ;\n"
403 " pxor 56(%2), %%mm7 ;\n"
404 " pxor 40(%4), %%mm5 ;\n"
405 " pxor 48(%3), %%mm6 ;\n"
406 " pxor 56(%3), %%mm7 ;\n"
407 " movq %%mm5, 40(%1) ;\n"
408 " pxor 48(%4), %%mm6 ;\n"
409 " pxor 56(%4), %%mm7 ;\n"
410 " movq %%mm6, 48(%1) ;\n"
411 " movq %%mm7, 56(%1) ;\n"
420 "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
428 xor_p5_mmx_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
429 unsigned long *p3, unsigned long *p4, unsigned long *p5)
431 unsigned long lines = bytes >> 6;
435 /* Make sure GCC forgets anything it knows about p4 or p5,
436 such that it won't pass to the asm volatile below a
437 register that is shared with any other variable. That's
438 because we modify p4 and p5 there, but we can't mark them
439 as read/write, otherwise we'd overflow the 10-asm-operands
440 limit of GCC < 3.1. */
441 __asm__ ("" : "+r" (p4), "+r" (p5));
443 __asm__ __volatile__ (
444 " .align 32,0x90 ;\n"
446 " movq (%1), %%mm0 ;\n"
447 " movq 8(%1), %%mm1 ;\n"
448 " pxor (%2), %%mm0 ;\n"
449 " pxor 8(%2), %%mm1 ;\n"
450 " movq 16(%1), %%mm2 ;\n"
451 " pxor (%3), %%mm0 ;\n"
452 " pxor 8(%3), %%mm1 ;\n"
453 " pxor 16(%2), %%mm2 ;\n"
454 " pxor (%4), %%mm0 ;\n"
455 " pxor 8(%4), %%mm1 ;\n"
456 " pxor 16(%3), %%mm2 ;\n"
457 " movq 24(%1), %%mm3 ;\n"
458 " pxor (%5), %%mm0 ;\n"
459 " pxor 8(%5), %%mm1 ;\n"
460 " movq %%mm0, (%1) ;\n"
461 " pxor 16(%4), %%mm2 ;\n"
462 " pxor 24(%2), %%mm3 ;\n"
463 " movq %%mm1, 8(%1) ;\n"
464 " pxor 16(%5), %%mm2 ;\n"
465 " pxor 24(%3), %%mm3 ;\n"
466 " movq 32(%1), %%mm4 ;\n"
467 " movq %%mm2, 16(%1) ;\n"
468 " pxor 24(%4), %%mm3 ;\n"
469 " pxor 32(%2), %%mm4 ;\n"
470 " movq 40(%1), %%mm5 ;\n"
471 " pxor 24(%5), %%mm3 ;\n"
472 " pxor 32(%3), %%mm4 ;\n"
473 " pxor 40(%2), %%mm5 ;\n"
474 " movq %%mm3, 24(%1) ;\n"
475 " pxor 32(%4), %%mm4 ;\n"
476 " pxor 40(%3), %%mm5 ;\n"
477 " movq 48(%1), %%mm6 ;\n"
478 " movq 56(%1), %%mm7 ;\n"
479 " pxor 32(%5), %%mm4 ;\n"
480 " pxor 40(%4), %%mm5 ;\n"
481 " pxor 48(%2), %%mm6 ;\n"
482 " pxor 56(%2), %%mm7 ;\n"
483 " movq %%mm4, 32(%1) ;\n"
484 " pxor 48(%3), %%mm6 ;\n"
485 " pxor 56(%3), %%mm7 ;\n"
486 " pxor 40(%5), %%mm5 ;\n"
487 " pxor 48(%4), %%mm6 ;\n"
488 " pxor 56(%4), %%mm7 ;\n"
489 " movq %%mm5, 40(%1) ;\n"
490 " pxor 48(%5), %%mm6 ;\n"
491 " pxor 56(%5), %%mm7 ;\n"
492 " movq %%mm6, 48(%1) ;\n"
493 " movq %%mm7, 56(%1) ;\n"
503 "+r" (p1), "+r" (p2), "+r" (p3)
507 /* p4 and p5 were modified, and now the variables are dead.
508 Clobber them just to be sure nobody does something stupid
509 like assuming they have some legal value. */
510 __asm__ ("" : "=r" (p4), "=r" (p5));
515 static struct xor_block_template xor_block_pII_mmx = {
517 .do_2 = xor_pII_mmx_2,
518 .do_3 = xor_pII_mmx_3,
519 .do_4 = xor_pII_mmx_4,
520 .do_5 = xor_pII_mmx_5,
523 static struct xor_block_template xor_block_p5_mmx = {
525 .do_2 = xor_p5_mmx_2,
526 .do_3 = xor_p5_mmx_3,
527 .do_4 = xor_p5_mmx_4,
528 .do_5 = xor_p5_mmx_5,
532 * Cache avoiding checksumming functions utilizing KNI instructions
533 * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
536 #define XMMS_SAVE do { \
538 __asm__ __volatile__ ( \
539 "movl %%cr0,%0 ;\n\t" \
541 "movups %%xmm0,(%1) ;\n\t" \
542 "movups %%xmm1,0x10(%1) ;\n\t" \
543 "movups %%xmm2,0x20(%1) ;\n\t" \
544 "movups %%xmm3,0x30(%1) ;\n\t" \
550 #define XMMS_RESTORE do { \
551 __asm__ __volatile__ ( \
553 "movups (%1),%%xmm0 ;\n\t" \
554 "movups 0x10(%1),%%xmm1 ;\n\t" \
555 "movups 0x20(%1),%%xmm2 ;\n\t" \
556 "movups 0x30(%1),%%xmm3 ;\n\t" \
557 "movl %0,%%cr0 ;\n\t" \
559 : "r" (cr0), "r" (xmm_save) \
564 #define ALIGN16 __attribute__((aligned(16)))
566 #define OFFS(x) "16*("#x")"
567 #define PF_OFFS(x) "256+16*("#x")"
568 #define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n"
569 #define LD(x,y) " movaps "OFFS(x)"(%1), %%xmm"#y" ;\n"
570 #define ST(x,y) " movaps %%xmm"#y", "OFFS(x)"(%1) ;\n"
571 #define PF1(x) " prefetchnta "PF_OFFS(x)"(%2) ;\n"
572 #define PF2(x) " prefetchnta "PF_OFFS(x)"(%3) ;\n"
573 #define PF3(x) " prefetchnta "PF_OFFS(x)"(%4) ;\n"
574 #define PF4(x) " prefetchnta "PF_OFFS(x)"(%5) ;\n"
575 #define PF5(x) " prefetchnta "PF_OFFS(x)"(%6) ;\n"
576 #define XO1(x,y) " xorps "OFFS(x)"(%2), %%xmm"#y" ;\n"
577 #define XO2(x,y) " xorps "OFFS(x)"(%3), %%xmm"#y" ;\n"
578 #define XO3(x,y) " xorps "OFFS(x)"(%4), %%xmm"#y" ;\n"
579 #define XO4(x,y) " xorps "OFFS(x)"(%5), %%xmm"#y" ;\n"
580 #define XO5(x,y) " xorps "OFFS(x)"(%6), %%xmm"#y" ;\n"
584 xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
586 unsigned long lines = bytes >> 8;
587 char xmm_save[16*4] ALIGN16;
592 __asm__ __volatile__ (
637 xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
640 unsigned long lines = bytes >> 8;
641 char xmm_save[16*4] ALIGN16;
646 __asm__ __volatile__ (
690 "+r" (p1), "+r"(p2), "+r"(p3)
698 xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
699 unsigned long *p3, unsigned long *p4)
701 unsigned long lines = bytes >> 8;
702 char xmm_save[16*4] ALIGN16;
707 __asm__ __volatile__ (
758 "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
766 xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
767 unsigned long *p3, unsigned long *p4, unsigned long *p5)
769 unsigned long lines = bytes >> 8;
770 char xmm_save[16*4] ALIGN16;
775 /* Make sure GCC forgets anything it knows about p4 or p5,
776 such that it won't pass to the asm volatile below a
777 register that is shared with any other variable. That's
778 because we modify p4 and p5 there, but we can't mark them
779 as read/write, otherwise we'd overflow the 10-asm-operands
780 limit of GCC < 3.1. */
781 __asm__ ("" : "+r" (p4), "+r" (p5));
783 __asm__ __volatile__ (
841 "+r" (p1), "+r" (p2), "+r" (p3)
845 /* p4 and p5 were modified, and now the variables are dead.
846 Clobber them just to be sure nobody does something stupid
847 like assuming they have some legal value. */
848 __asm__ ("" : "=r" (p4), "=r" (p5));
853 static struct xor_block_template xor_block_pIII_sse = {
861 /* Also try the generic routines. */
862 #include <asm-generic/xor.h>
864 #undef XOR_TRY_TEMPLATES
865 #define XOR_TRY_TEMPLATES \
867 xor_speed(&xor_block_8regs); \
868 xor_speed(&xor_block_8regs_p); \
869 xor_speed(&xor_block_32regs); \
870 xor_speed(&xor_block_32regs_p); \
872 xor_speed(&xor_block_pIII_sse); \
874 xor_speed(&xor_block_pII_mmx); \
875 xor_speed(&xor_block_p5_mmx); \
879 /* We force the use of the SSE xor block because it can write around L2.
880 We may also be able to load into the L1 only depending on how the cpu
881 deals with a load to a line that is being prefetched. */
882 #define XOR_SELECT_TEMPLATE(FASTEST) \
883 (cpu_has_xmm ? &xor_block_pIII_sse : FASTEST)