2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Quick'n'dirty IP checksum ...
8 * Copyright (C) 1998, 1999 Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc.
10 * Copyright (C) 2007 Maciej W. Rozycki
12 #include <linux/errno.h>
14 #include <asm/asm-offsets.h>
15 #include <asm/regdef.h>
19 * As we are sharing code base with the mips32 tree (which use the o32 ABI
20 * register definitions). We need to redefine the register definitions from
21 * the n64 ABI register naming to the o32 ABI register naming.
53 #endif /* USE_DOUBLE */
55 #define UNIT(unit) ((unit)*NBYTES)
57 #define ADDC(sum,reg) \
65 #define ADDC32(sum,reg) \
73 #define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \
74 LOAD _t0, (offset + UNIT(0))(src); \
75 LOAD _t1, (offset + UNIT(1))(src); \
76 LOAD _t2, (offset + UNIT(2))(src); \
77 LOAD _t3, (offset + UNIT(3))(src); \
84 #define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
85 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
87 #define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
88 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \
89 CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
94 * a1: length of the area to checksum
95 * a2: partial checksum
109 bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */
112 andi t7, src, 0x1 /* odd buffer? */
115 beqz t7, .Lword_align
119 LONG_SUBU a1, a1, 0x1
124 PTR_ADDU src, src, 0x1
128 beqz t8, .Ldword_align
132 LONG_SUBU a1, a1, 0x2
135 PTR_ADDU src, src, 0x2
138 bnez t8, .Ldo_end_words
142 beqz t8, .Lqword_align
146 LONG_SUBU a1, a1, 0x4
148 PTR_ADDU src, src, 0x4
152 beqz t8, .Loword_align
157 LONG_SUBU a1, a1, 0x8
162 LONG_SUBU a1, a1, 0x8
166 PTR_ADDU src, src, 0x8
170 beqz t8, .Lbegin_movement
179 CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
181 LONG_SUBU a1, a1, 0x10
182 PTR_ADDU src, src, 0x10
190 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
191 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
192 CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
193 CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
194 LONG_SUBU t8, t8, 0x01
195 .set reorder /* DADDI_WAR */
196 PTR_ADDU src, src, 0x80
197 bnez t8, .Lmove_128bytes
205 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
206 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
207 PTR_ADDU src, src, 0x40
210 beqz t2, .Ldo_end_words
214 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
216 PTR_ADDU src, src, 0x20
219 beqz t8, .Lsmall_csumcpy
225 LONG_SUBU t8, t8, 0x1
227 .set reorder /* DADDI_WAR */
228 PTR_ADDU src, src, 0x4
232 /* unknown src alignment and < 8 bytes to go */
240 /* Still a full word to go */
244 dsll t1, t1, 32 /* clear lower 32bit */
252 /* Still a halfword to go */
280 /* odd buffer alignment? */
290 /* Add the passed partial csum. */
298 * checksum and copy routines based on memcpy.S
300 * csum_partial_copy_nocheck(src, dst, len, sum)
301 * __csum_partial_copy_user(src, dst, len, sum, errp)
303 * See "Spec" in memcpy.S for details. Unlike __copy_user, all
304 * function in this file use the standard calling convention.
316 * The exception handler for loads requires that:
317 * 1- AT contain the address of the byte just past the end of the source
319 * 2- src_entry <= src < AT, and
320 * 3- (dst - src) == (dst_entry - src_entry),
321 * The _entry suffix denotes values when __copy_user was called.
323 * (1) is set up up by __csum_partial_copy_from_user and maintained by
324 * not writing AT in __csum_partial_copy
325 * (2) is met by incrementing src by the number of bytes copied
326 * (3) is met by not doing loads between a pair of increments of dst and src
328 * The exception handlers for stores stores -EFAULT to errptr and return.
329 * These handlers do not need to overwrite any data.
332 #define EXC(inst_reg,addr,handler) \
334 .section __ex_table,"a"; \
372 #endif /* USE_DOUBLE */
374 #ifdef CONFIG_CPU_LITTLE_ENDIAN
375 #define LDFIRST LOADR
377 #define STFIRST STORER
378 #define STREST STOREL
379 #define SHIFT_DISCARD SLLV
380 #define SHIFT_DISCARD_REVERT SRLV
382 #define LDFIRST LOADL
384 #define STFIRST STOREL
385 #define STREST STORER
386 #define SHIFT_DISCARD SRLV
387 #define SHIFT_DISCARD_REVERT SLLV
390 #define FIRST(unit) ((unit)*NBYTES)
391 #define REST(unit) (FIRST(unit)+NBYTES-1)
393 #define ADDRMASK (NBYTES-1)
395 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
401 LEAF(__csum_partial_copy_user)
402 PTR_ADDU AT, src, len /* See (1) above. */
408 FEXPORT(csum_partial_copy_nocheck)
412 * Note: dst & src may be unaligned, len may be 0
416 * The "issue break"s below are very approximate.
417 * Issue delays for dcache fills will perturb the schedule, as will
418 * load queue full replay traps, etc.
420 * If len < NBYTES use byte operations.
423 and t1, dst, ADDRMASK
424 bnez t2, .Lcopy_bytes_checklen
425 and t0, src, ADDRMASK
426 andi odd, dst, 0x1 /* odd buffer? */
427 bnez t1, .Ldst_unaligned
429 bnez t0, .Lsrc_unaligned_dst_aligned
431 * use delay slot for fall-through
432 * src and dst are aligned; need to compute rem
435 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
436 beqz t0, .Lcleanup_both_aligned # len < 8*NBYTES
438 SUB len, 8*NBYTES # subtract here for bgez loop
441 EXC( LOAD t0, UNIT(0)(src), .Ll_exc)
442 EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy)
443 EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy)
444 EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy)
445 EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy)
446 EXC( LOAD t5, UNIT(5)(src), .Ll_exc_copy)
447 EXC( LOAD t6, UNIT(6)(src), .Ll_exc_copy)
448 EXC( LOAD t7, UNIT(7)(src), .Ll_exc_copy)
449 SUB len, len, 8*NBYTES
450 ADD src, src, 8*NBYTES
451 EXC( STORE t0, UNIT(0)(dst), .Ls_exc)
453 EXC( STORE t1, UNIT(1)(dst), .Ls_exc)
455 EXC( STORE t2, UNIT(2)(dst), .Ls_exc)
457 EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
459 EXC( STORE t4, UNIT(4)(dst), .Ls_exc)
461 EXC( STORE t5, UNIT(5)(dst), .Ls_exc)
463 EXC( STORE t6, UNIT(6)(dst), .Ls_exc)
465 EXC( STORE t7, UNIT(7)(dst), .Ls_exc)
467 .set reorder /* DADDI_WAR */
468 ADD dst, dst, 8*NBYTES
471 ADD len, 8*NBYTES # revert len (see above)
474 * len == the number of bytes left to copy < 8*NBYTES
476 .Lcleanup_both_aligned:
479 sltu t0, len, 4*NBYTES
480 bnez t0, .Lless_than_4units
481 and rem, len, (NBYTES-1) # rem = len % NBYTES
485 EXC( LOAD t0, UNIT(0)(src), .Ll_exc)
486 EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy)
487 EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy)
488 EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy)
489 SUB len, len, 4*NBYTES
490 ADD src, src, 4*NBYTES
491 EXC( STORE t0, UNIT(0)(dst), .Ls_exc)
493 EXC( STORE t1, UNIT(1)(dst), .Ls_exc)
495 EXC( STORE t2, UNIT(2)(dst), .Ls_exc)
497 EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
499 .set reorder /* DADDI_WAR */
500 ADD dst, dst, 4*NBYTES
507 beq rem, len, .Lcopy_bytes
510 EXC( LOAD t0, 0(src), .Ll_exc)
513 EXC( STORE t0, 0(dst), .Ls_exc)
515 .set reorder /* DADDI_WAR */
521 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
522 * A loop would do only a byte at a time with possible branch
523 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
524 * because can't assume read-access to dst. Instead, use
525 * STREST dst, which doesn't require read access to dst.
527 * This code should perform better than a simple loop on modern,
528 * wide-issue mips processors because the code has fewer branches and
529 * more instruction-level parallelism.
533 ADD t1, dst, len # t1 is just past last byte of dst
535 SLL rem, len, 3 # rem = number of bits to keep
536 EXC( LOAD t0, 0(src), .Ll_exc)
537 SUB bits, bits, rem # bits = number of bits to discard
538 SHIFT_DISCARD t0, t0, bits
539 EXC( STREST t0, -1(t1), .Ls_exc)
540 SHIFT_DISCARD_REVERT t0, t0, bits
548 * t0 = src & ADDRMASK
549 * t1 = dst & ADDRMASK; T1 > 0
552 * Copy enough bytes to align dst
553 * Set match = (src and dst have same alignment)
556 EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc)
558 EXC( LDREST t3, REST(0)(src), .Ll_exc_copy)
559 SUB t2, t2, t1 # t2 = number of bytes copied
561 EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc)
562 SLL t4, t1, 3 # t4 = number of bits to discard
563 SHIFT_DISCARD t3, t3, t4
564 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
569 beqz match, .Lboth_aligned
572 .Lsrc_unaligned_dst_aligned:
573 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
574 beqz t0, .Lcleanup_src_unaligned
575 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
578 * Avoid consecutive LD*'s to the same register since some mips
579 * implementations can't issue them in the same cycle.
580 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
581 * are to the same unit (unless src is aligned, but it's not).
583 EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc)
584 EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy)
585 SUB len, len, 4*NBYTES
586 EXC( LDREST t0, REST(0)(src), .Ll_exc_copy)
587 EXC( LDREST t1, REST(1)(src), .Ll_exc_copy)
588 EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy)
589 EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy)
590 EXC( LDREST t2, REST(2)(src), .Ll_exc_copy)
591 EXC( LDREST t3, REST(3)(src), .Ll_exc_copy)
592 ADD src, src, 4*NBYTES
593 #ifdef CONFIG_CPU_SB1
594 nop # improves slotting
596 EXC( STORE t0, UNIT(0)(dst), .Ls_exc)
598 EXC( STORE t1, UNIT(1)(dst), .Ls_exc)
600 EXC( STORE t2, UNIT(2)(dst), .Ls_exc)
602 EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
604 .set reorder /* DADDI_WAR */
605 ADD dst, dst, 4*NBYTES
609 .Lcleanup_src_unaligned:
611 and rem, len, NBYTES-1 # rem = len % NBYTES
612 beq rem, len, .Lcopy_bytes
615 EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc)
616 EXC( LDREST t0, REST(0)(src), .Ll_exc_copy)
619 EXC( STORE t0, 0(dst), .Ls_exc)
621 .set reorder /* DADDI_WAR */
626 .Lcopy_bytes_checklen:
630 /* 0 < len < NBYTES */
631 #ifdef CONFIG_CPU_LITTLE_ENDIAN
632 #define SHIFT_START 0
635 #define SHIFT_START 8*(NBYTES-1)
638 move t2, zero # partial word
639 li t3, SHIFT_START # shift
640 /* use .Ll_exc_copy here to return correct sum on fault */
641 #define COPY_BYTE(N) \
642 EXC( lbu t0, N(src), .Ll_exc_copy); \
644 EXC( sb t0, N(dst), .Ls_exc); \
646 addu t3, SHIFT_INC; \
647 beqz len, .Lcopy_bytes_done; \
658 EXC( lbu t0, NBYTES-2(src), .Ll_exc_copy)
660 EXC( sb t0, NBYTES-2(dst), .Ls_exc)
677 /* odd buffer alignment? */
693 * Copy bytes from src until faulting load address (or until a
696 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
697 * may be more than a byte beyond the last address.
698 * Hence, the lb below may get an exception.
700 * Assumes src < THREAD_BUADDR($28)
702 LOAD t0, TI_TASK($28)
704 LOAD t0, THREAD_BUADDR(t0)
706 EXC( lbu t1, 0(src), .Ll_exc)
708 sb t1, 0(dst) # can't fault -- we're copy_from_user
712 .set reorder /* DADDI_WAR */
717 LOAD t0, TI_TASK($28)
719 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address
721 SUB len, AT, t0 # len number of uncopied bytes
723 * Here's where we rely on src and dst being incremented in tandem,
725 * dst += (fault addr - src) to put dst at first byte to clear
727 ADD dst, t0 # compute start address in a1
730 * Clear len bytes starting at dst. Can't call __bzero because it
731 * might modify len. An inefficient loop for these rare times...
733 .set reorder /* DADDI_WAR */
741 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
754 li v0, -1 /* invalid checksum */
759 END(__csum_partial_copy_user)