arch/arm/lib/uaccess.S

   1 /*
   2  *  linux/arch/arm/lib/uaccess.S
   3  *
   4  *  Copyright (C) 1995, 1996,1997,1998 Russell King
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  *  Routines to block copy data to/from user memory
  11  *   These are highly optimised both for the 4k page size
  12  *   and for various alignments.
  13  */
  14 #include <linux/linkage.h>
  15 #include <asm/assembler.h>
  16 #include <asm/errno.h>
  17 #include <asm/domain.h>
  18
  19                 .text
  20
  21 #define PAGE_SHIFT 12
  22
  23 /* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
  24  * Purpose  : copy a block to user memory from kernel memory
  25  * Params   : to   - user memory
  26  *          : from - kernel memory
  27  *          : n    - number of bytes to copy
  28  * Returns  : Number of bytes NOT copied.
  29  */
  30
  31 .Lc2u_dest_not_aligned:
  32                 rsb     ip, ip, #4
  33                 cmp     ip, #2
  34                 ldrb    r3, [r1], #1
  35 USER(           T(strb) r3, [r0], #1)                   @ May fault
  36                 ldrgeb  r3, [r1], #1
  37 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
  38                 ldrgtb  r3, [r1], #1
  39 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
  40                 sub     r2, r2, ip
  41                 b       .Lc2u_dest_aligned
  42
  43 ENTRY(__copy_to_user)
  44                 stmfd   sp!, {r2, r4 - r7, lr}
  45                 cmp     r2, #4
  46                 blt     .Lc2u_not_enough
  47                 ands    ip, r0, #3
  48                 bne     .Lc2u_dest_not_aligned
  49 .Lc2u_dest_aligned:
  50
  51                 ands    ip, r1, #3
  52                 bne     .Lc2u_src_not_aligned
  53 /*
  54  * Seeing as there has to be at least 8 bytes to copy, we can
  55  * copy one word, and force a user-mode page fault...
  56  */
  57
  58 .Lc2u_0fupi:    subs    r2, r2, #4
  59                 addmi   ip, r2, #4
  60                 bmi     .Lc2u_0nowords
  61                 ldr     r3, [r1], #4
  62 USER(           T(str)  r3, [r0], #4)                   @ May fault
  63                 mov     ip, r0, lsl #32 - PAGE_SHIFT    @ On each page, use a ld/st??t instruction
  64                 rsb     ip, ip, #0
  65                 movs    ip, ip, lsr #32 - PAGE_SHIFT
  66                 beq     .Lc2u_0fupi
  67 /*
  68  * ip = max no. of bytes to copy before needing another "strt" insn
  69  */
  70                 cmp     r2, ip
  71                 movlt   ip, r2
  72                 sub     r2, r2, ip
  73                 subs    ip, ip, #32
  74                 blt     .Lc2u_0rem8lp
  75
  76 .Lc2u_0cpy8lp:  ldmia   r1!, {r3 - r6}
  77                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
  78                 ldmia   r1!, {r3 - r6}
  79                 subs    ip, ip, #32
  80                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
  81                 bpl     .Lc2u_0cpy8lp
  82
  83 .Lc2u_0rem8lp:  cmn     ip, #16
  84                 ldmgeia r1!, {r3 - r6}
  85                 stmgeia r0!, {r3 - r6}                  @ Shouldnt fault
  86                 tst     ip, #8
  87                 ldmneia r1!, {r3 - r4}
  88                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
  89                 tst     ip, #4
  90                 ldrne   r3, [r1], #4
  91                 T(strne) r3, [r0], #4                   @ Shouldnt fault
  92                 ands    ip, ip, #3
  93                 beq     .Lc2u_0fupi
  94 .Lc2u_0nowords: teq     ip, #0
  95                 beq     .Lc2u_finished
  96 .Lc2u_nowords:  cmp     ip, #2
  97                 ldrb    r3, [r1], #1
  98 USER(           T(strb) r3, [r0], #1)                   @ May fault
  99                 ldrgeb  r3, [r1], #1
 100 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
 101                 ldrgtb  r3, [r1], #1
 102 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
 103                 b       .Lc2u_finished
 104
 105 .Lc2u_not_enough:
 106                 movs    ip, r2
 107                 bne     .Lc2u_nowords
 108 .Lc2u_finished: mov     r0, #0
 109                 ldmfd   sp!, {r2, r4 - r7, pc}
 110
 111 .Lc2u_src_not_aligned:
 112                 bic     r1, r1, #3
 113                 ldr     r7, [r1], #4
 114                 cmp     ip, #2
 115                 bgt     .Lc2u_3fupi
 116                 beq     .Lc2u_2fupi
 117 .Lc2u_1fupi:    subs    r2, r2, #4
 118                 addmi   ip, r2, #4
 119                 bmi     .Lc2u_1nowords
 120                 mov     r3, r7, pull #8
 121                 ldr     r7, [r1], #4
 122                 orr     r3, r3, r7, push #24
 123 USER(           T(str)  r3, [r0], #4)                   @ May fault
 124                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 125                 rsb     ip, ip, #0
 126                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 127                 beq     .Lc2u_1fupi
 128                 cmp     r2, ip
 129                 movlt   ip, r2
 130                 sub     r2, r2, ip
 131                 subs    ip, ip, #16
 132                 blt     .Lc2u_1rem8lp
 133
 134 .Lc2u_1cpy8lp:  mov     r3, r7, pull #8
 135                 ldmia   r1!, {r4 - r7}
 136                 subs    ip, ip, #16
 137                 orr     r3, r3, r4, push #24
 138                 mov     r4, r4, pull #8
 139                 orr     r4, r4, r5, push #24
 140                 mov     r5, r5, pull #8
 141                 orr     r5, r5, r6, push #24
 142                 mov     r6, r6, pull #8
 143                 orr     r6, r6, r7, push #24
 144                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 145                 bpl     .Lc2u_1cpy8lp
 146
 147 .Lc2u_1rem8lp:  tst     ip, #8
 148                 movne   r3, r7, pull #8
 149                 ldmneia r1!, {r4, r7}
 150                 orrne   r3, r3, r4, push #24
 151                 movne   r4, r4, pull #8
 152                 orrne   r4, r4, r7, push #24
 153                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 154                 tst     ip, #4
 155                 movne   r3, r7, pull #8
 156                 ldrne   r7, [r1], #4
 157                 orrne   r3, r3, r7, push #24
 158                 T(strne) r3, [r0], #4                   @ Shouldnt fault
 159                 ands    ip, ip, #3
 160                 beq     .Lc2u_1fupi
 161 .Lc2u_1nowords: mov     r3, r7, get_byte_1
 162                 teq     ip, #0
 163                 beq     .Lc2u_finished
 164                 cmp     ip, #2
 165 USER(           T(strb) r3, [r0], #1)                   @ May fault
 166                 movge   r3, r7, get_byte_2
 167 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
 168                 movgt   r3, r7, get_byte_3
 169 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
 170                 b       .Lc2u_finished
 171
 172 .Lc2u_2fupi:    subs    r2, r2, #4
 173                 addmi   ip, r2, #4
 174                 bmi     .Lc2u_2nowords
 175                 mov     r3, r7, pull #16
 176                 ldr     r7, [r1], #4
 177                 orr     r3, r3, r7, push #16
 178 USER(           T(str)  r3, [r0], #4)                   @ May fault
 179                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 180                 rsb     ip, ip, #0
 181                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 182                 beq     .Lc2u_2fupi
 183                 cmp     r2, ip
 184                 movlt   ip, r2
 185                 sub     r2, r2, ip
 186                 subs    ip, ip, #16
 187                 blt     .Lc2u_2rem8lp
 188
 189 .Lc2u_2cpy8lp:  mov     r3, r7, pull #16
 190                 ldmia   r1!, {r4 - r7}
 191                 subs    ip, ip, #16
 192                 orr     r3, r3, r4, push #16
 193                 mov     r4, r4, pull #16
 194                 orr     r4, r4, r5, push #16
 195                 mov     r5, r5, pull #16
 196                 orr     r5, r5, r6, push #16
 197                 mov     r6, r6, pull #16
 198                 orr     r6, r6, r7, push #16
 199                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 200                 bpl     .Lc2u_2cpy8lp
 201
 202 .Lc2u_2rem8lp:  tst     ip, #8
 203                 movne   r3, r7, pull #16
 204                 ldmneia r1!, {r4, r7}
 205                 orrne   r3, r3, r4, push #16
 206                 movne   r4, r4, pull #16
 207                 orrne   r4, r4, r7, push #16
 208                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 209                 tst     ip, #4
 210                 movne   r3, r7, pull #16
 211                 ldrne   r7, [r1], #4
 212                 orrne   r3, r3, r7, push #16
 213                 T(strne) r3, [r0], #4                   @ Shouldnt fault
 214                 ands    ip, ip, #3
 215                 beq     .Lc2u_2fupi
 216 .Lc2u_2nowords: mov     r3, r7, get_byte_2
 217                 teq     ip, #0
 218                 beq     .Lc2u_finished
 219                 cmp     ip, #2
 220 USER(           T(strb) r3, [r0], #1)                   @ May fault
 221                 movge   r3, r7, get_byte_3
 222 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
 223                 ldrgtb  r3, [r1], #0
 224 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
 225                 b       .Lc2u_finished
 226
 227 .Lc2u_3fupi:    subs    r2, r2, #4
 228                 addmi   ip, r2, #4
 229                 bmi     .Lc2u_3nowords
 230                 mov     r3, r7, pull #24
 231                 ldr     r7, [r1], #4
 232                 orr     r3, r3, r7, push #8
 233 USER(           T(str)  r3, [r0], #4)                   @ May fault
 234                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 235                 rsb     ip, ip, #0
 236                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 237                 beq     .Lc2u_3fupi
 238                 cmp     r2, ip
 239                 movlt   ip, r2
 240                 sub     r2, r2, ip
 241                 subs    ip, ip, #16
 242                 blt     .Lc2u_3rem8lp
 243
 244 .Lc2u_3cpy8lp:  mov     r3, r7, pull #24
 245                 ldmia   r1!, {r4 - r7}
 246                 subs    ip, ip, #16
 247                 orr     r3, r3, r4, push #8
 248                 mov     r4, r4, pull #24
 249                 orr     r4, r4, r5, push #8
 250                 mov     r5, r5, pull #24
 251                 orr     r5, r5, r6, push #8
 252                 mov     r6, r6, pull #24
 253                 orr     r6, r6, r7, push #8
 254                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 255                 bpl     .Lc2u_3cpy8lp
 256
 257 .Lc2u_3rem8lp:  tst     ip, #8
 258                 movne   r3, r7, pull #24
 259                 ldmneia r1!, {r4, r7}
 260                 orrne   r3, r3, r4, push #8
 261                 movne   r4, r4, pull #24
 262                 orrne   r4, r4, r7, push #8
 263                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 264                 tst     ip, #4
 265                 movne   r3, r7, pull #24
 266                 ldrne   r7, [r1], #4
 267                 orrne   r3, r3, r7, push #8
 268                 T(strne) r3, [r0], #4                   @ Shouldnt fault
 269                 ands    ip, ip, #3
 270                 beq     .Lc2u_3fupi
 271 .Lc2u_3nowords: mov     r3, r7, get_byte_3
 272                 teq     ip, #0
 273                 beq     .Lc2u_finished
 274                 cmp     ip, #2
 275 USER(           T(strb) r3, [r0], #1)                   @ May fault
 276                 ldrgeb  r3, [r1], #1
 277 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
 278                 ldrgtb  r3, [r1], #0
 279 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
 280                 b       .Lc2u_finished
 281 ENDPROC(__copy_to_user)
 282
 283                 .pushsection .fixup,"ax"
 284                 .align  0
 285 9001:           ldmfd   sp!, {r0, r4 - r7, pc}
 286                 .popsection
 287
 288 /* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
 289  * Purpose  : copy a block from user memory to kernel memory
 290  * Params   : to   - kernel memory
 291  *          : from - user memory
 292  *          : n    - number of bytes to copy
 293  * Returns  : Number of bytes NOT copied.
 294  */
 295 .Lcfu_dest_not_aligned:
 296                 rsb     ip, ip, #4
 297                 cmp     ip, #2
 298 USER(           T(ldrb) r3, [r1], #1)                   @ May fault
 299                 strb    r3, [r0], #1
 300 USER(           T(ldrgeb) r3, [r1], #1)                 @ May fault
 301                 strgeb  r3, [r0], #1
 302 USER(           T(ldrgtb) r3, [r1], #1)                 @ May fault
 303                 strgtb  r3, [r0], #1
 304                 sub     r2, r2, ip
 305                 b       .Lcfu_dest_aligned
 306
 307 ENTRY(__copy_from_user)
 308                 stmfd   sp!, {r0, r2, r4 - r7, lr}
 309                 cmp     r2, #4
 310                 blt     .Lcfu_not_enough
 311                 ands    ip, r0, #3
 312                 bne     .Lcfu_dest_not_aligned
 313 .Lcfu_dest_aligned:
 314                 ands    ip, r1, #3
 315                 bne     .Lcfu_src_not_aligned
 316
 317 /*
 318  * Seeing as there has to be at least 8 bytes to copy, we can
 319  * copy one word, and force a user-mode page fault...
 320  */
 321
 322 .Lcfu_0fupi:    subs    r2, r2, #4
 323                 addmi   ip, r2, #4
 324                 bmi     .Lcfu_0nowords
 325 USER(           T(ldr)  r3, [r1], #4)
 326                 str     r3, [r0], #4
 327                 mov     ip, r1, lsl #32 - PAGE_SHIFT    @ On each page, use a ld/st??t instruction
 328                 rsb     ip, ip, #0
 329                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 330                 beq     .Lcfu_0fupi
 331 /*
 332  * ip = max no. of bytes to copy before needing another "strt" insn
 333  */
 334                 cmp     r2, ip
 335                 movlt   ip, r2
 336                 sub     r2, r2, ip
 337                 subs    ip, ip, #32
 338                 blt     .Lcfu_0rem8lp
 339
 340 .Lcfu_0cpy8lp:  ldmia   r1!, {r3 - r6}                  @ Shouldnt fault
 341                 stmia   r0!, {r3 - r6}
 342                 ldmia   r1!, {r3 - r6}                  @ Shouldnt fault
 343                 subs    ip, ip, #32
 344                 stmia   r0!, {r3 - r6}
 345                 bpl     .Lcfu_0cpy8lp
 346
 347 .Lcfu_0rem8lp:  cmn     ip, #16
 348                 ldmgeia r1!, {r3 - r6}                  @ Shouldnt fault
 349                 stmgeia r0!, {r3 - r6}
 350                 tst     ip, #8
 351                 ldmneia r1!, {r3 - r4}                  @ Shouldnt fault
 352                 stmneia r0!, {r3 - r4}
 353                 tst     ip, #4
 354                 T(ldrne) r3, [r1], #4                   @ Shouldnt fault
 355                 strne   r3, [r0], #4
 356                 ands    ip, ip, #3
 357                 beq     .Lcfu_0fupi
 358 .Lcfu_0nowords: teq     ip, #0
 359                 beq     .Lcfu_finished
 360 .Lcfu_nowords:  cmp     ip, #2
 361 USER(           T(ldrb) r3, [r1], #1)                   @ May fault
 362                 strb    r3, [r0], #1
 363 USER(           T(ldrgeb) r3, [r1], #1)                 @ May fault
 364                 strgeb  r3, [r0], #1
 365 USER(           T(ldrgtb) r3, [r1], #1)                 @ May fault
 366                 strgtb  r3, [r0], #1
 367                 b       .Lcfu_finished
 368
 369 .Lcfu_not_enough:
 370                 movs    ip, r2
 371                 bne     .Lcfu_nowords
 372 .Lcfu_finished: mov     r0, #0
 373                 add     sp, sp, #8
 374                 ldmfd   sp!, {r4 - r7, pc}
 375
 376 .Lcfu_src_not_aligned:
 377                 bic     r1, r1, #3
 378 USER(           T(ldr)  r7, [r1], #4)                   @ May fault
 379                 cmp     ip, #2
 380                 bgt     .Lcfu_3fupi
 381                 beq     .Lcfu_2fupi
 382 .Lcfu_1fupi:    subs    r2, r2, #4
 383                 addmi   ip, r2, #4
 384                 bmi     .Lcfu_1nowords
 385                 mov     r3, r7, pull #8
 386 USER(           T(ldr)  r7, [r1], #4)                   @ May fault
 387                 orr     r3, r3, r7, push #24
 388                 str     r3, [r0], #4
 389                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 390                 rsb     ip, ip, #0
 391                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 392                 beq     .Lcfu_1fupi
 393                 cmp     r2, ip
 394                 movlt   ip, r2
 395                 sub     r2, r2, ip
 396                 subs    ip, ip, #16
 397                 blt     .Lcfu_1rem8lp
 398
 399 .Lcfu_1cpy8lp:  mov     r3, r7, pull #8
 400                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 401                 subs    ip, ip, #16
 402                 orr     r3, r3, r4, push #24
 403                 mov     r4, r4, pull #8
 404                 orr     r4, r4, r5, push #24
 405                 mov     r5, r5, pull #8
 406                 orr     r5, r5, r6, push #24
 407                 mov     r6, r6, pull #8
 408                 orr     r6, r6, r7, push #24
 409                 stmia   r0!, {r3 - r6}
 410                 bpl     .Lcfu_1cpy8lp
 411
 412 .Lcfu_1rem8lp:  tst     ip, #8
 413                 movne   r3, r7, pull #8
 414                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 415                 orrne   r3, r3, r4, push #24
 416                 movne   r4, r4, pull #8
 417                 orrne   r4, r4, r7, push #24
 418                 stmneia r0!, {r3 - r4}
 419                 tst     ip, #4
 420                 movne   r3, r7, pull #8
 421 USER(           T(ldrne) r7, [r1], #4)                  @ May fault
 422                 orrne   r3, r3, r7, push #24
 423                 strne   r3, [r0], #4
 424                 ands    ip, ip, #3
 425                 beq     .Lcfu_1fupi
 426 .Lcfu_1nowords: mov     r3, r7, get_byte_1
 427                 teq     ip, #0
 428                 beq     .Lcfu_finished
 429                 cmp     ip, #2
 430                 strb    r3, [r0], #1
 431                 movge   r3, r7, get_byte_2
 432                 strgeb  r3, [r0], #1
 433                 movgt   r3, r7, get_byte_3
 434                 strgtb  r3, [r0], #1
 435                 b       .Lcfu_finished
 436
 437 .Lcfu_2fupi:    subs    r2, r2, #4
 438                 addmi   ip, r2, #4
 439                 bmi     .Lcfu_2nowords
 440                 mov     r3, r7, pull #16
 441 USER(           T(ldr)  r7, [r1], #4)                   @ May fault
 442                 orr     r3, r3, r7, push #16
 443                 str     r3, [r0], #4
 444                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 445                 rsb     ip, ip, #0
 446                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 447                 beq     .Lcfu_2fupi
 448                 cmp     r2, ip
 449                 movlt   ip, r2
 450                 sub     r2, r2, ip
 451                 subs    ip, ip, #16
 452                 blt     .Lcfu_2rem8lp
 453
 454
 455 .Lcfu_2cpy8lp:  mov     r3, r7, pull #16
 456                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 457                 subs    ip, ip, #16
 458                 orr     r3, r3, r4, push #16
 459                 mov     r4, r4, pull #16
 460                 orr     r4, r4, r5, push #16
 461                 mov     r5, r5, pull #16
 462                 orr     r5, r5, r6, push #16
 463                 mov     r6, r6, pull #16
 464                 orr     r6, r6, r7, push #16
 465                 stmia   r0!, {r3 - r6}
 466                 bpl     .Lcfu_2cpy8lp
 467
 468 .Lcfu_2rem8lp:  tst     ip, #8
 469                 movne   r3, r7, pull #16
 470                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 471                 orrne   r3, r3, r4, push #16
 472                 movne   r4, r4, pull #16
 473                 orrne   r4, r4, r7, push #16
 474                 stmneia r0!, {r3 - r4}
 475                 tst     ip, #4
 476                 movne   r3, r7, pull #16
 477 USER(           T(ldrne) r7, [r1], #4)                  @ May fault
 478                 orrne   r3, r3, r7, push #16
 479                 strne   r3, [r0], #4
 480                 ands    ip, ip, #3
 481                 beq     .Lcfu_2fupi
 482 .Lcfu_2nowords: mov     r3, r7, get_byte_2
 483                 teq     ip, #0
 484                 beq     .Lcfu_finished
 485                 cmp     ip, #2
 486                 strb    r3, [r0], #1
 487                 movge   r3, r7, get_byte_3
 488                 strgeb  r3, [r0], #1
 489 USER(           T(ldrgtb) r3, [r1], #0)                 @ May fault
 490                 strgtb  r3, [r0], #1
 491                 b       .Lcfu_finished
 492
 493 .Lcfu_3fupi:    subs    r2, r2, #4
 494                 addmi   ip, r2, #4
 495                 bmi     .Lcfu_3nowords
 496                 mov     r3, r7, pull #24
 497 USER(           T(ldr)  r7, [r1], #4)                   @ May fault
 498                 orr     r3, r3, r7, push #8
 499                 str     r3, [r0], #4
 500                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 501                 rsb     ip, ip, #0
 502                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 503                 beq     .Lcfu_3fupi
 504                 cmp     r2, ip
 505                 movlt   ip, r2
 506                 sub     r2, r2, ip
 507                 subs    ip, ip, #16
 508                 blt     .Lcfu_3rem8lp
 509
 510 .Lcfu_3cpy8lp:  mov     r3, r7, pull #24
 511                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 512                 orr     r3, r3, r4, push #8
 513                 mov     r4, r4, pull #24
 514                 orr     r4, r4, r5, push #8
 515                 mov     r5, r5, pull #24
 516                 orr     r5, r5, r6, push #8
 517                 mov     r6, r6, pull #24
 518                 orr     r6, r6, r7, push #8
 519                 stmia   r0!, {r3 - r6}
 520                 subs    ip, ip, #16
 521                 bpl     .Lcfu_3cpy8lp
 522
 523 .Lcfu_3rem8lp:  tst     ip, #8
 524                 movne   r3, r7, pull #24
 525                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 526                 orrne   r3, r3, r4, push #8
 527                 movne   r4, r4, pull #24
 528                 orrne   r4, r4, r7, push #8
 529                 stmneia r0!, {r3 - r4}
 530                 tst     ip, #4
 531                 movne   r3, r7, pull #24
 532 USER(           T(ldrne) r7, [r1], #4)                  @ May fault
 533                 orrne   r3, r3, r7, push #8
 534                 strne   r3, [r0], #4
 535                 ands    ip, ip, #3
 536                 beq     .Lcfu_3fupi
 537 .Lcfu_3nowords: mov     r3, r7, get_byte_3
 538                 teq     ip, #0
 539                 beq     .Lcfu_finished
 540                 cmp     ip, #2
 541                 strb    r3, [r0], #1
 542 USER(           T(ldrgeb) r3, [r1], #1)                 @ May fault
 543                 strgeb  r3, [r0], #1
 544 USER(           T(ldrgtb) r3, [r1], #1)                 @ May fault
 545                 strgtb  r3, [r0], #1
 546                 b       .Lcfu_finished
 547 ENDPROC(__copy_from_user)
 548
 549                 .pushsection .fixup,"ax"
 550                 .align  0
 551                 /*
 552                  * We took an exception.  r0 contains a pointer to
 553                  * the byte not copied.
 554                  */
 555 9001:           ldr     r2, [sp], #4                    @ void *to
 556                 sub     r2, r0, r2                      @ bytes copied
 557                 ldr     r1, [sp], #4                    @ unsigned long count
 558                 subs    r4, r1, r2                      @ bytes left to copy
 559                 movne   r1, r4
 560                 blne    __memzero
 561                 mov     r0, r4
 562                 ldmfd   sp!, {r4 - r7, pc}
 563                 .popsection
 564