arch/arm/lib/uaccess.S

   1 /*
   2  *  linux/arch/arm/lib/uaccess.S
   3  *
   4  *  Copyright (C) 1995, 1996,1997,1998 Russell King
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  *  Routines to block copy data to/from user memory
  11  *   These are highly optimised both for the 4k page size
  12  *   and for various alignments.
  13  */
  14 #include <linux/linkage.h>
  15 #include <asm/assembler.h>
  16 #include <asm/errno.h>
  17
  18                 .text
  19
  20 #define PAGE_SHIFT 12
  21
  22 /* Prototype: int __arch_copy_to_user(void *to, const char *from, size_t n)
  23  * Purpose  : copy a block to user memory from kernel memory
  24  * Params   : to   - user memory
  25  *          : from - kernel memory
  26  *          : n    - number of bytes to copy
  27  * Returns  : Number of bytes NOT copied.
  28  */
  29
  30 .c2u_dest_not_aligned:
  31                 rsb     ip, ip, #4
  32                 cmp     ip, #2
  33                 ldrb    r3, [r1], #1
  34 USER(           strbt   r3, [r0], #1)                   @ May fault
  35                 ldrgeb  r3, [r1], #1
  36 USER(           strgebt r3, [r0], #1)                   @ May fault
  37                 ldrgtb  r3, [r1], #1
  38 USER(           strgtbt r3, [r0], #1)                   @ May fault
  39                 sub     r2, r2, ip
  40                 b       .c2u_dest_aligned
  41
  42 ENTRY(__arch_copy_to_user)
  43                 stmfd   sp!, {r2, r4 - r7, lr}
  44                 cmp     r2, #4
  45                 blt     .c2u_not_enough
  46         PLD(    pld     [r1, #0]                )
  47         PLD(    pld     [r0, #0]                )
  48                 ands    ip, r0, #3
  49                 bne     .c2u_dest_not_aligned
  50 .c2u_dest_aligned:
  51
  52                 ands    ip, r1, #3
  53                 bne     .c2u_src_not_aligned
  54 /*
  55  * Seeing as there has to be at least 8 bytes to copy, we can
  56  * copy one word, and force a user-mode page fault...
  57  */
  58
  59 .c2u_0fupi:     subs    r2, r2, #4
  60                 addmi   ip, r2, #4
  61                 bmi     .c2u_0nowords
  62                 ldr     r3, [r1], #4
  63 USER(           strt    r3, [r0], #4)                   @ May fault
  64                 mov     ip, r0, lsl #32 - PAGE_SHIFT    @ On each page, use a ld/st??t instruction
  65                 rsb     ip, ip, #0
  66                 movs    ip, ip, lsr #32 - PAGE_SHIFT
  67                 beq     .c2u_0fupi
  68 /*
  69  * ip = max no. of bytes to copy before needing another "strt" insn
  70  */
  71                 cmp     r2, ip
  72                 movlt   ip, r2
  73                 sub     r2, r2, ip
  74                 subs    ip, ip, #32
  75                 blt     .c2u_0rem8lp
  76         PLD(    pld     [r1, #28]               )
  77         PLD(    pld     [r0, #28]               )
  78         PLD(    subs    ip, ip, #64                     )
  79         PLD(    blt     .c2u_0cpynopld          )
  80         PLD(    pld     [r1, #60]               )
  81         PLD(    pld     [r0, #60]               )
  82
  83 .c2u_0cpy8lp:
  84         PLD(    pld     [r1, #92]               )
  85         PLD(    pld     [r0, #92]               )
  86 .c2u_0cpynopld: ldmia   r1!, {r3 - r6}
  87                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
  88                 ldmia   r1!, {r3 - r6}
  89                 subs    ip, ip, #32
  90                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
  91                 bpl     .c2u_0cpy8lp
  92         PLD(    cmn     ip, #64                 )
  93         PLD(    bge     .c2u_0cpynopld          )
  94         PLD(    add     ip, ip, #64             )
  95
  96 .c2u_0rem8lp:   cmn     ip, #16
  97                 ldmgeia r1!, {r3 - r6}
  98                 stmgeia r0!, {r3 - r6}                  @ Shouldnt fault
  99                 tst     ip, #8
 100                 ldmneia r1!, {r3 - r4}
 101                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 102                 tst     ip, #4
 103                 ldrne   r3, [r1], #4
 104                 strnet  r3, [r0], #4                    @ Shouldnt fault
 105                 ands    ip, ip, #3
 106                 beq     .c2u_0fupi
 107 .c2u_0nowords:  teq     ip, #0
 108                 beq     .c2u_finished
 109 .c2u_nowords:   cmp     ip, #2
 110                 ldrb    r3, [r1], #1
 111 USER(           strbt   r3, [r0], #1)                   @ May fault
 112                 ldrgeb  r3, [r1], #1
 113 USER(           strgebt r3, [r0], #1)                   @ May fault
 114                 ldrgtb  r3, [r1], #1
 115 USER(           strgtbt r3, [r0], #1)                   @ May fault
 116                 b       .c2u_finished
 117
 118 .c2u_not_enough:
 119                 movs    ip, r2
 120                 bne     .c2u_nowords
 121 .c2u_finished:  mov     r0, #0
 122                 LOADREGS(fd,sp!,{r2, r4 - r7, pc})
 123
 124 .c2u_src_not_aligned:
 125                 bic     r1, r1, #3
 126                 ldr     r7, [r1], #4
 127                 cmp     ip, #2
 128                 bgt     .c2u_3fupi
 129                 beq     .c2u_2fupi
 130 .c2u_1fupi:     subs    r2, r2, #4
 131                 addmi   ip, r2, #4
 132                 bmi     .c2u_1nowords
 133                 mov     r3, r7, pull #8
 134                 ldr     r7, [r1], #4
 135                 orr     r3, r3, r7, push #24
 136 USER(           strt    r3, [r0], #4)                   @ May fault
 137                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 138                 rsb     ip, ip, #0
 139                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 140                 beq     .c2u_1fupi
 141                 cmp     r2, ip
 142                 movlt   ip, r2
 143                 sub     r2, r2, ip
 144                 subs    ip, ip, #16
 145                 blt     .c2u_1rem8lp
 146         PLD(    pld     [r1, #12]               )
 147         PLD(    pld     [r0, #12]               )
 148         PLD(    subs    ip, ip, #32             )
 149         PLD(    blt     .c2u_1cpynopld          )
 150         PLD(    pld     [r1, #28]               )
 151         PLD(    pld     [r0, #28]               )
 152
 153 .c2u_1cpy8lp:
 154         PLD(    pld     [r1, #44]               )
 155         PLD(    pld     [r0, #44]               )
 156 .c2u_1cpynopld: mov     r3, r7, pull #8
 157                 ldmia   r1!, {r4 - r7}
 158                 subs    ip, ip, #16
 159                 orr     r3, r3, r4, push #24
 160                 mov     r4, r4, pull #8
 161                 orr     r4, r4, r5, push #24
 162                 mov     r5, r5, pull #8
 163                 orr     r5, r5, r6, push #24
 164                 mov     r6, r6, pull #8
 165                 orr     r6, r6, r7, push #24
 166                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 167                 bpl     .c2u_1cpy8lp
 168         PLD(    cmn     ip, #32                 )
 169         PLD(    bge     .c2u_1cpynopld          )
 170         PLD(    add     ip, ip, #32             )
 171
 172 .c2u_1rem8lp:   tst     ip, #8
 173                 movne   r3, r7, pull #8
 174                 ldmneia r1!, {r4, r7}
 175                 orrne   r3, r3, r4, push #24
 176                 movne   r4, r4, pull #8
 177                 orrne   r4, r4, r7, push #24
 178                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 179                 tst     ip, #4
 180                 movne   r3, r7, pull #8
 181                 ldrne   r7, [r1], #4
 182                 orrne   r3, r3, r7, push #24
 183                 strnet  r3, [r0], #4                    @ Shouldnt fault
 184                 ands    ip, ip, #3
 185                 beq     .c2u_1fupi
 186 .c2u_1nowords:  mov     r3, r7, get_byte_1
 187                 teq     ip, #0
 188                 beq     .c2u_finished
 189                 cmp     ip, #2
 190 USER(           strbt   r3, [r0], #1)                   @ May fault
 191                 movge   r3, r7, get_byte_2
 192 USER(           strgebt r3, [r0], #1)                   @ May fault
 193                 movgt   r3, r7, get_byte_3
 194 USER(           strgtbt r3, [r0], #1)                   @ May fault
 195                 b       .c2u_finished
 196
 197 .c2u_2fupi:     subs    r2, r2, #4
 198                 addmi   ip, r2, #4
 199                 bmi     .c2u_2nowords
 200                 mov     r3, r7, pull #16
 201                 ldr     r7, [r1], #4
 202                 orr     r3, r3, r7, push #16
 203 USER(           strt    r3, [r0], #4)                   @ May fault
 204                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 205                 rsb     ip, ip, #0
 206                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 207                 beq     .c2u_2fupi
 208                 cmp     r2, ip
 209                 movlt   ip, r2
 210                 sub     r2, r2, ip
 211                 subs    ip, ip, #16
 212                 blt     .c2u_2rem8lp
 213         PLD(    pld     [r1, #12]               )
 214         PLD(    pld     [r0, #12]               )
 215         PLD(    subs    ip, ip, #32             )
 216         PLD(    blt     .c2u_2cpynopld          )
 217         PLD(    pld     [r1, #28]               )
 218         PLD(    pld     [r0, #28]               )
 219
 220 .c2u_2cpy8lp:
 221         PLD(    pld     [r1, #44]               )
 222         PLD(    pld     [r0, #44]               )
 223 .c2u_2cpynopld: mov     r3, r7, pull #16
 224                 ldmia   r1!, {r4 - r7}
 225                 subs    ip, ip, #16
 226                 orr     r3, r3, r4, push #16
 227                 mov     r4, r4, pull #16
 228                 orr     r4, r4, r5, push #16
 229                 mov     r5, r5, pull #16
 230                 orr     r5, r5, r6, push #16
 231                 mov     r6, r6, pull #16
 232                 orr     r6, r6, r7, push #16
 233                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 234                 bpl     .c2u_2cpy8lp
 235         PLD(    cmn     ip, #32                 )
 236         PLD(    bge     .c2u_2cpynopld          )
 237         PLD(    add     ip, ip, #32             )
 238
 239 .c2u_2rem8lp:   tst     ip, #8
 240                 movne   r3, r7, pull #16
 241                 ldmneia r1!, {r4, r7}
 242                 orrne   r3, r3, r4, push #16
 243                 movne   r4, r4, pull #16
 244                 orrne   r4, r4, r7, push #16
 245                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 246                 tst     ip, #4
 247                 movne   r3, r7, pull #16
 248                 ldrne   r7, [r1], #4
 249                 orrne   r3, r3, r7, push #16
 250                 strnet  r3, [r0], #4                    @ Shouldnt fault
 251                 ands    ip, ip, #3
 252                 beq     .c2u_2fupi
 253 .c2u_2nowords:  mov     r3, r7, get_byte_2
 254                 teq     ip, #0
 255                 beq     .c2u_finished
 256                 cmp     ip, #2
 257 USER(           strbt   r3, [r0], #1)                   @ May fault
 258                 movge   r3, r7, get_byte_3
 259 USER(           strgebt r3, [r0], #1)                   @ May fault
 260                 ldrgtb  r3, [r1], #0
 261 USER(           strgtbt r3, [r0], #1)                   @ May fault
 262                 b       .c2u_finished
 263
 264 .c2u_3fupi:     subs    r2, r2, #4
 265                 addmi   ip, r2, #4
 266                 bmi     .c2u_3nowords
 267                 mov     r3, r7, pull #24
 268                 ldr     r7, [r1], #4
 269                 orr     r3, r3, r7, push #8
 270 USER(           strt    r3, [r0], #4)                   @ May fault
 271                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 272                 rsb     ip, ip, #0
 273                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 274                 beq     .c2u_3fupi
 275                 cmp     r2, ip
 276                 movlt   ip, r2
 277                 sub     r2, r2, ip
 278                 subs    ip, ip, #16
 279                 blt     .c2u_3rem8lp
 280         PLD(    pld     [r1, #12]               )
 281         PLD(    pld     [r0, #12]               )
 282         PLD(    subs    ip, ip, #32             )
 283         PLD(    blt     .c2u_3cpynopld          )
 284         PLD(    pld     [r1, #28]               )
 285         PLD(    pld     [r0, #28]               )
 286
 287 .c2u_3cpy8lp:
 288         PLD(    pld     [r1, #44]               )
 289         PLD(    pld     [r0, #44]               )
 290 .c2u_3cpynopld: mov     r3, r7, pull #24
 291                 ldmia   r1!, {r4 - r7}
 292                 subs    ip, ip, #16
 293                 orr     r3, r3, r4, push #8
 294                 mov     r4, r4, pull #24
 295                 orr     r4, r4, r5, push #8
 296                 mov     r5, r5, pull #24
 297                 orr     r5, r5, r6, push #8
 298                 mov     r6, r6, pull #24
 299                 orr     r6, r6, r7, push #8
 300                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 301                 bpl     .c2u_3cpy8lp
 302         PLD(    cmn     ip, #32                 )
 303         PLD(    bge     .c2u_3cpynopld          )
 304         PLD(    add     ip, ip, #32             )
 305
 306 .c2u_3rem8lp:   tst     ip, #8
 307                 movne   r3, r7, pull #24
 308                 ldmneia r1!, {r4, r7}
 309                 orrne   r3, r3, r4, push #8
 310                 movne   r4, r4, pull #24
 311                 orrne   r4, r4, r7, push #8
 312                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 313                 tst     ip, #4
 314                 movne   r3, r7, pull #24
 315                 ldrne   r7, [r1], #4
 316                 orrne   r3, r3, r7, push #8
 317                 strnet  r3, [r0], #4                    @ Shouldnt fault
 318                 ands    ip, ip, #3
 319                 beq     .c2u_3fupi
 320 .c2u_3nowords:  mov     r3, r7, get_byte_3
 321                 teq     ip, #0
 322                 beq     .c2u_finished
 323                 cmp     ip, #2
 324 USER(           strbt   r3, [r0], #1)                   @ May fault
 325                 ldrgeb  r3, [r1], #1
 326 USER(           strgebt r3, [r0], #1)                   @ May fault
 327                 ldrgtb  r3, [r1], #0
 328 USER(           strgtbt r3, [r0], #1)                   @ May fault
 329                 b       .c2u_finished
 330
 331                 .section .fixup,"ax"
 332                 .align  0
 333 9001:           LOADREGS(fd,sp!, {r0, r4 - r7, pc})
 334                 .previous
 335
 336 /* Prototype: unsigned long __arch_copy_from_user(void *to,const void *from,unsigned long n);
 337  * Purpose  : copy a block from user memory to kernel memory
 338  * Params   : to   - kernel memory
 339  *          : from - user memory
 340  *          : n    - number of bytes to copy
 341  * Returns  : Number of bytes NOT copied.
 342  */
 343 .cfu_dest_not_aligned:
 344                 rsb     ip, ip, #4
 345                 cmp     ip, #2
 346 USER(           ldrbt   r3, [r1], #1)                   @ May fault
 347                 strb    r3, [r0], #1
 348 USER(           ldrgebt r3, [r1], #1)                   @ May fault
 349                 strgeb  r3, [r0], #1
 350 USER(           ldrgtbt r3, [r1], #1)                   @ May fault
 351                 strgtb  r3, [r0], #1
 352                 sub     r2, r2, ip
 353                 b       .cfu_dest_aligned
 354
 355 ENTRY(__arch_copy_from_user)
 356                 stmfd   sp!, {r0, r2, r4 - r7, lr}
 357                 cmp     r2, #4
 358                 blt     .cfu_not_enough
 359         PLD(    pld     [r1, #0]                )
 360         PLD(    pld     [r0, #0]                )
 361                 ands    ip, r0, #3
 362                 bne     .cfu_dest_not_aligned
 363 .cfu_dest_aligned:
 364                 ands    ip, r1, #3
 365                 bne     .cfu_src_not_aligned
 366 /*
 367  * Seeing as there has to be at least 8 bytes to copy, we can
 368  * copy one word, and force a user-mode page fault...
 369  */
 370
 371 .cfu_0fupi:     subs    r2, r2, #4
 372                 addmi   ip, r2, #4
 373                 bmi     .cfu_0nowords
 374 USER(           ldrt    r3, [r1], #4)
 375                 str     r3, [r0], #4
 376                 mov     ip, r1, lsl #32 - PAGE_SHIFT    @ On each page, use a ld/st??t instruction
 377                 rsb     ip, ip, #0
 378                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 379                 beq     .cfu_0fupi
 380 /*
 381  * ip = max no. of bytes to copy before needing another "strt" insn
 382  */
 383                 cmp     r2, ip
 384                 movlt   ip, r2
 385                 sub     r2, r2, ip
 386                 subs    ip, ip, #32
 387                 blt     .cfu_0rem8lp
 388         PLD(    pld     [r1, #28]               )
 389         PLD(    pld     [r0, #28]               )
 390         PLD(    subs    ip, ip, #64                     )
 391         PLD(    blt     .cfu_0cpynopld          )
 392         PLD(    pld     [r1, #60]               )
 393         PLD(    pld     [r0, #60]               )
 394
 395 .cfu_0cpy8lp:
 396         PLD(    pld     [r1, #92]               )
 397         PLD(    pld     [r0, #92]               )
 398 .cfu_0cpynopld: ldmia   r1!, {r3 - r6}                  @ Shouldnt fault
 399                 stmia   r0!, {r3 - r6}
 400                 ldmia   r1!, {r3 - r6}                  @ Shouldnt fault
 401                 subs    ip, ip, #32
 402                 stmia   r0!, {r3 - r6}
 403                 bpl     .cfu_0cpy8lp
 404         PLD(    cmn     ip, #64                 )
 405         PLD(    bge     .cfu_0cpynopld          )
 406         PLD(    add     ip, ip, #64             )
 407
 408 .cfu_0rem8lp:   cmn     ip, #16
 409                 ldmgeia r1!, {r3 - r6}                  @ Shouldnt fault
 410                 stmgeia r0!, {r3 - r6}
 411                 tst     ip, #8
 412                 ldmneia r1!, {r3 - r4}                  @ Shouldnt fault
 413                 stmneia r0!, {r3 - r4}
 414                 tst     ip, #4
 415                 ldrnet  r3, [r1], #4                    @ Shouldnt fault
 416                 strne   r3, [r0], #4
 417                 ands    ip, ip, #3
 418                 beq     .cfu_0fupi
 419 .cfu_0nowords:  teq     ip, #0
 420                 beq     .cfu_finished
 421 .cfu_nowords:   cmp     ip, #2
 422 USER(           ldrbt   r3, [r1], #1)                   @ May fault
 423                 strb    r3, [r0], #1
 424 USER(           ldrgebt r3, [r1], #1)                   @ May fault
 425                 strgeb  r3, [r0], #1
 426 USER(           ldrgtbt r3, [r1], #1)                   @ May fault
 427                 strgtb  r3, [r0], #1
 428                 b       .cfu_finished
 429
 430 .cfu_not_enough:
 431                 movs    ip, r2
 432                 bne     .cfu_nowords
 433 .cfu_finished:  mov     r0, #0
 434                 add     sp, sp, #8
 435                 LOADREGS(fd,sp!,{r4 - r7, pc})
 436
 437 .cfu_src_not_aligned:
 438                 bic     r1, r1, #3
 439 USER(           ldrt    r7, [r1], #4)                   @ May fault
 440                 cmp     ip, #2
 441                 bgt     .cfu_3fupi
 442                 beq     .cfu_2fupi
 443 .cfu_1fupi:     subs    r2, r2, #4
 444                 addmi   ip, r2, #4
 445                 bmi     .cfu_1nowords
 446                 mov     r3, r7, pull #8
 447 USER(           ldrt    r7, [r1], #4)                   @ May fault
 448                 orr     r3, r3, r7, push #24
 449                 str     r3, [r0], #4
 450                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 451                 rsb     ip, ip, #0
 452                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 453                 beq     .cfu_1fupi
 454                 cmp     r2, ip
 455                 movlt   ip, r2
 456                 sub     r2, r2, ip
 457                 subs    ip, ip, #16
 458                 blt     .cfu_1rem8lp
 459         PLD(    pld     [r1, #12]               )
 460         PLD(    pld     [r0, #12]               )
 461         PLD(    subs    ip, ip, #32             )
 462         PLD(    blt     .cfu_1cpynopld          )
 463         PLD(    pld     [r1, #28]               )
 464         PLD(    pld     [r0, #28]               )
 465
 466 .cfu_1cpy8lp:
 467         PLD(    pld     [r1, #44]               )
 468         PLD(    pld     [r0, #44]               )
 469 .cfu_1cpynopld: mov     r3, r7, pull #8
 470                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 471                 subs    ip, ip, #16
 472                 orr     r3, r3, r4, push #24
 473                 mov     r4, r4, pull #8
 474                 orr     r4, r4, r5, push #24
 475                 mov     r5, r5, pull #8
 476                 orr     r5, r5, r6, push #24
 477                 mov     r6, r6, pull #8
 478                 orr     r6, r6, r7, push #24
 479                 stmia   r0!, {r3 - r6}
 480                 bpl     .cfu_1cpy8lp
 481         PLD(    cmn     ip, #32                 )
 482         PLD(    bge     .cfu_1cpynopld          )
 483         PLD(    add     ip, ip, #32             )
 484
 485 .cfu_1rem8lp:   tst     ip, #8
 486                 movne   r3, r7, pull #8
 487                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 488                 orrne   r3, r3, r4, push #24
 489                 movne   r4, r4, pull #8
 490                 orrne   r4, r4, r7, push #24
 491                 stmneia r0!, {r3 - r4}
 492                 tst     ip, #4
 493                 movne   r3, r7, pull #8
 494 USER(           ldrnet  r7, [r1], #4)                   @ May fault
 495                 orrne   r3, r3, r7, push #24
 496                 strne   r3, [r0], #4
 497                 ands    ip, ip, #3
 498                 beq     .cfu_1fupi
 499 .cfu_1nowords:  mov     r3, r7, get_byte_1
 500                 teq     ip, #0
 501                 beq     .cfu_finished
 502                 cmp     ip, #2
 503                 strb    r3, [r0], #1
 504                 movge   r3, r7, get_byte_2
 505                 strgeb  r3, [r0], #1
 506                 movgt   r3, r7, get_byte_3
 507                 strgtb  r3, [r0], #1
 508                 b       .cfu_finished
 509
 510 .cfu_2fupi:     subs    r2, r2, #4
 511                 addmi   ip, r2, #4
 512                 bmi     .cfu_2nowords
 513                 mov     r3, r7, pull #16
 514 USER(           ldrt    r7, [r1], #4)                   @ May fault
 515                 orr     r3, r3, r7, push #16
 516                 str     r3, [r0], #4
 517                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 518                 rsb     ip, ip, #0
 519                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 520                 beq     .cfu_2fupi
 521                 cmp     r2, ip
 522                 movlt   ip, r2
 523                 sub     r2, r2, ip
 524                 subs    ip, ip, #16
 525                 blt     .cfu_2rem8lp
 526         PLD(    pld     [r1, #12]               )
 527         PLD(    pld     [r0, #12]               )
 528         PLD(    subs    ip, ip, #32             )
 529         PLD(    blt     .cfu_2cpynopld          )
 530         PLD(    pld     [r1, #28]               )
 531         PLD(    pld     [r0, #28]               )
 532
 533 .cfu_2cpy8lp:
 534         PLD(    pld     [r1, #44]               )
 535         PLD(    pld     [r0, #44]               )
 536 .cfu_2cpynopld: mov     r3, r7, pull #16
 537                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 538                 subs    ip, ip, #16
 539                 orr     r3, r3, r4, push #16
 540                 mov     r4, r4, pull #16
 541                 orr     r4, r4, r5, push #16
 542                 mov     r5, r5, pull #16
 543                 orr     r5, r5, r6, push #16
 544                 mov     r6, r6, pull #16
 545                 orr     r6, r6, r7, push #16
 546                 stmia   r0!, {r3 - r6}
 547                 bpl     .cfu_2cpy8lp
 548         PLD(    cmn     ip, #32                 )
 549         PLD(    bge     .cfu_2cpynopld          )
 550         PLD(    add     ip, ip, #32             )
 551
 552 .cfu_2rem8lp:   tst     ip, #8
 553                 movne   r3, r7, pull #16
 554                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 555                 orrne   r3, r3, r4, push #16
 556                 movne   r4, r4, pull #16
 557                 orrne   r4, r4, r7, push #16
 558                 stmneia r0!, {r3 - r4}
 559                 tst     ip, #4
 560                 movne   r3, r7, pull #16
 561 USER(           ldrnet  r7, [r1], #4)                   @ May fault
 562                 orrne   r3, r3, r7, push #16
 563                 strne   r3, [r0], #4
 564                 ands    ip, ip, #3
 565                 beq     .cfu_2fupi
 566 .cfu_2nowords:  mov     r3, r7, get_byte_2
 567                 teq     ip, #0
 568                 beq     .cfu_finished
 569                 cmp     ip, #2
 570                 strb    r3, [r0], #1
 571                 movge   r3, r7, get_byte_3
 572                 strgeb  r3, [r0], #1
 573 USER(           ldrgtbt r3, [r1], #0)                   @ May fault
 574                 strgtb  r3, [r0], #1
 575                 b       .cfu_finished
 576
 577 .cfu_3fupi:     subs    r2, r2, #4
 578                 addmi   ip, r2, #4
 579                 bmi     .cfu_3nowords
 580                 mov     r3, r7, pull #24
 581 USER(           ldrt    r7, [r1], #4)                   @ May fault
 582                 orr     r3, r3, r7, push #8
 583                 str     r3, [r0], #4
 584                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 585                 rsb     ip, ip, #0
 586                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 587                 beq     .cfu_3fupi
 588                 cmp     r2, ip
 589                 movlt   ip, r2
 590                 sub     r2, r2, ip
 591                 subs    ip, ip, #16
 592                 blt     .cfu_3rem8lp
 593         PLD(    pld     [r1, #12]               )
 594         PLD(    pld     [r0, #12]               )
 595         PLD(    subs    ip, ip, #32             )
 596         PLD(    blt     .cfu_3cpynopld          )
 597         PLD(    pld     [r1, #28]               )
 598         PLD(    pld     [r0, #28]               )
 599
 600 .cfu_3cpy8lp:
 601         PLD(    pld     [r1, #44]               )
 602         PLD(    pld     [r0, #44]               )
 603 .cfu_3cpynopld: mov     r3, r7, pull #24
 604                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 605                 orr     r3, r3, r4, push #8
 606                 mov     r4, r4, pull #24
 607                 orr     r4, r4, r5, push #8
 608                 mov     r5, r5, pull #24
 609                 orr     r5, r5, r6, push #8
 610                 mov     r6, r6, pull #24
 611                 orr     r6, r6, r7, push #8
 612                 stmia   r0!, {r3 - r6}
 613                 subs    ip, ip, #16
 614                 bpl     .cfu_3cpy8lp
 615         PLD(    cmn     ip, #32                 )
 616         PLD(    bge     .cfu_3cpynopld          )
 617         PLD(    add     ip, ip, #32             )
 618
 619 .cfu_3rem8lp:   tst     ip, #8
 620                 movne   r3, r7, pull #24
 621                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 622                 orrne   r3, r3, r4, push #8
 623                 movne   r4, r4, pull #24
 624                 orrne   r4, r4, r7, push #8
 625                 stmneia r0!, {r3 - r4}
 626                 tst     ip, #4
 627                 movne   r3, r7, pull #24
 628 USER(           ldrnet  r7, [r1], #4)                   @ May fault
 629                 orrne   r3, r3, r7, push #8
 630                 strne   r3, [r0], #4
 631                 ands    ip, ip, #3
 632                 beq     .cfu_3fupi
 633 .cfu_3nowords:  mov     r3, r7, get_byte_3
 634                 teq     ip, #0
 635                 beq     .cfu_finished
 636                 cmp     ip, #2
 637                 strb    r3, [r0], #1
 638 USER(           ldrgebt r3, [r1], #1)                   @ May fault
 639                 strgeb  r3, [r0], #1
 640 USER(           ldrgtbt r3, [r1], #1)                   @ May fault
 641                 strgtb  r3, [r0], #1
 642                 b       .cfu_finished
 643
 644                 .section .fixup,"ax"
 645                 .align  0
 646                 /*
 647                  * We took an exception.  r0 contains a pointer to
 648                  * the byte not copied.
 649                  */
 650 9001:           ldr     r2, [sp], #4                    @ void *to
 651                 sub     r2, r0, r2                      @ bytes copied
 652                 ldr     r1, [sp], #4                    @ unsigned long count
 653                 subs    r4, r1, r2                      @ bytes left to copy
 654                 movne   r1, r4
 655                 blne    __memzero
 656                 mov     r0, r4
 657                 LOADREGS(fd,sp!, {r4 - r7, pc})
 658                 .previous
 659
 660 /* Prototype: int __arch_clear_user(void *addr, size_t sz)
 661  * Purpose  : clear some user memory
 662  * Params   : addr - user memory address to clear
 663  *          : sz   - number of bytes to clear
 664  * Returns  : number of bytes NOT cleared
 665  */
 666 ENTRY(__arch_clear_user)
 667                 stmfd   sp!, {r1, lr}
 668                 mov     r2, #0
 669                 cmp     r1, #4
 670                 blt     2f
 671                 ands    ip, r0, #3
 672                 beq     1f
 673                 cmp     ip, #2
 674 USER(           strbt   r2, [r0], #1)
 675 USER(           strlebt r2, [r0], #1)
 676 USER(           strltbt r2, [r0], #1)
 677                 rsb     ip, ip, #4
 678                 sub     r1, r1, ip              @  7  6  5  4  3  2  1
 679 1:              subs    r1, r1, #8              @ -1 -2 -3 -4 -5 -6 -7
 680 USER(           strplt  r2, [r0], #4)
 681 USER(           strplt  r2, [r0], #4)
 682                 bpl     1b
 683                 adds    r1, r1, #4              @  3  2  1  0 -1 -2 -3
 684 USER(           strplt  r2, [r0], #4)
 685 2:              tst     r1, #2                  @ 1x 1x 0x 0x 1x 1x 0x
 686 USER(           strnebt r2, [r0], #1)
 687 USER(           strnebt r2, [r0], #1)
 688                 tst     r1, #1                  @ x1 x0 x1 x0 x1 x0 x1
 689 USER(           strnebt r2, [r0], #1)
 690                 mov     r0, #0
 691                 LOADREGS(fd,sp!, {r1, pc})
 692
 693                 .section .fixup,"ax"
 694                 .align  0
 695 9001:           LOADREGS(fd,sp!, {r0, pc})
 696                 .previous
 697