arch/x86/lib/usercopy_32.c

   1 /*
   2  * User address space access functions.
   3  * The non inlined parts of asm-i386/uaccess.h are here.
   4  *
   5  * Copyright 1997 Andi Kleen <ak@muc.de>
   6  * Copyright 1997 Linus Torvalds
   7  */
   8 #include <linux/mm.h>
   9 #include <linux/highmem.h>
  10 #include <linux/blkdev.h>
  11 #include <linux/module.h>
  12 #include <linux/backing-dev.h>
  13 #include <linux/interrupt.h>
  14 #include <asm/uaccess.h>
  15 #include <asm/mmx.h>
  16
  17 #ifdef CONFIG_X86_INTEL_USERCOPY
  18 /*
  19  * Alignment at which movsl is preferred for bulk memory copies.
  20  */
  21 struct movsl_mask movsl_mask __read_mostly;
  22 #endif
  23
  24 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
  25 {
  26 #ifdef CONFIG_X86_INTEL_USERCOPY
  27         if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
  28                 return 0;
  29 #endif
  30         return 1;
  31 }
  32 #define movsl_is_ok(a1, a2, n) \
  33         __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
  34
  35 /*
  36  * Zero Userspace
  37  */
  38
  39 #define __do_clear_user(addr,size)                                      \
  40 do {                                                                    \
  41         int __d0;                                                       \
  42         might_fault();                                                  \
  43         __asm__ __volatile__(                                           \
  44                 "0:     rep; stosl\n"                                   \
  45                 "       movl %2,%0\n"                                   \
  46                 "1:     rep; stosb\n"                                   \
  47                 "2:\n"                                                  \
  48                 ".section .fixup,\"ax\"\n"                              \
  49                 "3:     lea 0(%2,%0,4),%0\n"                            \
  50                 "       jmp 2b\n"                                       \
  51                 ".previous\n"                                           \
  52                 _ASM_EXTABLE(0b,3b)                                     \
  53                 _ASM_EXTABLE(1b,2b)                                     \
  54                 : "=&c"(size), "=&D" (__d0)                             \
  55                 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));     \
  56 } while (0)
  57
  58 /**
  59  * clear_user: - Zero a block of memory in user space.
  60  * @to:   Destination address, in user space.
  61  * @n:    Number of bytes to zero.
  62  *
  63  * Zero a block of memory in user space.
  64  *
  65  * Returns number of bytes that could not be cleared.
  66  * On success, this will be zero.
  67  */
  68 unsigned long
  69 clear_user(void __user *to, unsigned long n)
  70 {
  71         might_fault();
  72         if (access_ok(VERIFY_WRITE, to, n))
  73                 __do_clear_user(to, n);
  74         return n;
  75 }
  76 EXPORT_SYMBOL(clear_user);
  77
  78 /**
  79  * __clear_user: - Zero a block of memory in user space, with less checking.
  80  * @to:   Destination address, in user space.
  81  * @n:    Number of bytes to zero.
  82  *
  83  * Zero a block of memory in user space.  Caller must check
  84  * the specified block with access_ok() before calling this function.
  85  *
  86  * Returns number of bytes that could not be cleared.
  87  * On success, this will be zero.
  88  */
  89 unsigned long
  90 __clear_user(void __user *to, unsigned long n)
  91 {
  92         __do_clear_user(to, n);
  93         return n;
  94 }
  95 EXPORT_SYMBOL(__clear_user);
  96
  97 /**
  98  * strnlen_user: - Get the size of a string in user space.
  99  * @s: The string to measure.
 100  * @n: The maximum valid length
 101  *
 102  * Get the size of a NUL-terminated string in user space.
 103  *
 104  * Returns the size of the string INCLUDING the terminating NUL.
 105  * On exception, returns 0.
 106  * If the string is too long, returns a value greater than @n.
 107  */
 108 long strnlen_user(const char __user *s, long n)
 109 {
 110         unsigned long mask = -__addr_ok(s);
 111         unsigned long res, tmp;
 112
 113         might_fault();
 114
 115         __asm__ __volatile__(
 116                 "       testl %0, %0\n"
 117                 "       jz 3f\n"
 118                 "       andl %0,%%ecx\n"
 119                 "0:     repne; scasb\n"
 120                 "       setne %%al\n"
 121                 "       subl %%ecx,%0\n"
 122                 "       addl %0,%%eax\n"
 123                 "1:\n"
 124                 ".section .fixup,\"ax\"\n"
 125                 "2:     xorl %%eax,%%eax\n"
 126                 "       jmp 1b\n"
 127                 "3:     movb $1,%%al\n"
 128                 "       jmp 1b\n"
 129                 ".previous\n"
 130                 ".section __ex_table,\"a\"\n"
 131                 "       .align 4\n"
 132                 "       .long 0b,2b\n"
 133                 ".previous"
 134                 :"=&r" (n), "=&D" (s), "=&a" (res), "=&c" (tmp)
 135                 :"0" (n), "1" (s), "2" (0), "3" (mask)
 136                 :"cc");
 137         return res & mask;
 138 }
 139 EXPORT_SYMBOL(strnlen_user);
 140
 141 #ifdef CONFIG_X86_INTEL_USERCOPY
 142 static unsigned long
 143 __copy_user_intel(void __user *to, const void *from, unsigned long size)
 144 {
 145         int d0, d1;
 146         __asm__ __volatile__(
 147                        "       .align 2,0x90\n"
 148                        "1:     movl 32(%4), %%eax\n"
 149                        "       cmpl $67, %0\n"
 150                        "       jbe 3f\n"
 151                        "2:     movl 64(%4), %%eax\n"
 152                        "       .align 2,0x90\n"
 153                        "3:     movl 0(%4), %%eax\n"
 154                        "4:     movl 4(%4), %%edx\n"
 155                        "5:     movl %%eax, 0(%3)\n"
 156                        "6:     movl %%edx, 4(%3)\n"
 157                        "7:     movl 8(%4), %%eax\n"
 158                        "8:     movl 12(%4),%%edx\n"
 159                        "9:     movl %%eax, 8(%3)\n"
 160                        "10:    movl %%edx, 12(%3)\n"
 161                        "11:    movl 16(%4), %%eax\n"
 162                        "12:    movl 20(%4), %%edx\n"
 163                        "13:    movl %%eax, 16(%3)\n"
 164                        "14:    movl %%edx, 20(%3)\n"
 165                        "15:    movl 24(%4), %%eax\n"
 166                        "16:    movl 28(%4), %%edx\n"
 167                        "17:    movl %%eax, 24(%3)\n"
 168                        "18:    movl %%edx, 28(%3)\n"
 169                        "19:    movl 32(%4), %%eax\n"
 170                        "20:    movl 36(%4), %%edx\n"
 171                        "21:    movl %%eax, 32(%3)\n"
 172                        "22:    movl %%edx, 36(%3)\n"
 173                        "23:    movl 40(%4), %%eax\n"
 174                        "24:    movl 44(%4), %%edx\n"
 175                        "25:    movl %%eax, 40(%3)\n"
 176                        "26:    movl %%edx, 44(%3)\n"
 177                        "27:    movl 48(%4), %%eax\n"
 178                        "28:    movl 52(%4), %%edx\n"
 179                        "29:    movl %%eax, 48(%3)\n"
 180                        "30:    movl %%edx, 52(%3)\n"
 181                        "31:    movl 56(%4), %%eax\n"
 182                        "32:    movl 60(%4), %%edx\n"
 183                        "33:    movl %%eax, 56(%3)\n"
 184                        "34:    movl %%edx, 60(%3)\n"
 185                        "       addl $-64, %0\n"
 186                        "       addl $64, %4\n"
 187                        "       addl $64, %3\n"
 188                        "       cmpl $63, %0\n"
 189                        "       ja  1b\n"
 190                        "35:    movl  %0, %%eax\n"
 191                        "       shrl  $2, %0\n"
 192                        "       andl  $3, %%eax\n"
 193                        "       cld\n"
 194                        "99:    rep; movsl\n"
 195                        "36:    movl %%eax, %0\n"
 196                        "37:    rep; movsb\n"
 197                        "100:\n"
 198                        ".section .fixup,\"ax\"\n"
 199                        "101:   lea 0(%%eax,%0,4),%0\n"
 200                        "       jmp 100b\n"
 201                        ".previous\n"
 202                        ".section __ex_table,\"a\"\n"
 203                        "       .align 4\n"
 204                        "       .long 1b,100b\n"
 205                        "       .long 2b,100b\n"
 206                        "       .long 3b,100b\n"
 207                        "       .long 4b,100b\n"
 208                        "       .long 5b,100b\n"
 209                        "       .long 6b,100b\n"
 210                        "       .long 7b,100b\n"
 211                        "       .long 8b,100b\n"
 212                        "       .long 9b,100b\n"
 213                        "       .long 10b,100b\n"
 214                        "       .long 11b,100b\n"
 215                        "       .long 12b,100b\n"
 216                        "       .long 13b,100b\n"
 217                        "       .long 14b,100b\n"
 218                        "       .long 15b,100b\n"
 219                        "       .long 16b,100b\n"
 220                        "       .long 17b,100b\n"
 221                        "       .long 18b,100b\n"
 222                        "       .long 19b,100b\n"
 223                        "       .long 20b,100b\n"
 224                        "       .long 21b,100b\n"
 225                        "       .long 22b,100b\n"
 226                        "       .long 23b,100b\n"
 227                        "       .long 24b,100b\n"
 228                        "       .long 25b,100b\n"
 229                        "       .long 26b,100b\n"
 230                        "       .long 27b,100b\n"
 231                        "       .long 28b,100b\n"
 232                        "       .long 29b,100b\n"
 233                        "       .long 30b,100b\n"
 234                        "       .long 31b,100b\n"
 235                        "       .long 32b,100b\n"
 236                        "       .long 33b,100b\n"
 237                        "       .long 34b,100b\n"
 238                        "       .long 35b,100b\n"
 239                        "       .long 36b,100b\n"
 240                        "       .long 37b,100b\n"
 241                        "       .long 99b,101b\n"
 242                        ".previous"
 243                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 244                        :  "1"(to), "2"(from), "0"(size)
 245                        : "eax", "edx", "memory");
 246         return size;
 247 }
 248
 249 static unsigned long
 250 __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size)
 251 {
 252         int d0, d1;
 253         __asm__ __volatile__(
 254                        "        .align 2,0x90\n"
 255                        "0:      movl 32(%4), %%eax\n"
 256                        "        cmpl $67, %0\n"
 257                        "        jbe 2f\n"
 258                        "1:      movl 64(%4), %%eax\n"
 259                        "        .align 2,0x90\n"
 260                        "2:      movl 0(%4), %%eax\n"
 261                        "21:     movl 4(%4), %%edx\n"
 262                        "        movl %%eax, 0(%3)\n"
 263                        "        movl %%edx, 4(%3)\n"
 264                        "3:      movl 8(%4), %%eax\n"
 265                        "31:     movl 12(%4),%%edx\n"
 266                        "        movl %%eax, 8(%3)\n"
 267                        "        movl %%edx, 12(%3)\n"
 268                        "4:      movl 16(%4), %%eax\n"
 269                        "41:     movl 20(%4), %%edx\n"
 270                        "        movl %%eax, 16(%3)\n"
 271                        "        movl %%edx, 20(%3)\n"
 272                        "10:     movl 24(%4), %%eax\n"
 273                        "51:     movl 28(%4), %%edx\n"
 274                        "        movl %%eax, 24(%3)\n"
 275                        "        movl %%edx, 28(%3)\n"
 276                        "11:     movl 32(%4), %%eax\n"
 277                        "61:     movl 36(%4), %%edx\n"
 278                        "        movl %%eax, 32(%3)\n"
 279                        "        movl %%edx, 36(%3)\n"
 280                        "12:     movl 40(%4), %%eax\n"
 281                        "71:     movl 44(%4), %%edx\n"
 282                        "        movl %%eax, 40(%3)\n"
 283                        "        movl %%edx, 44(%3)\n"
 284                        "13:     movl 48(%4), %%eax\n"
 285                        "81:     movl 52(%4), %%edx\n"
 286                        "        movl %%eax, 48(%3)\n"
 287                        "        movl %%edx, 52(%3)\n"
 288                        "14:     movl 56(%4), %%eax\n"
 289                        "91:     movl 60(%4), %%edx\n"
 290                        "        movl %%eax, 56(%3)\n"
 291                        "        movl %%edx, 60(%3)\n"
 292                        "        addl $-64, %0\n"
 293                        "        addl $64, %4\n"
 294                        "        addl $64, %3\n"
 295                        "        cmpl $63, %0\n"
 296                        "        ja  0b\n"
 297                        "5:      movl  %0, %%eax\n"
 298                        "        shrl  $2, %0\n"
 299                        "        andl $3, %%eax\n"
 300                        "        cld\n"
 301                        "6:      rep; movsl\n"
 302                        "        movl %%eax,%0\n"
 303                        "7:      rep; movsb\n"
 304                        "8:\n"
 305                        ".section .fixup,\"ax\"\n"
 306                        "9:      lea 0(%%eax,%0,4),%0\n"
 307                        "16:     pushl %0\n"
 308                        "        pushl %%eax\n"
 309                        "        xorl %%eax,%%eax\n"
 310                        "        rep; stosb\n"
 311                        "        popl %%eax\n"
 312                        "        popl %0\n"
 313                        "        jmp 8b\n"
 314                        ".previous\n"
 315                        ".section __ex_table,\"a\"\n"
 316                        "        .align 4\n"
 317                        "        .long 0b,16b\n"
 318                        "        .long 1b,16b\n"
 319                        "        .long 2b,16b\n"
 320                        "        .long 21b,16b\n"
 321                        "        .long 3b,16b\n"
 322                        "        .long 31b,16b\n"
 323                        "        .long 4b,16b\n"
 324                        "        .long 41b,16b\n"
 325                        "        .long 10b,16b\n"
 326                        "        .long 51b,16b\n"
 327                        "        .long 11b,16b\n"
 328                        "        .long 61b,16b\n"
 329                        "        .long 12b,16b\n"
 330                        "        .long 71b,16b\n"
 331                        "        .long 13b,16b\n"
 332                        "        .long 81b,16b\n"
 333                        "        .long 14b,16b\n"
 334                        "        .long 91b,16b\n"
 335                        "        .long 6b,9b\n"
 336                        "        .long 7b,16b\n"
 337                        ".previous"
 338                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 339                        :  "1"(to), "2"(from), "0"(size)
 340                        : "eax", "edx", "memory");
 341         return size;
 342 }
 343
 344 /*
 345  * Non Temporal Hint version of __copy_user_zeroing_intel.  It is cache aware.
 346  * hyoshiok@miraclelinux.com
 347  */
 348
 349 static unsigned long __copy_user_zeroing_intel_nocache(void *to,
 350                                 const void __user *from, unsigned long size)
 351 {
 352         int d0, d1;
 353
 354         __asm__ __volatile__(
 355                "        .align 2,0x90\n"
 356                "0:      movl 32(%4), %%eax\n"
 357                "        cmpl $67, %0\n"
 358                "        jbe 2f\n"
 359                "1:      movl 64(%4), %%eax\n"
 360                "        .align 2,0x90\n"
 361                "2:      movl 0(%4), %%eax\n"
 362                "21:     movl 4(%4), %%edx\n"
 363                "        movnti %%eax, 0(%3)\n"
 364                "        movnti %%edx, 4(%3)\n"
 365                "3:      movl 8(%4), %%eax\n"
 366                "31:     movl 12(%4),%%edx\n"
 367                "        movnti %%eax, 8(%3)\n"
 368                "        movnti %%edx, 12(%3)\n"
 369                "4:      movl 16(%4), %%eax\n"
 370                "41:     movl 20(%4), %%edx\n"
 371                "        movnti %%eax, 16(%3)\n"
 372                "        movnti %%edx, 20(%3)\n"
 373                "10:     movl 24(%4), %%eax\n"
 374                "51:     movl 28(%4), %%edx\n"
 375                "        movnti %%eax, 24(%3)\n"
 376                "        movnti %%edx, 28(%3)\n"
 377                "11:     movl 32(%4), %%eax\n"
 378                "61:     movl 36(%4), %%edx\n"
 379                "        movnti %%eax, 32(%3)\n"
 380                "        movnti %%edx, 36(%3)\n"
 381                "12:     movl 40(%4), %%eax\n"
 382                "71:     movl 44(%4), %%edx\n"
 383                "        movnti %%eax, 40(%3)\n"
 384                "        movnti %%edx, 44(%3)\n"
 385                "13:     movl 48(%4), %%eax\n"
 386                "81:     movl 52(%4), %%edx\n"
 387                "        movnti %%eax, 48(%3)\n"
 388                "        movnti %%edx, 52(%3)\n"
 389                "14:     movl 56(%4), %%eax\n"
 390                "91:     movl 60(%4), %%edx\n"
 391                "        movnti %%eax, 56(%3)\n"
 392                "        movnti %%edx, 60(%3)\n"
 393                "        addl $-64, %0\n"
 394                "        addl $64, %4\n"
 395                "        addl $64, %3\n"
 396                "        cmpl $63, %0\n"
 397                "        ja  0b\n"
 398                "        sfence \n"
 399                "5:      movl  %0, %%eax\n"
 400                "        shrl  $2, %0\n"
 401                "        andl $3, %%eax\n"
 402                "        cld\n"
 403                "6:      rep; movsl\n"
 404                "        movl %%eax,%0\n"
 405                "7:      rep; movsb\n"
 406                "8:\n"
 407                ".section .fixup,\"ax\"\n"
 408                "9:      lea 0(%%eax,%0,4),%0\n"
 409                "16:     pushl %0\n"
 410                "        pushl %%eax\n"
 411                "        xorl %%eax,%%eax\n"
 412                "        rep; stosb\n"
 413                "        popl %%eax\n"
 414                "        popl %0\n"
 415                "        jmp 8b\n"
 416                ".previous\n"
 417                ".section __ex_table,\"a\"\n"
 418                "        .align 4\n"
 419                "        .long 0b,16b\n"
 420                "        .long 1b,16b\n"
 421                "        .long 2b,16b\n"
 422                "        .long 21b,16b\n"
 423                "        .long 3b,16b\n"
 424                "        .long 31b,16b\n"
 425                "        .long 4b,16b\n"
 426                "        .long 41b,16b\n"
 427                "        .long 10b,16b\n"
 428                "        .long 51b,16b\n"
 429                "        .long 11b,16b\n"
 430                "        .long 61b,16b\n"
 431                "        .long 12b,16b\n"
 432                "        .long 71b,16b\n"
 433                "        .long 13b,16b\n"
 434                "        .long 81b,16b\n"
 435                "        .long 14b,16b\n"
 436                "        .long 91b,16b\n"
 437                "        .long 6b,9b\n"
 438                "        .long 7b,16b\n"
 439                ".previous"
 440                : "=&c"(size), "=&D" (d0), "=&S" (d1)
 441                :  "1"(to), "2"(from), "0"(size)
 442                : "eax", "edx", "memory");
 443         return size;
 444 }
 445
 446 static unsigned long __copy_user_intel_nocache(void *to,
 447                                 const void __user *from, unsigned long size)
 448 {
 449         int d0, d1;
 450
 451         __asm__ __volatile__(
 452                "        .align 2,0x90\n"
 453                "0:      movl 32(%4), %%eax\n"
 454                "        cmpl $67, %0\n"
 455                "        jbe 2f\n"
 456                "1:      movl 64(%4), %%eax\n"
 457                "        .align 2,0x90\n"
 458                "2:      movl 0(%4), %%eax\n"
 459                "21:     movl 4(%4), %%edx\n"
 460                "        movnti %%eax, 0(%3)\n"
 461                "        movnti %%edx, 4(%3)\n"
 462                "3:      movl 8(%4), %%eax\n"
 463                "31:     movl 12(%4),%%edx\n"
 464                "        movnti %%eax, 8(%3)\n"
 465                "        movnti %%edx, 12(%3)\n"
 466                "4:      movl 16(%4), %%eax\n"
 467                "41:     movl 20(%4), %%edx\n"
 468                "        movnti %%eax, 16(%3)\n"
 469                "        movnti %%edx, 20(%3)\n"
 470                "10:     movl 24(%4), %%eax\n"
 471                "51:     movl 28(%4), %%edx\n"
 472                "        movnti %%eax, 24(%3)\n"
 473                "        movnti %%edx, 28(%3)\n"
 474                "11:     movl 32(%4), %%eax\n"
 475                "61:     movl 36(%4), %%edx\n"
 476                "        movnti %%eax, 32(%3)\n"
 477                "        movnti %%edx, 36(%3)\n"
 478                "12:     movl 40(%4), %%eax\n"
 479                "71:     movl 44(%4), %%edx\n"
 480                "        movnti %%eax, 40(%3)\n"
 481                "        movnti %%edx, 44(%3)\n"
 482                "13:     movl 48(%4), %%eax\n"
 483                "81:     movl 52(%4), %%edx\n"
 484                "        movnti %%eax, 48(%3)\n"
 485                "        movnti %%edx, 52(%3)\n"
 486                "14:     movl 56(%4), %%eax\n"
 487                "91:     movl 60(%4), %%edx\n"
 488                "        movnti %%eax, 56(%3)\n"
 489                "        movnti %%edx, 60(%3)\n"
 490                "        addl $-64, %0\n"
 491                "        addl $64, %4\n"
 492                "        addl $64, %3\n"
 493                "        cmpl $63, %0\n"
 494                "        ja  0b\n"
 495                "        sfence \n"
 496                "5:      movl  %0, %%eax\n"
 497                "        shrl  $2, %0\n"
 498                "        andl $3, %%eax\n"
 499                "        cld\n"
 500                "6:      rep; movsl\n"
 501                "        movl %%eax,%0\n"
 502                "7:      rep; movsb\n"
 503                "8:\n"
 504                ".section .fixup,\"ax\"\n"
 505                "9:      lea 0(%%eax,%0,4),%0\n"
 506                "16:     jmp 8b\n"
 507                ".previous\n"
 508                ".section __ex_table,\"a\"\n"
 509                "        .align 4\n"
 510                "        .long 0b,16b\n"
 511                "        .long 1b,16b\n"
 512                "        .long 2b,16b\n"
 513                "        .long 21b,16b\n"
 514                "        .long 3b,16b\n"
 515                "        .long 31b,16b\n"
 516                "        .long 4b,16b\n"
 517                "        .long 41b,16b\n"
 518                "        .long 10b,16b\n"
 519                "        .long 51b,16b\n"
 520                "        .long 11b,16b\n"
 521                "        .long 61b,16b\n"
 522                "        .long 12b,16b\n"
 523                "        .long 71b,16b\n"
 524                "        .long 13b,16b\n"
 525                "        .long 81b,16b\n"
 526                "        .long 14b,16b\n"
 527                "        .long 91b,16b\n"
 528                "        .long 6b,9b\n"
 529                "        .long 7b,16b\n"
 530                ".previous"
 531                : "=&c"(size), "=&D" (d0), "=&S" (d1)
 532                :  "1"(to), "2"(from), "0"(size)
 533                : "eax", "edx", "memory");
 534         return size;
 535 }
 536
 537 #else
 538
 539 /*
 540  * Leave these declared but undefined.  They should not be any references to
 541  * them
 542  */
 543 unsigned long __copy_user_zeroing_intel(void *to, const void __user *from,
 544                                         unsigned long size);
 545 unsigned long __copy_user_intel(void __user *to, const void *from,
 546                                         unsigned long size);
 547 unsigned long __copy_user_zeroing_intel_nocache(void *to,
 548                                 const void __user *from, unsigned long size);
 549 #endif /* CONFIG_X86_INTEL_USERCOPY */
 550
 551 /* Generic arbitrary sized copy.  */
 552 #define __copy_user(to, from, size)                                     \
 553 do {                                                                    \
 554         int __d0, __d1, __d2;                                           \
 555         __asm__ __volatile__(                                           \
 556                 "       cmp  $7,%0\n"                                   \
 557                 "       jbe  1f\n"                                      \
 558                 "       movl %1,%0\n"                                   \
 559                 "       negl %0\n"                                      \
 560                 "       andl $7,%0\n"                                   \
 561                 "       subl %0,%3\n"                                   \
 562                 "4:     rep; movsb\n"                                   \
 563                 "       movl %3,%0\n"                                   \
 564                 "       shrl $2,%0\n"                                   \
 565                 "       andl $3,%3\n"                                   \
 566                 "       .align 2,0x90\n"                                \
 567                 "0:     rep; movsl\n"                                   \
 568                 "       movl %3,%0\n"                                   \
 569                 "1:     rep; movsb\n"                                   \
 570                 "2:\n"                                                  \
 571                 ".section .fixup,\"ax\"\n"                              \
 572                 "5:     addl %3,%0\n"                                   \
 573                 "       jmp 2b\n"                                       \
 574                 "3:     lea 0(%3,%0,4),%0\n"                            \
 575                 "       jmp 2b\n"                                       \
 576                 ".previous\n"                                           \
 577                 ".section __ex_table,\"a\"\n"                           \
 578                 "       .align 4\n"                                     \
 579                 "       .long 4b,5b\n"                                  \
 580                 "       .long 0b,3b\n"                                  \
 581                 "       .long 1b,2b\n"                                  \
 582                 ".previous"                                             \
 583                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 584                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 585                 : "memory");                                            \
 586 } while (0)
 587
 588 #define __copy_user_zeroing(to, from, size)                             \
 589 do {                                                                    \
 590         int __d0, __d1, __d2;                                           \
 591         __asm__ __volatile__(                                           \
 592                 "       cmp  $7,%0\n"                                   \
 593                 "       jbe  1f\n"                                      \
 594                 "       movl %1,%0\n"                                   \
 595                 "       negl %0\n"                                      \
 596                 "       andl $7,%0\n"                                   \
 597                 "       subl %0,%3\n"                                   \
 598                 "4:     rep; movsb\n"                                   \
 599                 "       movl %3,%0\n"                                   \
 600                 "       shrl $2,%0\n"                                   \
 601                 "       andl $3,%3\n"                                   \
 602                 "       .align 2,0x90\n"                                \
 603                 "0:     rep; movsl\n"                                   \
 604                 "       movl %3,%0\n"                                   \
 605                 "1:     rep; movsb\n"                                   \
 606                 "2:\n"                                                  \
 607                 ".section .fixup,\"ax\"\n"                              \
 608                 "5:     addl %3,%0\n"                                   \
 609                 "       jmp 6f\n"                                       \
 610                 "3:     lea 0(%3,%0,4),%0\n"                            \
 611                 "6:     pushl %0\n"                                     \
 612                 "       pushl %%eax\n"                                  \
 613                 "       xorl %%eax,%%eax\n"                             \
 614                 "       rep; stosb\n"                                   \
 615                 "       popl %%eax\n"                                   \
 616                 "       popl %0\n"                                      \
 617                 "       jmp 2b\n"                                       \
 618                 ".previous\n"                                           \
 619                 ".section __ex_table,\"a\"\n"                           \
 620                 "       .align 4\n"                                     \
 621                 "       .long 4b,5b\n"                                  \
 622                 "       .long 0b,3b\n"                                  \
 623                 "       .long 1b,6b\n"                                  \
 624                 ".previous"                                             \
 625                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 626                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 627                 : "memory");                                            \
 628 } while (0)
 629
 630 unsigned long __copy_to_user_ll(void __user *to, const void *from,
 631                                 unsigned long n)
 632 {
 633 #ifndef CONFIG_X86_WP_WORKS_OK
 634         if (unlikely(boot_cpu_data.wp_works_ok == 0) &&
 635                         ((unsigned long)to) < TASK_SIZE) {
 636                 /*
 637                  * When we are in an atomic section (see
 638                  * mm/filemap.c:file_read_actor), return the full
 639                  * length to take the slow path.
 640                  */
 641                 if (in_atomic())
 642                         return n;
 643
 644                 /*
 645                  * CPU does not honor the WP bit when writing
 646                  * from supervisory mode, and due to preemption or SMP,
 647                  * the page tables can change at any time.
 648                  * Do it manually.      Manfred <manfred@colorfullife.com>
 649                  */
 650                 while (n) {
 651                         unsigned long offset = ((unsigned long)to)%PAGE_SIZE;
 652                         unsigned long len = PAGE_SIZE - offset;
 653                         int retval;
 654                         struct page *pg;
 655                         void *maddr;
 656
 657                         if (len > n)
 658                                 len = n;
 659
 660 survive:
 661                         down_read(&current->mm->mmap_sem);
 662                         retval = get_user_pages(current, current->mm,
 663                                         (unsigned long)to, 1, 1, 0, &pg, NULL);
 664
 665                         if (retval == -ENOMEM && is_global_init(current)) {
 666                                 up_read(&current->mm->mmap_sem);
 667                                 congestion_wait(BLK_RW_ASYNC, HZ/50);
 668                                 goto survive;
 669                         }
 670
 671                         if (retval != 1) {
 672                                 up_read(&current->mm->mmap_sem);
 673                                 break;
 674                         }
 675
 676                         maddr = kmap_atomic(pg);
 677                         memcpy(maddr + offset, from, len);
 678                         kunmap_atomic(maddr);
 679                         set_page_dirty_lock(pg);
 680                         put_page(pg);
 681                         up_read(&current->mm->mmap_sem);
 682
 683                         from += len;
 684                         to += len;
 685                         n -= len;
 686                 }
 687                 return n;
 688         }
 689 #endif
 690         if (movsl_is_ok(to, from, n))
 691                 __copy_user(to, from, n);
 692         else
 693                 n = __copy_user_intel(to, from, n);
 694         return n;
 695 }
 696 EXPORT_SYMBOL(__copy_to_user_ll);
 697
 698 unsigned long __copy_from_user_ll(void *to, const void __user *from,
 699                                         unsigned long n)
 700 {
 701         if (movsl_is_ok(to, from, n))
 702                 __copy_user_zeroing(to, from, n);
 703         else
 704                 n = __copy_user_zeroing_intel(to, from, n);
 705         return n;
 706 }
 707 EXPORT_SYMBOL(__copy_from_user_ll);
 708
 709 unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from,
 710                                          unsigned long n)
 711 {
 712         if (movsl_is_ok(to, from, n))
 713                 __copy_user(to, from, n);
 714         else
 715                 n = __copy_user_intel((void __user *)to,
 716                                       (const void *)from, n);
 717         return n;
 718 }
 719 EXPORT_SYMBOL(__copy_from_user_ll_nozero);
 720
 721 unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from,
 722                                         unsigned long n)
 723 {
 724 #ifdef CONFIG_X86_INTEL_USERCOPY
 725         if (n > 64 && cpu_has_xmm2)
 726                 n = __copy_user_zeroing_intel_nocache(to, from, n);
 727         else
 728                 __copy_user_zeroing(to, from, n);
 729 #else
 730         __copy_user_zeroing(to, from, n);
 731 #endif
 732         return n;
 733 }
 734 EXPORT_SYMBOL(__copy_from_user_ll_nocache);
 735
 736 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
 737                                         unsigned long n)
 738 {
 739 #ifdef CONFIG_X86_INTEL_USERCOPY
 740         if (n > 64 && cpu_has_xmm2)
 741                 n = __copy_user_intel_nocache(to, from, n);
 742         else
 743                 __copy_user(to, from, n);
 744 #else
 745         __copy_user(to, from, n);
 746 #endif
 747         return n;
 748 }
 749 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
 750
 751 /**
 752  * copy_to_user: - Copy a block of data into user space.
 753  * @to:   Destination address, in user space.
 754  * @from: Source address, in kernel space.
 755  * @n:    Number of bytes to copy.
 756  *
 757  * Context: User context only.  This function may sleep.
 758  *
 759  * Copy data from kernel space to user space.
 760  *
 761  * Returns number of bytes that could not be copied.
 762  * On success, this will be zero.
 763  */
 764 unsigned long
 765 copy_to_user(void __user *to, const void *from, unsigned long n)
 766 {
 767         if (access_ok(VERIFY_WRITE, to, n))
 768                 n = __copy_to_user(to, from, n);
 769         return n;
 770 }
 771 EXPORT_SYMBOL(copy_to_user);
 772
 773 /**
 774  * copy_from_user: - Copy a block of data from user space.
 775  * @to:   Destination address, in kernel space.
 776  * @from: Source address, in user space.
 777  * @n:    Number of bytes to copy.
 778  *
 779  * Context: User context only.  This function may sleep.
 780  *
 781  * Copy data from user space to kernel space.
 782  *
 783  * Returns number of bytes that could not be copied.
 784  * On success, this will be zero.
 785  *
 786  * If some data could not be copied, this function will pad the copied
 787  * data to the requested size using zero bytes.
 788  */
 789 unsigned long
 790 _copy_from_user(void *to, const void __user *from, unsigned long n)
 791 {
 792         if (access_ok(VERIFY_READ, from, n))
 793                 n = __copy_from_user(to, from, n);
 794         else
 795                 memset(to, 0, n);
 796         return n;
 797 }
 798 EXPORT_SYMBOL(_copy_from_user);
 799
 800 void copy_from_user_overflow(void)
 801 {
 802         WARN(1, "Buffer overflow detected!\n");
 803 }
 804 EXPORT_SYMBOL(copy_from_user_overflow);