arch/mips/include/asm/uaccess.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  * Copyright (C) 2007  Maciej W. Rozycki
   9  */
  10 #ifndef _ASM_UACCESS_H
  11 #define _ASM_UACCESS_H
  12
  13 #include <linux/kernel.h>
  14 #include <linux/errno.h>
  15 #include <linux/thread_info.h>
  16
  17 /*
  18  * The fs value determines whether argument validity checking should be
  19  * performed or not.  If get_fs() == USER_DS, checking is performed, with
  20  * get_fs() == KERNEL_DS, checking is bypassed.
  21  *
  22  * For historical reasons, these macros are grossly misnamed.
  23  */
  24 #ifdef CONFIG_32BIT
  25
  26 #define __UA_LIMIT      0x80000000UL
  27
  28 #define __UA_ADDR       ".word"
  29 #define __UA_LA         "la"
  30 #define __UA_ADDU       "addu"
  31 #define __UA_t0         "$8"
  32 #define __UA_t1         "$9"
  33
  34 #endif /* CONFIG_32BIT */
  35
  36 #ifdef CONFIG_64BIT
  37
  38 extern u64 __ua_limit;
  39
  40 #define __UA_LIMIT      __ua_limit
  41
  42 #define __UA_ADDR       ".dword"
  43 #define __UA_LA         "dla"
  44 #define __UA_ADDU       "daddu"
  45 #define __UA_t0         "$12"
  46 #define __UA_t1         "$13"
  47
  48 #endif /* CONFIG_64BIT */
  49
  50 /*
  51  * USER_DS is a bitmask that has the bits set that may not be set in a valid
  52  * userspace address.  Note that we limit 32-bit userspace to 0x7fff8000 but
  53  * the arithmetic we're doing only works if the limit is a power of two, so
  54  * we use 0x80000000 here on 32-bit kernels.  If a process passes an invalid
  55  * address in this range it's the process's problem, not ours :-)
  56  */
  57
  58 #define KERNEL_DS       ((mm_segment_t) { 0UL })
  59 #define USER_DS         ((mm_segment_t) { __UA_LIMIT })
  60
  61 #define VERIFY_READ    0
  62 #define VERIFY_WRITE   1
  63
  64 #define get_ds()        (KERNEL_DS)
  65 #define get_fs()        (current_thread_info()->addr_limit)
  66 #define set_fs(x)       (current_thread_info()->addr_limit = (x))
  67
  68 #define segment_eq(a, b)        ((a).seg == (b).seg)
  69
  70
  71 /*
  72  * Is a address valid? This does a straighforward calculation rather
  73  * than tests.
  74  *
  75  * Address valid if:
  76  *  - "addr" doesn't have any high-bits set
  77  *  - AND "size" doesn't have any high-bits set
  78  *  - AND "addr+size" doesn't have any high-bits set
  79  *  - OR we are in kernel mode.
  80  *
  81  * __ua_size() is a trick to avoid runtime checking of positive constant
  82  * sizes; for those we already know at compile time that the size is ok.
  83  */
  84 #define __ua_size(size)                                                 \
  85         ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size))
  86
  87 /*
  88  * access_ok: - Checks if a user space pointer is valid
  89  * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
  90  *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
  91  *        to write to a block, it is always safe to read from it.
  92  * @addr: User space pointer to start of block to check
  93  * @size: Size of block to check
  94  *
  95  * Context: User context only.  This function may sleep.
  96  *
  97  * Checks if a pointer to a block of memory in user space is valid.
  98  *
  99  * Returns true (nonzero) if the memory block may be valid, false (zero)
 100  * if it is definitely invalid.
 101  *
 102  * Note that, depending on architecture, this function probably just
 103  * checks that the pointer is in the user space range - after calling
 104  * this function, memory access functions may still return -EFAULT.
 105  */
 106
 107 #define __access_mask get_fs().seg
 108
 109 #define __access_ok(addr, size, mask)                                   \
 110 ({                                                                      \
 111         unsigned long __addr = (unsigned long) (addr);                  \
 112         unsigned long __size = size;                                    \
 113         unsigned long __mask = mask;                                    \
 114         unsigned long __ok;                                             \
 115                                                                         \
 116         __chk_user_ptr(addr);                                           \
 117         __ok = (signed long)(__mask & (__addr | (__addr + __size) |     \
 118                 __ua_size(__size)));                                    \
 119         __ok == 0;                                                      \
 120 })
 121
 122 #define access_ok(type, addr, size)                                     \
 123         likely(__access_ok((addr), (size), __access_mask))
 124
 125 /*
 126  * put_user: - Write a simple value into user space.
 127  * @x:   Value to copy to user space.
 128  * @ptr: Destination address, in user space.
 129  *
 130  * Context: User context only.  This function may sleep.
 131  *
 132  * This macro copies a single simple value from kernel space to user
 133  * space.  It supports simple types like char and int, but not larger
 134  * data types like structures or arrays.
 135  *
 136  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 137  * to the result of dereferencing @ptr.
 138  *
 139  * Returns zero on success, or -EFAULT on error.
 140  */
 141 #define put_user(x,ptr) \
 142         __put_user_check((x), (ptr), sizeof(*(ptr)))
 143
 144 /*
 145  * get_user: - Get a simple variable from user space.
 146  * @x:   Variable to store result.
 147  * @ptr: Source address, in user space.
 148  *
 149  * Context: User context only.  This function may sleep.
 150  *
 151  * This macro copies a single simple variable from user space to kernel
 152  * space.  It supports simple types like char and int, but not larger
 153  * data types like structures or arrays.
 154  *
 155  * @ptr must have pointer-to-simple-variable type, and the result of
 156  * dereferencing @ptr must be assignable to @x without a cast.
 157  *
 158  * Returns zero on success, or -EFAULT on error.
 159  * On error, the variable @x is set to zero.
 160  */
 161 #define get_user(x,ptr) \
 162         __get_user_check((x), (ptr), sizeof(*(ptr)))
 163
 164 /*
 165  * __put_user: - Write a simple value into user space, with less checking.
 166  * @x:   Value to copy to user space.
 167  * @ptr: Destination address, in user space.
 168  *
 169  * Context: User context only.  This function may sleep.
 170  *
 171  * This macro copies a single simple value from kernel space to user
 172  * space.  It supports simple types like char and int, but not larger
 173  * data types like structures or arrays.
 174  *
 175  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 176  * to the result of dereferencing @ptr.
 177  *
 178  * Caller must check the pointer with access_ok() before calling this
 179  * function.
 180  *
 181  * Returns zero on success, or -EFAULT on error.
 182  */
 183 #define __put_user(x,ptr) \
 184         __put_user_nocheck((x), (ptr), sizeof(*(ptr)))
 185
 186 /*
 187  * __get_user: - Get a simple variable from user space, with less checking.
 188  * @x:   Variable to store result.
 189  * @ptr: Source address, in user space.
 190  *
 191  * Context: User context only.  This function may sleep.
 192  *
 193  * This macro copies a single simple variable from user space to kernel
 194  * space.  It supports simple types like char and int, but not larger
 195  * data types like structures or arrays.
 196  *
 197  * @ptr must have pointer-to-simple-variable type, and the result of
 198  * dereferencing @ptr must be assignable to @x without a cast.
 199  *
 200  * Caller must check the pointer with access_ok() before calling this
 201  * function.
 202  *
 203  * Returns zero on success, or -EFAULT on error.
 204  * On error, the variable @x is set to zero.
 205  */
 206 #define __get_user(x,ptr) \
 207         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
 208
 209 struct __large_struct { unsigned long buf[100]; };
 210 #define __m(x) (*(struct __large_struct __user *)(x))
 211
 212 /*
 213  * Yuck.  We need two variants, one for 64bit operation and one
 214  * for 32 bit mode and old iron.
 215  */
 216 #ifdef CONFIG_32BIT
 217 #define __GET_USER_DW(val, ptr) __get_user_asm_ll32(val, ptr)
 218 #endif
 219 #ifdef CONFIG_64BIT
 220 #define __GET_USER_DW(val, ptr) __get_user_asm(val, "ld", ptr)
 221 #endif
 222
 223 extern void __get_user_unknown(void);
 224
 225 #define __get_user_common(val, size, ptr)                               \
 226 do {                                                                    \
 227         switch (size) {                                                 \
 228         case 1: __get_user_asm(val, "lb", ptr); break;                  \
 229         case 2: __get_user_asm(val, "lh", ptr); break;                  \
 230         case 4: __get_user_asm(val, "lw", ptr); break;                  \
 231         case 8: __GET_USER_DW(val, ptr); break;                         \
 232         default: __get_user_unknown(); break;                           \
 233         }                                                               \
 234 } while (0)
 235
 236 #define __get_user_nocheck(x, ptr, size)                                \
 237 ({                                                                      \
 238         int __gu_err;                                                   \
 239                                                                         \
 240         __chk_user_ptr(ptr);                                            \
 241         __get_user_common((x), size, ptr);                              \
 242         __gu_err;                                                       \
 243 })
 244
 245 #define __get_user_check(x, ptr, size)                                  \
 246 ({                                                                      \
 247         int __gu_err = -EFAULT;                                         \
 248         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 249                                                                         \
 250         might_fault();                                                  \
 251         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size)))            \
 252                 __get_user_common((x), size, __gu_ptr);                 \
 253                                                                         \
 254         __gu_err;                                                       \
 255 })
 256
 257 #define __get_user_asm(val, insn, addr)                                 \
 258 {                                                                       \
 259         long __gu_tmp;                                                  \
 260                                                                         \
 261         __asm__ __volatile__(                                           \
 262         "1:     " insn "        %1, %3                          \n"     \
 263         "2:                                                     \n"     \
 264         "       .section .fixup,\"ax\"                          \n"     \
 265         "3:     li      %0, %4                                  \n"     \
 266         "       j       2b                                      \n"     \
 267         "       .previous                                       \n"     \
 268         "       .section __ex_table,\"a\"                       \n"     \
 269         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 270         "       .previous                                       \n"     \
 271         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 272         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 273                                                                         \
 274         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 275 }
 276
 277 /*
 278  * Get a long long 64 using 32 bit registers.
 279  */
 280 #define __get_user_asm_ll32(val, addr)                                  \
 281 {                                                                       \
 282         union {                                                         \
 283                 unsigned long long      l;                              \
 284                 __typeof__(*(addr))     t;                              \
 285         } __gu_tmp;                                                     \
 286                                                                         \
 287         __asm__ __volatile__(                                           \
 288         "1:     lw      %1, (%3)                                \n"     \
 289         "2:     lw      %D1, 4(%3)                              \n"     \
 290         "3:     .section        .fixup,\"ax\"                   \n"     \
 291         "4:     li      %0, %4                                  \n"     \
 292         "       move    %1, $0                                  \n"     \
 293         "       move    %D1, $0                                 \n"     \
 294         "       j       3b                                      \n"     \
 295         "       .previous                                       \n"     \
 296         "       .section        __ex_table,\"a\"                \n"     \
 297         "       " __UA_ADDR "   1b, 4b                          \n"     \
 298         "       " __UA_ADDR "   2b, 4b                          \n"     \
 299         "       .previous                                       \n"     \
 300         : "=r" (__gu_err), "=&r" (__gu_tmp.l)                           \
 301         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 302                                                                         \
 303         (val) = __gu_tmp.t;                                             \
 304 }
 305
 306 /*
 307  * Yuck.  We need two variants, one for 64bit operation and one
 308  * for 32 bit mode and old iron.
 309  */
 310 #ifdef CONFIG_32BIT
 311 #define __PUT_USER_DW(ptr) __put_user_asm_ll32(ptr)
 312 #endif
 313 #ifdef CONFIG_64BIT
 314 #define __PUT_USER_DW(ptr) __put_user_asm("sd", ptr)
 315 #endif
 316
 317 #define __put_user_nocheck(x, ptr, size)                                \
 318 ({                                                                      \
 319         __typeof__(*(ptr)) __pu_val;                                    \
 320         int __pu_err = 0;                                               \
 321                                                                         \
 322         __chk_user_ptr(ptr);                                            \
 323         __pu_val = (x);                                                 \
 324         switch (size) {                                                 \
 325         case 1: __put_user_asm("sb", ptr); break;                       \
 326         case 2: __put_user_asm("sh", ptr); break;                       \
 327         case 4: __put_user_asm("sw", ptr); break;                       \
 328         case 8: __PUT_USER_DW(ptr); break;                              \
 329         default: __put_user_unknown(); break;                           \
 330         }                                                               \
 331         __pu_err;                                                       \
 332 })
 333
 334 #define __put_user_check(x, ptr, size)                                  \
 335 ({                                                                      \
 336         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 337         __typeof__(*(ptr)) __pu_val = (x);                              \
 338         int __pu_err = -EFAULT;                                         \
 339                                                                         \
 340         might_fault();                                                  \
 341         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size))) {        \
 342                 switch (size) {                                         \
 343                 case 1: __put_user_asm("sb", __pu_addr); break;         \
 344                 case 2: __put_user_asm("sh", __pu_addr); break;         \
 345                 case 4: __put_user_asm("sw", __pu_addr); break;         \
 346                 case 8: __PUT_USER_DW(__pu_addr); break;                \
 347                 default: __put_user_unknown(); break;                   \
 348                 }                                                       \
 349         }                                                               \
 350         __pu_err;                                                       \
 351 })
 352
 353 #define __put_user_asm(insn, ptr)                                       \
 354 {                                                                       \
 355         __asm__ __volatile__(                                           \
 356         "1:     " insn "        %z2, %3         # __put_user_asm\n"     \
 357         "2:                                                     \n"     \
 358         "       .section        .fixup,\"ax\"                   \n"     \
 359         "3:     li      %0, %4                                  \n"     \
 360         "       j       2b                                      \n"     \
 361         "       .previous                                       \n"     \
 362         "       .section        __ex_table,\"a\"                \n"     \
 363         "       " __UA_ADDR "   1b, 3b                          \n"     \
 364         "       .previous                                       \n"     \
 365         : "=r" (__pu_err)                                               \
 366         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 367           "i" (-EFAULT));                                               \
 368 }
 369
 370 #define __put_user_asm_ll32(ptr)                                        \
 371 {                                                                       \
 372         __asm__ __volatile__(                                           \
 373         "1:     sw      %2, (%3)        # __put_user_asm_ll32   \n"     \
 374         "2:     sw      %D2, 4(%3)                              \n"     \
 375         "3:                                                     \n"     \
 376         "       .section        .fixup,\"ax\"                   \n"     \
 377         "4:     li      %0, %4                                  \n"     \
 378         "       j       3b                                      \n"     \
 379         "       .previous                                       \n"     \
 380         "       .section        __ex_table,\"a\"                \n"     \
 381         "       " __UA_ADDR "   1b, 4b                          \n"     \
 382         "       " __UA_ADDR "   2b, 4b                          \n"     \
 383         "       .previous"                                              \
 384         : "=r" (__pu_err)                                               \
 385         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 386           "i" (-EFAULT));                                               \
 387 }
 388
 389 extern void __put_user_unknown(void);
 390
 391 /*
 392  * put_user_unaligned: - Write a simple value into user space.
 393  * @x:   Value to copy to user space.
 394  * @ptr: Destination address, in user space.
 395  *
 396  * Context: User context only.  This function may sleep.
 397  *
 398  * This macro copies a single simple value from kernel space to user
 399  * space.  It supports simple types like char and int, but not larger
 400  * data types like structures or arrays.
 401  *
 402  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 403  * to the result of dereferencing @ptr.
 404  *
 405  * Returns zero on success, or -EFAULT on error.
 406  */
 407 #define put_user_unaligned(x,ptr)       \
 408         __put_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 409
 410 /*
 411  * get_user_unaligned: - Get a simple variable from user space.
 412  * @x:   Variable to store result.
 413  * @ptr: Source address, in user space.
 414  *
 415  * Context: User context only.  This function may sleep.
 416  *
 417  * This macro copies a single simple variable from user space to kernel
 418  * space.  It supports simple types like char and int, but not larger
 419  * data types like structures or arrays.
 420  *
 421  * @ptr must have pointer-to-simple-variable type, and the result of
 422  * dereferencing @ptr must be assignable to @x without a cast.
 423  *
 424  * Returns zero on success, or -EFAULT on error.
 425  * On error, the variable @x is set to zero.
 426  */
 427 #define get_user_unaligned(x,ptr) \
 428         __get_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 429
 430 /*
 431  * __put_user_unaligned: - Write a simple value into user space, with less checking.
 432  * @x:   Value to copy to user space.
 433  * @ptr: Destination address, in user space.
 434  *
 435  * Context: User context only.  This function may sleep.
 436  *
 437  * This macro copies a single simple value from kernel space to user
 438  * space.  It supports simple types like char and int, but not larger
 439  * data types like structures or arrays.
 440  *
 441  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 442  * to the result of dereferencing @ptr.
 443  *
 444  * Caller must check the pointer with access_ok() before calling this
 445  * function.
 446  *
 447  * Returns zero on success, or -EFAULT on error.
 448  */
 449 #define __put_user_unaligned(x,ptr) \
 450         __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
 451
 452 /*
 453  * __get_user_unaligned: - Get a simple variable from user space, with less checking.
 454  * @x:   Variable to store result.
 455  * @ptr: Source address, in user space.
 456  *
 457  * Context: User context only.  This function may sleep.
 458  *
 459  * This macro copies a single simple variable from user space to kernel
 460  * space.  It supports simple types like char and int, but not larger
 461  * data types like structures or arrays.
 462  *
 463  * @ptr must have pointer-to-simple-variable type, and the result of
 464  * dereferencing @ptr must be assignable to @x without a cast.
 465  *
 466  * Caller must check the pointer with access_ok() before calling this
 467  * function.
 468  *
 469  * Returns zero on success, or -EFAULT on error.
 470  * On error, the variable @x is set to zero.
 471  */
 472 #define __get_user_unaligned(x,ptr) \
 473         __get_user__unalignednocheck((x),(ptr),sizeof(*(ptr)))
 474
 475 /*
 476  * Yuck.  We need two variants, one for 64bit operation and one
 477  * for 32 bit mode and old iron.
 478  */
 479 #ifdef CONFIG_32BIT
 480 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 481         __get_user_unaligned_asm_ll32(val, ptr)
 482 #endif
 483 #ifdef CONFIG_64BIT
 484 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 485         __get_user_unaligned_asm(val, "uld", ptr)
 486 #endif
 487
 488 extern void __get_user_unaligned_unknown(void);
 489
 490 #define __get_user_unaligned_common(val, size, ptr)                     \
 491 do {                                                                    \
 492         switch (size) {                                                 \
 493         case 1: __get_user_asm(val, "lb", ptr); break;                  \
 494         case 2: __get_user_unaligned_asm(val, "ulh", ptr); break;       \
 495         case 4: __get_user_unaligned_asm(val, "ulw", ptr); break;       \
 496         case 8: __GET_USER_UNALIGNED_DW(val, ptr); break;               \
 497         default: __get_user_unaligned_unknown(); break;                 \
 498         }                                                               \
 499 } while (0)
 500
 501 #define __get_user_unaligned_nocheck(x,ptr,size)                        \
 502 ({                                                                      \
 503         int __gu_err;                                                   \
 504                                                                         \
 505         __get_user_unaligned_common((x), size, ptr);                    \
 506         __gu_err;                                                       \
 507 })
 508
 509 #define __get_user_unaligned_check(x,ptr,size)                          \
 510 ({                                                                      \
 511         int __gu_err = -EFAULT;                                         \
 512         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 513                                                                         \
 514         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size)))            \
 515                 __get_user_unaligned_common((x), size, __gu_ptr);       \
 516                                                                         \
 517         __gu_err;                                                       \
 518 })
 519
 520 #define __get_user_unaligned_asm(val, insn, addr)                       \
 521 {                                                                       \
 522         long __gu_tmp;                                                  \
 523                                                                         \
 524         __asm__ __volatile__(                                           \
 525         "1:     " insn "        %1, %3                          \n"     \
 526         "2:                                                     \n"     \
 527         "       .section .fixup,\"ax\"                          \n"     \
 528         "3:     li      %0, %4                                  \n"     \
 529         "       j       2b                                      \n"     \
 530         "       .previous                                       \n"     \
 531         "       .section __ex_table,\"a\"                       \n"     \
 532         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 533         "       "__UA_ADDR "\t1b + 4, 3b                        \n"     \
 534         "       .previous                                       \n"     \
 535         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 536         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 537                                                                         \
 538         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 539 }
 540
 541 /*
 542  * Get a long long 64 using 32 bit registers.
 543  */
 544 #define __get_user_unaligned_asm_ll32(val, addr)                        \
 545 {                                                                       \
 546         unsigned long long __gu_tmp;                                    \
 547                                                                         \
 548         __asm__ __volatile__(                                           \
 549         "1:     ulw     %1, (%3)                                \n"     \
 550         "2:     ulw     %D1, 4(%3)                              \n"     \
 551         "       move    %0, $0                                  \n"     \
 552         "3:     .section        .fixup,\"ax\"                   \n"     \
 553         "4:     li      %0, %4                                  \n"     \
 554         "       move    %1, $0                                  \n"     \
 555         "       move    %D1, $0                                 \n"     \
 556         "       j       3b                                      \n"     \
 557         "       .previous                                       \n"     \
 558         "       .section        __ex_table,\"a\"                \n"     \
 559         "       " __UA_ADDR "   1b, 4b                          \n"     \
 560         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 561         "       " __UA_ADDR "   2b, 4b                          \n"     \
 562         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 563         "       .previous                                       \n"     \
 564         : "=r" (__gu_err), "=&r" (__gu_tmp)                             \
 565         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 566         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 567 }
 568
 569 /*
 570  * Yuck.  We need two variants, one for 64bit operation and one
 571  * for 32 bit mode and old iron.
 572  */
 573 #ifdef CONFIG_32BIT
 574 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr)
 575 #endif
 576 #ifdef CONFIG_64BIT
 577 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr)
 578 #endif
 579
 580 #define __put_user_unaligned_nocheck(x,ptr,size)                        \
 581 ({                                                                      \
 582         __typeof__(*(ptr)) __pu_val;                                    \
 583         int __pu_err = 0;                                               \
 584                                                                         \
 585         __pu_val = (x);                                                 \
 586         switch (size) {                                                 \
 587         case 1: __put_user_asm("sb", ptr); break;                       \
 588         case 2: __put_user_unaligned_asm("ush", ptr); break;            \
 589         case 4: __put_user_unaligned_asm("usw", ptr); break;            \
 590         case 8: __PUT_USER_UNALIGNED_DW(ptr); break;                    \
 591         default: __put_user_unaligned_unknown(); break;                 \
 592         }                                                               \
 593         __pu_err;                                                       \
 594 })
 595
 596 #define __put_user_unaligned_check(x,ptr,size)                          \
 597 ({                                                                      \
 598         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 599         __typeof__(*(ptr)) __pu_val = (x);                              \
 600         int __pu_err = -EFAULT;                                         \
 601                                                                         \
 602         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size))) {        \
 603                 switch (size) {                                         \
 604                 case 1: __put_user_asm("sb", __pu_addr); break;         \
 605                 case 2: __put_user_unaligned_asm("ush", __pu_addr); break; \
 606                 case 4: __put_user_unaligned_asm("usw", __pu_addr); break; \
 607                 case 8: __PUT_USER_UNALGINED_DW(__pu_addr); break;      \
 608                 default: __put_user_unaligned_unknown(); break;         \
 609                 }                                                       \
 610         }                                                               \
 611         __pu_err;                                                       \
 612 })
 613
 614 #define __put_user_unaligned_asm(insn, ptr)                             \
 615 {                                                                       \
 616         __asm__ __volatile__(                                           \
 617         "1:     " insn "        %z2, %3         # __put_user_unaligned_asm\n" \
 618         "2:                                                     \n"     \
 619         "       .section        .fixup,\"ax\"                   \n"     \
 620         "3:     li      %0, %4                                  \n"     \
 621         "       j       2b                                      \n"     \
 622         "       .previous                                       \n"     \
 623         "       .section        __ex_table,\"a\"                \n"     \
 624         "       " __UA_ADDR "   1b, 3b                          \n"     \
 625         "       .previous                                       \n"     \
 626         : "=r" (__pu_err)                                               \
 627         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 628           "i" (-EFAULT));                                               \
 629 }
 630
 631 #define __put_user_unaligned_asm_ll32(ptr)                              \
 632 {                                                                       \
 633         __asm__ __volatile__(                                           \
 634         "1:     sw      %2, (%3)        # __put_user_unaligned_asm_ll32 \n" \
 635         "2:     sw      %D2, 4(%3)                              \n"     \
 636         "3:                                                     \n"     \
 637         "       .section        .fixup,\"ax\"                   \n"     \
 638         "4:     li      %0, %4                                  \n"     \
 639         "       j       3b                                      \n"     \
 640         "       .previous                                       \n"     \
 641         "       .section        __ex_table,\"a\"                \n"     \
 642         "       " __UA_ADDR "   1b, 4b                          \n"     \
 643         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 644         "       " __UA_ADDR "   2b, 4b                          \n"     \
 645         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 646         "       .previous"                                              \
 647         : "=r" (__pu_err)                                               \
 648         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 649           "i" (-EFAULT));                                               \
 650 }
 651
 652 extern void __put_user_unaligned_unknown(void);
 653
 654 /*
 655  * We're generating jump to subroutines which will be outside the range of
 656  * jump instructions
 657  */
 658 #ifdef MODULE
 659 #define __MODULE_JAL(destination)                                       \
 660         ".set\tnoat\n\t"                                                \
 661         __UA_LA "\t$1, " #destination "\n\t"                            \
 662         "jalr\t$1\n\t"                                                  \
 663         ".set\tat\n\t"
 664 #else
 665 #define __MODULE_JAL(destination)                                       \
 666         "jal\t" #destination "\n\t"
 667 #endif
 668
 669 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
 670 #define DADDI_SCRATCH "$0"
 671 #else
 672 #define DADDI_SCRATCH "$3"
 673 #endif
 674
 675 extern size_t __copy_user(void *__to, const void *__from, size_t __n);
 676
 677 #define __invoke_copy_to_user(to, from, n)                              \
 678 ({                                                                      \
 679         register void __user *__cu_to_r __asm__("$4");                  \
 680         register const void *__cu_from_r __asm__("$5");                 \
 681         register long __cu_len_r __asm__("$6");                         \
 682                                                                         \
 683         __cu_to_r = (to);                                               \
 684         __cu_from_r = (from);                                           \
 685         __cu_len_r = (n);                                               \
 686         __asm__ __volatile__(                                           \
 687         __MODULE_JAL(__copy_user)                                       \
 688         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 689         :                                                               \
 690         : "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31",         \
 691           DADDI_SCRATCH, "memory");                                     \
 692         __cu_len_r;                                                     \
 693 })
 694
 695 /*
 696  * __copy_to_user: - Copy a block of data into user space, with less checking.
 697  * @to:   Destination address, in user space.
 698  * @from: Source address, in kernel space.
 699  * @n:    Number of bytes to copy.
 700  *
 701  * Context: User context only.  This function may sleep.
 702  *
 703  * Copy data from kernel space to user space.  Caller must check
 704  * the specified block with access_ok() before calling this function.
 705  *
 706  * Returns number of bytes that could not be copied.
 707  * On success, this will be zero.
 708  */
 709 #define __copy_to_user(to, from, n)                                     \
 710 ({                                                                      \
 711         void __user *__cu_to;                                           \
 712         const void *__cu_from;                                          \
 713         long __cu_len;                                                  \
 714                                                                         \
 715         __cu_to = (to);                                                 \
 716         __cu_from = (from);                                             \
 717         __cu_len = (n);                                                 \
 718         might_fault();                                                  \
 719         __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, __cu_len); \
 720         __cu_len;                                                       \
 721 })
 722
 723 extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
 724
 725 #define __copy_to_user_inatomic(to, from, n)                            \
 726 ({                                                                      \
 727         void __user *__cu_to;                                           \
 728         const void *__cu_from;                                          \
 729         long __cu_len;                                                  \
 730                                                                         \
 731         __cu_to = (to);                                                 \
 732         __cu_from = (from);                                             \
 733         __cu_len = (n);                                                 \
 734         __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, __cu_len); \
 735         __cu_len;                                                       \
 736 })
 737
 738 #define __copy_from_user_inatomic(to, from, n)                          \
 739 ({                                                                      \
 740         void *__cu_to;                                                  \
 741         const void __user *__cu_from;                                   \
 742         long __cu_len;                                                  \
 743                                                                         \
 744         __cu_to = (to);                                                 \
 745         __cu_from = (from);                                             \
 746         __cu_len = (n);                                                 \
 747         __cu_len = __invoke_copy_from_user_inatomic(__cu_to, __cu_from, \
 748                                                     __cu_len);          \
 749         __cu_len;                                                       \
 750 })
 751
 752 /*
 753  * copy_to_user: - Copy a block of data into user space.
 754  * @to:   Destination address, in user space.
 755  * @from: Source address, in kernel space.
 756  * @n:    Number of bytes to copy.
 757  *
 758  * Context: User context only.  This function may sleep.
 759  *
 760  * Copy data from kernel space to user space.
 761  *
 762  * Returns number of bytes that could not be copied.
 763  * On success, this will be zero.
 764  */
 765 #define copy_to_user(to, from, n)                                       \
 766 ({                                                                      \
 767         void __user *__cu_to;                                           \
 768         const void *__cu_from;                                          \
 769         long __cu_len;                                                  \
 770                                                                         \
 771         __cu_to = (to);                                                 \
 772         __cu_from = (from);                                             \
 773         __cu_len = (n);                                                 \
 774         if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) {               \
 775                 might_fault();                                          \
 776                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 777                                                  __cu_len);             \
 778         }                                                               \
 779         __cu_len;                                                       \
 780 })
 781
 782 #define __invoke_copy_from_user(to, from, n)                            \
 783 ({                                                                      \
 784         register void *__cu_to_r __asm__("$4");                         \
 785         register const void __user *__cu_from_r __asm__("$5");          \
 786         register long __cu_len_r __asm__("$6");                         \
 787                                                                         \
 788         __cu_to_r = (to);                                               \
 789         __cu_from_r = (from);                                           \
 790         __cu_len_r = (n);                                               \
 791         __asm__ __volatile__(                                           \
 792         ".set\tnoreorder\n\t"                                           \
 793         __MODULE_JAL(__copy_user)                                       \
 794         ".set\tnoat\n\t"                                                \
 795         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 796         ".set\tat\n\t"                                                  \
 797         ".set\treorder"                                                 \
 798         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 799         :                                                               \
 800         : "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31",         \
 801           DADDI_SCRATCH, "memory");                                     \
 802         __cu_len_r;                                                     \
 803 })
 804
 805 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
 806 ({                                                                      \
 807         register void *__cu_to_r __asm__("$4");                         \
 808         register const void __user *__cu_from_r __asm__("$5");          \
 809         register long __cu_len_r __asm__("$6");                         \
 810                                                                         \
 811         __cu_to_r = (to);                                               \
 812         __cu_from_r = (from);                                           \
 813         __cu_len_r = (n);                                               \
 814         __asm__ __volatile__(                                           \
 815         ".set\tnoreorder\n\t"                                           \
 816         __MODULE_JAL(__copy_user_inatomic)                              \
 817         ".set\tnoat\n\t"                                                \
 818         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 819         ".set\tat\n\t"                                                  \
 820         ".set\treorder"                                                 \
 821         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 822         :                                                               \
 823         : "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31",         \
 824           DADDI_SCRATCH, "memory");                                     \
 825         __cu_len_r;                                                     \
 826 })
 827
 828 /*
 829  * __copy_from_user: - Copy a block of data from user space, with less checking.
 830  * @to:   Destination address, in kernel space.
 831  * @from: Source address, in user space.
 832  * @n:    Number of bytes to copy.
 833  *
 834  * Context: User context only.  This function may sleep.
 835  *
 836  * Copy data from user space to kernel space.  Caller must check
 837  * the specified block with access_ok() before calling this function.
 838  *
 839  * Returns number of bytes that could not be copied.
 840  * On success, this will be zero.
 841  *
 842  * If some data could not be copied, this function will pad the copied
 843  * data to the requested size using zero bytes.
 844  */
 845 #define __copy_from_user(to, from, n)                                   \
 846 ({                                                                      \
 847         void *__cu_to;                                                  \
 848         const void __user *__cu_from;                                   \
 849         long __cu_len;                                                  \
 850                                                                         \
 851         __cu_to = (to);                                                 \
 852         __cu_from = (from);                                             \
 853         __cu_len = (n);                                                 \
 854         might_fault();                                                  \
 855         __cu_len = __invoke_copy_from_user(__cu_to, __cu_from,          \
 856                                            __cu_len);                   \
 857         __cu_len;                                                       \
 858 })
 859
 860 /*
 861  * copy_from_user: - Copy a block of data from user space.
 862  * @to:   Destination address, in kernel space.
 863  * @from: Source address, in user space.
 864  * @n:    Number of bytes to copy.
 865  *
 866  * Context: User context only.  This function may sleep.
 867  *
 868  * Copy data from user space to kernel space.
 869  *
 870  * Returns number of bytes that could not be copied.
 871  * On success, this will be zero.
 872  *
 873  * If some data could not be copied, this function will pad the copied
 874  * data to the requested size using zero bytes.
 875  */
 876 #define copy_from_user(to, from, n)                                     \
 877 ({                                                                      \
 878         void *__cu_to;                                                  \
 879         const void __user *__cu_from;                                   \
 880         long __cu_len;                                                  \
 881                                                                         \
 882         __cu_to = (to);                                                 \
 883         __cu_from = (from);                                             \
 884         __cu_len = (n);                                                 \
 885         if (access_ok(VERIFY_READ, __cu_from, __cu_len)) {              \
 886                 might_fault();                                          \
 887                 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from,  \
 888                                                    __cu_len);           \
 889         }                                                               \
 890         __cu_len;                                                       \
 891 })
 892
 893 #define __copy_in_user(to, from, n)                                     \
 894 ({                                                                      \
 895         void __user *__cu_to;                                           \
 896         const void __user *__cu_from;                                   \
 897         long __cu_len;                                                  \
 898                                                                         \
 899         __cu_to = (to);                                                 \
 900         __cu_from = (from);                                             \
 901         __cu_len = (n);                                                 \
 902         might_fault();                                                  \
 903         __cu_len = __invoke_copy_from_user(__cu_to, __cu_from,          \
 904                                            __cu_len);                   \
 905         __cu_len;                                                       \
 906 })
 907
 908 #define copy_in_user(to, from, n)                                       \
 909 ({                                                                      \
 910         void __user *__cu_to;                                           \
 911         const void __user *__cu_from;                                   \
 912         long __cu_len;                                                  \
 913                                                                         \
 914         __cu_to = (to);                                                 \
 915         __cu_from = (from);                                             \
 916         __cu_len = (n);                                                 \
 917         if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&       \
 918                    access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {       \
 919                 might_fault();                                          \
 920                 __cu_len = __invoke_copy_from_user(__cu_to, __cu_from,  \
 921                                                    __cu_len);           \
 922         }                                                               \
 923         __cu_len;                                                       \
 924 })
 925
 926 /*
 927  * __clear_user: - Zero a block of memory in user space, with less checking.
 928  * @to:   Destination address, in user space.
 929  * @n:    Number of bytes to zero.
 930  *
 931  * Zero a block of memory in user space.  Caller must check
 932  * the specified block with access_ok() before calling this function.
 933  *
 934  * Returns number of bytes that could not be cleared.
 935  * On success, this will be zero.
 936  */
 937 static inline __kernel_size_t
 938 __clear_user(void __user *addr, __kernel_size_t size)
 939 {
 940         __kernel_size_t res;
 941
 942         might_fault();
 943         __asm__ __volatile__(
 944                 "move\t$4, %1\n\t"
 945                 "move\t$5, $0\n\t"
 946                 "move\t$6, %2\n\t"
 947                 __MODULE_JAL(__bzero)
 948                 "move\t%0, $6"
 949                 : "=r" (res)
 950                 : "r" (addr), "r" (size)
 951                 : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31");
 952
 953         return res;
 954 }
 955
 956 #define clear_user(addr,n)                                              \
 957 ({                                                                      \
 958         void __user * __cl_addr = (addr);                               \
 959         unsigned long __cl_size = (n);                                  \
 960         if (__cl_size && access_ok(VERIFY_WRITE,                        \
 961                                         __cl_addr, __cl_size))          \
 962                 __cl_size = __clear_user(__cl_addr, __cl_size);         \
 963         __cl_size;                                                      \
 964 })
 965
 966 /*
 967  * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
 968  * @dst:   Destination address, in kernel space.  This buffer must be at
 969  *         least @count bytes long.
 970  * @src:   Source address, in user space.
 971  * @count: Maximum number of bytes to copy, including the trailing NUL.
 972  *
 973  * Copies a NUL-terminated string from userspace to kernel space.
 974  * Caller must check the specified block with access_ok() before calling
 975  * this function.
 976  *
 977  * On success, returns the length of the string (not including the trailing
 978  * NUL).
 979  *
 980  * If access to userspace fails, returns -EFAULT (some data may have been
 981  * copied).
 982  *
 983  * If @count is smaller than the length of the string, copies @count bytes
 984  * and returns @count.
 985  */
 986 static inline long
 987 __strncpy_from_user(char *__to, const char __user *__from, long __len)
 988 {
 989         long res;
 990
 991         might_fault();
 992         __asm__ __volatile__(
 993                 "move\t$4, %1\n\t"
 994                 "move\t$5, %2\n\t"
 995                 "move\t$6, %3\n\t"
 996                 __MODULE_JAL(__strncpy_from_user_nocheck_asm)
 997                 "move\t%0, $2"
 998                 : "=r" (res)
 999                 : "r" (__to), "r" (__from), "r" (__len)
1000                 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1001
1002         return res;
1003 }
1004
1005 /*
1006  * strncpy_from_user: - Copy a NUL terminated string from userspace.
1007  * @dst:   Destination address, in kernel space.  This buffer must be at
1008  *         least @count bytes long.
1009  * @src:   Source address, in user space.
1010  * @count: Maximum number of bytes to copy, including the trailing NUL.
1011  *
1012  * Copies a NUL-terminated string from userspace to kernel space.
1013  *
1014  * On success, returns the length of the string (not including the trailing
1015  * NUL).
1016  *
1017  * If access to userspace fails, returns -EFAULT (some data may have been
1018  * copied).
1019  *
1020  * If @count is smaller than the length of the string, copies @count bytes
1021  * and returns @count.
1022  */
1023 static inline long
1024 strncpy_from_user(char *__to, const char __user *__from, long __len)
1025 {
1026         long res;
1027
1028         might_fault();
1029         __asm__ __volatile__(
1030                 "move\t$4, %1\n\t"
1031                 "move\t$5, %2\n\t"
1032                 "move\t$6, %3\n\t"
1033                 __MODULE_JAL(__strncpy_from_user_asm)
1034                 "move\t%0, $2"
1035                 : "=r" (res)
1036                 : "r" (__to), "r" (__from), "r" (__len)
1037                 : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1038
1039         return res;
1040 }
1041
1042 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */
1043 static inline long __strlen_user(const char __user *s)
1044 {
1045         long res;
1046
1047         might_fault();
1048         __asm__ __volatile__(
1049                 "move\t$4, %1\n\t"
1050                 __MODULE_JAL(__strlen_user_nocheck_asm)
1051                 "move\t%0, $2"
1052                 : "=r" (res)
1053                 : "r" (s)
1054                 : "$2", "$4", __UA_t0, "$31");
1055
1056         return res;
1057 }
1058
1059 /*
1060  * strlen_user: - Get the size of a string in user space.
1061  * @str: The string to measure.
1062  *
1063  * Context: User context only.  This function may sleep.
1064  *
1065  * Get the size of a NUL-terminated string in user space.
1066  *
1067  * Returns the size of the string INCLUDING the terminating NUL.
1068  * On exception, returns 0.
1069  *
1070  * If there is a limit on the length of a valid string, you may wish to
1071  * consider using strnlen_user() instead.
1072  */
1073 static inline long strlen_user(const char __user *s)
1074 {
1075         long res;
1076
1077         might_fault();
1078         __asm__ __volatile__(
1079                 "move\t$4, %1\n\t"
1080                 __MODULE_JAL(__strlen_user_asm)
1081                 "move\t%0, $2"
1082                 : "=r" (res)
1083                 : "r" (s)
1084                 : "$2", "$4", __UA_t0, "$31");
1085
1086         return res;
1087 }
1088
1089 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */
1090 static inline long __strnlen_user(const char __user *s, long n)
1091 {
1092         long res;
1093
1094         might_fault();
1095         __asm__ __volatile__(
1096                 "move\t$4, %1\n\t"
1097                 "move\t$5, %2\n\t"
1098                 __MODULE_JAL(__strnlen_user_nocheck_asm)
1099                 "move\t%0, $2"
1100                 : "=r" (res)
1101                 : "r" (s), "r" (n)
1102                 : "$2", "$4", "$5", __UA_t0, "$31");
1103
1104         return res;
1105 }
1106
1107 /*
1108  * strlen_user: - Get the size of a string in user space.
1109  * @str: The string to measure.
1110  *
1111  * Context: User context only.  This function may sleep.
1112  *
1113  * Get the size of a NUL-terminated string in user space.
1114  *
1115  * Returns the size of the string INCLUDING the terminating NUL.
1116  * On exception, returns 0.
1117  *
1118  * If there is a limit on the length of a valid string, you may wish to
1119  * consider using strnlen_user() instead.
1120  */
1121 static inline long strnlen_user(const char __user *s, long n)
1122 {
1123         long res;
1124
1125         might_fault();
1126         __asm__ __volatile__(
1127                 "move\t$4, %1\n\t"
1128                 "move\t$5, %2\n\t"
1129                 __MODULE_JAL(__strnlen_user_asm)
1130                 "move\t%0, $2"
1131                 : "=r" (res)
1132                 : "r" (s), "r" (n)
1133                 : "$2", "$4", "$5", __UA_t0, "$31");
1134
1135         return res;
1136 }
1137
1138 struct exception_table_entry
1139 {
1140         unsigned long insn;
1141         unsigned long nextinsn;
1142 };
1143
1144 extern int fixup_exception(struct pt_regs *regs);
1145
1146 #endif /* _ASM_UACCESS_H */