2 * linux/fs/read_write.c
4 * Copyright (C) 1991, 1992 Linus Torvalds
7 #include <linux/slab.h>
8 #include <linux/stat.h>
9 #include <linux/fcntl.h>
10 #include <linux/file.h>
11 #include <linux/uio.h>
12 #include <linux/fsnotify.h>
13 #include <linux/security.h>
14 #include <linux/module.h>
15 #include <linux/syscalls.h>
16 #include <linux/pagemap.h>
17 #include <linux/splice.h>
18 #include "read_write.h"
20 #include <asm/uaccess.h>
21 #include <asm/unistd.h>
23 const struct file_operations generic_ro_fops = {
24 .llseek = generic_file_llseek,
26 .aio_read = generic_file_aio_read,
27 .mmap = generic_file_readonly_mmap,
28 .splice_read = generic_file_splice_read,
31 EXPORT_SYMBOL(generic_ro_fops);
34 __negative_fpos_check(struct file *file, loff_t pos, size_t count)
37 * pos or pos+count is negative here, check overflow.
38 * too big "count" will be caught in rw_verify_area().
40 if ((pos < 0) && (pos + count < pos))
42 if (file->f_mode & FMODE_UNSIGNED_OFFSET)
48 * generic_file_llseek_unlocked - lockless generic llseek implementation
49 * @file: file structure to seek on
50 * @offset: file offset to seek to
51 * @origin: type of seek
53 * Updates the file offset to the value specified by @offset and @origin.
54 * Locking must be provided by the caller.
57 generic_file_llseek_unlocked(struct file *file, loff_t offset, int origin)
59 struct inode *inode = file->f_mapping->host;
63 offset += inode->i_size;
67 * Here we special-case the lseek(fd, 0, SEEK_CUR)
68 * position-querying operation. Avoid rewriting the "same"
69 * f_pos value back to the file because a concurrent read(),
70 * write() or lseek() might have altered it
74 offset += file->f_pos;
78 if (offset < 0 && __negative_fpos_check(file, offset, 0))
80 if (offset > inode->i_sb->s_maxbytes)
83 /* Special lock needed here? */
84 if (offset != file->f_pos) {
91 EXPORT_SYMBOL(generic_file_llseek_unlocked);
94 * generic_file_llseek - generic llseek implementation for regular files
95 * @file: file structure to seek on
96 * @offset: file offset to seek to
97 * @origin: type of seek
99 * This is a generic implemenation of ->llseek useable for all normal local
100 * filesystems. It just updates the file offset to the value specified by
101 * @offset and @origin under i_mutex.
103 loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
107 mutex_lock(&file->f_dentry->d_inode->i_mutex);
108 rval = generic_file_llseek_unlocked(file, offset, origin);
109 mutex_unlock(&file->f_dentry->d_inode->i_mutex);
113 EXPORT_SYMBOL(generic_file_llseek);
116 * noop_llseek - No Operation Performed llseek implementation
117 * @file: file structure to seek on
118 * @offset: file offset to seek to
119 * @origin: type of seek
121 * This is an implementation of ->llseek useable for the rare special case when
122 * userspace expects the seek to succeed but the (device) file is actually not
123 * able to perform the seek. In this case you use noop_llseek() instead of
124 * falling back to the default implementation of ->llseek.
126 loff_t noop_llseek(struct file *file, loff_t offset, int origin)
130 EXPORT_SYMBOL(noop_llseek);
132 loff_t no_llseek(struct file *file, loff_t offset, int origin)
136 EXPORT_SYMBOL(no_llseek);
138 loff_t default_llseek(struct file *file, loff_t offset, int origin)
142 mutex_lock(&file->f_dentry->d_inode->i_mutex);
145 offset += i_size_read(file->f_path.dentry->d_inode);
149 retval = file->f_pos;
152 offset += file->f_pos;
155 if (offset >= 0 || !__negative_fpos_check(file, offset, 0)) {
156 if (offset != file->f_pos) {
157 file->f_pos = offset;
163 mutex_unlock(&file->f_dentry->d_inode->i_mutex);
166 EXPORT_SYMBOL(default_llseek);
168 loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
170 loff_t (*fn)(struct file *, loff_t, int);
173 if (file->f_mode & FMODE_LSEEK) {
174 if (file->f_op && file->f_op->llseek)
175 fn = file->f_op->llseek;
177 return fn(file, offset, origin);
179 EXPORT_SYMBOL(vfs_llseek);
181 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, origin)
188 file = fget_light(fd, &fput_needed);
193 if (origin <= SEEK_MAX) {
194 loff_t res = vfs_llseek(file, offset, origin);
196 if (res != (loff_t)retval)
197 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
199 fput_light(file, fput_needed);
204 #ifdef __ARCH_WANT_SYS_LLSEEK
205 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
206 unsigned long, offset_low, loff_t __user *, result,
207 unsigned int, origin)
215 file = fget_light(fd, &fput_needed);
220 if (origin > SEEK_MAX)
223 offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
226 retval = (int)offset;
229 if (!copy_to_user(result, &offset, sizeof(offset)))
233 fput_light(file, fput_needed);
241 * rw_verify_area doesn't like huge counts. We limit
242 * them to something that fits in "int" so that others
243 * won't have to do range checks all the time.
245 int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
249 int retval = -EINVAL;
251 inode = file->f_path.dentry->d_inode;
252 if (unlikely((ssize_t) count < 0))
255 if (unlikely((pos < 0) || (loff_t) (pos + count) < 0)) {
256 retval = __negative_fpos_check(file, pos, count);
261 if (unlikely(inode->i_flock && mandatory_lock(inode))) {
262 retval = locks_mandatory_area(
263 read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
264 inode, file, pos, count);
268 retval = security_file_permission(file,
269 read_write == READ ? MAY_READ : MAY_WRITE);
272 return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
275 static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
277 set_current_state(TASK_UNINTERRUPTIBLE);
278 if (!kiocbIsKicked(iocb))
281 kiocbClearKicked(iocb);
282 __set_current_state(TASK_RUNNING);
285 ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
287 struct iovec iov = { .iov_base = buf, .iov_len = len };
291 init_sync_kiocb(&kiocb, filp);
292 kiocb.ki_pos = *ppos;
294 kiocb.ki_nbytes = len;
297 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
298 if (ret != -EIOCBRETRY)
300 wait_on_retry_sync_kiocb(&kiocb);
303 if (-EIOCBQUEUED == ret)
304 ret = wait_on_sync_kiocb(&kiocb);
305 *ppos = kiocb.ki_pos;
309 EXPORT_SYMBOL(do_sync_read);
311 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
315 if (!(file->f_mode & FMODE_READ))
317 if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
319 if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
322 ret = rw_verify_area(READ, file, pos, count);
325 if (file->f_op->read)
326 ret = file->f_op->read(file, buf, count, pos);
328 ret = do_sync_read(file, buf, count, pos);
330 fsnotify_access(file);
331 add_rchar(current, ret);
339 EXPORT_SYMBOL(vfs_read);
341 ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
343 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
347 init_sync_kiocb(&kiocb, filp);
348 kiocb.ki_pos = *ppos;
350 kiocb.ki_nbytes = len;
353 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
354 if (ret != -EIOCBRETRY)
356 wait_on_retry_sync_kiocb(&kiocb);
359 if (-EIOCBQUEUED == ret)
360 ret = wait_on_sync_kiocb(&kiocb);
361 *ppos = kiocb.ki_pos;
365 EXPORT_SYMBOL(do_sync_write);
367 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
371 if (!(file->f_mode & FMODE_WRITE))
373 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
375 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
378 ret = rw_verify_area(WRITE, file, pos, count);
381 if (file->f_op->write)
382 ret = file->f_op->write(file, buf, count, pos);
384 ret = do_sync_write(file, buf, count, pos);
386 fsnotify_modify(file);
387 add_wchar(current, ret);
395 EXPORT_SYMBOL(vfs_write);
397 static inline loff_t file_pos_read(struct file *file)
402 static inline void file_pos_write(struct file *file, loff_t pos)
407 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
410 ssize_t ret = -EBADF;
413 file = fget_light(fd, &fput_needed);
415 loff_t pos = file_pos_read(file);
416 ret = vfs_read(file, buf, count, &pos);
417 file_pos_write(file, pos);
418 fput_light(file, fput_needed);
424 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
428 ssize_t ret = -EBADF;
431 file = fget_light(fd, &fput_needed);
433 loff_t pos = file_pos_read(file);
434 ret = vfs_write(file, buf, count, &pos);
435 file_pos_write(file, pos);
436 fput_light(file, fput_needed);
442 SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
443 size_t count, loff_t pos)
446 ssize_t ret = -EBADF;
452 file = fget_light(fd, &fput_needed);
455 if (file->f_mode & FMODE_PREAD)
456 ret = vfs_read(file, buf, count, &pos);
457 fput_light(file, fput_needed);
462 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
463 asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
465 return SYSC_pread64((unsigned int) fd, (char __user *) buf,
466 (size_t) count, pos);
468 SYSCALL_ALIAS(sys_pread64, SyS_pread64);
471 SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
472 size_t count, loff_t pos)
475 ssize_t ret = -EBADF;
481 file = fget_light(fd, &fput_needed);
484 if (file->f_mode & FMODE_PWRITE)
485 ret = vfs_write(file, buf, count, &pos);
486 fput_light(file, fput_needed);
491 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
492 asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
494 return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
495 (size_t) count, pos);
497 SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
501 * Reduce an iovec's length in-place. Return the resulting number of segments
503 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
505 unsigned long seg = 0;
508 while (seg < nr_segs) {
510 if (len + iov->iov_len >= to) {
511 iov->iov_len = to - len;
519 EXPORT_SYMBOL(iov_shorten);
521 ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
522 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
527 init_sync_kiocb(&kiocb, filp);
528 kiocb.ki_pos = *ppos;
530 kiocb.ki_nbytes = len;
533 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
534 if (ret != -EIOCBRETRY)
536 wait_on_retry_sync_kiocb(&kiocb);
539 if (ret == -EIOCBQUEUED)
540 ret = wait_on_sync_kiocb(&kiocb);
541 *ppos = kiocb.ki_pos;
545 /* Do it by hand, with file-ops */
546 ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
547 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
549 struct iovec *vector = iov;
552 while (nr_segs > 0) {
557 base = vector->iov_base;
558 len = vector->iov_len;
562 nr = fn(filp, base, len, ppos);
577 /* A write operation does a read from user space and vice versa */
578 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
580 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
581 unsigned long nr_segs, unsigned long fast_segs,
582 struct iovec *fast_pointer,
583 struct iovec **ret_pointer)
587 struct iovec *iov = fast_pointer;
590 * SuS says "The readv() function *may* fail if the iovcnt argument
591 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
592 * traditionally returned zero for zero segments, so...
600 * First get the "struct iovec" from user memory and
601 * verify all the pointers
603 if (nr_segs > UIO_MAXIOV) {
607 if (nr_segs > fast_segs) {
608 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
614 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
620 * According to the Single Unix Specification we should return EINVAL
621 * if an element length is < 0 when cast to ssize_t or if the
622 * total length would overflow the ssize_t return value of the
625 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
629 for (seg = 0; seg < nr_segs; seg++) {
630 void __user *buf = iov[seg].iov_base;
631 ssize_t len = (ssize_t)iov[seg].iov_len;
633 /* see if we we're about to use an invalid len or if
634 * it's about to overflow ssize_t */
639 if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
643 if (len > MAX_RW_COUNT - ret) {
644 len = MAX_RW_COUNT - ret;
645 iov[seg].iov_len = len;
654 static ssize_t do_readv_writev(int type, struct file *file,
655 const struct iovec __user * uvector,
656 unsigned long nr_segs, loff_t *pos)
659 struct iovec iovstack[UIO_FASTIOV];
660 struct iovec *iov = iovstack;
670 ret = rw_copy_check_uvector(type, uvector, nr_segs,
671 ARRAY_SIZE(iovstack), iovstack, &iov);
676 ret = rw_verify_area(type, file, pos, tot_len);
682 fn = file->f_op->read;
683 fnv = file->f_op->aio_read;
685 fn = (io_fn_t)file->f_op->write;
686 fnv = file->f_op->aio_write;
690 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
693 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
698 if ((ret + (type == READ)) > 0) {
700 fsnotify_access(file);
702 fsnotify_modify(file);
707 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
708 unsigned long vlen, loff_t *pos)
710 if (!(file->f_mode & FMODE_READ))
712 if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
715 return do_readv_writev(READ, file, vec, vlen, pos);
718 EXPORT_SYMBOL(vfs_readv);
720 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
721 unsigned long vlen, loff_t *pos)
723 if (!(file->f_mode & FMODE_WRITE))
725 if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
728 return do_readv_writev(WRITE, file, vec, vlen, pos);
731 EXPORT_SYMBOL(vfs_writev);
733 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
737 ssize_t ret = -EBADF;
740 file = fget_light(fd, &fput_needed);
742 loff_t pos = file_pos_read(file);
743 ret = vfs_readv(file, vec, vlen, &pos);
744 file_pos_write(file, pos);
745 fput_light(file, fput_needed);
749 add_rchar(current, ret);
754 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
758 ssize_t ret = -EBADF;
761 file = fget_light(fd, &fput_needed);
763 loff_t pos = file_pos_read(file);
764 ret = vfs_writev(file, vec, vlen, &pos);
765 file_pos_write(file, pos);
766 fput_light(file, fput_needed);
770 add_wchar(current, ret);
775 static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
777 #define HALF_LONG_BITS (BITS_PER_LONG / 2)
778 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
781 SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
782 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
784 loff_t pos = pos_from_hilo(pos_h, pos_l);
786 ssize_t ret = -EBADF;
792 file = fget_light(fd, &fput_needed);
795 if (file->f_mode & FMODE_PREAD)
796 ret = vfs_readv(file, vec, vlen, &pos);
797 fput_light(file, fput_needed);
801 add_rchar(current, ret);
806 SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
807 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
809 loff_t pos = pos_from_hilo(pos_h, pos_l);
811 ssize_t ret = -EBADF;
817 file = fget_light(fd, &fput_needed);
820 if (file->f_mode & FMODE_PWRITE)
821 ret = vfs_writev(file, vec, vlen, &pos);
822 fput_light(file, fput_needed);
826 add_wchar(current, ret);
831 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
832 size_t count, loff_t max)
834 struct file * in_file, * out_file;
835 struct inode * in_inode, * out_inode;
838 int fput_needed_in, fput_needed_out, fl;
841 * Get input file, and verify that it is ok..
844 in_file = fget_light(in_fd, &fput_needed_in);
847 if (!(in_file->f_mode & FMODE_READ))
851 ppos = &in_file->f_pos;
853 if (!(in_file->f_mode & FMODE_PREAD))
855 retval = rw_verify_area(READ, in_file, ppos, count);
861 * Get output file, and verify that it is ok..
864 out_file = fget_light(out_fd, &fput_needed_out);
867 if (!(out_file->f_mode & FMODE_WRITE))
870 in_inode = in_file->f_path.dentry->d_inode;
871 out_inode = out_file->f_path.dentry->d_inode;
872 retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
878 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
881 if (unlikely(pos + count > max)) {
891 * We need to debate whether we can enable this or not. The
892 * man page documents EAGAIN return for the output at least,
893 * and the application is arguably buggy if it doesn't expect
894 * EAGAIN on a non-blocking file descriptor.
896 if (in_file->f_flags & O_NONBLOCK)
897 fl = SPLICE_F_NONBLOCK;
899 retval = do_splice_direct(in_file, ppos, out_file, count, fl);
902 add_rchar(current, retval);
903 add_wchar(current, retval);
912 fput_light(out_file, fput_needed_out);
914 fput_light(in_file, fput_needed_in);
919 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
926 if (unlikely(get_user(off, offset)))
929 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
930 if (unlikely(put_user(pos, offset)))
935 return do_sendfile(out_fd, in_fd, NULL, count, 0);
938 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
944 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
946 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
947 if (unlikely(put_user(pos, offset)))
952 return do_sendfile(out_fd, in_fd, NULL, count, 0);