fs/fuse/dev.c

   1 /*
   2   FUSE: Filesystem in Userspace
   3   Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
   4
   5   This program can be distributed under the terms of the GNU GPL.
   6   See the file COPYING.
   7 */
   8
   9 #include "fuse_i.h"
  10
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/poll.h>
  14 #include <linux/uio.h>
  15 #include <linux/miscdevice.h>
  16 #include <linux/pagemap.h>
  17 #include <linux/file.h>
  18 #include <linux/slab.h>
  19 #include <linux/pipe_fs_i.h>
  20 #include <linux/swap.h>
  21 #include <linux/splice.h>
  22
  23 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
  24
  25 static struct kmem_cache *fuse_req_cachep;
  26
  27 static struct fuse_conn *fuse_get_conn(struct file *file)
  28 {
  29         /*
  30          * Lockless access is OK, because file->private data is set
  31          * once during mount and is valid until the file is released.
  32          */
  33         return file->private_data;
  34 }
  35
  36 static void fuse_request_init(struct fuse_req *req)
  37 {
  38         memset(req, 0, sizeof(*req));
  39         INIT_LIST_HEAD(&req->list);
  40         INIT_LIST_HEAD(&req->intr_entry);
  41         init_waitqueue_head(&req->waitq);
  42         atomic_set(&req->count, 1);
  43 }
  44
  45 struct fuse_req *fuse_request_alloc(void)
  46 {
  47         struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, GFP_KERNEL);
  48         if (req)
  49                 fuse_request_init(req);
  50         return req;
  51 }
  52 EXPORT_SYMBOL_GPL(fuse_request_alloc);
  53
  54 struct fuse_req *fuse_request_alloc_nofs(void)
  55 {
  56         struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, GFP_NOFS);
  57         if (req)
  58                 fuse_request_init(req);
  59         return req;
  60 }
  61
  62 void fuse_request_free(struct fuse_req *req)
  63 {
  64         kmem_cache_free(fuse_req_cachep, req);
  65 }
  66
  67 static void block_sigs(sigset_t *oldset)
  68 {
  69         sigset_t mask;
  70
  71         siginitsetinv(&mask, sigmask(SIGKILL));
  72         sigprocmask(SIG_BLOCK, &mask, oldset);
  73 }
  74
  75 static void restore_sigs(sigset_t *oldset)
  76 {
  77         sigprocmask(SIG_SETMASK, oldset, NULL);
  78 }
  79
  80 static void __fuse_get_request(struct fuse_req *req)
  81 {
  82         atomic_inc(&req->count);
  83 }
  84
  85 /* Must be called with > 1 refcount */
  86 static void __fuse_put_request(struct fuse_req *req)
  87 {
  88         BUG_ON(atomic_read(&req->count) < 2);
  89         atomic_dec(&req->count);
  90 }
  91
  92 static void fuse_req_init_context(struct fuse_req *req)
  93 {
  94         req->in.h.uid = current_fsuid();
  95         req->in.h.gid = current_fsgid();
  96         req->in.h.pid = current->pid;
  97 }
  98
  99 struct fuse_req *fuse_get_req(struct fuse_conn *fc)
 100 {
 101         struct fuse_req *req;
 102         sigset_t oldset;
 103         int intr;
 104         int err;
 105
 106         atomic_inc(&fc->num_waiting);
 107         block_sigs(&oldset);
 108         intr = wait_event_interruptible(fc->blocked_waitq, !fc->blocked);
 109         restore_sigs(&oldset);
 110         err = -EINTR;
 111         if (intr)
 112                 goto out;
 113
 114         err = -ENOTCONN;
 115         if (!fc->connected)
 116                 goto out;
 117
 118         req = fuse_request_alloc();
 119         err = -ENOMEM;
 120         if (!req)
 121                 goto out;
 122
 123         fuse_req_init_context(req);
 124         req->waiting = 1;
 125         return req;
 126
 127  out:
 128         atomic_dec(&fc->num_waiting);
 129         return ERR_PTR(err);
 130 }
 131 EXPORT_SYMBOL_GPL(fuse_get_req);
 132
 133 /*
 134  * Return request in fuse_file->reserved_req.  However that may
 135  * currently be in use.  If that is the case, wait for it to become
 136  * available.
 137  */
 138 static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
 139                                          struct file *file)
 140 {
 141         struct fuse_req *req = NULL;
 142         struct fuse_file *ff = file->private_data;
 143
 144         do {
 145                 wait_event(fc->reserved_req_waitq, ff->reserved_req);
 146                 spin_lock(&fc->lock);
 147                 if (ff->reserved_req) {
 148                         req = ff->reserved_req;
 149                         ff->reserved_req = NULL;
 150                         get_file(file);
 151                         req->stolen_file = file;
 152                 }
 153                 spin_unlock(&fc->lock);
 154         } while (!req);
 155
 156         return req;
 157 }
 158
 159 /*
 160  * Put stolen request back into fuse_file->reserved_req
 161  */
 162 static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
 163 {
 164         struct file *file = req->stolen_file;
 165         struct fuse_file *ff = file->private_data;
 166
 167         spin_lock(&fc->lock);
 168         fuse_request_init(req);
 169         BUG_ON(ff->reserved_req);
 170         ff->reserved_req = req;
 171         wake_up_all(&fc->reserved_req_waitq);
 172         spin_unlock(&fc->lock);
 173         fput(file);
 174 }
 175
 176 /*
 177  * Gets a requests for a file operation, always succeeds
 178  *
 179  * This is used for sending the FLUSH request, which must get to
 180  * userspace, due to POSIX locks which may need to be unlocked.
 181  *
 182  * If allocation fails due to OOM, use the reserved request in
 183  * fuse_file.
 184  *
 185  * This is very unlikely to deadlock accidentally, since the
 186  * filesystem should not have it's own file open.  If deadlock is
 187  * intentional, it can still be broken by "aborting" the filesystem.
 188  */
 189 struct fuse_req *fuse_get_req_nofail(struct fuse_conn *fc, struct file *file)
 190 {
 191         struct fuse_req *req;
 192
 193         atomic_inc(&fc->num_waiting);
 194         wait_event(fc->blocked_waitq, !fc->blocked);
 195         req = fuse_request_alloc();
 196         if (!req)
 197                 req = get_reserved_req(fc, file);
 198
 199         fuse_req_init_context(req);
 200         req->waiting = 1;
 201         return req;
 202 }
 203
 204 void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
 205 {
 206         if (atomic_dec_and_test(&req->count)) {
 207                 if (req->waiting)
 208                         atomic_dec(&fc->num_waiting);
 209
 210                 if (req->stolen_file)
 211                         put_reserved_req(fc, req);
 212                 else
 213                         fuse_request_free(req);
 214         }
 215 }
 216 EXPORT_SYMBOL_GPL(fuse_put_request);
 217
 218 static unsigned len_args(unsigned numargs, struct fuse_arg *args)
 219 {
 220         unsigned nbytes = 0;
 221         unsigned i;
 222
 223         for (i = 0; i < numargs; i++)
 224                 nbytes += args[i].size;
 225
 226         return nbytes;
 227 }
 228
 229 static u64 fuse_get_unique(struct fuse_conn *fc)
 230 {
 231         fc->reqctr++;
 232         /* zero is special */
 233         if (fc->reqctr == 0)
 234                 fc->reqctr = 1;
 235
 236         return fc->reqctr;
 237 }
 238
 239 static void queue_request(struct fuse_conn *fc, struct fuse_req *req)
 240 {
 241         req->in.h.unique = fuse_get_unique(fc);
 242         req->in.h.len = sizeof(struct fuse_in_header) +
 243                 len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
 244         list_add_tail(&req->list, &fc->pending);
 245         req->state = FUSE_REQ_PENDING;
 246         if (!req->waiting) {
 247                 req->waiting = 1;
 248                 atomic_inc(&fc->num_waiting);
 249         }
 250         wake_up(&fc->waitq);
 251         kill_fasync(&fc->fasync, SIGIO, POLL_IN);
 252 }
 253
 254 static void flush_bg_queue(struct fuse_conn *fc)
 255 {
 256         while (fc->active_background < fc->max_background &&
 257                !list_empty(&fc->bg_queue)) {
 258                 struct fuse_req *req;
 259
 260                 req = list_entry(fc->bg_queue.next, struct fuse_req, list);
 261                 list_del(&req->list);
 262                 fc->active_background++;
 263                 queue_request(fc, req);
 264         }
 265 }
 266
 267 /*
 268  * This function is called when a request is finished.  Either a reply
 269  * has arrived or it was aborted (and not yet sent) or some error
 270  * occurred during communication with userspace, or the device file
 271  * was closed.  The requester thread is woken up (if still waiting),
 272  * the 'end' callback is called if given, else the reference to the
 273  * request is released
 274  *
 275  * Called with fc->lock, unlocks it
 276  */
 277 static void request_end(struct fuse_conn *fc, struct fuse_req *req)
 278 __releases(&fc->lock)
 279 {
 280         void (*end) (struct fuse_conn *, struct fuse_req *) = req->end;
 281         req->end = NULL;
 282         list_del(&req->list);
 283         list_del(&req->intr_entry);
 284         req->state = FUSE_REQ_FINISHED;
 285         if (req->background) {
 286                 if (fc->num_background == fc->max_background) {
 287                         fc->blocked = 0;
 288                         wake_up_all(&fc->blocked_waitq);
 289                 }
 290                 if (fc->num_background == fc->congestion_threshold &&
 291                     fc->connected && fc->bdi_initialized) {
 292                         clear_bdi_congested(&fc->bdi, BLK_RW_SYNC);
 293                         clear_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
 294                 }
 295                 fc->num_background--;
 296                 fc->active_background--;
 297                 flush_bg_queue(fc);
 298         }
 299         spin_unlock(&fc->lock);
 300         wake_up(&req->waitq);
 301         if (end)
 302                 end(fc, req);
 303         fuse_put_request(fc, req);
 304 }
 305
 306 static void wait_answer_interruptible(struct fuse_conn *fc,
 307                                       struct fuse_req *req)
 308 __releases(&fc->lock)
 309 __acquires(&fc->lock)
 310 {
 311         if (signal_pending(current))
 312                 return;
 313
 314         spin_unlock(&fc->lock);
 315         wait_event_interruptible(req->waitq, req->state == FUSE_REQ_FINISHED);
 316         spin_lock(&fc->lock);
 317 }
 318
 319 static void queue_interrupt(struct fuse_conn *fc, struct fuse_req *req)
 320 {
 321         list_add_tail(&req->intr_entry, &fc->interrupts);
 322         wake_up(&fc->waitq);
 323         kill_fasync(&fc->fasync, SIGIO, POLL_IN);
 324 }
 325
 326 static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
 327 __releases(&fc->lock)
 328 __acquires(&fc->lock)
 329 {
 330         if (!fc->no_interrupt) {
 331                 /* Any signal may interrupt this */
 332                 wait_answer_interruptible(fc, req);
 333
 334                 if (req->aborted)
 335                         goto aborted;
 336                 if (req->state == FUSE_REQ_FINISHED)
 337                         return;
 338
 339                 req->interrupted = 1;
 340                 if (req->state == FUSE_REQ_SENT)
 341                         queue_interrupt(fc, req);
 342         }
 343
 344         if (!req->force) {
 345                 sigset_t oldset;
 346
 347                 /* Only fatal signals may interrupt this */
 348                 block_sigs(&oldset);
 349                 wait_answer_interruptible(fc, req);
 350                 restore_sigs(&oldset);
 351
 352                 if (req->aborted)
 353                         goto aborted;
 354                 if (req->state == FUSE_REQ_FINISHED)
 355                         return;
 356
 357                 /* Request is not yet in userspace, bail out */
 358                 if (req->state == FUSE_REQ_PENDING) {
 359                         list_del(&req->list);
 360                         __fuse_put_request(req);
 361                         req->out.h.error = -EINTR;
 362                         return;
 363                 }
 364         }
 365
 366         /*
 367          * Either request is already in userspace, or it was forced.
 368          * Wait it out.
 369          */
 370         spin_unlock(&fc->lock);
 371         wait_event(req->waitq, req->state == FUSE_REQ_FINISHED);
 372         spin_lock(&fc->lock);
 373
 374         if (!req->aborted)
 375                 return;
 376
 377  aborted:
 378         BUG_ON(req->state != FUSE_REQ_FINISHED);
 379         if (req->locked) {
 380                 /* This is uninterruptible sleep, because data is
 381                    being copied to/from the buffers of req.  During
 382                    locked state, there mustn't be any filesystem
 383                    operation (e.g. page fault), since that could lead
 384                    to deadlock */
 385                 spin_unlock(&fc->lock);
 386                 wait_event(req->waitq, !req->locked);
 387                 spin_lock(&fc->lock);
 388         }
 389 }
 390
 391 void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
 392 {
 393         req->isreply = 1;
 394         spin_lock(&fc->lock);
 395         if (!fc->connected)
 396                 req->out.h.error = -ENOTCONN;
 397         else if (fc->conn_error)
 398                 req->out.h.error = -ECONNREFUSED;
 399         else {
 400                 queue_request(fc, req);
 401                 /* acquire extra reference, since request is still needed
 402                    after request_end() */
 403                 __fuse_get_request(req);
 404
 405                 request_wait_answer(fc, req);
 406         }
 407         spin_unlock(&fc->lock);
 408 }
 409 EXPORT_SYMBOL_GPL(fuse_request_send);
 410
 411 static void fuse_request_send_nowait_locked(struct fuse_conn *fc,
 412                                             struct fuse_req *req)
 413 {
 414         req->background = 1;
 415         fc->num_background++;
 416         if (fc->num_background == fc->max_background)
 417                 fc->blocked = 1;
 418         if (fc->num_background == fc->congestion_threshold &&
 419             fc->bdi_initialized) {
 420                 set_bdi_congested(&fc->bdi, BLK_RW_SYNC);
 421                 set_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
 422         }
 423         list_add_tail(&req->list, &fc->bg_queue);
 424         flush_bg_queue(fc);
 425 }
 426
 427 static void fuse_request_send_nowait(struct fuse_conn *fc, struct fuse_req *req)
 428 {
 429         spin_lock(&fc->lock);
 430         if (fc->connected) {
 431                 fuse_request_send_nowait_locked(fc, req);
 432                 spin_unlock(&fc->lock);
 433         } else {
 434                 req->out.h.error = -ENOTCONN;
 435                 request_end(fc, req);
 436         }
 437 }
 438
 439 void fuse_request_send_noreply(struct fuse_conn *fc, struct fuse_req *req)
 440 {
 441         req->isreply = 0;
 442         fuse_request_send_nowait(fc, req);
 443 }
 444
 445 void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
 446 {
 447         req->isreply = 1;
 448         fuse_request_send_nowait(fc, req);
 449 }
 450 EXPORT_SYMBOL_GPL(fuse_request_send_background);
 451
 452 /*
 453  * Called under fc->lock
 454  *
 455  * fc->connected must have been checked previously
 456  */
 457 void fuse_request_send_background_locked(struct fuse_conn *fc,
 458                                          struct fuse_req *req)
 459 {
 460         req->isreply = 1;
 461         fuse_request_send_nowait_locked(fc, req);
 462 }
 463
 464 /*
 465  * Lock the request.  Up to the next unlock_request() there mustn't be
 466  * anything that could cause a page-fault.  If the request was already
 467  * aborted bail out.
 468  */
 469 static int lock_request(struct fuse_conn *fc, struct fuse_req *req)
 470 {
 471         int err = 0;
 472         if (req) {
 473                 spin_lock(&fc->lock);
 474                 if (req->aborted)
 475                         err = -ENOENT;
 476                 else
 477                         req->locked = 1;
 478                 spin_unlock(&fc->lock);
 479         }
 480         return err;
 481 }
 482
 483 /*
 484  * Unlock request.  If it was aborted during being locked, the
 485  * requester thread is currently waiting for it to be unlocked, so
 486  * wake it up.
 487  */
 488 static void unlock_request(struct fuse_conn *fc, struct fuse_req *req)
 489 {
 490         if (req) {
 491                 spin_lock(&fc->lock);
 492                 req->locked = 0;
 493                 if (req->aborted)
 494                         wake_up(&req->waitq);
 495                 spin_unlock(&fc->lock);
 496         }
 497 }
 498
 499 struct fuse_copy_state {
 500         struct fuse_conn *fc;
 501         int write;
 502         struct fuse_req *req;
 503         const struct iovec *iov;
 504         struct pipe_buffer *pipebufs;
 505         struct pipe_buffer *currbuf;
 506         struct pipe_inode_info *pipe;
 507         unsigned long nr_segs;
 508         unsigned long seglen;
 509         unsigned long addr;
 510         struct page *pg;
 511         void *mapaddr;
 512         void *buf;
 513         unsigned len;
 514         unsigned move_pages:1;
 515 };
 516
 517 static void fuse_copy_init(struct fuse_copy_state *cs, struct fuse_conn *fc,
 518                            int write,
 519                            const struct iovec *iov, unsigned long nr_segs)
 520 {
 521         memset(cs, 0, sizeof(*cs));
 522         cs->fc = fc;
 523         cs->write = write;
 524         cs->iov = iov;
 525         cs->nr_segs = nr_segs;
 526 }
 527
 528 /* Unmap and put previous page of userspace buffer */
 529 static void fuse_copy_finish(struct fuse_copy_state *cs)
 530 {
 531         if (cs->currbuf) {
 532                 struct pipe_buffer *buf = cs->currbuf;
 533
 534                 if (!cs->write) {
 535                         buf->ops->unmap(cs->pipe, buf, cs->mapaddr);
 536                 } else {
 537                         kunmap_atomic(cs->mapaddr, KM_USER0);
 538                         buf->len = PAGE_SIZE - cs->len;
 539                 }
 540                 cs->currbuf = NULL;
 541                 cs->mapaddr = NULL;
 542         } else if (cs->mapaddr) {
 543                 kunmap_atomic(cs->mapaddr, KM_USER0);
 544                 if (cs->write) {
 545                         flush_dcache_page(cs->pg);
 546                         set_page_dirty_lock(cs->pg);
 547                 }
 548                 put_page(cs->pg);
 549                 cs->mapaddr = NULL;
 550         }
 551 }
 552
 553 /*
 554  * Get another pagefull of userspace buffer, and map it to kernel
 555  * address space, and lock request
 556  */
 557 static int fuse_copy_fill(struct fuse_copy_state *cs)
 558 {
 559         unsigned long offset;
 560         int err;
 561
 562         unlock_request(cs->fc, cs->req);
 563         fuse_copy_finish(cs);
 564         if (cs->pipebufs) {
 565                 struct pipe_buffer *buf = cs->pipebufs;
 566
 567                 if (!cs->write) {
 568                         err = buf->ops->confirm(cs->pipe, buf);
 569                         if (err)
 570                                 return err;
 571
 572                         BUG_ON(!cs->nr_segs);
 573                         cs->currbuf = buf;
 574                         cs->mapaddr = buf->ops->map(cs->pipe, buf, 1);
 575                         cs->len = buf->len;
 576                         cs->buf = cs->mapaddr + buf->offset;
 577                         cs->pipebufs++;
 578                         cs->nr_segs--;
 579                 } else {
 580                         struct page *page;
 581
 582                         if (cs->nr_segs == cs->pipe->buffers)
 583                                 return -EIO;
 584
 585                         page = alloc_page(GFP_HIGHUSER);
 586                         if (!page)
 587                                 return -ENOMEM;
 588
 589                         buf->page = page;
 590                         buf->offset = 0;
 591                         buf->len = 0;
 592
 593                         cs->currbuf = buf;
 594                         cs->mapaddr = kmap_atomic(page, KM_USER0);
 595                         cs->buf = cs->mapaddr;
 596                         cs->len = PAGE_SIZE;
 597                         cs->pipebufs++;
 598                         cs->nr_segs++;
 599                 }
 600         } else {
 601                 if (!cs->seglen) {
 602                         BUG_ON(!cs->nr_segs);
 603                         cs->seglen = cs->iov[0].iov_len;
 604                         cs->addr = (unsigned long) cs->iov[0].iov_base;
 605                         cs->iov++;
 606                         cs->nr_segs--;
 607                 }
 608                 err = get_user_pages_fast(cs->addr, 1, cs->write, &cs->pg);
 609                 if (err < 0)
 610                         return err;
 611                 BUG_ON(err != 1);
 612                 offset = cs->addr % PAGE_SIZE;
 613                 cs->mapaddr = kmap_atomic(cs->pg, KM_USER0);
 614                 cs->buf = cs->mapaddr + offset;
 615                 cs->len = min(PAGE_SIZE - offset, cs->seglen);
 616                 cs->seglen -= cs->len;
 617                 cs->addr += cs->len;
 618         }
 619
 620         return lock_request(cs->fc, cs->req);
 621 }
 622
 623 /* Do as much copy to/from userspace buffer as we can */
 624 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
 625 {
 626         unsigned ncpy = min(*size, cs->len);
 627         if (val) {
 628                 if (cs->write)
 629                         memcpy(cs->buf, *val, ncpy);
 630                 else
 631                         memcpy(*val, cs->buf, ncpy);
 632                 *val += ncpy;
 633         }
 634         *size -= ncpy;
 635         cs->len -= ncpy;
 636         cs->buf += ncpy;
 637         return ncpy;
 638 }
 639
 640 static int fuse_check_page(struct page *page)
 641 {
 642         if (page_mapcount(page) ||
 643             page->mapping != NULL ||
 644             page_count(page) != 1 ||
 645             (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
 646              ~(1 << PG_locked |
 647                1 << PG_referenced |
 648                1 << PG_uptodate |
 649                1 << PG_lru |
 650                1 << PG_active |
 651                1 << PG_reclaim))) {
 652                 printk(KERN_WARNING "fuse: trying to steal weird page\n");
 653                 printk(KERN_WARNING "  page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
 654                 return 1;
 655         }
 656         return 0;
 657 }
 658
 659 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 660 {
 661         int err;
 662         struct page *oldpage = *pagep;
 663         struct page *newpage;
 664         struct pipe_buffer *buf = cs->pipebufs;
 665         struct address_space *mapping;
 666         pgoff_t index;
 667
 668         unlock_request(cs->fc, cs->req);
 669         fuse_copy_finish(cs);
 670
 671         err = buf->ops->confirm(cs->pipe, buf);
 672         if (err)
 673                 return err;
 674
 675         BUG_ON(!cs->nr_segs);
 676         cs->currbuf = buf;
 677         cs->len = buf->len;
 678         cs->pipebufs++;
 679         cs->nr_segs--;
 680
 681         if (cs->len != PAGE_SIZE)
 682                 goto out_fallback;
 683
 684         if (buf->ops->steal(cs->pipe, buf) != 0)
 685                 goto out_fallback;
 686
 687         newpage = buf->page;
 688
 689         if (WARN_ON(!PageUptodate(newpage)))
 690                 return -EIO;
 691
 692         ClearPageMappedToDisk(newpage);
 693
 694         if (fuse_check_page(newpage) != 0)
 695                 goto out_fallback_unlock;
 696
 697         mapping = oldpage->mapping;
 698         index = oldpage->index;
 699
 700         /*
 701          * This is a new and locked page, it shouldn't be mapped or
 702          * have any special flags on it
 703          */
 704         if (WARN_ON(page_mapped(oldpage)))
 705                 goto out_fallback_unlock;
 706         if (WARN_ON(page_has_private(oldpage)))
 707                 goto out_fallback_unlock;
 708         if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
 709                 goto out_fallback_unlock;
 710         if (WARN_ON(PageMlocked(oldpage)))
 711                 goto out_fallback_unlock;
 712
 713         remove_from_page_cache(oldpage);
 714         page_cache_release(oldpage);
 715
 716         err = add_to_page_cache_locked(newpage, mapping, index, GFP_KERNEL);
 717         if (err) {
 718                 printk(KERN_WARNING "fuse_try_move_page: failed to add page");
 719                 goto out_fallback_unlock;
 720         }
 721         page_cache_get(newpage);
 722
 723         if (!(buf->flags & PIPE_BUF_FLAG_LRU))
 724                 lru_cache_add_file(newpage);
 725
 726         err = 0;
 727         spin_lock(&cs->fc->lock);
 728         if (cs->req->aborted)
 729                 err = -ENOENT;
 730         else
 731                 *pagep = newpage;
 732         spin_unlock(&cs->fc->lock);
 733
 734         if (err) {
 735                 unlock_page(newpage);
 736                 page_cache_release(newpage);
 737                 return err;
 738         }
 739
 740         unlock_page(oldpage);
 741         page_cache_release(oldpage);
 742         cs->len = 0;
 743
 744         return 0;
 745
 746 out_fallback_unlock:
 747         unlock_page(newpage);
 748 out_fallback:
 749         cs->mapaddr = buf->ops->map(cs->pipe, buf, 1);
 750         cs->buf = cs->mapaddr + buf->offset;
 751
 752         err = lock_request(cs->fc, cs->req);
 753         if (err)
 754                 return err;
 755
 756         return 1;
 757 }
 758
 759 static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
 760                          unsigned offset, unsigned count)
 761 {
 762         struct pipe_buffer *buf;
 763
 764         if (cs->nr_segs == cs->pipe->buffers)
 765                 return -EIO;
 766
 767         unlock_request(cs->fc, cs->req);
 768         fuse_copy_finish(cs);
 769
 770         buf = cs->pipebufs;
 771         page_cache_get(page);
 772         buf->page = page;
 773         buf->offset = offset;
 774         buf->len = count;
 775
 776         cs->pipebufs++;
 777         cs->nr_segs++;
 778         cs->len = 0;
 779
 780         return 0;
 781 }
 782
 783 /*
 784  * Copy a page in the request to/from the userspace buffer.  Must be
 785  * done atomically
 786  */
 787 static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
 788                           unsigned offset, unsigned count, int zeroing)
 789 {
 790         int err;
 791         struct page *page = *pagep;
 792
 793         if (page && zeroing && count < PAGE_SIZE) {
 794                 void *mapaddr = kmap_atomic(page, KM_USER1);
 795                 memset(mapaddr, 0, PAGE_SIZE);
 796                 kunmap_atomic(mapaddr, KM_USER1);
 797         }
 798         while (count) {
 799                 if (cs->write && cs->pipebufs && page) {
 800                         return fuse_ref_page(cs, page, offset, count);
 801                 } else if (!cs->len) {
 802                         if (cs->move_pages && page &&
 803                             offset == 0 && count == PAGE_SIZE) {
 804                                 err = fuse_try_move_page(cs, pagep);
 805                                 if (err <= 0)
 806                                         return err;
 807                         } else {
 808                                 err = fuse_copy_fill(cs);
 809                                 if (err)
 810                                         return err;
 811                         }
 812                 }
 813                 if (page) {
 814                         void *mapaddr = kmap_atomic(page, KM_USER1);
 815                         void *buf = mapaddr + offset;
 816                         offset += fuse_copy_do(cs, &buf, &count);
 817                         kunmap_atomic(mapaddr, KM_USER1);
 818                 } else
 819                         offset += fuse_copy_do(cs, NULL, &count);
 820         }
 821         if (page && !cs->write)
 822                 flush_dcache_page(page);
 823         return 0;
 824 }
 825
 826 /* Copy pages in the request to/from userspace buffer */
 827 static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
 828                            int zeroing)
 829 {
 830         unsigned i;
 831         struct fuse_req *req = cs->req;
 832         unsigned offset = req->page_offset;
 833         unsigned count = min(nbytes, (unsigned) PAGE_SIZE - offset);
 834
 835         for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
 836                 int err;
 837
 838                 err = fuse_copy_page(cs, &req->pages[i], offset, count,
 839                                      zeroing);
 840                 if (err)
 841                         return err;
 842
 843                 nbytes -= count;
 844                 count = min(nbytes, (unsigned) PAGE_SIZE);
 845                 offset = 0;
 846         }
 847         return 0;
 848 }
 849
 850 /* Copy a single argument in the request to/from userspace buffer */
 851 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
 852 {
 853         while (size) {
 854                 if (!cs->len) {
 855                         int err = fuse_copy_fill(cs);
 856                         if (err)
 857                                 return err;
 858                 }
 859                 fuse_copy_do(cs, &val, &size);
 860         }
 861         return 0;
 862 }
 863
 864 /* Copy request arguments to/from userspace buffer */
 865 static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
 866                           unsigned argpages, struct fuse_arg *args,
 867                           int zeroing)
 868 {
 869         int err = 0;
 870         unsigned i;
 871
 872         for (i = 0; !err && i < numargs; i++)  {
 873                 struct fuse_arg *arg = &args[i];
 874                 if (i == numargs - 1 && argpages)
 875                         err = fuse_copy_pages(cs, arg->size, zeroing);
 876                 else
 877                         err = fuse_copy_one(cs, arg->value, arg->size);
 878         }
 879         return err;
 880 }
 881
 882 static int request_pending(struct fuse_conn *fc)
 883 {
 884         return !list_empty(&fc->pending) || !list_empty(&fc->interrupts);
 885 }
 886
 887 /* Wait until a request is available on the pending list */
 888 static void request_wait(struct fuse_conn *fc)
 889 __releases(&fc->lock)
 890 __acquires(&fc->lock)
 891 {
 892         DECLARE_WAITQUEUE(wait, current);
 893
 894         add_wait_queue_exclusive(&fc->waitq, &wait);
 895         while (fc->connected && !request_pending(fc)) {
 896                 set_current_state(TASK_INTERRUPTIBLE);
 897                 if (signal_pending(current))
 898                         break;
 899
 900                 spin_unlock(&fc->lock);
 901                 schedule();
 902                 spin_lock(&fc->lock);
 903         }
 904         set_current_state(TASK_RUNNING);
 905         remove_wait_queue(&fc->waitq, &wait);
 906 }
 907
 908 /*
 909  * Transfer an interrupt request to userspace
 910  *
 911  * Unlike other requests this is assembled on demand, without a need
 912  * to allocate a separate fuse_req structure.
 913  *
 914  * Called with fc->lock held, releases it
 915  */
 916 static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_copy_state *cs,
 917                                size_t nbytes, struct fuse_req *req)
 918 __releases(&fc->lock)
 919 {
 920         struct fuse_in_header ih;
 921         struct fuse_interrupt_in arg;
 922         unsigned reqsize = sizeof(ih) + sizeof(arg);
 923         int err;
 924
 925         list_del_init(&req->intr_entry);
 926         req->intr_unique = fuse_get_unique(fc);
 927         memset(&ih, 0, sizeof(ih));
 928         memset(&arg, 0, sizeof(arg));
 929         ih.len = reqsize;
 930         ih.opcode = FUSE_INTERRUPT;
 931         ih.unique = req->intr_unique;
 932         arg.unique = req->in.h.unique;
 933
 934         spin_unlock(&fc->lock);
 935         if (nbytes < reqsize)
 936                 return -EINVAL;
 937
 938         err = fuse_copy_one(cs, &ih, sizeof(ih));
 939         if (!err)
 940                 err = fuse_copy_one(cs, &arg, sizeof(arg));
 941         fuse_copy_finish(cs);
 942
 943         return err ? err : reqsize;
 944 }
 945
 946 /*
 947  * Read a single request into the userspace filesystem's buffer.  This
 948  * function waits until a request is available, then removes it from
 949  * the pending list and copies request data to userspace buffer.  If
 950  * no reply is needed (FORGET) or request has been aborted or there
 951  * was an error during the copying then it's finished by calling
 952  * request_end().  Otherwise add it to the processing list, and set
 953  * the 'sent' flag.
 954  */
 955 static ssize_t fuse_dev_do_read(struct fuse_conn *fc, struct file *file,
 956                                 struct fuse_copy_state *cs, size_t nbytes)
 957 {
 958         int err;
 959         struct fuse_req *req;
 960         struct fuse_in *in;
 961         unsigned reqsize;
 962
 963  restart:
 964         spin_lock(&fc->lock);
 965         err = -EAGAIN;
 966         if ((file->f_flags & O_NONBLOCK) && fc->connected &&
 967             !request_pending(fc))
 968                 goto err_unlock;
 969
 970         request_wait(fc);
 971         err = -ENODEV;
 972         if (!fc->connected)
 973                 goto err_unlock;
 974         err = -ERESTARTSYS;
 975         if (!request_pending(fc))
 976                 goto err_unlock;
 977
 978         if (!list_empty(&fc->interrupts)) {
 979                 req = list_entry(fc->interrupts.next, struct fuse_req,
 980                                  intr_entry);
 981                 return fuse_read_interrupt(fc, cs, nbytes, req);
 982         }
 983
 984         req = list_entry(fc->pending.next, struct fuse_req, list);
 985         req->state = FUSE_REQ_READING;
 986         list_move(&req->list, &fc->io);
 987
 988         in = &req->in;
 989         reqsize = in->h.len;
 990         /* If request is too large, reply with an error and restart the read */
 991         if (nbytes < reqsize) {
 992                 req->out.h.error = -EIO;
 993                 /* SETXATTR is special, since it may contain too large data */
 994                 if (in->h.opcode == FUSE_SETXATTR)
 995                         req->out.h.error = -E2BIG;
 996                 request_end(fc, req);
 997                 goto restart;
 998         }
 999         spin_unlock(&fc->lock);
1000         cs->req = req;
1001         err = fuse_copy_one(cs, &in->h, sizeof(in->h));
1002         if (!err)
1003                 err = fuse_copy_args(cs, in->numargs, in->argpages,
1004                                      (struct fuse_arg *) in->args, 0);
1005         fuse_copy_finish(cs);
1006         spin_lock(&fc->lock);
1007         req->locked = 0;
1008         if (req->aborted) {
1009                 request_end(fc, req);
1010                 return -ENODEV;
1011         }
1012         if (err) {
1013                 req->out.h.error = -EIO;
1014                 request_end(fc, req);
1015                 return err;
1016         }
1017         if (!req->isreply)
1018                 request_end(fc, req);
1019         else {
1020                 req->state = FUSE_REQ_SENT;
1021                 list_move_tail(&req->list, &fc->processing);
1022                 if (req->interrupted)
1023                         queue_interrupt(fc, req);
1024                 spin_unlock(&fc->lock);
1025         }
1026         return reqsize;
1027
1028  err_unlock:
1029         spin_unlock(&fc->lock);
1030         return err;
1031 }
1032
1033 static ssize_t fuse_dev_read(struct kiocb *iocb, const struct iovec *iov,
1034                               unsigned long nr_segs, loff_t pos)
1035 {
1036         struct fuse_copy_state cs;
1037         struct file *file = iocb->ki_filp;
1038         struct fuse_conn *fc = fuse_get_conn(file);
1039         if (!fc)
1040                 return -EPERM;
1041
1042         fuse_copy_init(&cs, fc, 1, iov, nr_segs);
1043
1044         return fuse_dev_do_read(fc, file, &cs, iov_length(iov, nr_segs));
1045 }
1046
1047 static int fuse_dev_pipe_buf_steal(struct pipe_inode_info *pipe,
1048                                    struct pipe_buffer *buf)
1049 {
1050         return 1;
1051 }
1052
1053 static const struct pipe_buf_operations fuse_dev_pipe_buf_ops = {
1054         .can_merge = 0,
1055         .map = generic_pipe_buf_map,
1056         .unmap = generic_pipe_buf_unmap,
1057         .confirm = generic_pipe_buf_confirm,
1058         .release = generic_pipe_buf_release,
1059         .steal = fuse_dev_pipe_buf_steal,
1060         .get = generic_pipe_buf_get,
1061 };
1062
1063 static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1064                                     struct pipe_inode_info *pipe,
1065                                     size_t len, unsigned int flags)
1066 {
1067         int ret;
1068         int page_nr = 0;
1069         int do_wakeup = 0;
1070         struct pipe_buffer *bufs;
1071         struct fuse_copy_state cs;
1072         struct fuse_conn *fc = fuse_get_conn(in);
1073         if (!fc)
1074                 return -EPERM;
1075
1076         bufs = kmalloc(pipe->buffers * sizeof (struct pipe_buffer), GFP_KERNEL);
1077         if (!bufs)
1078                 return -ENOMEM;
1079
1080         fuse_copy_init(&cs, fc, 1, NULL, 0);
1081         cs.pipebufs = bufs;
1082         cs.pipe = pipe;
1083         ret = fuse_dev_do_read(fc, in, &cs, len);
1084         if (ret < 0)
1085                 goto out;
1086
1087         ret = 0;
1088         pipe_lock(pipe);
1089
1090         if (!pipe->readers) {
1091                 send_sig(SIGPIPE, current, 0);
1092                 if (!ret)
1093                         ret = -EPIPE;
1094                 goto out_unlock;
1095         }
1096
1097         if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
1098                 ret = -EIO;
1099                 goto out_unlock;
1100         }
1101
1102         while (page_nr < cs.nr_segs) {
1103                 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
1104                 struct pipe_buffer *buf = pipe->bufs + newbuf;
1105
1106                 buf->page = bufs[page_nr].page;
1107                 buf->offset = bufs[page_nr].offset;
1108                 buf->len = bufs[page_nr].len;
1109                 buf->ops = &fuse_dev_pipe_buf_ops;
1110
1111                 pipe->nrbufs++;
1112                 page_nr++;
1113                 ret += buf->len;
1114
1115                 if (pipe->inode)
1116                         do_wakeup = 1;
1117         }
1118
1119 out_unlock:
1120         pipe_unlock(pipe);
1121
1122         if (do_wakeup) {
1123                 smp_mb();
1124                 if (waitqueue_active(&pipe->wait))
1125                         wake_up_interruptible(&pipe->wait);
1126                 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
1127         }
1128
1129 out:
1130         for (; page_nr < cs.nr_segs; page_nr++)
1131                 page_cache_release(bufs[page_nr].page);
1132
1133         kfree(bufs);
1134         return ret;
1135 }
1136
1137 static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1138                             struct fuse_copy_state *cs)
1139 {
1140         struct fuse_notify_poll_wakeup_out outarg;
1141         int err = -EINVAL;
1142
1143         if (size != sizeof(outarg))
1144                 goto err;
1145
1146         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1147         if (err)
1148                 goto err;
1149
1150         fuse_copy_finish(cs);
1151         return fuse_notify_poll_wakeup(fc, &outarg);
1152
1153 err:
1154         fuse_copy_finish(cs);
1155         return err;
1156 }
1157
1158 static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1159                                    struct fuse_copy_state *cs)
1160 {
1161         struct fuse_notify_inval_inode_out outarg;
1162         int err = -EINVAL;
1163
1164         if (size != sizeof(outarg))
1165                 goto err;
1166
1167         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1168         if (err)
1169                 goto err;
1170         fuse_copy_finish(cs);
1171
1172         down_read(&fc->killsb);
1173         err = -ENOENT;
1174         if (fc->sb) {
1175                 err = fuse_reverse_inval_inode(fc->sb, outarg.ino,
1176                                                outarg.off, outarg.len);
1177         }
1178         up_read(&fc->killsb);
1179         return err;
1180
1181 err:
1182         fuse_copy_finish(cs);
1183         return err;
1184 }
1185
1186 static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1187                                    struct fuse_copy_state *cs)
1188 {
1189         struct fuse_notify_inval_entry_out outarg;
1190         int err = -ENOMEM;
1191         char *buf;
1192         struct qstr name;
1193
1194         buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1195         if (!buf)
1196                 goto err;
1197
1198         err = -EINVAL;
1199         if (size < sizeof(outarg))
1200                 goto err;
1201
1202         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1203         if (err)
1204                 goto err;
1205
1206         err = -ENAMETOOLONG;
1207         if (outarg.namelen > FUSE_NAME_MAX)
1208                 goto err;
1209
1210         name.name = buf;
1211         name.len = outarg.namelen;
1212         err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1213         if (err)
1214                 goto err;
1215         fuse_copy_finish(cs);
1216         buf[outarg.namelen] = 0;
1217         name.hash = full_name_hash(name.name, name.len);
1218
1219         down_read(&fc->killsb);
1220         err = -ENOENT;
1221         if (fc->sb)
1222                 err = fuse_reverse_inval_entry(fc->sb, outarg.parent, &name);
1223         up_read(&fc->killsb);
1224         kfree(buf);
1225         return err;
1226
1227 err:
1228         kfree(buf);
1229         fuse_copy_finish(cs);
1230         return err;
1231 }
1232
1233 static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1234                        unsigned int size, struct fuse_copy_state *cs)
1235 {
1236         switch (code) {
1237         case FUSE_NOTIFY_POLL:
1238                 return fuse_notify_poll(fc, size, cs);
1239
1240         case FUSE_NOTIFY_INVAL_INODE:
1241                 return fuse_notify_inval_inode(fc, size, cs);
1242
1243         case FUSE_NOTIFY_INVAL_ENTRY:
1244                 return fuse_notify_inval_entry(fc, size, cs);
1245
1246         default:
1247                 fuse_copy_finish(cs);
1248                 return -EINVAL;
1249         }
1250 }
1251
1252 /* Look up request on processing list by unique ID */
1253 static struct fuse_req *request_find(struct fuse_conn *fc, u64 unique)
1254 {
1255         struct list_head *entry;
1256
1257         list_for_each(entry, &fc->processing) {
1258                 struct fuse_req *req;
1259                 req = list_entry(entry, struct fuse_req, list);
1260                 if (req->in.h.unique == unique || req->intr_unique == unique)
1261                         return req;
1262         }
1263         return NULL;
1264 }
1265
1266 static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out,
1267                          unsigned nbytes)
1268 {
1269         unsigned reqsize = sizeof(struct fuse_out_header);
1270
1271         if (out->h.error)
1272                 return nbytes != reqsize ? -EINVAL : 0;
1273
1274         reqsize += len_args(out->numargs, out->args);
1275
1276         if (reqsize < nbytes || (reqsize > nbytes && !out->argvar))
1277                 return -EINVAL;
1278         else if (reqsize > nbytes) {
1279                 struct fuse_arg *lastarg = &out->args[out->numargs-1];
1280                 unsigned diffsize = reqsize - nbytes;
1281                 if (diffsize > lastarg->size)
1282                         return -EINVAL;
1283                 lastarg->size -= diffsize;
1284         }
1285         return fuse_copy_args(cs, out->numargs, out->argpages, out->args,
1286                               out->page_zeroing);
1287 }
1288
1289 /*
1290  * Write a single reply to a request.  First the header is copied from
1291  * the write buffer.  The request is then searched on the processing
1292  * list by the unique ID found in the header.  If found, then remove
1293  * it from the list and copy the rest of the buffer to the request.
1294  * The request is finished by calling request_end()
1295  */
1296 static ssize_t fuse_dev_do_write(struct fuse_conn *fc,
1297                                  struct fuse_copy_state *cs, size_t nbytes)
1298 {
1299         int err;
1300         struct fuse_req *req;
1301         struct fuse_out_header oh;
1302
1303         if (nbytes < sizeof(struct fuse_out_header))
1304                 return -EINVAL;
1305
1306         err = fuse_copy_one(cs, &oh, sizeof(oh));
1307         if (err)
1308                 goto err_finish;
1309
1310         err = -EINVAL;
1311         if (oh.len != nbytes)
1312                 goto err_finish;
1313
1314         /*
1315          * Zero oh.unique indicates unsolicited notification message
1316          * and error contains notification code.
1317          */
1318         if (!oh.unique) {
1319                 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
1320                 return err ? err : nbytes;
1321         }
1322
1323         err = -EINVAL;
1324         if (oh.error <= -1000 || oh.error > 0)
1325                 goto err_finish;
1326
1327         spin_lock(&fc->lock);
1328         err = -ENOENT;
1329         if (!fc->connected)
1330                 goto err_unlock;
1331
1332         req = request_find(fc, oh.unique);
1333         if (!req)
1334                 goto err_unlock;
1335
1336         if (req->aborted) {
1337                 spin_unlock(&fc->lock);
1338                 fuse_copy_finish(cs);
1339                 spin_lock(&fc->lock);
1340                 request_end(fc, req);
1341                 return -ENOENT;
1342         }
1343         /* Is it an interrupt reply? */
1344         if (req->intr_unique == oh.unique) {
1345                 err = -EINVAL;
1346                 if (nbytes != sizeof(struct fuse_out_header))
1347                         goto err_unlock;
1348
1349                 if (oh.error == -ENOSYS)
1350                         fc->no_interrupt = 1;
1351                 else if (oh.error == -EAGAIN)
1352                         queue_interrupt(fc, req);
1353
1354                 spin_unlock(&fc->lock);
1355                 fuse_copy_finish(cs);
1356                 return nbytes;
1357         }
1358
1359         req->state = FUSE_REQ_WRITING;
1360         list_move(&req->list, &fc->io);
1361         req->out.h = oh;
1362         req->locked = 1;
1363         cs->req = req;
1364         if (!req->out.page_replace)
1365                 cs->move_pages = 0;
1366         spin_unlock(&fc->lock);
1367
1368         err = copy_out_args(cs, &req->out, nbytes);
1369         fuse_copy_finish(cs);
1370
1371         spin_lock(&fc->lock);
1372         req->locked = 0;
1373         if (!err) {
1374                 if (req->aborted)
1375                         err = -ENOENT;
1376         } else if (!req->aborted)
1377                 req->out.h.error = -EIO;
1378         request_end(fc, req);
1379
1380         return err ? err : nbytes;
1381
1382  err_unlock:
1383         spin_unlock(&fc->lock);
1384  err_finish:
1385         fuse_copy_finish(cs);
1386         return err;
1387 }
1388
1389 static ssize_t fuse_dev_write(struct kiocb *iocb, const struct iovec *iov,
1390                               unsigned long nr_segs, loff_t pos)
1391 {
1392         struct fuse_copy_state cs;
1393         struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp);
1394         if (!fc)
1395                 return -EPERM;
1396
1397         fuse_copy_init(&cs, fc, 0, iov, nr_segs);
1398
1399         return fuse_dev_do_write(fc, &cs, iov_length(iov, nr_segs));
1400 }
1401
1402 static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
1403                                      struct file *out, loff_t *ppos,
1404                                      size_t len, unsigned int flags)
1405 {
1406         unsigned nbuf;
1407         unsigned idx;
1408         struct pipe_buffer *bufs;
1409         struct fuse_copy_state cs;
1410         struct fuse_conn *fc;
1411         size_t rem;
1412         ssize_t ret;
1413
1414         fc = fuse_get_conn(out);
1415         if (!fc)
1416                 return -EPERM;
1417
1418         bufs = kmalloc(pipe->buffers * sizeof (struct pipe_buffer), GFP_KERNEL);
1419         if (!bufs)
1420                 return -ENOMEM;
1421
1422         pipe_lock(pipe);
1423         nbuf = 0;
1424         rem = 0;
1425         for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
1426                 rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;
1427
1428         ret = -EINVAL;
1429         if (rem < len) {
1430                 pipe_unlock(pipe);
1431                 goto out;
1432         }
1433
1434         rem = len;
1435         while (rem) {
1436                 struct pipe_buffer *ibuf;
1437                 struct pipe_buffer *obuf;
1438
1439                 BUG_ON(nbuf >= pipe->buffers);
1440                 BUG_ON(!pipe->nrbufs);
1441                 ibuf = &pipe->bufs[pipe->curbuf];
1442                 obuf = &bufs[nbuf];
1443
1444                 if (rem >= ibuf->len) {
1445                         *obuf = *ibuf;
1446                         ibuf->ops = NULL;
1447                         pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
1448                         pipe->nrbufs--;
1449                 } else {
1450                         ibuf->ops->get(pipe, ibuf);
1451                         *obuf = *ibuf;
1452                         obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1453                         obuf->len = rem;
1454                         ibuf->offset += obuf->len;
1455                         ibuf->len -= obuf->len;
1456                 }
1457                 nbuf++;
1458                 rem -= obuf->len;
1459         }
1460         pipe_unlock(pipe);
1461
1462         fuse_copy_init(&cs, fc, 0, NULL, nbuf);
1463         cs.pipebufs = bufs;
1464         cs.pipe = pipe;
1465
1466         if (flags & SPLICE_F_MOVE)
1467                 cs.move_pages = 1;
1468
1469         ret = fuse_dev_do_write(fc, &cs, len);
1470
1471         for (idx = 0; idx < nbuf; idx++) {
1472                 struct pipe_buffer *buf = &bufs[idx];
1473                 buf->ops->release(pipe, buf);
1474         }
1475 out:
1476         kfree(bufs);
1477         return ret;
1478 }
1479
1480 static unsigned fuse_dev_poll(struct file *file, poll_table *wait)
1481 {
1482         unsigned mask = POLLOUT | POLLWRNORM;
1483         struct fuse_conn *fc = fuse_get_conn(file);
1484         if (!fc)
1485                 return POLLERR;
1486
1487         poll_wait(file, &fc->waitq, wait);
1488
1489         spin_lock(&fc->lock);
1490         if (!fc->connected)
1491                 mask = POLLERR;
1492         else if (request_pending(fc))
1493                 mask |= POLLIN | POLLRDNORM;
1494         spin_unlock(&fc->lock);
1495
1496         return mask;
1497 }
1498
1499 /*
1500  * Abort all requests on the given list (pending or processing)
1501  *
1502  * This function releases and reacquires fc->lock
1503  */
1504 static void end_requests(struct fuse_conn *fc, struct list_head *head)
1505 __releases(&fc->lock)
1506 __acquires(&fc->lock)
1507 {
1508         while (!list_empty(head)) {
1509                 struct fuse_req *req;
1510                 req = list_entry(head->next, struct fuse_req, list);
1511                 req->out.h.error = -ECONNABORTED;
1512                 request_end(fc, req);
1513                 spin_lock(&fc->lock);
1514         }
1515 }
1516
1517 /*
1518  * Abort requests under I/O
1519  *
1520  * The requests are set to aborted and finished, and the request
1521  * waiter is woken up.  This will make request_wait_answer() wait
1522  * until the request is unlocked and then return.
1523  *
1524  * If the request is asynchronous, then the end function needs to be
1525  * called after waiting for the request to be unlocked (if it was
1526  * locked).
1527  */
1528 static void end_io_requests(struct fuse_conn *fc)
1529 __releases(&fc->lock)
1530 __acquires(&fc->lock)
1531 {
1532         while (!list_empty(&fc->io)) {
1533                 struct fuse_req *req =
1534                         list_entry(fc->io.next, struct fuse_req, list);
1535                 void (*end) (struct fuse_conn *, struct fuse_req *) = req->end;
1536
1537                 req->aborted = 1;
1538                 req->out.h.error = -ECONNABORTED;
1539                 req->state = FUSE_REQ_FINISHED;
1540                 list_del_init(&req->list);
1541                 wake_up(&req->waitq);
1542                 if (end) {
1543                         req->end = NULL;
1544                         __fuse_get_request(req);
1545                         spin_unlock(&fc->lock);
1546                         wait_event(req->waitq, !req->locked);
1547                         end(fc, req);
1548                         fuse_put_request(fc, req);
1549                         spin_lock(&fc->lock);
1550                 }
1551         }
1552 }
1553
1554 /*
1555  * Abort all requests.
1556  *
1557  * Emergency exit in case of a malicious or accidental deadlock, or
1558  * just a hung filesystem.
1559  *
1560  * The same effect is usually achievable through killing the
1561  * filesystem daemon and all users of the filesystem.  The exception
1562  * is the combination of an asynchronous request and the tricky
1563  * deadlock (see Documentation/filesystems/fuse.txt).
1564  *
1565  * During the aborting, progression of requests from the pending and
1566  * processing lists onto the io list, and progression of new requests
1567  * onto the pending list is prevented by req->connected being false.
1568  *
1569  * Progression of requests under I/O to the processing list is
1570  * prevented by the req->aborted flag being true for these requests.
1571  * For this reason requests on the io list must be aborted first.
1572  */
1573 void fuse_abort_conn(struct fuse_conn *fc)
1574 {
1575         spin_lock(&fc->lock);
1576         if (fc->connected) {
1577                 fc->connected = 0;
1578                 fc->blocked = 0;
1579                 end_io_requests(fc);
1580                 end_requests(fc, &fc->pending);
1581                 end_requests(fc, &fc->processing);
1582                 wake_up_all(&fc->waitq);
1583                 wake_up_all(&fc->blocked_waitq);
1584                 kill_fasync(&fc->fasync, SIGIO, POLL_IN);
1585         }
1586         spin_unlock(&fc->lock);
1587 }
1588 EXPORT_SYMBOL_GPL(fuse_abort_conn);
1589
1590 int fuse_dev_release(struct inode *inode, struct file *file)
1591 {
1592         struct fuse_conn *fc = fuse_get_conn(file);
1593         if (fc) {
1594                 spin_lock(&fc->lock);
1595                 fc->connected = 0;
1596                 end_requests(fc, &fc->pending);
1597                 end_requests(fc, &fc->processing);
1598                 spin_unlock(&fc->lock);
1599                 fuse_conn_put(fc);
1600         }
1601
1602         return 0;
1603 }
1604 EXPORT_SYMBOL_GPL(fuse_dev_release);
1605
1606 static int fuse_dev_fasync(int fd, struct file *file, int on)
1607 {
1608         struct fuse_conn *fc = fuse_get_conn(file);
1609         if (!fc)
1610                 return -EPERM;
1611
1612         /* No locking - fasync_helper does its own locking */
1613         return fasync_helper(fd, file, on, &fc->fasync);
1614 }
1615
1616 const struct file_operations fuse_dev_operations = {
1617         .owner          = THIS_MODULE,
1618         .llseek         = no_llseek,
1619         .read           = do_sync_read,
1620         .aio_read       = fuse_dev_read,
1621         .splice_read    = fuse_dev_splice_read,
1622         .write          = do_sync_write,
1623         .aio_write      = fuse_dev_write,
1624         .splice_write   = fuse_dev_splice_write,
1625         .poll           = fuse_dev_poll,
1626         .release        = fuse_dev_release,
1627         .fasync         = fuse_dev_fasync,
1628 };
1629 EXPORT_SYMBOL_GPL(fuse_dev_operations);
1630
1631 static struct miscdevice fuse_miscdevice = {
1632         .minor = FUSE_MINOR,
1633         .name  = "fuse",
1634         .fops = &fuse_dev_operations,
1635 };
1636
1637 int __init fuse_dev_init(void)
1638 {
1639         int err = -ENOMEM;
1640         fuse_req_cachep = kmem_cache_create("fuse_request",
1641                                             sizeof(struct fuse_req),
1642                                             0, 0, NULL);
1643         if (!fuse_req_cachep)
1644                 goto out;
1645
1646         err = misc_register(&fuse_miscdevice);
1647         if (err)
1648                 goto out_cache_clean;
1649
1650         return 0;
1651
1652  out_cache_clean:
1653         kmem_cache_destroy(fuse_req_cachep);
1654  out:
1655         return err;
1656 }
1657
1658 void fuse_dev_cleanup(void)
1659 {
1660         misc_deregister(&fuse_miscdevice);
1661         kmem_cache_destroy(fuse_req_cachep);
1662 }