fs/fscache/page.c

   1 /* Cache page management and data I/O routines
   2  *
   3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define FSCACHE_DEBUG_LEVEL PAGE
  13 #include <linux/module.h>
  14 #include <linux/fscache-cache.h>
  15 #include <linux/buffer_head.h>
  16 #include <linux/pagevec.h>
  17 #include "internal.h"
  18
  19 /*
  20  * check to see if a page is being written to the cache
  21  */
  22 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  23 {
  24         void *val;
  25
  26         rcu_read_lock();
  27         val = radix_tree_lookup(&cookie->stores, page->index);
  28         rcu_read_unlock();
  29
  30         return val != NULL;
  31 }
  32 EXPORT_SYMBOL(__fscache_check_page_write);
  33
  34 /*
  35  * wait for a page to finish being written to the cache
  36  */
  37 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  38 {
  39         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  40
  41         wait_event(*wq, !__fscache_check_page_write(cookie, page));
  42 }
  43 EXPORT_SYMBOL(__fscache_wait_on_page_write);
  44
  45 /*
  46  * note that a page has finished being written to the cache
  47  */
  48 static void fscache_end_page_write(struct fscache_object *object,
  49                                    struct page *page)
  50 {
  51         struct fscache_cookie *cookie;
  52         struct page *xpage = NULL;
  53
  54         spin_lock(&object->lock);
  55         cookie = object->cookie;
  56         if (cookie) {
  57                 /* delete the page from the tree if it is now no longer
  58                  * pending */
  59                 spin_lock(&cookie->stores_lock);
  60                 if (!radix_tree_tag_get(&cookie->stores, page->index,
  61                                         FSCACHE_COOKIE_PENDING_TAG)) {
  62                         fscache_stat(&fscache_n_store_radix_deletes);
  63                         xpage = radix_tree_delete(&cookie->stores, page->index);
  64                 }
  65                 spin_unlock(&cookie->stores_lock);
  66                 wake_up_bit(&cookie->flags, 0);
  67         }
  68         spin_unlock(&object->lock);
  69         if (xpage)
  70                 page_cache_release(xpage);
  71 }
  72
  73 /*
  74  * actually apply the changed attributes to a cache object
  75  */
  76 static void fscache_attr_changed_op(struct fscache_operation *op)
  77 {
  78         struct fscache_object *object = op->object;
  79         int ret;
  80
  81         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
  82
  83         fscache_stat(&fscache_n_attr_changed_calls);
  84
  85         if (fscache_object_is_active(object)) {
  86                 fscache_set_op_state(op, "CallFS");
  87                 fscache_stat(&fscache_n_cop_attr_changed);
  88                 ret = object->cache->ops->attr_changed(object);
  89                 fscache_stat_d(&fscache_n_cop_attr_changed);
  90                 fscache_set_op_state(op, "Done");
  91                 if (ret < 0)
  92                         fscache_abort_object(object);
  93         }
  94
  95         _leave("");
  96 }
  97
  98 /*
  99  * notification that the attributes on an object have changed
 100  */
 101 int __fscache_attr_changed(struct fscache_cookie *cookie)
 102 {
 103         struct fscache_operation *op;
 104         struct fscache_object *object;
 105
 106         _enter("%p", cookie);
 107
 108         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 109
 110         fscache_stat(&fscache_n_attr_changed);
 111
 112         op = kzalloc(sizeof(*op), GFP_KERNEL);
 113         if (!op) {
 114                 fscache_stat(&fscache_n_attr_changed_nomem);
 115                 _leave(" = -ENOMEM");
 116                 return -ENOMEM;
 117         }
 118
 119         fscache_operation_init(op, NULL);
 120         fscache_operation_init_slow(op, fscache_attr_changed_op);
 121         op->flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_EXCLUSIVE);
 122         fscache_set_op_name(op, "Attr");
 123
 124         spin_lock(&cookie->lock);
 125
 126         if (hlist_empty(&cookie->backing_objects))
 127                 goto nobufs;
 128         object = hlist_entry(cookie->backing_objects.first,
 129                              struct fscache_object, cookie_link);
 130
 131         if (fscache_submit_exclusive_op(object, op) < 0)
 132                 goto nobufs;
 133         spin_unlock(&cookie->lock);
 134         fscache_stat(&fscache_n_attr_changed_ok);
 135         fscache_put_operation(op);
 136         _leave(" = 0");
 137         return 0;
 138
 139 nobufs:
 140         spin_unlock(&cookie->lock);
 141         kfree(op);
 142         fscache_stat(&fscache_n_attr_changed_nobufs);
 143         _leave(" = %d", -ENOBUFS);
 144         return -ENOBUFS;
 145 }
 146 EXPORT_SYMBOL(__fscache_attr_changed);
 147
 148 /*
 149  * handle secondary execution given to a retrieval op on behalf of the
 150  * cache
 151  */
 152 static void fscache_retrieval_work(struct work_struct *work)
 153 {
 154         struct fscache_retrieval *op =
 155                 container_of(work, struct fscache_retrieval, op.fast_work);
 156         unsigned long start;
 157
 158         _enter("{OP%x}", op->op.debug_id);
 159
 160         start = jiffies;
 161         op->op.processor(&op->op);
 162         fscache_hist(fscache_ops_histogram, start);
 163         fscache_put_operation(&op->op);
 164 }
 165
 166 /*
 167  * release a retrieval op reference
 168  */
 169 static void fscache_release_retrieval_op(struct fscache_operation *_op)
 170 {
 171         struct fscache_retrieval *op =
 172                 container_of(_op, struct fscache_retrieval, op);
 173
 174         _enter("{OP%x}", op->op.debug_id);
 175
 176         fscache_hist(fscache_retrieval_histogram, op->start_time);
 177         if (op->context)
 178                 fscache_put_context(op->op.object->cookie, op->context);
 179
 180         _leave("");
 181 }
 182
 183 /*
 184  * allocate a retrieval op
 185  */
 186 static struct fscache_retrieval *fscache_alloc_retrieval(
 187         struct address_space *mapping,
 188         fscache_rw_complete_t end_io_func,
 189         void *context)
 190 {
 191         struct fscache_retrieval *op;
 192
 193         /* allocate a retrieval operation and attempt to submit it */
 194         op = kzalloc(sizeof(*op), GFP_NOIO);
 195         if (!op) {
 196                 fscache_stat(&fscache_n_retrievals_nomem);
 197                 return NULL;
 198         }
 199
 200         fscache_operation_init(&op->op, fscache_release_retrieval_op);
 201         op->op.flags    = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING);
 202         op->mapping     = mapping;
 203         op->end_io_func = end_io_func;
 204         op->context     = context;
 205         op->start_time  = jiffies;
 206         INIT_WORK(&op->op.fast_work, fscache_retrieval_work);
 207         INIT_LIST_HEAD(&op->to_do);
 208         fscache_set_op_name(&op->op, "Retr");
 209         return op;
 210 }
 211
 212 /*
 213  * wait for a deferred lookup to complete
 214  */
 215 static int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 216 {
 217         unsigned long jif;
 218
 219         _enter("");
 220
 221         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 222                 _leave(" = 0 [imm]");
 223                 return 0;
 224         }
 225
 226         fscache_stat(&fscache_n_retrievals_wait);
 227
 228         jif = jiffies;
 229         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 230                         fscache_wait_bit_interruptible,
 231                         TASK_INTERRUPTIBLE) != 0) {
 232                 fscache_stat(&fscache_n_retrievals_intr);
 233                 _leave(" = -ERESTARTSYS");
 234                 return -ERESTARTSYS;
 235         }
 236
 237         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 238
 239         smp_rmb();
 240         fscache_hist(fscache_retrieval_delay_histogram, jif);
 241         _leave(" = 0 [dly]");
 242         return 0;
 243 }
 244
 245 /*
 246  * read a page from the cache or allocate a block in which to store it
 247  * - we return:
 248  *   -ENOMEM    - out of memory, nothing done
 249  *   -ERESTARTSYS - interrupted
 250  *   -ENOBUFS   - no backing object available in which to cache the block
 251  *   -ENODATA   - no data available in the backing object for this block
 252  *   0          - dispatched a read - it'll call end_io_func() when finished
 253  */
 254 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 255                                  struct page *page,
 256                                  fscache_rw_complete_t end_io_func,
 257                                  void *context,
 258                                  gfp_t gfp)
 259 {
 260         struct fscache_retrieval *op;
 261         struct fscache_object *object;
 262         int ret;
 263
 264         _enter("%p,%p,,,", cookie, page);
 265
 266         fscache_stat(&fscache_n_retrievals);
 267
 268         if (hlist_empty(&cookie->backing_objects))
 269                 goto nobufs;
 270
 271         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 272         ASSERTCMP(page, !=, NULL);
 273
 274         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 275                 return -ERESTARTSYS;
 276
 277         op = fscache_alloc_retrieval(page->mapping, end_io_func, context);
 278         if (!op) {
 279                 _leave(" = -ENOMEM");
 280                 return -ENOMEM;
 281         }
 282         fscache_set_op_name(&op->op, "RetrRA1");
 283
 284         spin_lock(&cookie->lock);
 285
 286         if (hlist_empty(&cookie->backing_objects))
 287                 goto nobufs_unlock;
 288         object = hlist_entry(cookie->backing_objects.first,
 289                              struct fscache_object, cookie_link);
 290
 291         ASSERTCMP(object->state, >, FSCACHE_OBJECT_LOOKING_UP);
 292
 293         atomic_inc(&object->n_reads);
 294         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 295
 296         if (fscache_submit_op(object, &op->op) < 0)
 297                 goto nobufs_unlock;
 298         spin_unlock(&cookie->lock);
 299
 300         fscache_stat(&fscache_n_retrieval_ops);
 301
 302         /* pin the netfs read context in case we need to do the actual netfs
 303          * read because we've encountered a cache read failure */
 304         fscache_get_context(object->cookie, op->context);
 305
 306         /* we wait for the operation to become active, and then process it
 307          * *here*, in this thread, and not in the thread pool */
 308         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 309                 _debug(">>> WT");
 310                 fscache_stat(&fscache_n_retrieval_op_waits);
 311                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 312                                 fscache_wait_bit_interruptible,
 313                                 TASK_INTERRUPTIBLE) < 0) {
 314                         ret = fscache_cancel_op(&op->op);
 315                         if (ret == 0) {
 316                                 ret = -ERESTARTSYS;
 317                                 goto error;
 318                         }
 319
 320                         /* it's been removed from the pending queue by another
 321                          * party, so we should get to run shortly */
 322                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 323                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 324                 }
 325                 _debug("<<< GO");
 326         }
 327
 328         /* ask the cache to honour the operation */
 329         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 330                 fscache_stat(&fscache_n_cop_allocate_page);
 331                 ret = object->cache->ops->allocate_page(op, page, gfp);
 332                 fscache_stat_d(&fscache_n_cop_allocate_page);
 333                 if (ret == 0)
 334                         ret = -ENODATA;
 335         } else {
 336                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
 337                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 338                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
 339         }
 340
 341 error:
 342         if (ret == -ENOMEM)
 343                 fscache_stat(&fscache_n_retrievals_nomem);
 344         else if (ret == -ERESTARTSYS)
 345                 fscache_stat(&fscache_n_retrievals_intr);
 346         else if (ret == -ENODATA)
 347                 fscache_stat(&fscache_n_retrievals_nodata);
 348         else if (ret < 0)
 349                 fscache_stat(&fscache_n_retrievals_nobufs);
 350         else
 351                 fscache_stat(&fscache_n_retrievals_ok);
 352
 353         fscache_put_retrieval(op);
 354         _leave(" = %d", ret);
 355         return ret;
 356
 357 nobufs_unlock:
 358         spin_unlock(&cookie->lock);
 359         kfree(op);
 360 nobufs:
 361         fscache_stat(&fscache_n_retrievals_nobufs);
 362         _leave(" = -ENOBUFS");
 363         return -ENOBUFS;
 364 }
 365 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 366
 367 /*
 368  * read a list of page from the cache or allocate a block in which to store
 369  * them
 370  * - we return:
 371  *   -ENOMEM    - out of memory, some pages may be being read
 372  *   -ERESTARTSYS - interrupted, some pages may be being read
 373  *   -ENOBUFS   - no backing object or space available in which to cache any
 374  *                pages not being read
 375  *   -ENODATA   - no data available in the backing object for some or all of
 376  *                the pages
 377  *   0          - dispatched a read on all pages
 378  *
 379  * end_io_func() will be called for each page read from the cache as it is
 380  * finishes being read
 381  *
 382  * any pages for which a read is dispatched will be removed from pages and
 383  * nr_pages
 384  */
 385 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 386                                   struct address_space *mapping,
 387                                   struct list_head *pages,
 388                                   unsigned *nr_pages,
 389                                   fscache_rw_complete_t end_io_func,
 390                                   void *context,
 391                                   gfp_t gfp)
 392 {
 393         struct fscache_retrieval *op;
 394         struct fscache_object *object;
 395         int ret;
 396
 397         _enter("%p,,%d,,,", cookie, *nr_pages);
 398
 399         fscache_stat(&fscache_n_retrievals);
 400
 401         if (hlist_empty(&cookie->backing_objects))
 402                 goto nobufs;
 403
 404         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 405         ASSERTCMP(*nr_pages, >, 0);
 406         ASSERT(!list_empty(pages));
 407
 408         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 409                 return -ERESTARTSYS;
 410
 411         op = fscache_alloc_retrieval(mapping, end_io_func, context);
 412         if (!op)
 413                 return -ENOMEM;
 414         fscache_set_op_name(&op->op, "RetrRAN");
 415
 416         spin_lock(&cookie->lock);
 417
 418         if (hlist_empty(&cookie->backing_objects))
 419                 goto nobufs_unlock;
 420         object = hlist_entry(cookie->backing_objects.first,
 421                              struct fscache_object, cookie_link);
 422
 423         atomic_inc(&object->n_reads);
 424         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 425
 426         if (fscache_submit_op(object, &op->op) < 0)
 427                 goto nobufs_unlock;
 428         spin_unlock(&cookie->lock);
 429
 430         fscache_stat(&fscache_n_retrieval_ops);
 431
 432         /* pin the netfs read context in case we need to do the actual netfs
 433          * read because we've encountered a cache read failure */
 434         fscache_get_context(object->cookie, op->context);
 435
 436         /* we wait for the operation to become active, and then process it
 437          * *here*, in this thread, and not in the thread pool */
 438         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 439                 _debug(">>> WT");
 440                 fscache_stat(&fscache_n_retrieval_op_waits);
 441                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 442                                 fscache_wait_bit_interruptible,
 443                                 TASK_INTERRUPTIBLE) < 0) {
 444                         ret = fscache_cancel_op(&op->op);
 445                         if (ret == 0) {
 446                                 ret = -ERESTARTSYS;
 447                                 goto error;
 448                         }
 449
 450                         /* it's been removed from the pending queue by another
 451                          * party, so we should get to run shortly */
 452                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 453                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 454                 }
 455                 _debug("<<< GO");
 456         }
 457
 458         /* ask the cache to honour the operation */
 459         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 460                 fscache_stat(&fscache_n_cop_allocate_pages);
 461                 ret = object->cache->ops->allocate_pages(
 462                         op, pages, nr_pages, gfp);
 463                 fscache_stat_d(&fscache_n_cop_allocate_pages);
 464         } else {
 465                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
 466                 ret = object->cache->ops->read_or_alloc_pages(
 467                         op, pages, nr_pages, gfp);
 468                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
 469         }
 470
 471 error:
 472         if (ret == -ENOMEM)
 473                 fscache_stat(&fscache_n_retrievals_nomem);
 474         else if (ret == -ERESTARTSYS)
 475                 fscache_stat(&fscache_n_retrievals_intr);
 476         else if (ret == -ENODATA)
 477                 fscache_stat(&fscache_n_retrievals_nodata);
 478         else if (ret < 0)
 479                 fscache_stat(&fscache_n_retrievals_nobufs);
 480         else
 481                 fscache_stat(&fscache_n_retrievals_ok);
 482
 483         fscache_put_retrieval(op);
 484         _leave(" = %d", ret);
 485         return ret;
 486
 487 nobufs_unlock:
 488         spin_unlock(&cookie->lock);
 489         kfree(op);
 490 nobufs:
 491         fscache_stat(&fscache_n_retrievals_nobufs);
 492         _leave(" = -ENOBUFS");
 493         return -ENOBUFS;
 494 }
 495 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 496
 497 /*
 498  * allocate a block in the cache on which to store a page
 499  * - we return:
 500  *   -ENOMEM    - out of memory, nothing done
 501  *   -ERESTARTSYS - interrupted
 502  *   -ENOBUFS   - no backing object available in which to cache the block
 503  *   0          - block allocated
 504  */
 505 int __fscache_alloc_page(struct fscache_cookie *cookie,
 506                          struct page *page,
 507                          gfp_t gfp)
 508 {
 509         struct fscache_retrieval *op;
 510         struct fscache_object *object;
 511         int ret;
 512
 513         _enter("%p,%p,,,", cookie, page);
 514
 515         fscache_stat(&fscache_n_allocs);
 516
 517         if (hlist_empty(&cookie->backing_objects))
 518                 goto nobufs;
 519
 520         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 521         ASSERTCMP(page, !=, NULL);
 522
 523         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 524                 return -ERESTARTSYS;
 525
 526         op = fscache_alloc_retrieval(page->mapping, NULL, NULL);
 527         if (!op)
 528                 return -ENOMEM;
 529         fscache_set_op_name(&op->op, "RetrAL1");
 530
 531         spin_lock(&cookie->lock);
 532
 533         if (hlist_empty(&cookie->backing_objects))
 534                 goto nobufs_unlock;
 535         object = hlist_entry(cookie->backing_objects.first,
 536                              struct fscache_object, cookie_link);
 537
 538         if (fscache_submit_op(object, &op->op) < 0)
 539                 goto nobufs_unlock;
 540         spin_unlock(&cookie->lock);
 541
 542         fscache_stat(&fscache_n_alloc_ops);
 543
 544         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 545                 _debug(">>> WT");
 546                 fscache_stat(&fscache_n_alloc_op_waits);
 547                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 548                                 fscache_wait_bit_interruptible,
 549                                 TASK_INTERRUPTIBLE) < 0) {
 550                         ret = fscache_cancel_op(&op->op);
 551                         if (ret == 0) {
 552                                 ret = -ERESTARTSYS;
 553                                 goto error;
 554                         }
 555
 556                         /* it's been removed from the pending queue by another
 557                          * party, so we should get to run shortly */
 558                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 559                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 560                 }
 561                 _debug("<<< GO");
 562         }
 563
 564         /* ask the cache to honour the operation */
 565         fscache_stat(&fscache_n_cop_allocate_page);
 566         ret = object->cache->ops->allocate_page(op, page, gfp);
 567         fscache_stat_d(&fscache_n_cop_allocate_page);
 568
 569 error:
 570         if (ret == -ERESTARTSYS)
 571                 fscache_stat(&fscache_n_allocs_intr);
 572         else if (ret < 0)
 573                 fscache_stat(&fscache_n_allocs_nobufs);
 574         else
 575                 fscache_stat(&fscache_n_allocs_ok);
 576
 577         fscache_put_retrieval(op);
 578         _leave(" = %d", ret);
 579         return ret;
 580
 581 nobufs_unlock:
 582         spin_unlock(&cookie->lock);
 583         kfree(op);
 584 nobufs:
 585         fscache_stat(&fscache_n_allocs_nobufs);
 586         _leave(" = -ENOBUFS");
 587         return -ENOBUFS;
 588 }
 589 EXPORT_SYMBOL(__fscache_alloc_page);
 590
 591 /*
 592  * release a write op reference
 593  */
 594 static void fscache_release_write_op(struct fscache_operation *_op)
 595 {
 596         _enter("{OP%x}", _op->debug_id);
 597 }
 598
 599 /*
 600  * perform the background storage of a page into the cache
 601  */
 602 static void fscache_write_op(struct fscache_operation *_op)
 603 {
 604         struct fscache_storage *op =
 605                 container_of(_op, struct fscache_storage, op);
 606         struct fscache_object *object = op->op.object;
 607         struct fscache_cookie *cookie;
 608         struct page *page;
 609         unsigned n;
 610         void *results[1];
 611         int ret;
 612
 613         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 614
 615         fscache_set_op_state(&op->op, "GetPage");
 616
 617         spin_lock(&object->lock);
 618         cookie = object->cookie;
 619
 620         if (!fscache_object_is_active(object) || !cookie) {
 621                 spin_unlock(&object->lock);
 622                 _leave("");
 623                 return;
 624         }
 625
 626         spin_lock(&cookie->stores_lock);
 627
 628         fscache_stat(&fscache_n_store_calls);
 629
 630         /* find a page to store */
 631         page = NULL;
 632         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 633                                        FSCACHE_COOKIE_PENDING_TAG);
 634         if (n != 1)
 635                 goto superseded;
 636         page = results[0];
 637         _debug("gang %d [%lx]", n, page->index);
 638         if (page->index > op->store_limit) {
 639                 fscache_stat(&fscache_n_store_pages_over_limit);
 640                 goto superseded;
 641         }
 642
 643         radix_tree_tag_clear(&cookie->stores, page->index,
 644                              FSCACHE_COOKIE_PENDING_TAG);
 645
 646         spin_unlock(&cookie->stores_lock);
 647         spin_unlock(&object->lock);
 648
 649         if (page) {
 650                 fscache_set_op_state(&op->op, "Store");
 651                 fscache_stat(&fscache_n_store_pages);
 652                 fscache_stat(&fscache_n_cop_write_page);
 653                 ret = object->cache->ops->write_page(op, page);
 654                 fscache_stat_d(&fscache_n_cop_write_page);
 655                 fscache_set_op_state(&op->op, "EndWrite");
 656                 fscache_end_page_write(object, page);
 657                 if (ret < 0) {
 658                         fscache_set_op_state(&op->op, "Abort");
 659                         fscache_abort_object(object);
 660                 } else {
 661                         fscache_enqueue_operation(&op->op);
 662                 }
 663         }
 664
 665         _leave("");
 666         return;
 667
 668 superseded:
 669         /* this writer is going away and there aren't any more things to
 670          * write */
 671         _debug("cease");
 672         spin_unlock(&cookie->stores_lock);
 673         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 674         spin_unlock(&object->lock);
 675         _leave("");
 676 }
 677
 678 /*
 679  * request a page be stored in the cache
 680  * - returns:
 681  *   -ENOMEM    - out of memory, nothing done
 682  *   -ENOBUFS   - no backing object available in which to cache the page
 683  *   0          - dispatched a write - it'll call end_io_func() when finished
 684  *
 685  * if the cookie still has a backing object at this point, that object can be
 686  * in one of a few states with respect to storage processing:
 687  *
 688  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 689  *      set)
 690  *
 691  *      (a) no writes yet (set FSCACHE_COOKIE_PENDING_FILL and queue deferred
 692  *          fill op)
 693  *
 694  *      (b) writes deferred till post-creation (mark page for writing and
 695  *          return immediately)
 696  *
 697  *  (2) negative lookup, object created, initial fill being made from netfs
 698  *      (FSCACHE_COOKIE_INITIAL_FILL is set)
 699  *
 700  *      (a) fill point not yet reached this page (mark page for writing and
 701  *          return)
 702  *
 703  *      (b) fill point passed this page (queue op to store this page)
 704  *
 705  *  (3) object extant (queue op to store this page)
 706  *
 707  * any other state is invalid
 708  */
 709 int __fscache_write_page(struct fscache_cookie *cookie,
 710                          struct page *page,
 711                          gfp_t gfp)
 712 {
 713         struct fscache_storage *op;
 714         struct fscache_object *object;
 715         int ret;
 716
 717         _enter("%p,%x,", cookie, (u32) page->flags);
 718
 719         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 720         ASSERT(PageFsCache(page));
 721
 722         fscache_stat(&fscache_n_stores);
 723
 724         op = kzalloc(sizeof(*op), GFP_NOIO);
 725         if (!op)
 726                 goto nomem;
 727
 728         fscache_operation_init(&op->op, fscache_release_write_op);
 729         fscache_operation_init_slow(&op->op, fscache_write_op);
 730         op->op.flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_WAITING);
 731         fscache_set_op_name(&op->op, "Write1");
 732
 733         ret = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
 734         if (ret < 0)
 735                 goto nomem_free;
 736
 737         ret = -ENOBUFS;
 738         spin_lock(&cookie->lock);
 739
 740         if (hlist_empty(&cookie->backing_objects))
 741                 goto nobufs;
 742         object = hlist_entry(cookie->backing_objects.first,
 743                              struct fscache_object, cookie_link);
 744         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 745                 goto nobufs;
 746
 747         /* add the page to the pending-storage radix tree on the backing
 748          * object */
 749         spin_lock(&object->lock);
 750         spin_lock(&cookie->stores_lock);
 751
 752         _debug("store limit %llx", (unsigned long long) object->store_limit);
 753
 754         ret = radix_tree_insert(&cookie->stores, page->index, page);
 755         if (ret < 0) {
 756                 if (ret == -EEXIST)
 757                         goto already_queued;
 758                 _debug("insert failed %d", ret);
 759                 goto nobufs_unlock_obj;
 760         }
 761
 762         radix_tree_tag_set(&cookie->stores, page->index,
 763                            FSCACHE_COOKIE_PENDING_TAG);
 764         page_cache_get(page);
 765
 766         /* we only want one writer at a time, but we do need to queue new
 767          * writers after exclusive ops */
 768         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 769                 goto already_pending;
 770
 771         spin_unlock(&cookie->stores_lock);
 772         spin_unlock(&object->lock);
 773
 774         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 775         op->store_limit = object->store_limit;
 776
 777         if (fscache_submit_op(object, &op->op) < 0)
 778                 goto submit_failed;
 779
 780         spin_unlock(&cookie->lock);
 781         radix_tree_preload_end();
 782         fscache_stat(&fscache_n_store_ops);
 783         fscache_stat(&fscache_n_stores_ok);
 784
 785         /* the slow work queue now carries its own ref on the object */
 786         fscache_put_operation(&op->op);
 787         _leave(" = 0");
 788         return 0;
 789
 790 already_queued:
 791         fscache_stat(&fscache_n_stores_again);
 792 already_pending:
 793         spin_unlock(&cookie->stores_lock);
 794         spin_unlock(&object->lock);
 795         spin_unlock(&cookie->lock);
 796         radix_tree_preload_end();
 797         kfree(op);
 798         fscache_stat(&fscache_n_stores_ok);
 799         _leave(" = 0");
 800         return 0;
 801
 802 submit_failed:
 803         spin_lock(&cookie->stores_lock);
 804         radix_tree_delete(&cookie->stores, page->index);
 805         spin_unlock(&cookie->stores_lock);
 806         page_cache_release(page);
 807         ret = -ENOBUFS;
 808         goto nobufs;
 809
 810 nobufs_unlock_obj:
 811         spin_unlock(&object->lock);
 812 nobufs:
 813         spin_unlock(&cookie->lock);
 814         radix_tree_preload_end();
 815         kfree(op);
 816         fscache_stat(&fscache_n_stores_nobufs);
 817         _leave(" = -ENOBUFS");
 818         return -ENOBUFS;
 819
 820 nomem_free:
 821         kfree(op);
 822 nomem:
 823         fscache_stat(&fscache_n_stores_oom);
 824         _leave(" = -ENOMEM");
 825         return -ENOMEM;
 826 }
 827 EXPORT_SYMBOL(__fscache_write_page);
 828
 829 /*
 830  * remove a page from the cache
 831  */
 832 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
 833 {
 834         struct fscache_object *object;
 835
 836         _enter(",%p", page);
 837
 838         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 839         ASSERTCMP(page, !=, NULL);
 840
 841         fscache_stat(&fscache_n_uncaches);
 842
 843         /* cache withdrawal may beat us to it */
 844         if (!PageFsCache(page))
 845                 goto done;
 846
 847         /* get the object */
 848         spin_lock(&cookie->lock);
 849
 850         if (hlist_empty(&cookie->backing_objects)) {
 851                 ClearPageFsCache(page);
 852                 goto done_unlock;
 853         }
 854
 855         object = hlist_entry(cookie->backing_objects.first,
 856                              struct fscache_object, cookie_link);
 857
 858         /* there might now be stuff on disk we could read */
 859         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 860
 861         /* only invoke the cache backend if we managed to mark the page
 862          * uncached here; this deals with synchronisation vs withdrawal */
 863         if (TestClearPageFsCache(page) &&
 864             object->cache->ops->uncache_page) {
 865                 /* the cache backend releases the cookie lock */
 866                 fscache_stat(&fscache_n_cop_uncache_page);
 867                 object->cache->ops->uncache_page(object, page);
 868                 fscache_stat_d(&fscache_n_cop_uncache_page);
 869                 goto done;
 870         }
 871
 872 done_unlock:
 873         spin_unlock(&cookie->lock);
 874 done:
 875         _leave("");
 876 }
 877 EXPORT_SYMBOL(__fscache_uncache_page);
 878
 879 /**
 880  * fscache_mark_pages_cached - Mark pages as being cached
 881  * @op: The retrieval op pages are being marked for
 882  * @pagevec: The pages to be marked
 883  *
 884  * Mark a bunch of netfs pages as being cached.  After this is called,
 885  * the netfs must call fscache_uncache_page() to remove the mark.
 886  */
 887 void fscache_mark_pages_cached(struct fscache_retrieval *op,
 888                                struct pagevec *pagevec)
 889 {
 890         struct fscache_cookie *cookie = op->op.object->cookie;
 891         unsigned long loop;
 892
 893 #ifdef CONFIG_FSCACHE_STATS
 894         atomic_add(pagevec->nr, &fscache_n_marks);
 895 #endif
 896
 897         for (loop = 0; loop < pagevec->nr; loop++) {
 898                 struct page *page = pagevec->pages[loop];
 899
 900                 _debug("- mark %p{%lx}", page, page->index);
 901                 if (TestSetPageFsCache(page)) {
 902                         static bool once_only;
 903                         if (!once_only) {
 904                                 once_only = true;
 905                                 printk(KERN_WARNING "FS-Cache:"
 906                                        " Cookie type %s marked page %lx"
 907                                        " multiple times\n",
 908                                        cookie->def->name, page->index);
 909                         }
 910                 }
 911         }
 912
 913         if (cookie->def->mark_pages_cached)
 914                 cookie->def->mark_pages_cached(cookie->netfs_data,
 915                                                op->mapping, pagevec);
 916         pagevec_reinit(pagevec);
 917 }
 918 EXPORT_SYMBOL(fscache_mark_pages_cached);