fs/xfs/linux-2.6/xfs_buf.c

   1 /*
   2  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
   3  * All Rights Reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it would be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write the Free Software Foundation,
  16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 #include "xfs.h"
  19 #include <linux/stddef.h>
  20 #include <linux/errno.h>
  21 #include <linux/slab.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/init.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/bio.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/proc_fs.h>
  28 #include <linux/workqueue.h>
  29 #include <linux/percpu.h>
  30 #include <linux/blkdev.h>
  31 #include <linux/hash.h>
  32 #include <linux/kthread.h>
  33 #include <linux/migrate.h>
  34 #include <linux/backing-dev.h>
  35
  36 STATIC kmem_zone_t *xfs_buf_zone;
  37 STATIC kmem_shaker_t xfs_buf_shake;
  38 STATIC int xfsbufd(void *);
  39 STATIC int xfsbufd_wakeup(int, gfp_t);
  40 STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
  41
  42 STATIC struct workqueue_struct *xfslogd_workqueue;
  43 struct workqueue_struct *xfsdatad_workqueue;
  44
  45 #ifdef XFS_BUF_TRACE
  46 void
  47 xfs_buf_trace(
  48         xfs_buf_t       *bp,
  49         char            *id,
  50         void            *data,
  51         void            *ra)
  52 {
  53         ktrace_enter(xfs_buf_trace_buf,
  54                 bp, id,
  55                 (void *)(unsigned long)bp->b_flags,
  56                 (void *)(unsigned long)bp->b_hold.counter,
  57                 (void *)(unsigned long)bp->b_sema.count.counter,
  58                 (void *)current,
  59                 data, ra,
  60                 (void *)(unsigned long)((bp->b_file_offset>>32) & 0xffffffff),
  61                 (void *)(unsigned long)(bp->b_file_offset & 0xffffffff),
  62                 (void *)(unsigned long)bp->b_buffer_length,
  63                 NULL, NULL, NULL, NULL, NULL);
  64 }
  65 ktrace_t *xfs_buf_trace_buf;
  66 #define XFS_BUF_TRACE_SIZE      4096
  67 #define XB_TRACE(bp, id, data)  \
  68         xfs_buf_trace(bp, id, (void *)data, (void *)__builtin_return_address(0))
  69 #else
  70 #define XB_TRACE(bp, id, data)  do { } while (0)
  71 #endif
  72
  73 #ifdef XFS_BUF_LOCK_TRACKING
  74 # define XB_SET_OWNER(bp)       ((bp)->b_last_holder = current->pid)
  75 # define XB_CLEAR_OWNER(bp)     ((bp)->b_last_holder = -1)
  76 # define XB_GET_OWNER(bp)       ((bp)->b_last_holder)
  77 #else
  78 # define XB_SET_OWNER(bp)       do { } while (0)
  79 # define XB_CLEAR_OWNER(bp)     do { } while (0)
  80 # define XB_GET_OWNER(bp)       do { } while (0)
  81 #endif
  82
  83 #define xb_to_gfp(flags) \
  84         ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
  85           ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
  86
  87 #define xb_to_km(flags) \
  88          (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
  89
  90 #define xfs_buf_allocate(flags) \
  91         kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
  92 #define xfs_buf_deallocate(bp) \
  93         kmem_zone_free(xfs_buf_zone, (bp));
  94
  95 /*
  96  *      Page Region interfaces.
  97  *
  98  *      For pages in filesystems where the blocksize is smaller than the
  99  *      pagesize, we use the page->private field (long) to hold a bitmap
 100  *      of uptodate regions within the page.
 101  *
 102  *      Each such region is "bytes per page / bits per long" bytes long.
 103  *
 104  *      NBPPR == number-of-bytes-per-page-region
 105  *      BTOPR == bytes-to-page-region (rounded up)
 106  *      BTOPRT == bytes-to-page-region-truncated (rounded down)
 107  */
 108 #if (BITS_PER_LONG == 32)
 109 #define PRSHIFT         (PAGE_CACHE_SHIFT - 5)  /* (32 == 1<<5) */
 110 #elif (BITS_PER_LONG == 64)
 111 #define PRSHIFT         (PAGE_CACHE_SHIFT - 6)  /* (64 == 1<<6) */
 112 #else
 113 #error BITS_PER_LONG must be 32 or 64
 114 #endif
 115 #define NBPPR           (PAGE_CACHE_SIZE/BITS_PER_LONG)
 116 #define BTOPR(b)        (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
 117 #define BTOPRT(b)       (((unsigned int)(b) >> PRSHIFT))
 118
 119 STATIC unsigned long
 120 page_region_mask(
 121         size_t          offset,
 122         size_t          length)
 123 {
 124         unsigned long   mask;
 125         int             first, final;
 126
 127         first = BTOPR(offset);
 128         final = BTOPRT(offset + length - 1);
 129         first = min(first, final);
 130
 131         mask = ~0UL;
 132         mask <<= BITS_PER_LONG - (final - first);
 133         mask >>= BITS_PER_LONG - (final);
 134
 135         ASSERT(offset + length <= PAGE_CACHE_SIZE);
 136         ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
 137
 138         return mask;
 139 }
 140
 141 STATIC inline void
 142 set_page_region(
 143         struct page     *page,
 144         size_t          offset,
 145         size_t          length)
 146 {
 147         set_page_private(page,
 148                 page_private(page) | page_region_mask(offset, length));
 149         if (page_private(page) == ~0UL)
 150                 SetPageUptodate(page);
 151 }
 152
 153 STATIC inline int
 154 test_page_region(
 155         struct page     *page,
 156         size_t          offset,
 157         size_t          length)
 158 {
 159         unsigned long   mask = page_region_mask(offset, length);
 160
 161         return (mask && (page_private(page) & mask) == mask);
 162 }
 163
 164 /*
 165  *      Mapping of multi-page buffers into contiguous virtual space
 166  */
 167
 168 typedef struct a_list {
 169         void            *vm_addr;
 170         struct a_list   *next;
 171 } a_list_t;
 172
 173 STATIC a_list_t         *as_free_head;
 174 STATIC int              as_list_len;
 175 STATIC DEFINE_SPINLOCK(as_lock);
 176
 177 /*
 178  *      Try to batch vunmaps because they are costly.
 179  */
 180 STATIC void
 181 free_address(
 182         void            *addr)
 183 {
 184         a_list_t        *aentry;
 185
 186         aentry = kmalloc(sizeof(a_list_t), GFP_NOWAIT);
 187         if (likely(aentry)) {
 188                 spin_lock(&as_lock);
 189                 aentry->next = as_free_head;
 190                 aentry->vm_addr = addr;
 191                 as_free_head = aentry;
 192                 as_list_len++;
 193                 spin_unlock(&as_lock);
 194         } else {
 195                 vunmap(addr);
 196         }
 197 }
 198
 199 STATIC void
 200 purge_addresses(void)
 201 {
 202         a_list_t        *aentry, *old;
 203
 204         if (as_free_head == NULL)
 205                 return;
 206
 207         spin_lock(&as_lock);
 208         aentry = as_free_head;
 209         as_free_head = NULL;
 210         as_list_len = 0;
 211         spin_unlock(&as_lock);
 212
 213         while ((old = aentry) != NULL) {
 214                 vunmap(aentry->vm_addr);
 215                 aentry = aentry->next;
 216                 kfree(old);
 217         }
 218 }
 219
 220 /*
 221  *      Internal xfs_buf_t object manipulation
 222  */
 223
 224 STATIC void
 225 _xfs_buf_initialize(
 226         xfs_buf_t               *bp,
 227         xfs_buftarg_t           *target,
 228         xfs_off_t               range_base,
 229         size_t                  range_length,
 230         xfs_buf_flags_t         flags)
 231 {
 232         /*
 233          * We don't want certain flags to appear in b_flags.
 234          */
 235         flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
 236
 237         memset(bp, 0, sizeof(xfs_buf_t));
 238         atomic_set(&bp->b_hold, 1);
 239         init_MUTEX_LOCKED(&bp->b_iodonesema);
 240         INIT_LIST_HEAD(&bp->b_list);
 241         INIT_LIST_HEAD(&bp->b_hash_list);
 242         init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
 243         XB_SET_OWNER(bp);
 244         bp->b_target = target;
 245         bp->b_file_offset = range_base;
 246         /*
 247          * Set buffer_length and count_desired to the same value initially.
 248          * I/O routines should use count_desired, which will be the same in
 249          * most cases but may be reset (e.g. XFS recovery).
 250          */
 251         bp->b_buffer_length = bp->b_count_desired = range_length;
 252         bp->b_flags = flags;
 253         bp->b_bn = XFS_BUF_DADDR_NULL;
 254         atomic_set(&bp->b_pin_count, 0);
 255         init_waitqueue_head(&bp->b_waiters);
 256
 257         XFS_STATS_INC(xb_create);
 258         XB_TRACE(bp, "initialize", target);
 259 }
 260
 261 /*
 262  *      Allocate a page array capable of holding a specified number
 263  *      of pages, and point the page buf at it.
 264  */
 265 STATIC int
 266 _xfs_buf_get_pages(
 267         xfs_buf_t               *bp,
 268         int                     page_count,
 269         xfs_buf_flags_t         flags)
 270 {
 271         /* Make sure that we have a page list */
 272         if (bp->b_pages == NULL) {
 273                 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
 274                 bp->b_page_count = page_count;
 275                 if (page_count <= XB_PAGES) {
 276                         bp->b_pages = bp->b_page_array;
 277                 } else {
 278                         bp->b_pages = kmem_alloc(sizeof(struct page *) *
 279                                         page_count, xb_to_km(flags));
 280                         if (bp->b_pages == NULL)
 281                                 return -ENOMEM;
 282                 }
 283                 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
 284         }
 285         return 0;
 286 }
 287
 288 /*
 289  *      Frees b_pages if it was allocated.
 290  */
 291 STATIC void
 292 _xfs_buf_free_pages(
 293         xfs_buf_t       *bp)
 294 {
 295         if (bp->b_pages != bp->b_page_array) {
 296                 kmem_free(bp->b_pages,
 297                           bp->b_page_count * sizeof(struct page *));
 298         }
 299 }
 300
 301 /*
 302  *      Releases the specified buffer.
 303  *
 304  *      The modification state of any associated pages is left unchanged.
 305  *      The buffer most not be on any hash - use xfs_buf_rele instead for
 306  *      hashed and refcounted buffers
 307  */
 308 void
 309 xfs_buf_free(
 310         xfs_buf_t               *bp)
 311 {
 312         XB_TRACE(bp, "free", 0);
 313
 314         ASSERT(list_empty(&bp->b_hash_list));
 315
 316         if (bp->b_flags & _XBF_PAGE_CACHE) {
 317                 uint            i;
 318
 319                 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
 320                         free_address(bp->b_addr - bp->b_offset);
 321
 322                 for (i = 0; i < bp->b_page_count; i++) {
 323                         struct page     *page = bp->b_pages[i];
 324
 325                         ASSERT(!PagePrivate(page));
 326                         page_cache_release(page);
 327                 }
 328                 _xfs_buf_free_pages(bp);
 329         } else if (bp->b_flags & _XBF_KMEM_ALLOC) {
 330                  /*
 331                   * XXX(hch): bp->b_count_desired might be incorrect (see
 332                   * xfs_buf_associate_memory for details), but fortunately
 333                   * the Linux version of kmem_free ignores the len argument..
 334                   */
 335                 kmem_free(bp->b_addr, bp->b_count_desired);
 336                 _xfs_buf_free_pages(bp);
 337         }
 338
 339         xfs_buf_deallocate(bp);
 340 }
 341
 342 /*
 343  *      Finds all pages for buffer in question and builds it's page list.
 344  */
 345 STATIC int
 346 _xfs_buf_lookup_pages(
 347         xfs_buf_t               *bp,
 348         uint                    flags)
 349 {
 350         struct address_space    *mapping = bp->b_target->bt_mapping;
 351         size_t                  blocksize = bp->b_target->bt_bsize;
 352         size_t                  size = bp->b_count_desired;
 353         size_t                  nbytes, offset;
 354         gfp_t                   gfp_mask = xb_to_gfp(flags);
 355         unsigned short          page_count, i;
 356         pgoff_t                 first;
 357         xfs_off_t               end;
 358         int                     error;
 359
 360         end = bp->b_file_offset + bp->b_buffer_length;
 361         page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
 362
 363         error = _xfs_buf_get_pages(bp, page_count, flags);
 364         if (unlikely(error))
 365                 return error;
 366         bp->b_flags |= _XBF_PAGE_CACHE;
 367
 368         offset = bp->b_offset;
 369         first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
 370
 371         for (i = 0; i < bp->b_page_count; i++) {
 372                 struct page     *page;
 373                 uint            retries = 0;
 374
 375               retry:
 376                 page = find_or_create_page(mapping, first + i, gfp_mask);
 377                 if (unlikely(page == NULL)) {
 378                         if (flags & XBF_READ_AHEAD) {
 379                                 bp->b_page_count = i;
 380                                 for (i = 0; i < bp->b_page_count; i++)
 381                                         unlock_page(bp->b_pages[i]);
 382                                 return -ENOMEM;
 383                         }
 384
 385                         /*
 386                          * This could deadlock.
 387                          *
 388                          * But until all the XFS lowlevel code is revamped to
 389                          * handle buffer allocation failures we can't do much.
 390                          */
 391                         if (!(++retries % 100))
 392                                 printk(KERN_ERR
 393                                         "XFS: possible memory allocation "
 394                                         "deadlock in %s (mode:0x%x)\n",
 395                                         __FUNCTION__, gfp_mask);
 396
 397                         XFS_STATS_INC(xb_page_retries);
 398                         xfsbufd_wakeup(0, gfp_mask);
 399                         congestion_wait(WRITE, HZ/50);
 400                         goto retry;
 401                 }
 402
 403                 XFS_STATS_INC(xb_page_found);
 404
 405                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
 406                 size -= nbytes;
 407
 408                 ASSERT(!PagePrivate(page));
 409                 if (!PageUptodate(page)) {
 410                         page_count--;
 411                         if (blocksize >= PAGE_CACHE_SIZE) {
 412                                 if (flags & XBF_READ)
 413                                         bp->b_locked = 1;
 414                         } else if (!PagePrivate(page)) {
 415                                 if (test_page_region(page, offset, nbytes))
 416                                         page_count++;
 417                         }
 418                 }
 419
 420                 bp->b_pages[i] = page;
 421                 offset = 0;
 422         }
 423
 424         if (!bp->b_locked) {
 425                 for (i = 0; i < bp->b_page_count; i++)
 426                         unlock_page(bp->b_pages[i]);
 427         }
 428
 429         if (page_count == bp->b_page_count)
 430                 bp->b_flags |= XBF_DONE;
 431
 432         XB_TRACE(bp, "lookup_pages", (long)page_count);
 433         return error;
 434 }
 435
 436 /*
 437  *      Map buffer into kernel address-space if nessecary.
 438  */
 439 STATIC int
 440 _xfs_buf_map_pages(
 441         xfs_buf_t               *bp,
 442         uint                    flags)
 443 {
 444         /* A single page buffer is always mappable */
 445         if (bp->b_page_count == 1) {
 446                 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
 447                 bp->b_flags |= XBF_MAPPED;
 448         } else if (flags & XBF_MAPPED) {
 449                 if (as_list_len > 64)
 450                         purge_addresses();
 451                 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
 452                                         VM_MAP, PAGE_KERNEL);
 453                 if (unlikely(bp->b_addr == NULL))
 454                         return -ENOMEM;
 455                 bp->b_addr += bp->b_offset;
 456                 bp->b_flags |= XBF_MAPPED;
 457         }
 458
 459         return 0;
 460 }
 461
 462 /*
 463  *      Finding and Reading Buffers
 464  */
 465
 466 /*
 467  *      Look up, and creates if absent, a lockable buffer for
 468  *      a given range of an inode.  The buffer is returned
 469  *      locked.  If other overlapping buffers exist, they are
 470  *      released before the new buffer is created and locked,
 471  *      which may imply that this call will block until those buffers
 472  *      are unlocked.  No I/O is implied by this call.
 473  */
 474 xfs_buf_t *
 475 _xfs_buf_find(
 476         xfs_buftarg_t           *btp,   /* block device target          */
 477         xfs_off_t               ioff,   /* starting offset of range     */
 478         size_t                  isize,  /* length of range              */
 479         xfs_buf_flags_t         flags,
 480         xfs_buf_t               *new_bp)
 481 {
 482         xfs_off_t               range_base;
 483         size_t                  range_length;
 484         xfs_bufhash_t           *hash;
 485         xfs_buf_t               *bp, *n;
 486
 487         range_base = (ioff << BBSHIFT);
 488         range_length = (isize << BBSHIFT);
 489
 490         /* Check for IOs smaller than the sector size / not sector aligned */
 491         ASSERT(!(range_length < (1 << btp->bt_sshift)));
 492         ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
 493
 494         hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
 495
 496         spin_lock(&hash->bh_lock);
 497
 498         list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
 499                 ASSERT(btp == bp->b_target);
 500                 if (bp->b_file_offset == range_base &&
 501                     bp->b_buffer_length == range_length) {
 502                         /*
 503                          * If we look at something, bring it to the
 504                          * front of the list for next time.
 505                          */
 506                         atomic_inc(&bp->b_hold);
 507                         list_move(&bp->b_hash_list, &hash->bh_list);
 508                         goto found;
 509                 }
 510         }
 511
 512         /* No match found */
 513         if (new_bp) {
 514                 _xfs_buf_initialize(new_bp, btp, range_base,
 515                                 range_length, flags);
 516                 new_bp->b_hash = hash;
 517                 list_add(&new_bp->b_hash_list, &hash->bh_list);
 518         } else {
 519                 XFS_STATS_INC(xb_miss_locked);
 520         }
 521
 522         spin_unlock(&hash->bh_lock);
 523         return new_bp;
 524
 525 found:
 526         spin_unlock(&hash->bh_lock);
 527
 528         /* Attempt to get the semaphore without sleeping,
 529          * if this does not work then we need to drop the
 530          * spinlock and do a hard attempt on the semaphore.
 531          */
 532         if (down_trylock(&bp->b_sema)) {
 533                 if (!(flags & XBF_TRYLOCK)) {
 534                         /* wait for buffer ownership */
 535                         XB_TRACE(bp, "get_lock", 0);
 536                         xfs_buf_lock(bp);
 537                         XFS_STATS_INC(xb_get_locked_waited);
 538                 } else {
 539                         /* We asked for a trylock and failed, no need
 540                          * to look at file offset and length here, we
 541                          * know that this buffer at least overlaps our
 542                          * buffer and is locked, therefore our buffer
 543                          * either does not exist, or is this buffer.
 544                          */
 545                         xfs_buf_rele(bp);
 546                         XFS_STATS_INC(xb_busy_locked);
 547                         return NULL;
 548                 }
 549         } else {
 550                 /* trylock worked */
 551                 XB_SET_OWNER(bp);
 552         }
 553
 554         if (bp->b_flags & XBF_STALE) {
 555                 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
 556                 bp->b_flags &= XBF_MAPPED;
 557         }
 558         XB_TRACE(bp, "got_lock", 0);
 559         XFS_STATS_INC(xb_get_locked);
 560         return bp;
 561 }
 562
 563 /*
 564  *      Assembles a buffer covering the specified range.
 565  *      Storage in memory for all portions of the buffer will be allocated,
 566  *      although backing storage may not be.
 567  */
 568 xfs_buf_t *
 569 xfs_buf_get_flags(
 570         xfs_buftarg_t           *target,/* target for buffer            */
 571         xfs_off_t               ioff,   /* starting offset of range     */
 572         size_t                  isize,  /* length of range              */
 573         xfs_buf_flags_t         flags)
 574 {
 575         xfs_buf_t               *bp, *new_bp;
 576         int                     error = 0, i;
 577
 578         new_bp = xfs_buf_allocate(flags);
 579         if (unlikely(!new_bp))
 580                 return NULL;
 581
 582         bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
 583         if (bp == new_bp) {
 584                 error = _xfs_buf_lookup_pages(bp, flags);
 585                 if (error)
 586                         goto no_buffer;
 587         } else {
 588                 xfs_buf_deallocate(new_bp);
 589                 if (unlikely(bp == NULL))
 590                         return NULL;
 591         }
 592
 593         for (i = 0; i < bp->b_page_count; i++)
 594                 mark_page_accessed(bp->b_pages[i]);
 595
 596         if (!(bp->b_flags & XBF_MAPPED)) {
 597                 error = _xfs_buf_map_pages(bp, flags);
 598                 if (unlikely(error)) {
 599                         printk(KERN_WARNING "%s: failed to map pages\n",
 600                                         __FUNCTION__);
 601                         goto no_buffer;
 602                 }
 603         }
 604
 605         XFS_STATS_INC(xb_get);
 606
 607         /*
 608          * Always fill in the block number now, the mapped cases can do
 609          * their own overlay of this later.
 610          */
 611         bp->b_bn = ioff;
 612         bp->b_count_desired = bp->b_buffer_length;
 613
 614         XB_TRACE(bp, "get", (unsigned long)flags);
 615         return bp;
 616
 617  no_buffer:
 618         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 619                 xfs_buf_unlock(bp);
 620         xfs_buf_rele(bp);
 621         return NULL;
 622 }
 623
 624 xfs_buf_t *
 625 xfs_buf_read_flags(
 626         xfs_buftarg_t           *target,
 627         xfs_off_t               ioff,
 628         size_t                  isize,
 629         xfs_buf_flags_t         flags)
 630 {
 631         xfs_buf_t               *bp;
 632
 633         flags |= XBF_READ;
 634
 635         bp = xfs_buf_get_flags(target, ioff, isize, flags);
 636         if (bp) {
 637                 if (!XFS_BUF_ISDONE(bp)) {
 638                         XB_TRACE(bp, "read", (unsigned long)flags);
 639                         XFS_STATS_INC(xb_get_read);
 640                         xfs_buf_iostart(bp, flags);
 641                 } else if (flags & XBF_ASYNC) {
 642                         XB_TRACE(bp, "read_async", (unsigned long)flags);
 643                         /*
 644                          * Read ahead call which is already satisfied,
 645                          * drop the buffer
 646                          */
 647                         goto no_buffer;
 648                 } else {
 649                         XB_TRACE(bp, "read_done", (unsigned long)flags);
 650                         /* We do not want read in the flags */
 651                         bp->b_flags &= ~XBF_READ;
 652                 }
 653         }
 654
 655         return bp;
 656
 657  no_buffer:
 658         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 659                 xfs_buf_unlock(bp);
 660         xfs_buf_rele(bp);
 661         return NULL;
 662 }
 663
 664 /*
 665  *      If we are not low on memory then do the readahead in a deadlock
 666  *      safe manner.
 667  */
 668 void
 669 xfs_buf_readahead(
 670         xfs_buftarg_t           *target,
 671         xfs_off_t               ioff,
 672         size_t                  isize,
 673         xfs_buf_flags_t         flags)
 674 {
 675         struct backing_dev_info *bdi;
 676
 677         bdi = target->bt_mapping->backing_dev_info;
 678         if (bdi_read_congested(bdi))
 679                 return;
 680
 681         flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
 682         xfs_buf_read_flags(target, ioff, isize, flags);
 683 }
 684
 685 xfs_buf_t *
 686 xfs_buf_get_empty(
 687         size_t                  len,
 688         xfs_buftarg_t           *target)
 689 {
 690         xfs_buf_t               *bp;
 691
 692         bp = xfs_buf_allocate(0);
 693         if (bp)
 694                 _xfs_buf_initialize(bp, target, 0, len, 0);
 695         return bp;
 696 }
 697
 698 static inline struct page *
 699 mem_to_page(
 700         void                    *addr)
 701 {
 702         if (((unsigned long)addr < VMALLOC_START) ||
 703             ((unsigned long)addr >= VMALLOC_END)) {
 704                 return virt_to_page(addr);
 705         } else {
 706                 return vmalloc_to_page(addr);
 707         }
 708 }
 709
 710 int
 711 xfs_buf_associate_memory(
 712         xfs_buf_t               *bp,
 713         void                    *mem,
 714         size_t                  len)
 715 {
 716         int                     rval;
 717         int                     i = 0;
 718         size_t                  ptr;
 719         size_t                  end, end_cur;
 720         off_t                   offset;
 721         int                     page_count;
 722
 723         page_count = PAGE_CACHE_ALIGN(len) >> PAGE_CACHE_SHIFT;
 724         offset = (off_t) mem - ((off_t)mem & PAGE_CACHE_MASK);
 725         if (offset && (len > PAGE_CACHE_SIZE))
 726                 page_count++;
 727
 728         /* Free any previous set of page pointers */
 729         if (bp->b_pages)
 730                 _xfs_buf_free_pages(bp);
 731
 732         bp->b_pages = NULL;
 733         bp->b_addr = mem;
 734
 735         rval = _xfs_buf_get_pages(bp, page_count, 0);
 736         if (rval)
 737                 return rval;
 738
 739         bp->b_offset = offset;
 740         ptr = (size_t) mem & PAGE_CACHE_MASK;
 741         end = PAGE_CACHE_ALIGN((size_t) mem + len);
 742         end_cur = end;
 743         /* set up first page */
 744         bp->b_pages[0] = mem_to_page(mem);
 745
 746         ptr += PAGE_CACHE_SIZE;
 747         bp->b_page_count = ++i;
 748         while (ptr < end) {
 749                 bp->b_pages[i] = mem_to_page((void *)ptr);
 750                 bp->b_page_count = ++i;
 751                 ptr += PAGE_CACHE_SIZE;
 752         }
 753         bp->b_locked = 0;
 754
 755         bp->b_count_desired = bp->b_buffer_length = len;
 756         bp->b_flags |= XBF_MAPPED;
 757
 758         return 0;
 759 }
 760
 761 xfs_buf_t *
 762 xfs_buf_get_noaddr(
 763         size_t                  len,
 764         xfs_buftarg_t           *target)
 765 {
 766         size_t                  malloc_len = len;
 767         xfs_buf_t               *bp;
 768         void                    *data;
 769         int                     error;
 770
 771         bp = xfs_buf_allocate(0);
 772         if (unlikely(bp == NULL))
 773                 goto fail;
 774         _xfs_buf_initialize(bp, target, 0, len, 0);
 775
 776  try_again:
 777         data = kmem_alloc(malloc_len, KM_SLEEP | KM_MAYFAIL | KM_LARGE);
 778         if (unlikely(data == NULL))
 779                 goto fail_free_buf;
 780
 781         /* check whether alignment matches.. */
 782         if ((__psunsigned_t)data !=
 783             ((__psunsigned_t)data & ~target->bt_smask)) {
 784                 /* .. else double the size and try again */
 785                 kmem_free(data, malloc_len);
 786                 malloc_len <<= 1;
 787                 goto try_again;
 788         }
 789
 790         error = xfs_buf_associate_memory(bp, data, len);
 791         if (error)
 792                 goto fail_free_mem;
 793         bp->b_flags |= _XBF_KMEM_ALLOC;
 794
 795         xfs_buf_unlock(bp);
 796
 797         XB_TRACE(bp, "no_daddr", data);
 798         return bp;
 799  fail_free_mem:
 800         kmem_free(data, malloc_len);
 801  fail_free_buf:
 802         xfs_buf_free(bp);
 803  fail:
 804         return NULL;
 805 }
 806
 807 /*
 808  *      Increment reference count on buffer, to hold the buffer concurrently
 809  *      with another thread which may release (free) the buffer asynchronously.
 810  *      Must hold the buffer already to call this function.
 811  */
 812 void
 813 xfs_buf_hold(
 814         xfs_buf_t               *bp)
 815 {
 816         atomic_inc(&bp->b_hold);
 817         XB_TRACE(bp, "hold", 0);
 818 }
 819
 820 /*
 821  *      Releases a hold on the specified buffer.  If the
 822  *      the hold count is 1, calls xfs_buf_free.
 823  */
 824 void
 825 xfs_buf_rele(
 826         xfs_buf_t               *bp)
 827 {
 828         xfs_bufhash_t           *hash = bp->b_hash;
 829
 830         XB_TRACE(bp, "rele", bp->b_relse);
 831
 832         if (unlikely(!hash)) {
 833                 ASSERT(!bp->b_relse);
 834                 if (atomic_dec_and_test(&bp->b_hold))
 835                         xfs_buf_free(bp);
 836                 return;
 837         }
 838
 839         if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
 840                 if (bp->b_relse) {
 841                         atomic_inc(&bp->b_hold);
 842                         spin_unlock(&hash->bh_lock);
 843                         (*(bp->b_relse)) (bp);
 844                 } else if (bp->b_flags & XBF_FS_MANAGED) {
 845                         spin_unlock(&hash->bh_lock);
 846                 } else {
 847                         ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
 848                         list_del_init(&bp->b_hash_list);
 849                         spin_unlock(&hash->bh_lock);
 850                         xfs_buf_free(bp);
 851                 }
 852         } else {
 853                 /*
 854                  * Catch reference count leaks
 855                  */
 856                 ASSERT(atomic_read(&bp->b_hold) >= 0);
 857         }
 858 }
 859
 860
 861 /*
 862  *      Mutual exclusion on buffers.  Locking model:
 863  *
 864  *      Buffers associated with inodes for which buffer locking
 865  *      is not enabled are not protected by semaphores, and are
 866  *      assumed to be exclusively owned by the caller.  There is a
 867  *      spinlock in the buffer, used by the caller when concurrent
 868  *      access is possible.
 869  */
 870
 871 /*
 872  *      Locks a buffer object, if it is not already locked.
 873  *      Note that this in no way locks the underlying pages, so it is only
 874  *      useful for synchronizing concurrent use of buffer objects, not for
 875  *      synchronizing independent access to the underlying pages.
 876  */
 877 int
 878 xfs_buf_cond_lock(
 879         xfs_buf_t               *bp)
 880 {
 881         int                     locked;
 882
 883         locked = down_trylock(&bp->b_sema) == 0;
 884         if (locked) {
 885                 XB_SET_OWNER(bp);
 886         }
 887         XB_TRACE(bp, "cond_lock", (long)locked);
 888         return locked ? 0 : -EBUSY;
 889 }
 890
 891 #if defined(DEBUG) || defined(XFS_BLI_TRACE)
 892 int
 893 xfs_buf_lock_value(
 894         xfs_buf_t               *bp)
 895 {
 896         return atomic_read(&bp->b_sema.count);
 897 }
 898 #endif
 899
 900 /*
 901  *      Locks a buffer object.
 902  *      Note that this in no way locks the underlying pages, so it is only
 903  *      useful for synchronizing concurrent use of buffer objects, not for
 904  *      synchronizing independent access to the underlying pages.
 905  */
 906 void
 907 xfs_buf_lock(
 908         xfs_buf_t               *bp)
 909 {
 910         XB_TRACE(bp, "lock", 0);
 911         if (atomic_read(&bp->b_io_remaining))
 912                 blk_run_address_space(bp->b_target->bt_mapping);
 913         down(&bp->b_sema);
 914         XB_SET_OWNER(bp);
 915         XB_TRACE(bp, "locked", 0);
 916 }
 917
 918 /*
 919  *      Releases the lock on the buffer object.
 920  *      If the buffer is marked delwri but is not queued, do so before we
 921  *      unlock the buffer as we need to set flags correctly.  We also need to
 922  *      take a reference for the delwri queue because the unlocker is going to
 923  *      drop their's and they don't know we just queued it.
 924  */
 925 void
 926 xfs_buf_unlock(
 927         xfs_buf_t               *bp)
 928 {
 929         if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
 930                 atomic_inc(&bp->b_hold);
 931                 bp->b_flags |= XBF_ASYNC;
 932                 xfs_buf_delwri_queue(bp, 0);
 933         }
 934
 935         XB_CLEAR_OWNER(bp);
 936         up(&bp->b_sema);
 937         XB_TRACE(bp, "unlock", 0);
 938 }
 939
 940
 941 /*
 942  *      Pinning Buffer Storage in Memory
 943  *      Ensure that no attempt to force a buffer to disk will succeed.
 944  */
 945 void
 946 xfs_buf_pin(
 947         xfs_buf_t               *bp)
 948 {
 949         atomic_inc(&bp->b_pin_count);
 950         XB_TRACE(bp, "pin", (long)bp->b_pin_count.counter);
 951 }
 952
 953 void
 954 xfs_buf_unpin(
 955         xfs_buf_t               *bp)
 956 {
 957         if (atomic_dec_and_test(&bp->b_pin_count))
 958                 wake_up_all(&bp->b_waiters);
 959         XB_TRACE(bp, "unpin", (long)bp->b_pin_count.counter);
 960 }
 961
 962 int
 963 xfs_buf_ispin(
 964         xfs_buf_t               *bp)
 965 {
 966         return atomic_read(&bp->b_pin_count);
 967 }
 968
 969 STATIC void
 970 xfs_buf_wait_unpin(
 971         xfs_buf_t               *bp)
 972 {
 973         DECLARE_WAITQUEUE       (wait, current);
 974
 975         if (atomic_read(&bp->b_pin_count) == 0)
 976                 return;
 977
 978         add_wait_queue(&bp->b_waiters, &wait);
 979         for (;;) {
 980                 set_current_state(TASK_UNINTERRUPTIBLE);
 981                 if (atomic_read(&bp->b_pin_count) == 0)
 982                         break;
 983                 if (atomic_read(&bp->b_io_remaining))
 984                         blk_run_address_space(bp->b_target->bt_mapping);
 985                 schedule();
 986         }
 987         remove_wait_queue(&bp->b_waiters, &wait);
 988         set_current_state(TASK_RUNNING);
 989 }
 990
 991 /*
 992  *      Buffer Utility Routines
 993  */
 994
 995 STATIC void
 996 xfs_buf_iodone_work(
 997         void                    *v)
 998 {
 999         xfs_buf_t               *bp = (xfs_buf_t *)v;
1000
1001         if (bp->b_iodone)
1002                 (*(bp->b_iodone))(bp);
1003         else if (bp->b_flags & XBF_ASYNC)
1004                 xfs_buf_relse(bp);
1005 }
1006
1007 void
1008 xfs_buf_ioend(
1009         xfs_buf_t               *bp,
1010         int                     schedule)
1011 {
1012         bp->b_flags &= ~(XBF_READ | XBF_WRITE);
1013         if (bp->b_error == 0)
1014                 bp->b_flags |= XBF_DONE;
1015
1016         XB_TRACE(bp, "iodone", bp->b_iodone);
1017
1018         if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1019                 if (schedule) {
1020                         INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work, bp);
1021                         queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1022                 } else {
1023                         xfs_buf_iodone_work(bp);
1024                 }
1025         } else {
1026                 up(&bp->b_iodonesema);
1027         }
1028 }
1029
1030 void
1031 xfs_buf_ioerror(
1032         xfs_buf_t               *bp,
1033         int                     error)
1034 {
1035         ASSERT(error >= 0 && error <= 0xffff);
1036         bp->b_error = (unsigned short)error;
1037         XB_TRACE(bp, "ioerror", (unsigned long)error);
1038 }
1039
1040 /*
1041  *      Initiate I/O on a buffer, based on the flags supplied.
1042  *      The b_iodone routine in the buffer supplied will only be called
1043  *      when all of the subsidiary I/O requests, if any, have been completed.
1044  */
1045 int
1046 xfs_buf_iostart(
1047         xfs_buf_t               *bp,
1048         xfs_buf_flags_t         flags)
1049 {
1050         int                     status = 0;
1051
1052         XB_TRACE(bp, "iostart", (unsigned long)flags);
1053
1054         if (flags & XBF_DELWRI) {
1055                 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC);
1056                 bp->b_flags |= flags & (XBF_DELWRI | XBF_ASYNC);
1057                 xfs_buf_delwri_queue(bp, 1);
1058                 return status;
1059         }
1060
1061         bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
1062                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1063         bp->b_flags |= flags & (XBF_READ | XBF_WRITE | XBF_ASYNC | \
1064                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1065
1066         BUG_ON(bp->b_bn == XFS_BUF_DADDR_NULL);
1067
1068         /* For writes allow an alternate strategy routine to precede
1069          * the actual I/O request (which may not be issued at all in
1070          * a shutdown situation, for example).
1071          */
1072         status = (flags & XBF_WRITE) ?
1073                 xfs_buf_iostrategy(bp) : xfs_buf_iorequest(bp);
1074
1075         /* Wait for I/O if we are not an async request.
1076          * Note: async I/O request completion will release the buffer,
1077          * and that can already be done by this point.  So using the
1078          * buffer pointer from here on, after async I/O, is invalid.
1079          */
1080         if (!status && !(flags & XBF_ASYNC))
1081                 status = xfs_buf_iowait(bp);
1082
1083         return status;
1084 }
1085
1086 STATIC __inline__ int
1087 _xfs_buf_iolocked(
1088         xfs_buf_t               *bp)
1089 {
1090         ASSERT(bp->b_flags & (XBF_READ | XBF_WRITE));
1091         if (bp->b_flags & XBF_READ)
1092                 return bp->b_locked;
1093         return 0;
1094 }
1095
1096 STATIC __inline__ void
1097 _xfs_buf_ioend(
1098         xfs_buf_t               *bp,
1099         int                     schedule)
1100 {
1101         if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1102                 bp->b_locked = 0;
1103                 xfs_buf_ioend(bp, schedule);
1104         }
1105 }
1106
1107 STATIC int
1108 xfs_buf_bio_end_io(
1109         struct bio              *bio,
1110         unsigned int            bytes_done,
1111         int                     error)
1112 {
1113         xfs_buf_t               *bp = (xfs_buf_t *)bio->bi_private;
1114         unsigned int            blocksize = bp->b_target->bt_bsize;
1115         struct bio_vec          *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1116
1117         if (bio->bi_size)
1118                 return 1;
1119
1120         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1121                 bp->b_error = EIO;
1122
1123         do {
1124                 struct page     *page = bvec->bv_page;
1125
1126                 ASSERT(!PagePrivate(page));
1127                 if (unlikely(bp->b_error)) {
1128                         if (bp->b_flags & XBF_READ)
1129                                 ClearPageUptodate(page);
1130                 } else if (blocksize >= PAGE_CACHE_SIZE) {
1131                         SetPageUptodate(page);
1132                 } else if (!PagePrivate(page) &&
1133                                 (bp->b_flags & _XBF_PAGE_CACHE)) {
1134                         set_page_region(page, bvec->bv_offset, bvec->bv_len);
1135                 }
1136
1137                 if (--bvec >= bio->bi_io_vec)
1138                         prefetchw(&bvec->bv_page->flags);
1139
1140                 if (_xfs_buf_iolocked(bp)) {
1141                         unlock_page(page);
1142                 }
1143         } while (bvec >= bio->bi_io_vec);
1144
1145         _xfs_buf_ioend(bp, 1);
1146         bio_put(bio);
1147         return 0;
1148 }
1149
1150 STATIC void
1151 _xfs_buf_ioapply(
1152         xfs_buf_t               *bp)
1153 {
1154         int                     i, rw, map_i, total_nr_pages, nr_pages;
1155         struct bio              *bio;
1156         int                     offset = bp->b_offset;
1157         int                     size = bp->b_count_desired;
1158         sector_t                sector = bp->b_bn;
1159         unsigned int            blocksize = bp->b_target->bt_bsize;
1160         int                     locking = _xfs_buf_iolocked(bp);
1161
1162         total_nr_pages = bp->b_page_count;
1163         map_i = 0;
1164
1165         if (bp->b_flags & XBF_ORDERED) {
1166                 ASSERT(!(bp->b_flags & XBF_READ));
1167                 rw = WRITE_BARRIER;
1168         } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1169                 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1170                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1171                 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
1172         } else {
1173                 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1174                      (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
1175         }
1176
1177         /* Special code path for reading a sub page size buffer in --
1178          * we populate up the whole page, and hence the other metadata
1179          * in the same page.  This optimization is only valid when the
1180          * filesystem block size is not smaller than the page size.
1181          */
1182         if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1183             (bp->b_flags & XBF_READ) && locking &&
1184             (blocksize >= PAGE_CACHE_SIZE)) {
1185                 bio = bio_alloc(GFP_NOIO, 1);
1186
1187                 bio->bi_bdev = bp->b_target->bt_bdev;
1188                 bio->bi_sector = sector - (offset >> BBSHIFT);
1189                 bio->bi_end_io = xfs_buf_bio_end_io;
1190                 bio->bi_private = bp;
1191
1192                 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1193                 size = 0;
1194
1195                 atomic_inc(&bp->b_io_remaining);
1196
1197                 goto submit_io;
1198         }
1199
1200         /* Lock down the pages which we need to for the request */
1201         if (locking && (bp->b_flags & XBF_WRITE) && (bp->b_locked == 0)) {
1202                 for (i = 0; size; i++) {
1203                         int             nbytes = PAGE_CACHE_SIZE - offset;
1204                         struct page     *page = bp->b_pages[i];
1205
1206                         if (nbytes > size)
1207                                 nbytes = size;
1208
1209                         lock_page(page);
1210
1211                         size -= nbytes;
1212                         offset = 0;
1213                 }
1214                 offset = bp->b_offset;
1215                 size = bp->b_count_desired;
1216         }
1217
1218 next_chunk:
1219         atomic_inc(&bp->b_io_remaining);
1220         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1221         if (nr_pages > total_nr_pages)
1222                 nr_pages = total_nr_pages;
1223
1224         bio = bio_alloc(GFP_NOIO, nr_pages);
1225         bio->bi_bdev = bp->b_target->bt_bdev;
1226         bio->bi_sector = sector;
1227         bio->bi_end_io = xfs_buf_bio_end_io;
1228         bio->bi_private = bp;
1229
1230         for (; size && nr_pages; nr_pages--, map_i++) {
1231                 int     rbytes, nbytes = PAGE_CACHE_SIZE - offset;
1232
1233                 if (nbytes > size)
1234                         nbytes = size;
1235
1236                 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1237                 if (rbytes < nbytes)
1238                         break;
1239
1240                 offset = 0;
1241                 sector += nbytes >> BBSHIFT;
1242                 size -= nbytes;
1243                 total_nr_pages--;
1244         }
1245
1246 submit_io:
1247         if (likely(bio->bi_size)) {
1248                 submit_bio(rw, bio);
1249                 if (size)
1250                         goto next_chunk;
1251         } else {
1252                 bio_put(bio);
1253                 xfs_buf_ioerror(bp, EIO);
1254         }
1255 }
1256
1257 int
1258 xfs_buf_iorequest(
1259         xfs_buf_t               *bp)
1260 {
1261         XB_TRACE(bp, "iorequest", 0);
1262
1263         if (bp->b_flags & XBF_DELWRI) {
1264                 xfs_buf_delwri_queue(bp, 1);
1265                 return 0;
1266         }
1267
1268         if (bp->b_flags & XBF_WRITE) {
1269                 xfs_buf_wait_unpin(bp);
1270         }
1271
1272         xfs_buf_hold(bp);
1273
1274         /* Set the count to 1 initially, this will stop an I/O
1275          * completion callout which happens before we have started
1276          * all the I/O from calling xfs_buf_ioend too early.
1277          */
1278         atomic_set(&bp->b_io_remaining, 1);
1279         _xfs_buf_ioapply(bp);
1280         _xfs_buf_ioend(bp, 0);
1281
1282         xfs_buf_rele(bp);
1283         return 0;
1284 }
1285
1286 /*
1287  *      Waits for I/O to complete on the buffer supplied.
1288  *      It returns immediately if no I/O is pending.
1289  *      It returns the I/O error code, if any, or 0 if there was no error.
1290  */
1291 int
1292 xfs_buf_iowait(
1293         xfs_buf_t               *bp)
1294 {
1295         XB_TRACE(bp, "iowait", 0);
1296         if (atomic_read(&bp->b_io_remaining))
1297                 blk_run_address_space(bp->b_target->bt_mapping);
1298         down(&bp->b_iodonesema);
1299         XB_TRACE(bp, "iowaited", (long)bp->b_error);
1300         return bp->b_error;
1301 }
1302
1303 xfs_caddr_t
1304 xfs_buf_offset(
1305         xfs_buf_t               *bp,
1306         size_t                  offset)
1307 {
1308         struct page             *page;
1309
1310         if (bp->b_flags & XBF_MAPPED)
1311                 return XFS_BUF_PTR(bp) + offset;
1312
1313         offset += bp->b_offset;
1314         page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1315         return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
1316 }
1317
1318 /*
1319  *      Move data into or out of a buffer.
1320  */
1321 void
1322 xfs_buf_iomove(
1323         xfs_buf_t               *bp,    /* buffer to process            */
1324         size_t                  boff,   /* starting buffer offset       */
1325         size_t                  bsize,  /* length to copy               */
1326         caddr_t                 data,   /* data address                 */
1327         xfs_buf_rw_t            mode)   /* read/write/zero flag         */
1328 {
1329         size_t                  bend, cpoff, csize;
1330         struct page             *page;
1331
1332         bend = boff + bsize;
1333         while (boff < bend) {
1334                 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1335                 cpoff = xfs_buf_poff(boff + bp->b_offset);
1336                 csize = min_t(size_t,
1337                               PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
1338
1339                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1340
1341                 switch (mode) {
1342                 case XBRW_ZERO:
1343                         memset(page_address(page) + cpoff, 0, csize);
1344                         break;
1345                 case XBRW_READ:
1346                         memcpy(data, page_address(page) + cpoff, csize);
1347                         break;
1348                 case XBRW_WRITE:
1349                         memcpy(page_address(page) + cpoff, data, csize);
1350                 }
1351
1352                 boff += csize;
1353                 data += csize;
1354         }
1355 }
1356
1357 /*
1358  *      Handling of buffer targets (buftargs).
1359  */
1360
1361 /*
1362  *      Wait for any bufs with callbacks that have been submitted but
1363  *      have not yet returned... walk the hash list for the target.
1364  */
1365 void
1366 xfs_wait_buftarg(
1367         xfs_buftarg_t   *btp)
1368 {
1369         xfs_buf_t       *bp, *n;
1370         xfs_bufhash_t   *hash;
1371         uint            i;
1372
1373         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1374                 hash = &btp->bt_hash[i];
1375 again:
1376                 spin_lock(&hash->bh_lock);
1377                 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1378                         ASSERT(btp == bp->b_target);
1379                         if (!(bp->b_flags & XBF_FS_MANAGED)) {
1380                                 spin_unlock(&hash->bh_lock);
1381                                 /*
1382                                  * Catch superblock reference count leaks
1383                                  * immediately
1384                                  */
1385                                 BUG_ON(bp->b_bn == 0);
1386                                 delay(100);
1387                                 goto again;
1388                         }
1389                 }
1390                 spin_unlock(&hash->bh_lock);
1391         }
1392 }
1393
1394 /*
1395  *      Allocate buffer hash table for a given target.
1396  *      For devices containing metadata (i.e. not the log/realtime devices)
1397  *      we need to allocate a much larger hash table.
1398  */
1399 STATIC void
1400 xfs_alloc_bufhash(
1401         xfs_buftarg_t           *btp,
1402         int                     external)
1403 {
1404         unsigned int            i;
1405
1406         btp->bt_hashshift = external ? 3 : 8;   /* 8 or 256 buckets */
1407         btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1408         btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
1409                                         sizeof(xfs_bufhash_t), KM_SLEEP | KM_LARGE);
1410         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1411                 spin_lock_init(&btp->bt_hash[i].bh_lock);
1412                 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1413         }
1414 }
1415
1416 STATIC void
1417 xfs_free_bufhash(
1418         xfs_buftarg_t           *btp)
1419 {
1420         kmem_free(btp->bt_hash, (1<<btp->bt_hashshift) * sizeof(xfs_bufhash_t));
1421         btp->bt_hash = NULL;
1422 }
1423
1424 /*
1425  *      buftarg list for delwrite queue processing
1426  */
1427 STATIC LIST_HEAD(xfs_buftarg_list);
1428 STATIC DEFINE_SPINLOCK(xfs_buftarg_lock);
1429
1430 STATIC void
1431 xfs_register_buftarg(
1432         xfs_buftarg_t           *btp)
1433 {
1434         spin_lock(&xfs_buftarg_lock);
1435         list_add(&btp->bt_list, &xfs_buftarg_list);
1436         spin_unlock(&xfs_buftarg_lock);
1437 }
1438
1439 STATIC void
1440 xfs_unregister_buftarg(
1441         xfs_buftarg_t           *btp)
1442 {
1443         spin_lock(&xfs_buftarg_lock);
1444         list_del(&btp->bt_list);
1445         spin_unlock(&xfs_buftarg_lock);
1446 }
1447
1448 void
1449 xfs_free_buftarg(
1450         xfs_buftarg_t           *btp,
1451         int                     external)
1452 {
1453         xfs_flush_buftarg(btp, 1);
1454         if (external)
1455                 xfs_blkdev_put(btp->bt_bdev);
1456         xfs_free_bufhash(btp);
1457         iput(btp->bt_mapping->host);
1458
1459         /* Unregister the buftarg first so that we don't get a
1460          * wakeup finding a non-existent task
1461          */
1462         xfs_unregister_buftarg(btp);
1463         kthread_stop(btp->bt_task);
1464
1465         kmem_free(btp, sizeof(*btp));
1466 }
1467
1468 STATIC int
1469 xfs_setsize_buftarg_flags(
1470         xfs_buftarg_t           *btp,
1471         unsigned int            blocksize,
1472         unsigned int            sectorsize,
1473         int                     verbose)
1474 {
1475         btp->bt_bsize = blocksize;
1476         btp->bt_sshift = ffs(sectorsize) - 1;
1477         btp->bt_smask = sectorsize - 1;
1478
1479         if (set_blocksize(btp->bt_bdev, sectorsize)) {
1480                 printk(KERN_WARNING
1481                         "XFS: Cannot set_blocksize to %u on device %s\n",
1482                         sectorsize, XFS_BUFTARG_NAME(btp));
1483                 return EINVAL;
1484         }
1485
1486         if (verbose &&
1487             (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1488                 printk(KERN_WARNING
1489                         "XFS: %u byte sectors in use on device %s.  "
1490                         "This is suboptimal; %u or greater is ideal.\n",
1491                         sectorsize, XFS_BUFTARG_NAME(btp),
1492                         (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1493         }
1494
1495         return 0;
1496 }
1497
1498 /*
1499  *      When allocating the initial buffer target we have not yet
1500  *      read in the superblock, so don't know what sized sectors
1501  *      are being used is at this early stage.  Play safe.
1502  */
1503 STATIC int
1504 xfs_setsize_buftarg_early(
1505         xfs_buftarg_t           *btp,
1506         struct block_device     *bdev)
1507 {
1508         return xfs_setsize_buftarg_flags(btp,
1509                         PAGE_CACHE_SIZE, bdev_hardsect_size(bdev), 0);
1510 }
1511
1512 int
1513 xfs_setsize_buftarg(
1514         xfs_buftarg_t           *btp,
1515         unsigned int            blocksize,
1516         unsigned int            sectorsize)
1517 {
1518         return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1519 }
1520
1521 STATIC int
1522 xfs_mapping_buftarg(
1523         xfs_buftarg_t           *btp,
1524         struct block_device     *bdev)
1525 {
1526         struct backing_dev_info *bdi;
1527         struct inode            *inode;
1528         struct address_space    *mapping;
1529         static const struct address_space_operations mapping_aops = {
1530                 .sync_page = block_sync_page,
1531                 .migratepage = fail_migrate_page,
1532         };
1533
1534         inode = new_inode(bdev->bd_inode->i_sb);
1535         if (!inode) {
1536                 printk(KERN_WARNING
1537                         "XFS: Cannot allocate mapping inode for device %s\n",
1538                         XFS_BUFTARG_NAME(btp));
1539                 return ENOMEM;
1540         }
1541         inode->i_mode = S_IFBLK;
1542         inode->i_bdev = bdev;
1543         inode->i_rdev = bdev->bd_dev;
1544         bdi = blk_get_backing_dev_info(bdev);
1545         if (!bdi)
1546                 bdi = &default_backing_dev_info;
1547         mapping = &inode->i_data;
1548         mapping->a_ops = &mapping_aops;
1549         mapping->backing_dev_info = bdi;
1550         mapping_set_gfp_mask(mapping, GFP_NOFS);
1551         btp->bt_mapping = mapping;
1552         return 0;
1553 }
1554
1555 STATIC int
1556 xfs_alloc_delwrite_queue(
1557         xfs_buftarg_t           *btp)
1558 {
1559         int     error = 0;
1560
1561         INIT_LIST_HEAD(&btp->bt_list);
1562         INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1563         spinlock_init(&btp->bt_delwrite_lock, "delwri_lock");
1564         btp->bt_flags = 0;
1565         btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1566         if (IS_ERR(btp->bt_task)) {
1567                 error = PTR_ERR(btp->bt_task);
1568                 goto out_error;
1569         }
1570         xfs_register_buftarg(btp);
1571 out_error:
1572         return error;
1573 }
1574
1575 xfs_buftarg_t *
1576 xfs_alloc_buftarg(
1577         struct block_device     *bdev,
1578         int                     external)
1579 {
1580         xfs_buftarg_t           *btp;
1581
1582         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1583
1584         btp->bt_dev =  bdev->bd_dev;
1585         btp->bt_bdev = bdev;
1586         if (xfs_setsize_buftarg_early(btp, bdev))
1587                 goto error;
1588         if (xfs_mapping_buftarg(btp, bdev))
1589                 goto error;
1590         if (xfs_alloc_delwrite_queue(btp))
1591                 goto error;
1592         xfs_alloc_bufhash(btp, external);
1593         return btp;
1594
1595 error:
1596         kmem_free(btp, sizeof(*btp));
1597         return NULL;
1598 }
1599
1600
1601 /*
1602  *      Delayed write buffer handling
1603  */
1604 STATIC void
1605 xfs_buf_delwri_queue(
1606         xfs_buf_t               *bp,
1607         int                     unlock)
1608 {
1609         struct list_head        *dwq = &bp->b_target->bt_delwrite_queue;
1610         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1611
1612         XB_TRACE(bp, "delwri_q", (long)unlock);
1613         ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1614
1615         spin_lock(dwlk);
1616         /* If already in the queue, dequeue and place at tail */
1617         if (!list_empty(&bp->b_list)) {
1618                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1619                 if (unlock)
1620                         atomic_dec(&bp->b_hold);
1621                 list_del(&bp->b_list);
1622         }
1623
1624         bp->b_flags |= _XBF_DELWRI_Q;
1625         list_add_tail(&bp->b_list, dwq);
1626         bp->b_queuetime = jiffies;
1627         spin_unlock(dwlk);
1628
1629         if (unlock)
1630                 xfs_buf_unlock(bp);
1631 }
1632
1633 void
1634 xfs_buf_delwri_dequeue(
1635         xfs_buf_t               *bp)
1636 {
1637         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1638         int                     dequeued = 0;
1639
1640         spin_lock(dwlk);
1641         if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1642                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1643                 list_del_init(&bp->b_list);
1644                 dequeued = 1;
1645         }
1646         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1647         spin_unlock(dwlk);
1648
1649         if (dequeued)
1650                 xfs_buf_rele(bp);
1651
1652         XB_TRACE(bp, "delwri_dq", (long)dequeued);
1653 }
1654
1655 STATIC void
1656 xfs_buf_runall_queues(
1657         struct workqueue_struct *queue)
1658 {
1659         flush_workqueue(queue);
1660 }
1661
1662 STATIC int
1663 xfsbufd_wakeup(
1664         int                     priority,
1665         gfp_t                   mask)
1666 {
1667         xfs_buftarg_t           *btp;
1668
1669         spin_lock(&xfs_buftarg_lock);
1670         list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
1671                 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
1672                         continue;
1673                 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
1674                 wake_up_process(btp->bt_task);
1675         }
1676         spin_unlock(&xfs_buftarg_lock);
1677         return 0;
1678 }
1679
1680 STATIC int
1681 xfsbufd(
1682         void                    *data)
1683 {
1684         struct list_head        tmp;
1685         unsigned long           age;
1686         xfs_buftarg_t           *target = (xfs_buftarg_t *)data;
1687         xfs_buf_t               *bp, *n;
1688         struct list_head        *dwq = &target->bt_delwrite_queue;
1689         spinlock_t              *dwlk = &target->bt_delwrite_lock;
1690         int                     count;
1691
1692         current->flags |= PF_MEMALLOC;
1693
1694         INIT_LIST_HEAD(&tmp);
1695         do {
1696                 if (unlikely(freezing(current))) {
1697                         set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1698                         refrigerator();
1699                 } else {
1700                         clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1701                 }
1702
1703                 schedule_timeout_interruptible(
1704                         xfs_buf_timer_centisecs * msecs_to_jiffies(10));
1705
1706                 count = 0;
1707                 age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1708                 spin_lock(dwlk);
1709                 list_for_each_entry_safe(bp, n, dwq, b_list) {
1710                         XB_TRACE(bp, "walkq1", (long)xfs_buf_ispin(bp));
1711                         ASSERT(bp->b_flags & XBF_DELWRI);
1712
1713                         if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
1714                                 if (!test_bit(XBT_FORCE_FLUSH,
1715                                                 &target->bt_flags) &&
1716                                     time_before(jiffies,
1717                                                 bp->b_queuetime + age)) {
1718                                         xfs_buf_unlock(bp);
1719                                         break;
1720                                 }
1721
1722                                 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1723                                                  _XBF_RUN_QUEUES);
1724                                 bp->b_flags |= XBF_WRITE;
1725                                 list_move_tail(&bp->b_list, &tmp);
1726                                 count++;
1727                         }
1728                 }
1729                 spin_unlock(dwlk);
1730
1731                 while (!list_empty(&tmp)) {
1732                         bp = list_entry(tmp.next, xfs_buf_t, b_list);
1733                         ASSERT(target == bp->b_target);
1734
1735                         list_del_init(&bp->b_list);
1736                         xfs_buf_iostrategy(bp);
1737                 }
1738
1739                 if (as_list_len > 0)
1740                         purge_addresses();
1741                 if (count)
1742                         blk_run_address_space(target->bt_mapping);
1743
1744                 clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1745         } while (!kthread_should_stop());
1746
1747         return 0;
1748 }
1749
1750 /*
1751  *      Go through all incore buffers, and release buffers if they belong to
1752  *      the given device. This is used in filesystem error handling to
1753  *      preserve the consistency of its metadata.
1754  */
1755 int
1756 xfs_flush_buftarg(
1757         xfs_buftarg_t           *target,
1758         int                     wait)
1759 {
1760         struct list_head        tmp;
1761         xfs_buf_t               *bp, *n;
1762         int                     pincount = 0;
1763         struct list_head        *dwq = &target->bt_delwrite_queue;
1764         spinlock_t              *dwlk = &target->bt_delwrite_lock;
1765
1766         xfs_buf_runall_queues(xfsdatad_workqueue);
1767         xfs_buf_runall_queues(xfslogd_workqueue);
1768
1769         INIT_LIST_HEAD(&tmp);
1770         spin_lock(dwlk);
1771         list_for_each_entry_safe(bp, n, dwq, b_list) {
1772                 ASSERT(bp->b_target == target);
1773                 ASSERT(bp->b_flags & (XBF_DELWRI | _XBF_DELWRI_Q));
1774                 XB_TRACE(bp, "walkq2", (long)xfs_buf_ispin(bp));
1775                 if (xfs_buf_ispin(bp)) {
1776                         pincount++;
1777                         continue;
1778                 }
1779
1780                 list_move_tail(&bp->b_list, &tmp);
1781         }
1782         spin_unlock(dwlk);
1783
1784         /*
1785          * Dropped the delayed write list lock, now walk the temporary list
1786          */
1787         list_for_each_entry_safe(bp, n, &tmp, b_list) {
1788                 xfs_buf_lock(bp);
1789                 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|_XBF_RUN_QUEUES);
1790                 bp->b_flags |= XBF_WRITE;
1791                 if (wait)
1792                         bp->b_flags &= ~XBF_ASYNC;
1793                 else
1794                         list_del_init(&bp->b_list);
1795
1796                 xfs_buf_iostrategy(bp);
1797         }
1798
1799         if (wait)
1800                 blk_run_address_space(target->bt_mapping);
1801
1802         /*
1803          * Remaining list items must be flushed before returning
1804          */
1805         while (!list_empty(&tmp)) {
1806                 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1807
1808                 list_del_init(&bp->b_list);
1809                 xfs_iowait(bp);
1810                 xfs_buf_relse(bp);
1811         }
1812
1813         return pincount;
1814 }
1815
1816 int __init
1817 xfs_buf_init(void)
1818 {
1819 #ifdef XFS_BUF_TRACE
1820         xfs_buf_trace_buf = ktrace_alloc(XFS_BUF_TRACE_SIZE, KM_SLEEP);
1821 #endif
1822
1823         xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1824                                                 KM_ZONE_HWALIGN, NULL);
1825         if (!xfs_buf_zone)
1826                 goto out_free_trace_buf;
1827
1828         xfslogd_workqueue = create_workqueue("xfslogd");
1829         if (!xfslogd_workqueue)
1830                 goto out_free_buf_zone;
1831
1832         xfsdatad_workqueue = create_workqueue("xfsdatad");
1833         if (!xfsdatad_workqueue)
1834                 goto out_destroy_xfslogd_workqueue;
1835
1836         xfs_buf_shake = kmem_shake_register(xfsbufd_wakeup);
1837         if (!xfs_buf_shake)
1838                 goto out_destroy_xfsdatad_workqueue;
1839
1840         return 0;
1841
1842  out_destroy_xfsdatad_workqueue:
1843         destroy_workqueue(xfsdatad_workqueue);
1844  out_destroy_xfslogd_workqueue:
1845         destroy_workqueue(xfslogd_workqueue);
1846  out_free_buf_zone:
1847         kmem_zone_destroy(xfs_buf_zone);
1848  out_free_trace_buf:
1849 #ifdef XFS_BUF_TRACE
1850         ktrace_free(xfs_buf_trace_buf);
1851 #endif
1852         return -ENOMEM;
1853 }
1854
1855 void
1856 xfs_buf_terminate(void)
1857 {
1858         kmem_shake_deregister(xfs_buf_shake);
1859         destroy_workqueue(xfsdatad_workqueue);
1860         destroy_workqueue(xfslogd_workqueue);
1861         kmem_zone_destroy(xfs_buf_zone);
1862 #ifdef XFS_BUF_TRACE
1863         ktrace_free(xfs_buf_trace_buf);
1864 #endif
1865 }