Merge branch 'upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/vitb/linux...
[pandora-kernel.git] / fs / buffer.c
1 /*
2  *  linux/fs/buffer.c
3  *
4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
5  */
6
7 /*
8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9  *
10  * Removed a lot of unnecessary code and simplified things now that
11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12  *
13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
15  *
16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17  *
18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/smp_lock.h>
28 #include <linux/capability.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/module.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
44
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46 static void invalidate_bh_lrus(void);
47
48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
49
50 inline void
51 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52 {
53         bh->b_end_io = handler;
54         bh->b_private = private;
55 }
56
57 static int sync_buffer(void *word)
58 {
59         struct block_device *bd;
60         struct buffer_head *bh
61                 = container_of(word, struct buffer_head, b_state);
62
63         smp_mb();
64         bd = bh->b_bdev;
65         if (bd)
66                 blk_run_address_space(bd->bd_inode->i_mapping);
67         io_schedule();
68         return 0;
69 }
70
71 void fastcall __lock_buffer(struct buffer_head *bh)
72 {
73         wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74                                                         TASK_UNINTERRUPTIBLE);
75 }
76 EXPORT_SYMBOL(__lock_buffer);
77
78 void fastcall unlock_buffer(struct buffer_head *bh)
79 {
80         clear_buffer_locked(bh);
81         smp_mb__after_clear_bit();
82         wake_up_bit(&bh->b_state, BH_Lock);
83 }
84
85 /*
86  * Block until a buffer comes unlocked.  This doesn't stop it
87  * from becoming locked again - you have to lock it yourself
88  * if you want to preserve its state.
89  */
90 void __wait_on_buffer(struct buffer_head * bh)
91 {
92         wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
93 }
94
95 static void
96 __clear_page_buffers(struct page *page)
97 {
98         ClearPagePrivate(page);
99         set_page_private(page, 0);
100         page_cache_release(page);
101 }
102
103 static void buffer_io_error(struct buffer_head *bh)
104 {
105         char b[BDEVNAME_SIZE];
106
107         printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
108                         bdevname(bh->b_bdev, b),
109                         (unsigned long long)bh->b_blocknr);
110 }
111
112 /*
113  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
114  * unlock the buffer. This is what ll_rw_block uses too.
115  */
116 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
117 {
118         if (uptodate) {
119                 set_buffer_uptodate(bh);
120         } else {
121                 /* This happens, due to failed READA attempts. */
122                 clear_buffer_uptodate(bh);
123         }
124         unlock_buffer(bh);
125         put_bh(bh);
126 }
127
128 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
129 {
130         char b[BDEVNAME_SIZE];
131
132         if (uptodate) {
133                 set_buffer_uptodate(bh);
134         } else {
135                 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
136                         buffer_io_error(bh);
137                         printk(KERN_WARNING "lost page write due to "
138                                         "I/O error on %s\n",
139                                        bdevname(bh->b_bdev, b));
140                 }
141                 set_buffer_write_io_error(bh);
142                 clear_buffer_uptodate(bh);
143         }
144         unlock_buffer(bh);
145         put_bh(bh);
146 }
147
148 /*
149  * Write out and wait upon all the dirty data associated with a block
150  * device via its mapping.  Does not take the superblock lock.
151  */
152 int sync_blockdev(struct block_device *bdev)
153 {
154         int ret = 0;
155
156         if (bdev)
157                 ret = filemap_write_and_wait(bdev->bd_inode->i_mapping);
158         return ret;
159 }
160 EXPORT_SYMBOL(sync_blockdev);
161
162 /*
163  * Write out and wait upon all dirty data associated with this
164  * device.   Filesystem data as well as the underlying block
165  * device.  Takes the superblock lock.
166  */
167 int fsync_bdev(struct block_device *bdev)
168 {
169         struct super_block *sb = get_super(bdev);
170         if (sb) {
171                 int res = fsync_super(sb);
172                 drop_super(sb);
173                 return res;
174         }
175         return sync_blockdev(bdev);
176 }
177
178 /**
179  * freeze_bdev  --  lock a filesystem and force it into a consistent state
180  * @bdev:       blockdevice to lock
181  *
182  * This takes the block device bd_mount_mutex to make sure no new mounts
183  * happen on bdev until thaw_bdev() is called.
184  * If a superblock is found on this device, we take the s_umount semaphore
185  * on it to make sure nobody unmounts until the snapshot creation is done.
186  */
187 struct super_block *freeze_bdev(struct block_device *bdev)
188 {
189         struct super_block *sb;
190
191         mutex_lock(&bdev->bd_mount_mutex);
192         sb = get_super(bdev);
193         if (sb && !(sb->s_flags & MS_RDONLY)) {
194                 sb->s_frozen = SB_FREEZE_WRITE;
195                 smp_wmb();
196
197                 __fsync_super(sb);
198
199                 sb->s_frozen = SB_FREEZE_TRANS;
200                 smp_wmb();
201
202                 sync_blockdev(sb->s_bdev);
203
204                 if (sb->s_op->write_super_lockfs)
205                         sb->s_op->write_super_lockfs(sb);
206         }
207
208         sync_blockdev(bdev);
209         return sb;      /* thaw_bdev releases s->s_umount and bd_mount_sem */
210 }
211 EXPORT_SYMBOL(freeze_bdev);
212
213 /**
214  * thaw_bdev  -- unlock filesystem
215  * @bdev:       blockdevice to unlock
216  * @sb:         associated superblock
217  *
218  * Unlocks the filesystem and marks it writeable again after freeze_bdev().
219  */
220 void thaw_bdev(struct block_device *bdev, struct super_block *sb)
221 {
222         if (sb) {
223                 BUG_ON(sb->s_bdev != bdev);
224
225                 if (sb->s_op->unlockfs)
226                         sb->s_op->unlockfs(sb);
227                 sb->s_frozen = SB_UNFROZEN;
228                 smp_wmb();
229                 wake_up(&sb->s_wait_unfrozen);
230                 drop_super(sb);
231         }
232
233         mutex_unlock(&bdev->bd_mount_mutex);
234 }
235 EXPORT_SYMBOL(thaw_bdev);
236
237 /*
238  * Various filesystems appear to want __find_get_block to be non-blocking.
239  * But it's the page lock which protects the buffers.  To get around this,
240  * we get exclusion from try_to_free_buffers with the blockdev mapping's
241  * private_lock.
242  *
243  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
244  * may be quite high.  This code could TryLock the page, and if that
245  * succeeds, there is no need to take private_lock. (But if
246  * private_lock is contended then so is mapping->tree_lock).
247  */
248 static struct buffer_head *
249 __find_get_block_slow(struct block_device *bdev, sector_t block)
250 {
251         struct inode *bd_inode = bdev->bd_inode;
252         struct address_space *bd_mapping = bd_inode->i_mapping;
253         struct buffer_head *ret = NULL;
254         pgoff_t index;
255         struct buffer_head *bh;
256         struct buffer_head *head;
257         struct page *page;
258         int all_mapped = 1;
259
260         index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
261         page = find_get_page(bd_mapping, index);
262         if (!page)
263                 goto out;
264
265         spin_lock(&bd_mapping->private_lock);
266         if (!page_has_buffers(page))
267                 goto out_unlock;
268         head = page_buffers(page);
269         bh = head;
270         do {
271                 if (bh->b_blocknr == block) {
272                         ret = bh;
273                         get_bh(bh);
274                         goto out_unlock;
275                 }
276                 if (!buffer_mapped(bh))
277                         all_mapped = 0;
278                 bh = bh->b_this_page;
279         } while (bh != head);
280
281         /* we might be here because some of the buffers on this page are
282          * not mapped.  This is due to various races between
283          * file io on the block device and getblk.  It gets dealt with
284          * elsewhere, don't buffer_error if we had some unmapped buffers
285          */
286         if (all_mapped) {
287                 printk("__find_get_block_slow() failed. "
288                         "block=%llu, b_blocknr=%llu\n",
289                         (unsigned long long)block,
290                         (unsigned long long)bh->b_blocknr);
291                 printk("b_state=0x%08lx, b_size=%zu\n",
292                         bh->b_state, bh->b_size);
293                 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
294         }
295 out_unlock:
296         spin_unlock(&bd_mapping->private_lock);
297         page_cache_release(page);
298 out:
299         return ret;
300 }
301
302 /* If invalidate_buffers() will trash dirty buffers, it means some kind
303    of fs corruption is going on. Trashing dirty data always imply losing
304    information that was supposed to be just stored on the physical layer
305    by the user.
306
307    Thus invalidate_buffers in general usage is not allwowed to trash
308    dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
309    be preserved.  These buffers are simply skipped.
310   
311    We also skip buffers which are still in use.  For example this can
312    happen if a userspace program is reading the block device.
313
314    NOTE: In the case where the user removed a removable-media-disk even if
315    there's still dirty data not synced on disk (due a bug in the device driver
316    or due an error of the user), by not destroying the dirty buffers we could
317    generate corruption also on the next media inserted, thus a parameter is
318    necessary to handle this case in the most safe way possible (trying
319    to not corrupt also the new disk inserted with the data belonging to
320    the old now corrupted disk). Also for the ramdisk the natural thing
321    to do in order to release the ramdisk memory is to destroy dirty buffers.
322
323    These are two special cases. Normal usage imply the device driver
324    to issue a sync on the device (without waiting I/O completion) and
325    then an invalidate_buffers call that doesn't trash dirty buffers.
326
327    For handling cache coherency with the blkdev pagecache the 'update' case
328    is been introduced. It is needed to re-read from disk any pinned
329    buffer. NOTE: re-reading from disk is destructive so we can do it only
330    when we assume nobody is changing the buffercache under our I/O and when
331    we think the disk contains more recent information than the buffercache.
332    The update == 1 pass marks the buffers we need to update, the update == 2
333    pass does the actual I/O. */
334 void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
335 {
336         struct address_space *mapping = bdev->bd_inode->i_mapping;
337
338         if (mapping->nrpages == 0)
339                 return;
340
341         invalidate_bh_lrus();
342         /*
343          * FIXME: what about destroy_dirty_buffers?
344          * We really want to use invalidate_inode_pages2() for
345          * that, but not until that's cleaned up.
346          */
347         invalidate_inode_pages(mapping);
348 }
349
350 /*
351  * Kick pdflush then try to free up some ZONE_NORMAL memory.
352  */
353 static void free_more_memory(void)
354 {
355         struct zone **zones;
356         pg_data_t *pgdat;
357
358         wakeup_pdflush(1024);
359         yield();
360
361         for_each_online_pgdat(pgdat) {
362                 zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
363                 if (*zones)
364                         try_to_free_pages(zones, GFP_NOFS);
365         }
366 }
367
368 /*
369  * I/O completion handler for block_read_full_page() - pages
370  * which come unlocked at the end of I/O.
371  */
372 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
373 {
374         unsigned long flags;
375         struct buffer_head *first;
376         struct buffer_head *tmp;
377         struct page *page;
378         int page_uptodate = 1;
379
380         BUG_ON(!buffer_async_read(bh));
381
382         page = bh->b_page;
383         if (uptodate) {
384                 set_buffer_uptodate(bh);
385         } else {
386                 clear_buffer_uptodate(bh);
387                 if (printk_ratelimit())
388                         buffer_io_error(bh);
389                 SetPageError(page);
390         }
391
392         /*
393          * Be _very_ careful from here on. Bad things can happen if
394          * two buffer heads end IO at almost the same time and both
395          * decide that the page is now completely done.
396          */
397         first = page_buffers(page);
398         local_irq_save(flags);
399         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
400         clear_buffer_async_read(bh);
401         unlock_buffer(bh);
402         tmp = bh;
403         do {
404                 if (!buffer_uptodate(tmp))
405                         page_uptodate = 0;
406                 if (buffer_async_read(tmp)) {
407                         BUG_ON(!buffer_locked(tmp));
408                         goto still_busy;
409                 }
410                 tmp = tmp->b_this_page;
411         } while (tmp != bh);
412         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
413         local_irq_restore(flags);
414
415         /*
416          * If none of the buffers had errors and they are all
417          * uptodate then we can set the page uptodate.
418          */
419         if (page_uptodate && !PageError(page))
420                 SetPageUptodate(page);
421         unlock_page(page);
422         return;
423
424 still_busy:
425         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
426         local_irq_restore(flags);
427         return;
428 }
429
430 /*
431  * Completion handler for block_write_full_page() - pages which are unlocked
432  * during I/O, and which have PageWriteback cleared upon I/O completion.
433  */
434 static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
435 {
436         char b[BDEVNAME_SIZE];
437         unsigned long flags;
438         struct buffer_head *first;
439         struct buffer_head *tmp;
440         struct page *page;
441
442         BUG_ON(!buffer_async_write(bh));
443
444         page = bh->b_page;
445         if (uptodate) {
446                 set_buffer_uptodate(bh);
447         } else {
448                 if (printk_ratelimit()) {
449                         buffer_io_error(bh);
450                         printk(KERN_WARNING "lost page write due to "
451                                         "I/O error on %s\n",
452                                bdevname(bh->b_bdev, b));
453                 }
454                 set_bit(AS_EIO, &page->mapping->flags);
455                 clear_buffer_uptodate(bh);
456                 SetPageError(page);
457         }
458
459         first = page_buffers(page);
460         local_irq_save(flags);
461         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
462
463         clear_buffer_async_write(bh);
464         unlock_buffer(bh);
465         tmp = bh->b_this_page;
466         while (tmp != bh) {
467                 if (buffer_async_write(tmp)) {
468                         BUG_ON(!buffer_locked(tmp));
469                         goto still_busy;
470                 }
471                 tmp = tmp->b_this_page;
472         }
473         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
474         local_irq_restore(flags);
475         end_page_writeback(page);
476         return;
477
478 still_busy:
479         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
480         local_irq_restore(flags);
481         return;
482 }
483
484 /*
485  * If a page's buffers are under async readin (end_buffer_async_read
486  * completion) then there is a possibility that another thread of
487  * control could lock one of the buffers after it has completed
488  * but while some of the other buffers have not completed.  This
489  * locked buffer would confuse end_buffer_async_read() into not unlocking
490  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
491  * that this buffer is not under async I/O.
492  *
493  * The page comes unlocked when it has no locked buffer_async buffers
494  * left.
495  *
496  * PageLocked prevents anyone starting new async I/O reads any of
497  * the buffers.
498  *
499  * PageWriteback is used to prevent simultaneous writeout of the same
500  * page.
501  *
502  * PageLocked prevents anyone from starting writeback of a page which is
503  * under read I/O (PageWriteback is only ever set against a locked page).
504  */
505 static void mark_buffer_async_read(struct buffer_head *bh)
506 {
507         bh->b_end_io = end_buffer_async_read;
508         set_buffer_async_read(bh);
509 }
510
511 void mark_buffer_async_write(struct buffer_head *bh)
512 {
513         bh->b_end_io = end_buffer_async_write;
514         set_buffer_async_write(bh);
515 }
516 EXPORT_SYMBOL(mark_buffer_async_write);
517
518
519 /*
520  * fs/buffer.c contains helper functions for buffer-backed address space's
521  * fsync functions.  A common requirement for buffer-based filesystems is
522  * that certain data from the backing blockdev needs to be written out for
523  * a successful fsync().  For example, ext2 indirect blocks need to be
524  * written back and waited upon before fsync() returns.
525  *
526  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
527  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
528  * management of a list of dependent buffers at ->i_mapping->private_list.
529  *
530  * Locking is a little subtle: try_to_free_buffers() will remove buffers
531  * from their controlling inode's queue when they are being freed.  But
532  * try_to_free_buffers() will be operating against the *blockdev* mapping
533  * at the time, not against the S_ISREG file which depends on those buffers.
534  * So the locking for private_list is via the private_lock in the address_space
535  * which backs the buffers.  Which is different from the address_space 
536  * against which the buffers are listed.  So for a particular address_space,
537  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
538  * mapping->private_list will always be protected by the backing blockdev's
539  * ->private_lock.
540  *
541  * Which introduces a requirement: all buffers on an address_space's
542  * ->private_list must be from the same address_space: the blockdev's.
543  *
544  * address_spaces which do not place buffers at ->private_list via these
545  * utility functions are free to use private_lock and private_list for
546  * whatever they want.  The only requirement is that list_empty(private_list)
547  * be true at clear_inode() time.
548  *
549  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
550  * filesystems should do that.  invalidate_inode_buffers() should just go
551  * BUG_ON(!list_empty).
552  *
553  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
554  * take an address_space, not an inode.  And it should be called
555  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
556  * queued up.
557  *
558  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
559  * list if it is already on a list.  Because if the buffer is on a list,
560  * it *must* already be on the right one.  If not, the filesystem is being
561  * silly.  This will save a ton of locking.  But first we have to ensure
562  * that buffers are taken *off* the old inode's list when they are freed
563  * (presumably in truncate).  That requires careful auditing of all
564  * filesystems (do it inside bforget()).  It could also be done by bringing
565  * b_inode back.
566  */
567
568 /*
569  * The buffer's backing address_space's private_lock must be held
570  */
571 static inline void __remove_assoc_queue(struct buffer_head *bh)
572 {
573         list_del_init(&bh->b_assoc_buffers);
574 }
575
576 int inode_has_buffers(struct inode *inode)
577 {
578         return !list_empty(&inode->i_data.private_list);
579 }
580
581 /*
582  * osync is designed to support O_SYNC io.  It waits synchronously for
583  * all already-submitted IO to complete, but does not queue any new
584  * writes to the disk.
585  *
586  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
587  * you dirty the buffers, and then use osync_inode_buffers to wait for
588  * completion.  Any other dirty buffers which are not yet queued for
589  * write will not be flushed to disk by the osync.
590  */
591 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
592 {
593         struct buffer_head *bh;
594         struct list_head *p;
595         int err = 0;
596
597         spin_lock(lock);
598 repeat:
599         list_for_each_prev(p, list) {
600                 bh = BH_ENTRY(p);
601                 if (buffer_locked(bh)) {
602                         get_bh(bh);
603                         spin_unlock(lock);
604                         wait_on_buffer(bh);
605                         if (!buffer_uptodate(bh))
606                                 err = -EIO;
607                         brelse(bh);
608                         spin_lock(lock);
609                         goto repeat;
610                 }
611         }
612         spin_unlock(lock);
613         return err;
614 }
615
616 /**
617  * sync_mapping_buffers - write out and wait upon a mapping's "associated"
618  *                        buffers
619  * @mapping: the mapping which wants those buffers written
620  *
621  * Starts I/O against the buffers at mapping->private_list, and waits upon
622  * that I/O.
623  *
624  * Basically, this is a convenience function for fsync().
625  * @mapping is a file or directory which needs those buffers to be written for
626  * a successful fsync().
627  */
628 int sync_mapping_buffers(struct address_space *mapping)
629 {
630         struct address_space *buffer_mapping = mapping->assoc_mapping;
631
632         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
633                 return 0;
634
635         return fsync_buffers_list(&buffer_mapping->private_lock,
636                                         &mapping->private_list);
637 }
638 EXPORT_SYMBOL(sync_mapping_buffers);
639
640 /*
641  * Called when we've recently written block `bblock', and it is known that
642  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
643  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
644  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
645  */
646 void write_boundary_block(struct block_device *bdev,
647                         sector_t bblock, unsigned blocksize)
648 {
649         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
650         if (bh) {
651                 if (buffer_dirty(bh))
652                         ll_rw_block(WRITE, 1, &bh);
653                 put_bh(bh);
654         }
655 }
656
657 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
658 {
659         struct address_space *mapping = inode->i_mapping;
660         struct address_space *buffer_mapping = bh->b_page->mapping;
661
662         mark_buffer_dirty(bh);
663         if (!mapping->assoc_mapping) {
664                 mapping->assoc_mapping = buffer_mapping;
665         } else {
666                 BUG_ON(mapping->assoc_mapping != buffer_mapping);
667         }
668         if (list_empty(&bh->b_assoc_buffers)) {
669                 spin_lock(&buffer_mapping->private_lock);
670                 list_move_tail(&bh->b_assoc_buffers,
671                                 &mapping->private_list);
672                 spin_unlock(&buffer_mapping->private_lock);
673         }
674 }
675 EXPORT_SYMBOL(mark_buffer_dirty_inode);
676
677 /*
678  * Add a page to the dirty page list.
679  *
680  * It is a sad fact of life that this function is called from several places
681  * deeply under spinlocking.  It may not sleep.
682  *
683  * If the page has buffers, the uptodate buffers are set dirty, to preserve
684  * dirty-state coherency between the page and the buffers.  It the page does
685  * not have buffers then when they are later attached they will all be set
686  * dirty.
687  *
688  * The buffers are dirtied before the page is dirtied.  There's a small race
689  * window in which a writepage caller may see the page cleanness but not the
690  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
691  * before the buffers, a concurrent writepage caller could clear the page dirty
692  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
693  * page on the dirty page list.
694  *
695  * We use private_lock to lock against try_to_free_buffers while using the
696  * page's buffer list.  Also use this to protect against clean buffers being
697  * added to the page after it was set dirty.
698  *
699  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
700  * address_space though.
701  */
702 int __set_page_dirty_buffers(struct page *page)
703 {
704         struct address_space * const mapping = page->mapping;
705
706         spin_lock(&mapping->private_lock);
707         if (page_has_buffers(page)) {
708                 struct buffer_head *head = page_buffers(page);
709                 struct buffer_head *bh = head;
710
711                 do {
712                         set_buffer_dirty(bh);
713                         bh = bh->b_this_page;
714                 } while (bh != head);
715         }
716         spin_unlock(&mapping->private_lock);
717
718         if (!TestSetPageDirty(page)) {
719                 write_lock_irq(&mapping->tree_lock);
720                 if (page->mapping) {    /* Race with truncate? */
721                         if (mapping_cap_account_dirty(mapping))
722                                 __inc_zone_page_state(page, NR_FILE_DIRTY);
723                         radix_tree_tag_set(&mapping->page_tree,
724                                                 page_index(page),
725                                                 PAGECACHE_TAG_DIRTY);
726                 }
727                 write_unlock_irq(&mapping->tree_lock);
728                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
729                 return 1;
730         }
731         return 0;
732 }
733 EXPORT_SYMBOL(__set_page_dirty_buffers);
734
735 /*
736  * Write out and wait upon a list of buffers.
737  *
738  * We have conflicting pressures: we want to make sure that all
739  * initially dirty buffers get waited on, but that any subsequently
740  * dirtied buffers don't.  After all, we don't want fsync to last
741  * forever if somebody is actively writing to the file.
742  *
743  * Do this in two main stages: first we copy dirty buffers to a
744  * temporary inode list, queueing the writes as we go.  Then we clean
745  * up, waiting for those writes to complete.
746  * 
747  * During this second stage, any subsequent updates to the file may end
748  * up refiling the buffer on the original inode's dirty list again, so
749  * there is a chance we will end up with a buffer queued for write but
750  * not yet completed on that list.  So, as a final cleanup we go through
751  * the osync code to catch these locked, dirty buffers without requeuing
752  * any newly dirty buffers for write.
753  */
754 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
755 {
756         struct buffer_head *bh;
757         struct list_head tmp;
758         int err = 0, err2;
759
760         INIT_LIST_HEAD(&tmp);
761
762         spin_lock(lock);
763         while (!list_empty(list)) {
764                 bh = BH_ENTRY(list->next);
765                 list_del_init(&bh->b_assoc_buffers);
766                 if (buffer_dirty(bh) || buffer_locked(bh)) {
767                         list_add(&bh->b_assoc_buffers, &tmp);
768                         if (buffer_dirty(bh)) {
769                                 get_bh(bh);
770                                 spin_unlock(lock);
771                                 /*
772                                  * Ensure any pending I/O completes so that
773                                  * ll_rw_block() actually writes the current
774                                  * contents - it is a noop if I/O is still in
775                                  * flight on potentially older contents.
776                                  */
777                                 ll_rw_block(SWRITE, 1, &bh);
778                                 brelse(bh);
779                                 spin_lock(lock);
780                         }
781                 }
782         }
783
784         while (!list_empty(&tmp)) {
785                 bh = BH_ENTRY(tmp.prev);
786                 __remove_assoc_queue(bh);
787                 get_bh(bh);
788                 spin_unlock(lock);
789                 wait_on_buffer(bh);
790                 if (!buffer_uptodate(bh))
791                         err = -EIO;
792                 brelse(bh);
793                 spin_lock(lock);
794         }
795         
796         spin_unlock(lock);
797         err2 = osync_buffers_list(lock, list);
798         if (err)
799                 return err;
800         else
801                 return err2;
802 }
803
804 /*
805  * Invalidate any and all dirty buffers on a given inode.  We are
806  * probably unmounting the fs, but that doesn't mean we have already
807  * done a sync().  Just drop the buffers from the inode list.
808  *
809  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
810  * assumes that all the buffers are against the blockdev.  Not true
811  * for reiserfs.
812  */
813 void invalidate_inode_buffers(struct inode *inode)
814 {
815         if (inode_has_buffers(inode)) {
816                 struct address_space *mapping = &inode->i_data;
817                 struct list_head *list = &mapping->private_list;
818                 struct address_space *buffer_mapping = mapping->assoc_mapping;
819
820                 spin_lock(&buffer_mapping->private_lock);
821                 while (!list_empty(list))
822                         __remove_assoc_queue(BH_ENTRY(list->next));
823                 spin_unlock(&buffer_mapping->private_lock);
824         }
825 }
826
827 /*
828  * Remove any clean buffers from the inode's buffer list.  This is called
829  * when we're trying to free the inode itself.  Those buffers can pin it.
830  *
831  * Returns true if all buffers were removed.
832  */
833 int remove_inode_buffers(struct inode *inode)
834 {
835         int ret = 1;
836
837         if (inode_has_buffers(inode)) {
838                 struct address_space *mapping = &inode->i_data;
839                 struct list_head *list = &mapping->private_list;
840                 struct address_space *buffer_mapping = mapping->assoc_mapping;
841
842                 spin_lock(&buffer_mapping->private_lock);
843                 while (!list_empty(list)) {
844                         struct buffer_head *bh = BH_ENTRY(list->next);
845                         if (buffer_dirty(bh)) {
846                                 ret = 0;
847                                 break;
848                         }
849                         __remove_assoc_queue(bh);
850                 }
851                 spin_unlock(&buffer_mapping->private_lock);
852         }
853         return ret;
854 }
855
856 /*
857  * Create the appropriate buffers when given a page for data area and
858  * the size of each buffer.. Use the bh->b_this_page linked list to
859  * follow the buffers created.  Return NULL if unable to create more
860  * buffers.
861  *
862  * The retry flag is used to differentiate async IO (paging, swapping)
863  * which may not fail from ordinary buffer allocations.
864  */
865 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
866                 int retry)
867 {
868         struct buffer_head *bh, *head;
869         long offset;
870
871 try_again:
872         head = NULL;
873         offset = PAGE_SIZE;
874         while ((offset -= size) >= 0) {
875                 bh = alloc_buffer_head(GFP_NOFS);
876                 if (!bh)
877                         goto no_grow;
878
879                 bh->b_bdev = NULL;
880                 bh->b_this_page = head;
881                 bh->b_blocknr = -1;
882                 head = bh;
883
884                 bh->b_state = 0;
885                 atomic_set(&bh->b_count, 0);
886                 bh->b_private = NULL;
887                 bh->b_size = size;
888
889                 /* Link the buffer to its page */
890                 set_bh_page(bh, page, offset);
891
892                 init_buffer(bh, NULL, NULL);
893         }
894         return head;
895 /*
896  * In case anything failed, we just free everything we got.
897  */
898 no_grow:
899         if (head) {
900                 do {
901                         bh = head;
902                         head = head->b_this_page;
903                         free_buffer_head(bh);
904                 } while (head);
905         }
906
907         /*
908          * Return failure for non-async IO requests.  Async IO requests
909          * are not allowed to fail, so we have to wait until buffer heads
910          * become available.  But we don't want tasks sleeping with 
911          * partially complete buffers, so all were released above.
912          */
913         if (!retry)
914                 return NULL;
915
916         /* We're _really_ low on memory. Now we just
917          * wait for old buffer heads to become free due to
918          * finishing IO.  Since this is an async request and
919          * the reserve list is empty, we're sure there are 
920          * async buffer heads in use.
921          */
922         free_more_memory();
923         goto try_again;
924 }
925 EXPORT_SYMBOL_GPL(alloc_page_buffers);
926
927 static inline void
928 link_dev_buffers(struct page *page, struct buffer_head *head)
929 {
930         struct buffer_head *bh, *tail;
931
932         bh = head;
933         do {
934                 tail = bh;
935                 bh = bh->b_this_page;
936         } while (bh);
937         tail->b_this_page = head;
938         attach_page_buffers(page, head);
939 }
940
941 /*
942  * Initialise the state of a blockdev page's buffers.
943  */ 
944 static void
945 init_page_buffers(struct page *page, struct block_device *bdev,
946                         sector_t block, int size)
947 {
948         struct buffer_head *head = page_buffers(page);
949         struct buffer_head *bh = head;
950         int uptodate = PageUptodate(page);
951
952         do {
953                 if (!buffer_mapped(bh)) {
954                         init_buffer(bh, NULL, NULL);
955                         bh->b_bdev = bdev;
956                         bh->b_blocknr = block;
957                         if (uptodate)
958                                 set_buffer_uptodate(bh);
959                         set_buffer_mapped(bh);
960                 }
961                 block++;
962                 bh = bh->b_this_page;
963         } while (bh != head);
964 }
965
966 /*
967  * Create the page-cache page that contains the requested block.
968  *
969  * This is user purely for blockdev mappings.
970  */
971 static struct page *
972 grow_dev_page(struct block_device *bdev, sector_t block,
973                 pgoff_t index, int size)
974 {
975         struct inode *inode = bdev->bd_inode;
976         struct page *page;
977         struct buffer_head *bh;
978
979         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
980         if (!page)
981                 return NULL;
982
983         BUG_ON(!PageLocked(page));
984
985         if (page_has_buffers(page)) {
986                 bh = page_buffers(page);
987                 if (bh->b_size == size) {
988                         init_page_buffers(page, bdev, block, size);
989                         return page;
990                 }
991                 if (!try_to_free_buffers(page))
992                         goto failed;
993         }
994
995         /*
996          * Allocate some buffers for this page
997          */
998         bh = alloc_page_buffers(page, size, 0);
999         if (!bh)
1000                 goto failed;
1001
1002         /*
1003          * Link the page to the buffers and initialise them.  Take the
1004          * lock to be atomic wrt __find_get_block(), which does not
1005          * run under the page lock.
1006          */
1007         spin_lock(&inode->i_mapping->private_lock);
1008         link_dev_buffers(page, bh);
1009         init_page_buffers(page, bdev, block, size);
1010         spin_unlock(&inode->i_mapping->private_lock);
1011         return page;
1012
1013 failed:
1014         BUG();
1015         unlock_page(page);
1016         page_cache_release(page);
1017         return NULL;
1018 }
1019
1020 /*
1021  * Create buffers for the specified block device block's page.  If
1022  * that page was dirty, the buffers are set dirty also.
1023  *
1024  * Except that's a bug.  Attaching dirty buffers to a dirty
1025  * blockdev's page can result in filesystem corruption, because
1026  * some of those buffers may be aliases of filesystem data.
1027  * grow_dev_page() will go BUG() if this happens.
1028  */
1029 static int
1030 grow_buffers(struct block_device *bdev, sector_t block, int size)
1031 {
1032         struct page *page;
1033         pgoff_t index;
1034         int sizebits;
1035
1036         sizebits = -1;
1037         do {
1038                 sizebits++;
1039         } while ((size << sizebits) < PAGE_SIZE);
1040
1041         index = block >> sizebits;
1042         block = index << sizebits;
1043
1044         /* Create a page with the proper size buffers.. */
1045         page = grow_dev_page(bdev, block, index, size);
1046         if (!page)
1047                 return 0;
1048         unlock_page(page);
1049         page_cache_release(page);
1050         return 1;
1051 }
1052
1053 static struct buffer_head *
1054 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1055 {
1056         /* Size must be multiple of hard sectorsize */
1057         if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1058                         (size < 512 || size > PAGE_SIZE))) {
1059                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1060                                         size);
1061                 printk(KERN_ERR "hardsect size: %d\n",
1062                                         bdev_hardsect_size(bdev));
1063
1064                 dump_stack();
1065                 return NULL;
1066         }
1067
1068         for (;;) {
1069                 struct buffer_head * bh;
1070
1071                 bh = __find_get_block(bdev, block, size);
1072                 if (bh)
1073                         return bh;
1074
1075                 if (!grow_buffers(bdev, block, size))
1076                         free_more_memory();
1077         }
1078 }
1079
1080 /*
1081  * The relationship between dirty buffers and dirty pages:
1082  *
1083  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1084  * the page is tagged dirty in its radix tree.
1085  *
1086  * At all times, the dirtiness of the buffers represents the dirtiness of
1087  * subsections of the page.  If the page has buffers, the page dirty bit is
1088  * merely a hint about the true dirty state.
1089  *
1090  * When a page is set dirty in its entirety, all its buffers are marked dirty
1091  * (if the page has buffers).
1092  *
1093  * When a buffer is marked dirty, its page is dirtied, but the page's other
1094  * buffers are not.
1095  *
1096  * Also.  When blockdev buffers are explicitly read with bread(), they
1097  * individually become uptodate.  But their backing page remains not
1098  * uptodate - even if all of its buffers are uptodate.  A subsequent
1099  * block_read_full_page() against that page will discover all the uptodate
1100  * buffers, will set the page uptodate and will perform no I/O.
1101  */
1102
1103 /**
1104  * mark_buffer_dirty - mark a buffer_head as needing writeout
1105  * @bh: the buffer_head to mark dirty
1106  *
1107  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1108  * backing page dirty, then tag the page as dirty in its address_space's radix
1109  * tree and then attach the address_space's inode to its superblock's dirty
1110  * inode list.
1111  *
1112  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1113  * mapping->tree_lock and the global inode_lock.
1114  */
1115 void fastcall mark_buffer_dirty(struct buffer_head *bh)
1116 {
1117         if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1118                 __set_page_dirty_nobuffers(bh->b_page);
1119 }
1120
1121 /*
1122  * Decrement a buffer_head's reference count.  If all buffers against a page
1123  * have zero reference count, are clean and unlocked, and if the page is clean
1124  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1125  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1126  * a page but it ends up not being freed, and buffers may later be reattached).
1127  */
1128 void __brelse(struct buffer_head * buf)
1129 {
1130         if (atomic_read(&buf->b_count)) {
1131                 put_bh(buf);
1132                 return;
1133         }
1134         printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1135         WARN_ON(1);
1136 }
1137
1138 /*
1139  * bforget() is like brelse(), except it discards any
1140  * potentially dirty data.
1141  */
1142 void __bforget(struct buffer_head *bh)
1143 {
1144         clear_buffer_dirty(bh);
1145         if (!list_empty(&bh->b_assoc_buffers)) {
1146                 struct address_space *buffer_mapping = bh->b_page->mapping;
1147
1148                 spin_lock(&buffer_mapping->private_lock);
1149                 list_del_init(&bh->b_assoc_buffers);
1150                 spin_unlock(&buffer_mapping->private_lock);
1151         }
1152         __brelse(bh);
1153 }
1154
1155 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1156 {
1157         lock_buffer(bh);
1158         if (buffer_uptodate(bh)) {
1159                 unlock_buffer(bh);
1160                 return bh;
1161         } else {
1162                 get_bh(bh);
1163                 bh->b_end_io = end_buffer_read_sync;
1164                 submit_bh(READ, bh);
1165                 wait_on_buffer(bh);
1166                 if (buffer_uptodate(bh))
1167                         return bh;
1168         }
1169         brelse(bh);
1170         return NULL;
1171 }
1172
1173 /*
1174  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1175  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1176  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1177  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1178  * CPU's LRUs at the same time.
1179  *
1180  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1181  * sb_find_get_block().
1182  *
1183  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1184  * a local interrupt disable for that.
1185  */
1186
1187 #define BH_LRU_SIZE     8
1188
1189 struct bh_lru {
1190         struct buffer_head *bhs[BH_LRU_SIZE];
1191 };
1192
1193 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1194
1195 #ifdef CONFIG_SMP
1196 #define bh_lru_lock()   local_irq_disable()
1197 #define bh_lru_unlock() local_irq_enable()
1198 #else
1199 #define bh_lru_lock()   preempt_disable()
1200 #define bh_lru_unlock() preempt_enable()
1201 #endif
1202
1203 static inline void check_irqs_on(void)
1204 {
1205 #ifdef irqs_disabled
1206         BUG_ON(irqs_disabled());
1207 #endif
1208 }
1209
1210 /*
1211  * The LRU management algorithm is dopey-but-simple.  Sorry.
1212  */
1213 static void bh_lru_install(struct buffer_head *bh)
1214 {
1215         struct buffer_head *evictee = NULL;
1216         struct bh_lru *lru;
1217
1218         check_irqs_on();
1219         bh_lru_lock();
1220         lru = &__get_cpu_var(bh_lrus);
1221         if (lru->bhs[0] != bh) {
1222                 struct buffer_head *bhs[BH_LRU_SIZE];
1223                 int in;
1224                 int out = 0;
1225
1226                 get_bh(bh);
1227                 bhs[out++] = bh;
1228                 for (in = 0; in < BH_LRU_SIZE; in++) {
1229                         struct buffer_head *bh2 = lru->bhs[in];
1230
1231                         if (bh2 == bh) {
1232                                 __brelse(bh2);
1233                         } else {
1234                                 if (out >= BH_LRU_SIZE) {
1235                                         BUG_ON(evictee != NULL);
1236                                         evictee = bh2;
1237                                 } else {
1238                                         bhs[out++] = bh2;
1239                                 }
1240                         }
1241                 }
1242                 while (out < BH_LRU_SIZE)
1243                         bhs[out++] = NULL;
1244                 memcpy(lru->bhs, bhs, sizeof(bhs));
1245         }
1246         bh_lru_unlock();
1247
1248         if (evictee)
1249                 __brelse(evictee);
1250 }
1251
1252 /*
1253  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1254  */
1255 static struct buffer_head *
1256 lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1257 {
1258         struct buffer_head *ret = NULL;
1259         struct bh_lru *lru;
1260         int i;
1261
1262         check_irqs_on();
1263         bh_lru_lock();
1264         lru = &__get_cpu_var(bh_lrus);
1265         for (i = 0; i < BH_LRU_SIZE; i++) {
1266                 struct buffer_head *bh = lru->bhs[i];
1267
1268                 if (bh && bh->b_bdev == bdev &&
1269                                 bh->b_blocknr == block && bh->b_size == size) {
1270                         if (i) {
1271                                 while (i) {
1272                                         lru->bhs[i] = lru->bhs[i - 1];
1273                                         i--;
1274                                 }
1275                                 lru->bhs[0] = bh;
1276                         }
1277                         get_bh(bh);
1278                         ret = bh;
1279                         break;
1280                 }
1281         }
1282         bh_lru_unlock();
1283         return ret;
1284 }
1285
1286 /*
1287  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1288  * it in the LRU and mark it as accessed.  If it is not present then return
1289  * NULL
1290  */
1291 struct buffer_head *
1292 __find_get_block(struct block_device *bdev, sector_t block, int size)
1293 {
1294         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1295
1296         if (bh == NULL) {
1297                 bh = __find_get_block_slow(bdev, block);
1298                 if (bh)
1299                         bh_lru_install(bh);
1300         }
1301         if (bh)
1302                 touch_buffer(bh);
1303         return bh;
1304 }
1305 EXPORT_SYMBOL(__find_get_block);
1306
1307 /*
1308  * __getblk will locate (and, if necessary, create) the buffer_head
1309  * which corresponds to the passed block_device, block and size. The
1310  * returned buffer has its reference count incremented.
1311  *
1312  * __getblk() cannot fail - it just keeps trying.  If you pass it an
1313  * illegal block number, __getblk() will happily return a buffer_head
1314  * which represents the non-existent block.  Very weird.
1315  *
1316  * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1317  * attempt is failing.  FIXME, perhaps?
1318  */
1319 struct buffer_head *
1320 __getblk(struct block_device *bdev, sector_t block, int size)
1321 {
1322         struct buffer_head *bh = __find_get_block(bdev, block, size);
1323
1324         might_sleep();
1325         if (bh == NULL)
1326                 bh = __getblk_slow(bdev, block, size);
1327         return bh;
1328 }
1329 EXPORT_SYMBOL(__getblk);
1330
1331 /*
1332  * Do async read-ahead on a buffer..
1333  */
1334 void __breadahead(struct block_device *bdev, sector_t block, int size)
1335 {
1336         struct buffer_head *bh = __getblk(bdev, block, size);
1337         if (likely(bh)) {
1338                 ll_rw_block(READA, 1, &bh);
1339                 brelse(bh);
1340         }
1341 }
1342 EXPORT_SYMBOL(__breadahead);
1343
1344 /**
1345  *  __bread() - reads a specified block and returns the bh
1346  *  @bdev: the block_device to read from
1347  *  @block: number of block
1348  *  @size: size (in bytes) to read
1349  * 
1350  *  Reads a specified block, and returns buffer head that contains it.
1351  *  It returns NULL if the block was unreadable.
1352  */
1353 struct buffer_head *
1354 __bread(struct block_device *bdev, sector_t block, int size)
1355 {
1356         struct buffer_head *bh = __getblk(bdev, block, size);
1357
1358         if (likely(bh) && !buffer_uptodate(bh))
1359                 bh = __bread_slow(bh);
1360         return bh;
1361 }
1362 EXPORT_SYMBOL(__bread);
1363
1364 /*
1365  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1366  * This doesn't race because it runs in each cpu either in irq
1367  * or with preempt disabled.
1368  */
1369 static void invalidate_bh_lru(void *arg)
1370 {
1371         struct bh_lru *b = &get_cpu_var(bh_lrus);
1372         int i;
1373
1374         for (i = 0; i < BH_LRU_SIZE; i++) {
1375                 brelse(b->bhs[i]);
1376                 b->bhs[i] = NULL;
1377         }
1378         put_cpu_var(bh_lrus);
1379 }
1380         
1381 static void invalidate_bh_lrus(void)
1382 {
1383         on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1384 }
1385
1386 void set_bh_page(struct buffer_head *bh,
1387                 struct page *page, unsigned long offset)
1388 {
1389         bh->b_page = page;
1390         BUG_ON(offset >= PAGE_SIZE);
1391         if (PageHighMem(page))
1392                 /*
1393                  * This catches illegal uses and preserves the offset:
1394                  */
1395                 bh->b_data = (char *)(0 + offset);
1396         else
1397                 bh->b_data = page_address(page) + offset;
1398 }
1399 EXPORT_SYMBOL(set_bh_page);
1400
1401 /*
1402  * Called when truncating a buffer on a page completely.
1403  */
1404 static void discard_buffer(struct buffer_head * bh)
1405 {
1406         lock_buffer(bh);
1407         clear_buffer_dirty(bh);
1408         bh->b_bdev = NULL;
1409         clear_buffer_mapped(bh);
1410         clear_buffer_req(bh);
1411         clear_buffer_new(bh);
1412         clear_buffer_delay(bh);
1413         unlock_buffer(bh);
1414 }
1415
1416 /**
1417  * block_invalidatepage - invalidate part of all of a buffer-backed page
1418  *
1419  * @page: the page which is affected
1420  * @offset: the index of the truncation point
1421  *
1422  * block_invalidatepage() is called when all or part of the page has become
1423  * invalidatedby a truncate operation.
1424  *
1425  * block_invalidatepage() does not have to release all buffers, but it must
1426  * ensure that no dirty buffer is left outside @offset and that no I/O
1427  * is underway against any of the blocks which are outside the truncation
1428  * point.  Because the caller is about to free (and possibly reuse) those
1429  * blocks on-disk.
1430  */
1431 void block_invalidatepage(struct page *page, unsigned long offset)
1432 {
1433         struct buffer_head *head, *bh, *next;
1434         unsigned int curr_off = 0;
1435
1436         BUG_ON(!PageLocked(page));
1437         if (!page_has_buffers(page))
1438                 goto out;
1439
1440         head = page_buffers(page);
1441         bh = head;
1442         do {
1443                 unsigned int next_off = curr_off + bh->b_size;
1444                 next = bh->b_this_page;
1445
1446                 /*
1447                  * is this block fully invalidated?
1448                  */
1449                 if (offset <= curr_off)
1450                         discard_buffer(bh);
1451                 curr_off = next_off;
1452                 bh = next;
1453         } while (bh != head);
1454
1455         /*
1456          * We release buffers only if the entire page is being invalidated.
1457          * The get_block cached value has been unconditionally invalidated,
1458          * so real IO is not possible anymore.
1459          */
1460         if (offset == 0)
1461                 try_to_release_page(page, 0);
1462 out:
1463         return;
1464 }
1465 EXPORT_SYMBOL(block_invalidatepage);
1466
1467 /*
1468  * We attach and possibly dirty the buffers atomically wrt
1469  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1470  * is already excluded via the page lock.
1471  */
1472 void create_empty_buffers(struct page *page,
1473                         unsigned long blocksize, unsigned long b_state)
1474 {
1475         struct buffer_head *bh, *head, *tail;
1476
1477         head = alloc_page_buffers(page, blocksize, 1);
1478         bh = head;
1479         do {
1480                 bh->b_state |= b_state;
1481                 tail = bh;
1482                 bh = bh->b_this_page;
1483         } while (bh);
1484         tail->b_this_page = head;
1485
1486         spin_lock(&page->mapping->private_lock);
1487         if (PageUptodate(page) || PageDirty(page)) {
1488                 bh = head;
1489                 do {
1490                         if (PageDirty(page))
1491                                 set_buffer_dirty(bh);
1492                         if (PageUptodate(page))
1493                                 set_buffer_uptodate(bh);
1494                         bh = bh->b_this_page;
1495                 } while (bh != head);
1496         }
1497         attach_page_buffers(page, head);
1498         spin_unlock(&page->mapping->private_lock);
1499 }
1500 EXPORT_SYMBOL(create_empty_buffers);
1501
1502 /*
1503  * We are taking a block for data and we don't want any output from any
1504  * buffer-cache aliases starting from return from that function and
1505  * until the moment when something will explicitly mark the buffer
1506  * dirty (hopefully that will not happen until we will free that block ;-)
1507  * We don't even need to mark it not-uptodate - nobody can expect
1508  * anything from a newly allocated buffer anyway. We used to used
1509  * unmap_buffer() for such invalidation, but that was wrong. We definitely
1510  * don't want to mark the alias unmapped, for example - it would confuse
1511  * anyone who might pick it with bread() afterwards...
1512  *
1513  * Also..  Note that bforget() doesn't lock the buffer.  So there can
1514  * be writeout I/O going on against recently-freed buffers.  We don't
1515  * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1516  * only if we really need to.  That happens here.
1517  */
1518 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1519 {
1520         struct buffer_head *old_bh;
1521
1522         might_sleep();
1523
1524         old_bh = __find_get_block_slow(bdev, block);
1525         if (old_bh) {
1526                 clear_buffer_dirty(old_bh);
1527                 wait_on_buffer(old_bh);
1528                 clear_buffer_req(old_bh);
1529                 __brelse(old_bh);
1530         }
1531 }
1532 EXPORT_SYMBOL(unmap_underlying_metadata);
1533
1534 /*
1535  * NOTE! All mapped/uptodate combinations are valid:
1536  *
1537  *      Mapped  Uptodate        Meaning
1538  *
1539  *      No      No              "unknown" - must do get_block()
1540  *      No      Yes             "hole" - zero-filled
1541  *      Yes     No              "allocated" - allocated on disk, not read in
1542  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1543  *
1544  * "Dirty" is valid only with the last case (mapped+uptodate).
1545  */
1546
1547 /*
1548  * While block_write_full_page is writing back the dirty buffers under
1549  * the page lock, whoever dirtied the buffers may decide to clean them
1550  * again at any time.  We handle that by only looking at the buffer
1551  * state inside lock_buffer().
1552  *
1553  * If block_write_full_page() is called for regular writeback
1554  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1555  * locked buffer.   This only can happen if someone has written the buffer
1556  * directly, with submit_bh().  At the address_space level PageWriteback
1557  * prevents this contention from occurring.
1558  */
1559 static int __block_write_full_page(struct inode *inode, struct page *page,
1560                         get_block_t *get_block, struct writeback_control *wbc)
1561 {
1562         int err;
1563         sector_t block;
1564         sector_t last_block;
1565         struct buffer_head *bh, *head;
1566         const unsigned blocksize = 1 << inode->i_blkbits;
1567         int nr_underway = 0;
1568
1569         BUG_ON(!PageLocked(page));
1570
1571         last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1572
1573         if (!page_has_buffers(page)) {
1574                 create_empty_buffers(page, blocksize,
1575                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1576         }
1577
1578         /*
1579          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1580          * here, and the (potentially unmapped) buffers may become dirty at
1581          * any time.  If a buffer becomes dirty here after we've inspected it
1582          * then we just miss that fact, and the page stays dirty.
1583          *
1584          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1585          * handle that here by just cleaning them.
1586          */
1587
1588         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1589         head = page_buffers(page);
1590         bh = head;
1591
1592         /*
1593          * Get all the dirty buffers mapped to disk addresses and
1594          * handle any aliases from the underlying blockdev's mapping.
1595          */
1596         do {
1597                 if (block > last_block) {
1598                         /*
1599                          * mapped buffers outside i_size will occur, because
1600                          * this page can be outside i_size when there is a
1601                          * truncate in progress.
1602                          */
1603                         /*
1604                          * The buffer was zeroed by block_write_full_page()
1605                          */
1606                         clear_buffer_dirty(bh);
1607                         set_buffer_uptodate(bh);
1608                 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1609                         WARN_ON(bh->b_size != blocksize);
1610                         err = get_block(inode, block, bh, 1);
1611                         if (err)
1612                                 goto recover;
1613                         if (buffer_new(bh)) {
1614                                 /* blockdev mappings never come here */
1615                                 clear_buffer_new(bh);
1616                                 unmap_underlying_metadata(bh->b_bdev,
1617                                                         bh->b_blocknr);
1618                         }
1619                 }
1620                 bh = bh->b_this_page;
1621                 block++;
1622         } while (bh != head);
1623
1624         do {
1625                 if (!buffer_mapped(bh))
1626                         continue;
1627                 /*
1628                  * If it's a fully non-blocking write attempt and we cannot
1629                  * lock the buffer then redirty the page.  Note that this can
1630                  * potentially cause a busy-wait loop from pdflush and kswapd
1631                  * activity, but those code paths have their own higher-level
1632                  * throttling.
1633                  */
1634                 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1635                         lock_buffer(bh);
1636                 } else if (test_set_buffer_locked(bh)) {
1637                         redirty_page_for_writepage(wbc, page);
1638                         continue;
1639                 }
1640                 if (test_clear_buffer_dirty(bh)) {
1641                         mark_buffer_async_write(bh);
1642                 } else {
1643                         unlock_buffer(bh);
1644                 }
1645         } while ((bh = bh->b_this_page) != head);
1646
1647         /*
1648          * The page and its buffers are protected by PageWriteback(), so we can
1649          * drop the bh refcounts early.
1650          */
1651         BUG_ON(PageWriteback(page));
1652         set_page_writeback(page);
1653
1654         do {
1655                 struct buffer_head *next = bh->b_this_page;
1656                 if (buffer_async_write(bh)) {
1657                         submit_bh(WRITE, bh);
1658                         nr_underway++;
1659                 }
1660                 bh = next;
1661         } while (bh != head);
1662         unlock_page(page);
1663
1664         err = 0;
1665 done:
1666         if (nr_underway == 0) {
1667                 /*
1668                  * The page was marked dirty, but the buffers were
1669                  * clean.  Someone wrote them back by hand with
1670                  * ll_rw_block/submit_bh.  A rare case.
1671                  */
1672                 int uptodate = 1;
1673                 do {
1674                         if (!buffer_uptodate(bh)) {
1675                                 uptodate = 0;
1676                                 break;
1677                         }
1678                         bh = bh->b_this_page;
1679                 } while (bh != head);
1680                 if (uptodate)
1681                         SetPageUptodate(page);
1682                 end_page_writeback(page);
1683                 /*
1684                  * The page and buffer_heads can be released at any time from
1685                  * here on.
1686                  */
1687                 wbc->pages_skipped++;   /* We didn't write this page */
1688         }
1689         return err;
1690
1691 recover:
1692         /*
1693          * ENOSPC, or some other error.  We may already have added some
1694          * blocks to the file, so we need to write these out to avoid
1695          * exposing stale data.
1696          * The page is currently locked and not marked for writeback
1697          */
1698         bh = head;
1699         /* Recovery: lock and submit the mapped buffers */
1700         do {
1701                 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1702                         lock_buffer(bh);
1703                         mark_buffer_async_write(bh);
1704                 } else {
1705                         /*
1706                          * The buffer may have been set dirty during
1707                          * attachment to a dirty page.
1708                          */
1709                         clear_buffer_dirty(bh);
1710                 }
1711         } while ((bh = bh->b_this_page) != head);
1712         SetPageError(page);
1713         BUG_ON(PageWriteback(page));
1714         set_page_writeback(page);
1715         unlock_page(page);
1716         do {
1717                 struct buffer_head *next = bh->b_this_page;
1718                 if (buffer_async_write(bh)) {
1719                         clear_buffer_dirty(bh);
1720                         submit_bh(WRITE, bh);
1721                         nr_underway++;
1722                 }
1723                 bh = next;
1724         } while (bh != head);
1725         goto done;
1726 }
1727
1728 static int __block_prepare_write(struct inode *inode, struct page *page,
1729                 unsigned from, unsigned to, get_block_t *get_block)
1730 {
1731         unsigned block_start, block_end;
1732         sector_t block;
1733         int err = 0;
1734         unsigned blocksize, bbits;
1735         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1736
1737         BUG_ON(!PageLocked(page));
1738         BUG_ON(from > PAGE_CACHE_SIZE);
1739         BUG_ON(to > PAGE_CACHE_SIZE);
1740         BUG_ON(from > to);
1741
1742         blocksize = 1 << inode->i_blkbits;
1743         if (!page_has_buffers(page))
1744                 create_empty_buffers(page, blocksize, 0);
1745         head = page_buffers(page);
1746
1747         bbits = inode->i_blkbits;
1748         block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1749
1750         for(bh = head, block_start = 0; bh != head || !block_start;
1751             block++, block_start=block_end, bh = bh->b_this_page) {
1752                 block_end = block_start + blocksize;
1753                 if (block_end <= from || block_start >= to) {
1754                         if (PageUptodate(page)) {
1755                                 if (!buffer_uptodate(bh))
1756                                         set_buffer_uptodate(bh);
1757                         }
1758                         continue;
1759                 }
1760                 if (buffer_new(bh))
1761                         clear_buffer_new(bh);
1762                 if (!buffer_mapped(bh)) {
1763                         WARN_ON(bh->b_size != blocksize);
1764                         err = get_block(inode, block, bh, 1);
1765                         if (err)
1766                                 break;
1767                         if (buffer_new(bh)) {
1768                                 unmap_underlying_metadata(bh->b_bdev,
1769                                                         bh->b_blocknr);
1770                                 if (PageUptodate(page)) {
1771                                         set_buffer_uptodate(bh);
1772                                         continue;
1773                                 }
1774                                 if (block_end > to || block_start < from) {
1775                                         void *kaddr;
1776
1777                                         kaddr = kmap_atomic(page, KM_USER0);
1778                                         if (block_end > to)
1779                                                 memset(kaddr+to, 0,
1780                                                         block_end-to);
1781                                         if (block_start < from)
1782                                                 memset(kaddr+block_start,
1783                                                         0, from-block_start);
1784                                         flush_dcache_page(page);
1785                                         kunmap_atomic(kaddr, KM_USER0);
1786                                 }
1787                                 continue;
1788                         }
1789                 }
1790                 if (PageUptodate(page)) {
1791                         if (!buffer_uptodate(bh))
1792                                 set_buffer_uptodate(bh);
1793                         continue; 
1794                 }
1795                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1796                      (block_start < from || block_end > to)) {
1797                         ll_rw_block(READ, 1, &bh);
1798                         *wait_bh++=bh;
1799                 }
1800         }
1801         /*
1802          * If we issued read requests - let them complete.
1803          */
1804         while(wait_bh > wait) {
1805                 wait_on_buffer(*--wait_bh);
1806                 if (!buffer_uptodate(*wait_bh))
1807                         err = -EIO;
1808         }
1809         if (!err) {
1810                 bh = head;
1811                 do {
1812                         if (buffer_new(bh))
1813                                 clear_buffer_new(bh);
1814                 } while ((bh = bh->b_this_page) != head);
1815                 return 0;
1816         }
1817         /* Error case: */
1818         /*
1819          * Zero out any newly allocated blocks to avoid exposing stale
1820          * data.  If BH_New is set, we know that the block was newly
1821          * allocated in the above loop.
1822          */
1823         bh = head;
1824         block_start = 0;
1825         do {
1826                 block_end = block_start+blocksize;
1827                 if (block_end <= from)
1828                         goto next_bh;
1829                 if (block_start >= to)
1830                         break;
1831                 if (buffer_new(bh)) {
1832                         void *kaddr;
1833
1834                         clear_buffer_new(bh);
1835                         kaddr = kmap_atomic(page, KM_USER0);
1836                         memset(kaddr+block_start, 0, bh->b_size);
1837                         kunmap_atomic(kaddr, KM_USER0);
1838                         set_buffer_uptodate(bh);
1839                         mark_buffer_dirty(bh);
1840                 }
1841 next_bh:
1842                 block_start = block_end;
1843                 bh = bh->b_this_page;
1844         } while (bh != head);
1845         return err;
1846 }
1847
1848 static int __block_commit_write(struct inode *inode, struct page *page,
1849                 unsigned from, unsigned to)
1850 {
1851         unsigned block_start, block_end;
1852         int partial = 0;
1853         unsigned blocksize;
1854         struct buffer_head *bh, *head;
1855
1856         blocksize = 1 << inode->i_blkbits;
1857
1858         for(bh = head = page_buffers(page), block_start = 0;
1859             bh != head || !block_start;
1860             block_start=block_end, bh = bh->b_this_page) {
1861                 block_end = block_start + blocksize;
1862                 if (block_end <= from || block_start >= to) {
1863                         if (!buffer_uptodate(bh))
1864                                 partial = 1;
1865                 } else {
1866                         set_buffer_uptodate(bh);
1867                         mark_buffer_dirty(bh);
1868                 }
1869         }
1870
1871         /*
1872          * If this is a partial write which happened to make all buffers
1873          * uptodate then we can optimize away a bogus readpage() for
1874          * the next read(). Here we 'discover' whether the page went
1875          * uptodate as a result of this (potentially partial) write.
1876          */
1877         if (!partial)
1878                 SetPageUptodate(page);
1879         return 0;
1880 }
1881
1882 /*
1883  * Generic "read page" function for block devices that have the normal
1884  * get_block functionality. This is most of the block device filesystems.
1885  * Reads the page asynchronously --- the unlock_buffer() and
1886  * set/clear_buffer_uptodate() functions propagate buffer state into the
1887  * page struct once IO has completed.
1888  */
1889 int block_read_full_page(struct page *page, get_block_t *get_block)
1890 {
1891         struct inode *inode = page->mapping->host;
1892         sector_t iblock, lblock;
1893         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
1894         unsigned int blocksize;
1895         int nr, i;
1896         int fully_mapped = 1;
1897
1898         BUG_ON(!PageLocked(page));
1899         blocksize = 1 << inode->i_blkbits;
1900         if (!page_has_buffers(page))
1901                 create_empty_buffers(page, blocksize, 0);
1902         head = page_buffers(page);
1903
1904         iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1905         lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
1906         bh = head;
1907         nr = 0;
1908         i = 0;
1909
1910         do {
1911                 if (buffer_uptodate(bh))
1912                         continue;
1913
1914                 if (!buffer_mapped(bh)) {
1915                         int err = 0;
1916
1917                         fully_mapped = 0;
1918                         if (iblock < lblock) {
1919                                 WARN_ON(bh->b_size != blocksize);
1920                                 err = get_block(inode, iblock, bh, 0);
1921                                 if (err)
1922                                         SetPageError(page);
1923                         }
1924                         if (!buffer_mapped(bh)) {
1925                                 void *kaddr = kmap_atomic(page, KM_USER0);
1926                                 memset(kaddr + i * blocksize, 0, blocksize);
1927                                 flush_dcache_page(page);
1928                                 kunmap_atomic(kaddr, KM_USER0);
1929                                 if (!err)
1930                                         set_buffer_uptodate(bh);
1931                                 continue;
1932                         }
1933                         /*
1934                          * get_block() might have updated the buffer
1935                          * synchronously
1936                          */
1937                         if (buffer_uptodate(bh))
1938                                 continue;
1939                 }
1940                 arr[nr++] = bh;
1941         } while (i++, iblock++, (bh = bh->b_this_page) != head);
1942
1943         if (fully_mapped)
1944                 SetPageMappedToDisk(page);
1945
1946         if (!nr) {
1947                 /*
1948                  * All buffers are uptodate - we can set the page uptodate
1949                  * as well. But not if get_block() returned an error.
1950                  */
1951                 if (!PageError(page))
1952                         SetPageUptodate(page);
1953                 unlock_page(page);
1954                 return 0;
1955         }
1956
1957         /* Stage two: lock the buffers */
1958         for (i = 0; i < nr; i++) {
1959                 bh = arr[i];
1960                 lock_buffer(bh);
1961                 mark_buffer_async_read(bh);
1962         }
1963
1964         /*
1965          * Stage 3: start the IO.  Check for uptodateness
1966          * inside the buffer lock in case another process reading
1967          * the underlying blockdev brought it uptodate (the sct fix).
1968          */
1969         for (i = 0; i < nr; i++) {
1970                 bh = arr[i];
1971                 if (buffer_uptodate(bh))
1972                         end_buffer_async_read(bh, 1);
1973                 else
1974                         submit_bh(READ, bh);
1975         }
1976         return 0;
1977 }
1978
1979 /* utility function for filesystems that need to do work on expanding
1980  * truncates.  Uses prepare/commit_write to allow the filesystem to
1981  * deal with the hole.  
1982  */
1983 static int __generic_cont_expand(struct inode *inode, loff_t size,
1984                                  pgoff_t index, unsigned int offset)
1985 {
1986         struct address_space *mapping = inode->i_mapping;
1987         struct page *page;
1988         unsigned long limit;
1989         int err;
1990
1991         err = -EFBIG;
1992         limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
1993         if (limit != RLIM_INFINITY && size > (loff_t)limit) {
1994                 send_sig(SIGXFSZ, current, 0);
1995                 goto out;
1996         }
1997         if (size > inode->i_sb->s_maxbytes)
1998                 goto out;
1999
2000         err = -ENOMEM;
2001         page = grab_cache_page(mapping, index);
2002         if (!page)
2003                 goto out;
2004         err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2005         if (err) {
2006                 /*
2007                  * ->prepare_write() may have instantiated a few blocks
2008                  * outside i_size.  Trim these off again.
2009                  */
2010                 unlock_page(page);
2011                 page_cache_release(page);
2012                 vmtruncate(inode, inode->i_size);
2013                 goto out;
2014         }
2015
2016         err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2017
2018         unlock_page(page);
2019         page_cache_release(page);
2020         if (err > 0)
2021                 err = 0;
2022 out:
2023         return err;
2024 }
2025
2026 int generic_cont_expand(struct inode *inode, loff_t size)
2027 {
2028         pgoff_t index;
2029         unsigned int offset;
2030
2031         offset = (size & (PAGE_CACHE_SIZE - 1)); /* Within page */
2032
2033         /* ugh.  in prepare/commit_write, if from==to==start of block, we
2034         ** skip the prepare.  make sure we never send an offset for the start
2035         ** of a block
2036         */
2037         if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2038                 /* caller must handle this extra byte. */
2039                 offset++;
2040         }
2041         index = size >> PAGE_CACHE_SHIFT;
2042
2043         return __generic_cont_expand(inode, size, index, offset);
2044 }
2045
2046 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2047 {
2048         loff_t pos = size - 1;
2049         pgoff_t index = pos >> PAGE_CACHE_SHIFT;
2050         unsigned int offset = (pos & (PAGE_CACHE_SIZE - 1)) + 1;
2051
2052         /* prepare/commit_write can handle even if from==to==start of block. */
2053         return __generic_cont_expand(inode, size, index, offset);
2054 }
2055
2056 /*
2057  * For moronic filesystems that do not allow holes in file.
2058  * We may have to extend the file.
2059  */
2060
2061 int cont_prepare_write(struct page *page, unsigned offset,
2062                 unsigned to, get_block_t *get_block, loff_t *bytes)
2063 {
2064         struct address_space *mapping = page->mapping;
2065         struct inode *inode = mapping->host;
2066         struct page *new_page;
2067         pgoff_t pgpos;
2068         long status;
2069         unsigned zerofrom;
2070         unsigned blocksize = 1 << inode->i_blkbits;
2071         void *kaddr;
2072
2073         while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2074                 status = -ENOMEM;
2075                 new_page = grab_cache_page(mapping, pgpos);
2076                 if (!new_page)
2077                         goto out;
2078                 /* we might sleep */
2079                 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2080                         unlock_page(new_page);
2081                         page_cache_release(new_page);
2082                         continue;
2083                 }
2084                 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2085                 if (zerofrom & (blocksize-1)) {
2086                         *bytes |= (blocksize-1);
2087                         (*bytes)++;
2088                 }
2089                 status = __block_prepare_write(inode, new_page, zerofrom,
2090                                                 PAGE_CACHE_SIZE, get_block);
2091                 if (status)
2092                         goto out_unmap;
2093                 kaddr = kmap_atomic(new_page, KM_USER0);
2094                 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2095                 flush_dcache_page(new_page);
2096                 kunmap_atomic(kaddr, KM_USER0);
2097                 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2098                 unlock_page(new_page);
2099                 page_cache_release(new_page);
2100         }
2101
2102         if (page->index < pgpos) {
2103                 /* completely inside the area */
2104                 zerofrom = offset;
2105         } else {
2106                 /* page covers the boundary, find the boundary offset */
2107                 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2108
2109                 /* if we will expand the thing last block will be filled */
2110                 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2111                         *bytes |= (blocksize-1);
2112                         (*bytes)++;
2113                 }
2114
2115                 /* starting below the boundary? Nothing to zero out */
2116                 if (offset <= zerofrom)
2117                         zerofrom = offset;
2118         }
2119         status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2120         if (status)
2121                 goto out1;
2122         if (zerofrom < offset) {
2123                 kaddr = kmap_atomic(page, KM_USER0);
2124                 memset(kaddr+zerofrom, 0, offset-zerofrom);
2125                 flush_dcache_page(page);
2126                 kunmap_atomic(kaddr, KM_USER0);
2127                 __block_commit_write(inode, page, zerofrom, offset);
2128         }
2129         return 0;
2130 out1:
2131         ClearPageUptodate(page);
2132         return status;
2133
2134 out_unmap:
2135         ClearPageUptodate(new_page);
2136         unlock_page(new_page);
2137         page_cache_release(new_page);
2138 out:
2139         return status;
2140 }
2141
2142 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2143                         get_block_t *get_block)
2144 {
2145         struct inode *inode = page->mapping->host;
2146         int err = __block_prepare_write(inode, page, from, to, get_block);
2147         if (err)
2148                 ClearPageUptodate(page);
2149         return err;
2150 }
2151
2152 int block_commit_write(struct page *page, unsigned from, unsigned to)
2153 {
2154         struct inode *inode = page->mapping->host;
2155         __block_commit_write(inode,page,from,to);
2156         return 0;
2157 }
2158
2159 int generic_commit_write(struct file *file, struct page *page,
2160                 unsigned from, unsigned to)
2161 {
2162         struct inode *inode = page->mapping->host;
2163         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2164         __block_commit_write(inode,page,from,to);
2165         /*
2166          * No need to use i_size_read() here, the i_size
2167          * cannot change under us because we hold i_mutex.
2168          */
2169         if (pos > inode->i_size) {
2170                 i_size_write(inode, pos);
2171                 mark_inode_dirty(inode);
2172         }
2173         return 0;
2174 }
2175
2176
2177 /*
2178  * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2179  * immediately, while under the page lock.  So it needs a special end_io
2180  * handler which does not touch the bh after unlocking it.
2181  *
2182  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2183  * a race there is benign: unlock_buffer() only use the bh's address for
2184  * hashing after unlocking the buffer, so it doesn't actually touch the bh
2185  * itself.
2186  */
2187 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2188 {
2189         if (uptodate) {
2190                 set_buffer_uptodate(bh);
2191         } else {
2192                 /* This happens, due to failed READA attempts. */
2193                 clear_buffer_uptodate(bh);
2194         }
2195         unlock_buffer(bh);
2196 }
2197
2198 /*
2199  * On entry, the page is fully not uptodate.
2200  * On exit the page is fully uptodate in the areas outside (from,to)
2201  */
2202 int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2203                         get_block_t *get_block)
2204 {
2205         struct inode *inode = page->mapping->host;
2206         const unsigned blkbits = inode->i_blkbits;
2207         const unsigned blocksize = 1 << blkbits;
2208         struct buffer_head map_bh;
2209         struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2210         unsigned block_in_page;
2211         unsigned block_start;
2212         sector_t block_in_file;
2213         char *kaddr;
2214         int nr_reads = 0;
2215         int i;
2216         int ret = 0;
2217         int is_mapped_to_disk = 1;
2218         int dirtied_it = 0;
2219
2220         if (PageMappedToDisk(page))
2221                 return 0;
2222
2223         block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2224         map_bh.b_page = page;
2225
2226         /*
2227          * We loop across all blocks in the page, whether or not they are
2228          * part of the affected region.  This is so we can discover if the
2229          * page is fully mapped-to-disk.
2230          */
2231         for (block_start = 0, block_in_page = 0;
2232                   block_start < PAGE_CACHE_SIZE;
2233                   block_in_page++, block_start += blocksize) {
2234                 unsigned block_end = block_start + blocksize;
2235                 int create;
2236
2237                 map_bh.b_state = 0;
2238                 create = 1;
2239                 if (block_start >= to)
2240                         create = 0;
2241                 map_bh.b_size = blocksize;
2242                 ret = get_block(inode, block_in_file + block_in_page,
2243                                         &map_bh, create);
2244                 if (ret)
2245                         goto failed;
2246                 if (!buffer_mapped(&map_bh))
2247                         is_mapped_to_disk = 0;
2248                 if (buffer_new(&map_bh))
2249                         unmap_underlying_metadata(map_bh.b_bdev,
2250                                                         map_bh.b_blocknr);
2251                 if (PageUptodate(page))
2252                         continue;
2253                 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2254                         kaddr = kmap_atomic(page, KM_USER0);
2255                         if (block_start < from) {
2256                                 memset(kaddr+block_start, 0, from-block_start);
2257                                 dirtied_it = 1;
2258                         }
2259                         if (block_end > to) {
2260                                 memset(kaddr + to, 0, block_end - to);
2261                                 dirtied_it = 1;
2262                         }
2263                         flush_dcache_page(page);
2264                         kunmap_atomic(kaddr, KM_USER0);
2265                         continue;
2266                 }
2267                 if (buffer_uptodate(&map_bh))
2268                         continue;       /* reiserfs does this */
2269                 if (block_start < from || block_end > to) {
2270                         struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2271
2272                         if (!bh) {
2273                                 ret = -ENOMEM;
2274                                 goto failed;
2275                         }
2276                         bh->b_state = map_bh.b_state;
2277                         atomic_set(&bh->b_count, 0);
2278                         bh->b_this_page = NULL;
2279                         bh->b_page = page;
2280                         bh->b_blocknr = map_bh.b_blocknr;
2281                         bh->b_size = blocksize;
2282                         bh->b_data = (char *)(long)block_start;
2283                         bh->b_bdev = map_bh.b_bdev;
2284                         bh->b_private = NULL;
2285                         read_bh[nr_reads++] = bh;
2286                 }
2287         }
2288
2289         if (nr_reads) {
2290                 struct buffer_head *bh;
2291
2292                 /*
2293                  * The page is locked, so these buffers are protected from
2294                  * any VM or truncate activity.  Hence we don't need to care
2295                  * for the buffer_head refcounts.
2296                  */
2297                 for (i = 0; i < nr_reads; i++) {
2298                         bh = read_bh[i];
2299                         lock_buffer(bh);
2300                         bh->b_end_io = end_buffer_read_nobh;
2301                         submit_bh(READ, bh);
2302                 }
2303                 for (i = 0; i < nr_reads; i++) {
2304                         bh = read_bh[i];
2305                         wait_on_buffer(bh);
2306                         if (!buffer_uptodate(bh))
2307                                 ret = -EIO;
2308                         free_buffer_head(bh);
2309                         read_bh[i] = NULL;
2310                 }
2311                 if (ret)
2312                         goto failed;
2313         }
2314
2315         if (is_mapped_to_disk)
2316                 SetPageMappedToDisk(page);
2317         SetPageUptodate(page);
2318
2319         /*
2320          * Setting the page dirty here isn't necessary for the prepare_write
2321          * function - commit_write will do that.  But if/when this function is
2322          * used within the pagefault handler to ensure that all mmapped pages
2323          * have backing space in the filesystem, we will need to dirty the page
2324          * if its contents were altered.
2325          */
2326         if (dirtied_it)
2327                 set_page_dirty(page);
2328
2329         return 0;
2330
2331 failed:
2332         for (i = 0; i < nr_reads; i++) {
2333                 if (read_bh[i])
2334                         free_buffer_head(read_bh[i]);
2335         }
2336
2337         /*
2338          * Error recovery is pretty slack.  Clear the page and mark it dirty
2339          * so we'll later zero out any blocks which _were_ allocated.
2340          */
2341         kaddr = kmap_atomic(page, KM_USER0);
2342         memset(kaddr, 0, PAGE_CACHE_SIZE);
2343         kunmap_atomic(kaddr, KM_USER0);
2344         SetPageUptodate(page);
2345         set_page_dirty(page);
2346         return ret;
2347 }
2348 EXPORT_SYMBOL(nobh_prepare_write);
2349
2350 int nobh_commit_write(struct file *file, struct page *page,
2351                 unsigned from, unsigned to)
2352 {
2353         struct inode *inode = page->mapping->host;
2354         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2355
2356         set_page_dirty(page);
2357         if (pos > inode->i_size) {
2358                 i_size_write(inode, pos);
2359                 mark_inode_dirty(inode);
2360         }
2361         return 0;
2362 }
2363 EXPORT_SYMBOL(nobh_commit_write);
2364
2365 /*
2366  * nobh_writepage() - based on block_full_write_page() except
2367  * that it tries to operate without attaching bufferheads to
2368  * the page.
2369  */
2370 int nobh_writepage(struct page *page, get_block_t *get_block,
2371                         struct writeback_control *wbc)
2372 {
2373         struct inode * const inode = page->mapping->host;
2374         loff_t i_size = i_size_read(inode);
2375         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2376         unsigned offset;
2377         void *kaddr;
2378         int ret;
2379
2380         /* Is the page fully inside i_size? */
2381         if (page->index < end_index)
2382                 goto out;
2383
2384         /* Is the page fully outside i_size? (truncate in progress) */
2385         offset = i_size & (PAGE_CACHE_SIZE-1);
2386         if (page->index >= end_index+1 || !offset) {
2387                 /*
2388                  * The page may have dirty, unmapped buffers.  For example,
2389                  * they may have been added in ext3_writepage().  Make them
2390                  * freeable here, so the page does not leak.
2391                  */
2392 #if 0
2393                 /* Not really sure about this  - do we need this ? */
2394                 if (page->mapping->a_ops->invalidatepage)
2395                         page->mapping->a_ops->invalidatepage(page, offset);
2396 #endif
2397                 unlock_page(page);
2398                 return 0; /* don't care */
2399         }
2400
2401         /*
2402          * The page straddles i_size.  It must be zeroed out on each and every
2403          * writepage invocation because it may be mmapped.  "A file is mapped
2404          * in multiples of the page size.  For a file that is not a multiple of
2405          * the  page size, the remaining memory is zeroed when mapped, and
2406          * writes to that region are not written out to the file."
2407          */
2408         kaddr = kmap_atomic(page, KM_USER0);
2409         memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2410         flush_dcache_page(page);
2411         kunmap_atomic(kaddr, KM_USER0);
2412 out:
2413         ret = mpage_writepage(page, get_block, wbc);
2414         if (ret == -EAGAIN)
2415                 ret = __block_write_full_page(inode, page, get_block, wbc);
2416         return ret;
2417 }
2418 EXPORT_SYMBOL(nobh_writepage);
2419
2420 /*
2421  * This function assumes that ->prepare_write() uses nobh_prepare_write().
2422  */
2423 int nobh_truncate_page(struct address_space *mapping, loff_t from)
2424 {
2425         struct inode *inode = mapping->host;
2426         unsigned blocksize = 1 << inode->i_blkbits;
2427         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2428         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2429         unsigned to;
2430         struct page *page;
2431         const struct address_space_operations *a_ops = mapping->a_ops;
2432         char *kaddr;
2433         int ret = 0;
2434
2435         if ((offset & (blocksize - 1)) == 0)
2436                 goto out;
2437
2438         ret = -ENOMEM;
2439         page = grab_cache_page(mapping, index);
2440         if (!page)
2441                 goto out;
2442
2443         to = (offset + blocksize) & ~(blocksize - 1);
2444         ret = a_ops->prepare_write(NULL, page, offset, to);
2445         if (ret == 0) {
2446                 kaddr = kmap_atomic(page, KM_USER0);
2447                 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2448                 flush_dcache_page(page);
2449                 kunmap_atomic(kaddr, KM_USER0);
2450                 set_page_dirty(page);
2451         }
2452         unlock_page(page);
2453         page_cache_release(page);
2454 out:
2455         return ret;
2456 }
2457 EXPORT_SYMBOL(nobh_truncate_page);
2458
2459 int block_truncate_page(struct address_space *mapping,
2460                         loff_t from, get_block_t *get_block)
2461 {
2462         pgoff_t index = from >> PAGE_CACHE_SHIFT;
2463         unsigned offset = from & (PAGE_CACHE_SIZE-1);
2464         unsigned blocksize;
2465         sector_t iblock;
2466         unsigned length, pos;
2467         struct inode *inode = mapping->host;
2468         struct page *page;
2469         struct buffer_head *bh;
2470         void *kaddr;
2471         int err;
2472
2473         blocksize = 1 << inode->i_blkbits;
2474         length = offset & (blocksize - 1);
2475
2476         /* Block boundary? Nothing to do */
2477         if (!length)
2478                 return 0;
2479
2480         length = blocksize - length;
2481         iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2482         
2483         page = grab_cache_page(mapping, index);
2484         err = -ENOMEM;
2485         if (!page)
2486                 goto out;
2487
2488         if (!page_has_buffers(page))
2489                 create_empty_buffers(page, blocksize, 0);
2490
2491         /* Find the buffer that contains "offset" */
2492         bh = page_buffers(page);
2493         pos = blocksize;
2494         while (offset >= pos) {
2495                 bh = bh->b_this_page;
2496                 iblock++;
2497                 pos += blocksize;
2498         }
2499
2500         err = 0;
2501         if (!buffer_mapped(bh)) {
2502                 WARN_ON(bh->b_size != blocksize);
2503                 err = get_block(inode, iblock, bh, 0);
2504                 if (err)
2505                         goto unlock;
2506                 /* unmapped? It's a hole - nothing to do */
2507                 if (!buffer_mapped(bh))
2508                         goto unlock;
2509         }
2510
2511         /* Ok, it's mapped. Make sure it's up-to-date */
2512         if (PageUptodate(page))
2513                 set_buffer_uptodate(bh);
2514
2515         if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2516                 err = -EIO;
2517                 ll_rw_block(READ, 1, &bh);
2518                 wait_on_buffer(bh);
2519                 /* Uhhuh. Read error. Complain and punt. */
2520                 if (!buffer_uptodate(bh))
2521                         goto unlock;
2522         }
2523
2524         kaddr = kmap_atomic(page, KM_USER0);
2525         memset(kaddr + offset, 0, length);
2526         flush_dcache_page(page);
2527         kunmap_atomic(kaddr, KM_USER0);
2528
2529         mark_buffer_dirty(bh);
2530         err = 0;
2531
2532 unlock:
2533         unlock_page(page);
2534         page_cache_release(page);
2535 out:
2536         return err;
2537 }
2538
2539 /*
2540  * The generic ->writepage function for buffer-backed address_spaces
2541  */
2542 int block_write_full_page(struct page *page, get_block_t *get_block,
2543                         struct writeback_control *wbc)
2544 {
2545         struct inode * const inode = page->mapping->host;
2546         loff_t i_size = i_size_read(inode);
2547         const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2548         unsigned offset;
2549         void *kaddr;
2550
2551         /* Is the page fully inside i_size? */
2552         if (page->index < end_index)
2553                 return __block_write_full_page(inode, page, get_block, wbc);
2554
2555         /* Is the page fully outside i_size? (truncate in progress) */
2556         offset = i_size & (PAGE_CACHE_SIZE-1);
2557         if (page->index >= end_index+1 || !offset) {
2558                 /*
2559                  * The page may have dirty, unmapped buffers.  For example,
2560                  * they may have been added in ext3_writepage().  Make them
2561                  * freeable here, so the page does not leak.
2562                  */
2563                 do_invalidatepage(page, 0);
2564                 unlock_page(page);
2565                 return 0; /* don't care */
2566         }
2567
2568         /*
2569          * The page straddles i_size.  It must be zeroed out on each and every
2570          * writepage invokation because it may be mmapped.  "A file is mapped
2571          * in multiples of the page size.  For a file that is not a multiple of
2572          * the  page size, the remaining memory is zeroed when mapped, and
2573          * writes to that region are not written out to the file."
2574          */
2575         kaddr = kmap_atomic(page, KM_USER0);
2576         memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2577         flush_dcache_page(page);
2578         kunmap_atomic(kaddr, KM_USER0);
2579         return __block_write_full_page(inode, page, get_block, wbc);
2580 }
2581
2582 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2583                             get_block_t *get_block)
2584 {
2585         struct buffer_head tmp;
2586         struct inode *inode = mapping->host;
2587         tmp.b_state = 0;
2588         tmp.b_blocknr = 0;
2589         tmp.b_size = 1 << inode->i_blkbits;
2590         get_block(inode, block, &tmp, 0);
2591         return tmp.b_blocknr;
2592 }
2593
2594 static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2595 {
2596         struct buffer_head *bh = bio->bi_private;
2597
2598         if (bio->bi_size)
2599                 return 1;
2600
2601         if (err == -EOPNOTSUPP) {
2602                 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2603                 set_bit(BH_Eopnotsupp, &bh->b_state);
2604         }
2605
2606         bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2607         bio_put(bio);
2608         return 0;
2609 }
2610
2611 int submit_bh(int rw, struct buffer_head * bh)
2612 {
2613         struct bio *bio;
2614         int ret = 0;
2615
2616         BUG_ON(!buffer_locked(bh));
2617         BUG_ON(!buffer_mapped(bh));
2618         BUG_ON(!bh->b_end_io);
2619
2620         if (buffer_ordered(bh) && (rw == WRITE))
2621                 rw = WRITE_BARRIER;
2622
2623         /*
2624          * Only clear out a write error when rewriting, should this
2625          * include WRITE_SYNC as well?
2626          */
2627         if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2628                 clear_buffer_write_io_error(bh);
2629
2630         /*
2631          * from here on down, it's all bio -- do the initial mapping,
2632          * submit_bio -> generic_make_request may further map this bio around
2633          */
2634         bio = bio_alloc(GFP_NOIO, 1);
2635
2636         bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2637         bio->bi_bdev = bh->b_bdev;
2638         bio->bi_io_vec[0].bv_page = bh->b_page;
2639         bio->bi_io_vec[0].bv_len = bh->b_size;
2640         bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2641
2642         bio->bi_vcnt = 1;
2643         bio->bi_idx = 0;
2644         bio->bi_size = bh->b_size;
2645
2646         bio->bi_end_io = end_bio_bh_io_sync;
2647         bio->bi_private = bh;
2648
2649         bio_get(bio);
2650         submit_bio(rw, bio);
2651
2652         if (bio_flagged(bio, BIO_EOPNOTSUPP))
2653                 ret = -EOPNOTSUPP;
2654
2655         bio_put(bio);
2656         return ret;
2657 }
2658
2659 /**
2660  * ll_rw_block: low-level access to block devices (DEPRECATED)
2661  * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
2662  * @nr: number of &struct buffer_heads in the array
2663  * @bhs: array of pointers to &struct buffer_head
2664  *
2665  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2666  * requests an I/O operation on them, either a %READ or a %WRITE.  The third
2667  * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2668  * are sent to disk. The fourth %READA option is described in the documentation
2669  * for generic_make_request() which ll_rw_block() calls.
2670  *
2671  * This function drops any buffer that it cannot get a lock on (with the
2672  * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2673  * clean when doing a write request, and any buffer that appears to be
2674  * up-to-date when doing read request.  Further it marks as clean buffers that
2675  * are processed for writing (the buffer cache won't assume that they are
2676  * actually clean until the buffer gets unlocked).
2677  *
2678  * ll_rw_block sets b_end_io to simple completion handler that marks
2679  * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2680  * any waiters. 
2681  *
2682  * All of the buffers must be for the same device, and must also be a
2683  * multiple of the current approved size for the device.
2684  */
2685 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2686 {
2687         int i;
2688
2689         for (i = 0; i < nr; i++) {
2690                 struct buffer_head *bh = bhs[i];
2691
2692                 if (rw == SWRITE)
2693                         lock_buffer(bh);
2694                 else if (test_set_buffer_locked(bh))
2695                         continue;
2696
2697                 if (rw == WRITE || rw == SWRITE) {
2698                         if (test_clear_buffer_dirty(bh)) {
2699                                 bh->b_end_io = end_buffer_write_sync;
2700                                 get_bh(bh);
2701                                 submit_bh(WRITE, bh);
2702                                 continue;
2703                         }
2704                 } else {
2705                         if (!buffer_uptodate(bh)) {
2706                                 bh->b_end_io = end_buffer_read_sync;
2707                                 get_bh(bh);
2708                                 submit_bh(rw, bh);
2709                                 continue;
2710                         }
2711                 }
2712                 unlock_buffer(bh);
2713         }
2714 }
2715
2716 /*
2717  * For a data-integrity writeout, we need to wait upon any in-progress I/O
2718  * and then start new I/O and then wait upon it.  The caller must have a ref on
2719  * the buffer_head.
2720  */
2721 int sync_dirty_buffer(struct buffer_head *bh)
2722 {
2723         int ret = 0;
2724
2725         WARN_ON(atomic_read(&bh->b_count) < 1);
2726         lock_buffer(bh);
2727         if (test_clear_buffer_dirty(bh)) {
2728                 get_bh(bh);
2729                 bh->b_end_io = end_buffer_write_sync;
2730                 ret = submit_bh(WRITE, bh);
2731                 wait_on_buffer(bh);
2732                 if (buffer_eopnotsupp(bh)) {
2733                         clear_buffer_eopnotsupp(bh);
2734                         ret = -EOPNOTSUPP;
2735                 }
2736                 if (!ret && !buffer_uptodate(bh))
2737                         ret = -EIO;
2738         } else {
2739                 unlock_buffer(bh);
2740         }
2741         return ret;
2742 }
2743
2744 /*
2745  * try_to_free_buffers() checks if all the buffers on this particular page
2746  * are unused, and releases them if so.
2747  *
2748  * Exclusion against try_to_free_buffers may be obtained by either
2749  * locking the page or by holding its mapping's private_lock.
2750  *
2751  * If the page is dirty but all the buffers are clean then we need to
2752  * be sure to mark the page clean as well.  This is because the page
2753  * may be against a block device, and a later reattachment of buffers
2754  * to a dirty page will set *all* buffers dirty.  Which would corrupt
2755  * filesystem data on the same device.
2756  *
2757  * The same applies to regular filesystem pages: if all the buffers are
2758  * clean then we set the page clean and proceed.  To do that, we require
2759  * total exclusion from __set_page_dirty_buffers().  That is obtained with
2760  * private_lock.
2761  *
2762  * try_to_free_buffers() is non-blocking.
2763  */
2764 static inline int buffer_busy(struct buffer_head *bh)
2765 {
2766         return atomic_read(&bh->b_count) |
2767                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2768 }
2769
2770 static int
2771 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2772 {
2773         struct buffer_head *head = page_buffers(page);
2774         struct buffer_head *bh;
2775
2776         bh = head;
2777         do {
2778                 if (buffer_write_io_error(bh) && page->mapping)
2779                         set_bit(AS_EIO, &page->mapping->flags);
2780                 if (buffer_busy(bh))
2781                         goto failed;
2782                 bh = bh->b_this_page;
2783         } while (bh != head);
2784
2785         do {
2786                 struct buffer_head *next = bh->b_this_page;
2787
2788                 if (!list_empty(&bh->b_assoc_buffers))
2789                         __remove_assoc_queue(bh);
2790                 bh = next;
2791         } while (bh != head);
2792         *buffers_to_free = head;
2793         __clear_page_buffers(page);
2794         return 1;
2795 failed:
2796         return 0;
2797 }
2798
2799 int try_to_free_buffers(struct page *page)
2800 {
2801         struct address_space * const mapping = page->mapping;
2802         struct buffer_head *buffers_to_free = NULL;
2803         int ret = 0;
2804
2805         BUG_ON(!PageLocked(page));
2806         if (PageWriteback(page))
2807                 return 0;
2808
2809         if (mapping == NULL) {          /* can this still happen? */
2810                 ret = drop_buffers(page, &buffers_to_free);
2811                 goto out;
2812         }
2813
2814         spin_lock(&mapping->private_lock);
2815         ret = drop_buffers(page, &buffers_to_free);
2816         spin_unlock(&mapping->private_lock);
2817         if (ret) {
2818                 /*
2819                  * If the filesystem writes its buffers by hand (eg ext3)
2820                  * then we can have clean buffers against a dirty page.  We
2821                  * clean the page here; otherwise later reattachment of buffers
2822                  * could encounter a non-uptodate page, which is unresolvable.
2823                  * This only applies in the rare case where try_to_free_buffers
2824                  * succeeds but the page is not freed.
2825                  */
2826                 clear_page_dirty(page);
2827         }
2828 out:
2829         if (buffers_to_free) {
2830                 struct buffer_head *bh = buffers_to_free;
2831
2832                 do {
2833                         struct buffer_head *next = bh->b_this_page;
2834                         free_buffer_head(bh);
2835                         bh = next;
2836                 } while (bh != buffers_to_free);
2837         }
2838         return ret;
2839 }
2840 EXPORT_SYMBOL(try_to_free_buffers);
2841
2842 void block_sync_page(struct page *page)
2843 {
2844         struct address_space *mapping;
2845
2846         smp_mb();
2847         mapping = page_mapping(page);
2848         if (mapping)
2849                 blk_run_backing_dev(mapping->backing_dev_info, page);
2850 }
2851
2852 /*
2853  * There are no bdflush tunables left.  But distributions are
2854  * still running obsolete flush daemons, so we terminate them here.
2855  *
2856  * Use of bdflush() is deprecated and will be removed in a future kernel.
2857  * The `pdflush' kernel threads fully replace bdflush daemons and this call.
2858  */
2859 asmlinkage long sys_bdflush(int func, long data)
2860 {
2861         static int msg_count;
2862
2863         if (!capable(CAP_SYS_ADMIN))
2864                 return -EPERM;
2865
2866         if (msg_count < 5) {
2867                 msg_count++;
2868                 printk(KERN_INFO
2869                         "warning: process `%s' used the obsolete bdflush"
2870                         " system call\n", current->comm);
2871                 printk(KERN_INFO "Fix your initscripts?\n");
2872         }
2873
2874         if (func == 1)
2875                 do_exit(0);
2876         return 0;
2877 }
2878
2879 /*
2880  * Buffer-head allocation
2881  */
2882 static kmem_cache_t *bh_cachep;
2883
2884 /*
2885  * Once the number of bh's in the machine exceeds this level, we start
2886  * stripping them in writeback.
2887  */
2888 static int max_buffer_heads;
2889
2890 int buffer_heads_over_limit;
2891
2892 struct bh_accounting {
2893         int nr;                 /* Number of live bh's */
2894         int ratelimit;          /* Limit cacheline bouncing */
2895 };
2896
2897 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
2898
2899 static void recalc_bh_state(void)
2900 {
2901         int i;
2902         int tot = 0;
2903
2904         if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
2905                 return;
2906         __get_cpu_var(bh_accounting).ratelimit = 0;
2907         for_each_online_cpu(i)
2908                 tot += per_cpu(bh_accounting, i).nr;
2909         buffer_heads_over_limit = (tot > max_buffer_heads);
2910 }
2911         
2912 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
2913 {
2914         struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
2915         if (ret) {
2916                 get_cpu_var(bh_accounting).nr++;
2917                 recalc_bh_state();
2918                 put_cpu_var(bh_accounting);
2919         }
2920         return ret;
2921 }
2922 EXPORT_SYMBOL(alloc_buffer_head);
2923
2924 void free_buffer_head(struct buffer_head *bh)
2925 {
2926         BUG_ON(!list_empty(&bh->b_assoc_buffers));
2927         kmem_cache_free(bh_cachep, bh);
2928         get_cpu_var(bh_accounting).nr--;
2929         recalc_bh_state();
2930         put_cpu_var(bh_accounting);
2931 }
2932 EXPORT_SYMBOL(free_buffer_head);
2933
2934 static void
2935 init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
2936 {
2937         if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
2938                             SLAB_CTOR_CONSTRUCTOR) {
2939                 struct buffer_head * bh = (struct buffer_head *)data;
2940
2941                 memset(bh, 0, sizeof(*bh));
2942                 INIT_LIST_HEAD(&bh->b_assoc_buffers);
2943         }
2944 }
2945
2946 #ifdef CONFIG_HOTPLUG_CPU
2947 static void buffer_exit_cpu(int cpu)
2948 {
2949         int i;
2950         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
2951
2952         for (i = 0; i < BH_LRU_SIZE; i++) {
2953                 brelse(b->bhs[i]);
2954                 b->bhs[i] = NULL;
2955         }
2956         get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
2957         per_cpu(bh_accounting, cpu).nr = 0;
2958         put_cpu_var(bh_accounting);
2959 }
2960
2961 static int buffer_cpu_notify(struct notifier_block *self,
2962                               unsigned long action, void *hcpu)
2963 {
2964         if (action == CPU_DEAD)
2965                 buffer_exit_cpu((unsigned long)hcpu);
2966         return NOTIFY_OK;
2967 }
2968 #endif /* CONFIG_HOTPLUG_CPU */
2969
2970 void __init buffer_init(void)
2971 {
2972         int nrpages;
2973
2974         bh_cachep = kmem_cache_create("buffer_head",
2975                                         sizeof(struct buffer_head), 0,
2976                                         (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
2977                                         SLAB_MEM_SPREAD),
2978                                         init_buffer_head,
2979                                         NULL);
2980
2981         /*
2982          * Limit the bh occupancy to 10% of ZONE_NORMAL
2983          */
2984         nrpages = (nr_free_buffer_pages() * 10) / 100;
2985         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
2986         hotcpu_notifier(buffer_cpu_notify, 0);
2987 }
2988
2989 EXPORT_SYMBOL(__bforget);
2990 EXPORT_SYMBOL(__brelse);
2991 EXPORT_SYMBOL(__wait_on_buffer);
2992 EXPORT_SYMBOL(block_commit_write);
2993 EXPORT_SYMBOL(block_prepare_write);
2994 EXPORT_SYMBOL(block_read_full_page);
2995 EXPORT_SYMBOL(block_sync_page);
2996 EXPORT_SYMBOL(block_truncate_page);
2997 EXPORT_SYMBOL(block_write_full_page);
2998 EXPORT_SYMBOL(cont_prepare_write);
2999 EXPORT_SYMBOL(end_buffer_read_sync);
3000 EXPORT_SYMBOL(end_buffer_write_sync);
3001 EXPORT_SYMBOL(file_fsync);
3002 EXPORT_SYMBOL(fsync_bdev);
3003 EXPORT_SYMBOL(generic_block_bmap);
3004 EXPORT_SYMBOL(generic_commit_write);
3005 EXPORT_SYMBOL(generic_cont_expand);
3006 EXPORT_SYMBOL(generic_cont_expand_simple);
3007 EXPORT_SYMBOL(init_buffer);
3008 EXPORT_SYMBOL(invalidate_bdev);
3009 EXPORT_SYMBOL(ll_rw_block);
3010 EXPORT_SYMBOL(mark_buffer_dirty);
3011 EXPORT_SYMBOL(submit_bh);
3012 EXPORT_SYMBOL(sync_dirty_buffer);
3013 EXPORT_SYMBOL(unlock_buffer);