4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/smp_lock.h>
28 #include <linux/capability.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/module.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46 static void invalidate_bh_lrus(void);
48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
51 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
53 bh->b_end_io = handler;
54 bh->b_private = private;
57 static int sync_buffer(void *word)
59 struct block_device *bd;
60 struct buffer_head *bh
61 = container_of(word, struct buffer_head, b_state);
66 blk_run_address_space(bd->bd_inode->i_mapping);
71 void fastcall __lock_buffer(struct buffer_head *bh)
73 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74 TASK_UNINTERRUPTIBLE);
76 EXPORT_SYMBOL(__lock_buffer);
78 void fastcall unlock_buffer(struct buffer_head *bh)
80 clear_buffer_locked(bh);
81 smp_mb__after_clear_bit();
82 wake_up_bit(&bh->b_state, BH_Lock);
86 * Block until a buffer comes unlocked. This doesn't stop it
87 * from becoming locked again - you have to lock it yourself
88 * if you want to preserve its state.
90 void __wait_on_buffer(struct buffer_head * bh)
92 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
96 __clear_page_buffers(struct page *page)
98 ClearPagePrivate(page);
99 set_page_private(page, 0);
100 page_cache_release(page);
103 static void buffer_io_error(struct buffer_head *bh)
105 char b[BDEVNAME_SIZE];
107 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
108 bdevname(bh->b_bdev, b),
109 (unsigned long long)bh->b_blocknr);
113 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
114 * unlock the buffer. This is what ll_rw_block uses too.
116 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
119 set_buffer_uptodate(bh);
121 /* This happens, due to failed READA attempts. */
122 clear_buffer_uptodate(bh);
128 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
130 char b[BDEVNAME_SIZE];
133 set_buffer_uptodate(bh);
135 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
137 printk(KERN_WARNING "lost page write due to "
139 bdevname(bh->b_bdev, b));
141 set_buffer_write_io_error(bh);
142 clear_buffer_uptodate(bh);
149 * Write out and wait upon all the dirty data associated with a block
150 * device via its mapping. Does not take the superblock lock.
152 int sync_blockdev(struct block_device *bdev)
157 ret = filemap_write_and_wait(bdev->bd_inode->i_mapping);
160 EXPORT_SYMBOL(sync_blockdev);
163 * Write out and wait upon all dirty data associated with this
164 * device. Filesystem data as well as the underlying block
165 * device. Takes the superblock lock.
167 int fsync_bdev(struct block_device *bdev)
169 struct super_block *sb = get_super(bdev);
171 int res = fsync_super(sb);
175 return sync_blockdev(bdev);
179 * freeze_bdev -- lock a filesystem and force it into a consistent state
180 * @bdev: blockdevice to lock
182 * This takes the block device bd_mount_mutex to make sure no new mounts
183 * happen on bdev until thaw_bdev() is called.
184 * If a superblock is found on this device, we take the s_umount semaphore
185 * on it to make sure nobody unmounts until the snapshot creation is done.
187 struct super_block *freeze_bdev(struct block_device *bdev)
189 struct super_block *sb;
191 mutex_lock(&bdev->bd_mount_mutex);
192 sb = get_super(bdev);
193 if (sb && !(sb->s_flags & MS_RDONLY)) {
194 sb->s_frozen = SB_FREEZE_WRITE;
199 sb->s_frozen = SB_FREEZE_TRANS;
202 sync_blockdev(sb->s_bdev);
204 if (sb->s_op->write_super_lockfs)
205 sb->s_op->write_super_lockfs(sb);
209 return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */
211 EXPORT_SYMBOL(freeze_bdev);
214 * thaw_bdev -- unlock filesystem
215 * @bdev: blockdevice to unlock
216 * @sb: associated superblock
218 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
220 void thaw_bdev(struct block_device *bdev, struct super_block *sb)
223 BUG_ON(sb->s_bdev != bdev);
225 if (sb->s_op->unlockfs)
226 sb->s_op->unlockfs(sb);
227 sb->s_frozen = SB_UNFROZEN;
229 wake_up(&sb->s_wait_unfrozen);
233 mutex_unlock(&bdev->bd_mount_mutex);
235 EXPORT_SYMBOL(thaw_bdev);
238 * Various filesystems appear to want __find_get_block to be non-blocking.
239 * But it's the page lock which protects the buffers. To get around this,
240 * we get exclusion from try_to_free_buffers with the blockdev mapping's
243 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
244 * may be quite high. This code could TryLock the page, and if that
245 * succeeds, there is no need to take private_lock. (But if
246 * private_lock is contended then so is mapping->tree_lock).
248 static struct buffer_head *
249 __find_get_block_slow(struct block_device *bdev, sector_t block)
251 struct inode *bd_inode = bdev->bd_inode;
252 struct address_space *bd_mapping = bd_inode->i_mapping;
253 struct buffer_head *ret = NULL;
255 struct buffer_head *bh;
256 struct buffer_head *head;
260 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
261 page = find_get_page(bd_mapping, index);
265 spin_lock(&bd_mapping->private_lock);
266 if (!page_has_buffers(page))
268 head = page_buffers(page);
271 if (bh->b_blocknr == block) {
276 if (!buffer_mapped(bh))
278 bh = bh->b_this_page;
279 } while (bh != head);
281 /* we might be here because some of the buffers on this page are
282 * not mapped. This is due to various races between
283 * file io on the block device and getblk. It gets dealt with
284 * elsewhere, don't buffer_error if we had some unmapped buffers
287 printk("__find_get_block_slow() failed. "
288 "block=%llu, b_blocknr=%llu\n",
289 (unsigned long long)block,
290 (unsigned long long)bh->b_blocknr);
291 printk("b_state=0x%08lx, b_size=%zu\n",
292 bh->b_state, bh->b_size);
293 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
296 spin_unlock(&bd_mapping->private_lock);
297 page_cache_release(page);
302 /* If invalidate_buffers() will trash dirty buffers, it means some kind
303 of fs corruption is going on. Trashing dirty data always imply losing
304 information that was supposed to be just stored on the physical layer
307 Thus invalidate_buffers in general usage is not allwowed to trash
308 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
309 be preserved. These buffers are simply skipped.
311 We also skip buffers which are still in use. For example this can
312 happen if a userspace program is reading the block device.
314 NOTE: In the case where the user removed a removable-media-disk even if
315 there's still dirty data not synced on disk (due a bug in the device driver
316 or due an error of the user), by not destroying the dirty buffers we could
317 generate corruption also on the next media inserted, thus a parameter is
318 necessary to handle this case in the most safe way possible (trying
319 to not corrupt also the new disk inserted with the data belonging to
320 the old now corrupted disk). Also for the ramdisk the natural thing
321 to do in order to release the ramdisk memory is to destroy dirty buffers.
323 These are two special cases. Normal usage imply the device driver
324 to issue a sync on the device (without waiting I/O completion) and
325 then an invalidate_buffers call that doesn't trash dirty buffers.
327 For handling cache coherency with the blkdev pagecache the 'update' case
328 is been introduced. It is needed to re-read from disk any pinned
329 buffer. NOTE: re-reading from disk is destructive so we can do it only
330 when we assume nobody is changing the buffercache under our I/O and when
331 we think the disk contains more recent information than the buffercache.
332 The update == 1 pass marks the buffers we need to update, the update == 2
333 pass does the actual I/O. */
334 void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
336 struct address_space *mapping = bdev->bd_inode->i_mapping;
338 if (mapping->nrpages == 0)
341 invalidate_bh_lrus();
343 * FIXME: what about destroy_dirty_buffers?
344 * We really want to use invalidate_inode_pages2() for
345 * that, but not until that's cleaned up.
347 invalidate_inode_pages(mapping);
351 * Kick pdflush then try to free up some ZONE_NORMAL memory.
353 static void free_more_memory(void)
358 wakeup_pdflush(1024);
361 for_each_online_pgdat(pgdat) {
362 zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
364 try_to_free_pages(zones, GFP_NOFS);
369 * I/O completion handler for block_read_full_page() - pages
370 * which come unlocked at the end of I/O.
372 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
375 struct buffer_head *first;
376 struct buffer_head *tmp;
378 int page_uptodate = 1;
380 BUG_ON(!buffer_async_read(bh));
384 set_buffer_uptodate(bh);
386 clear_buffer_uptodate(bh);
387 if (printk_ratelimit())
393 * Be _very_ careful from here on. Bad things can happen if
394 * two buffer heads end IO at almost the same time and both
395 * decide that the page is now completely done.
397 first = page_buffers(page);
398 local_irq_save(flags);
399 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
400 clear_buffer_async_read(bh);
404 if (!buffer_uptodate(tmp))
406 if (buffer_async_read(tmp)) {
407 BUG_ON(!buffer_locked(tmp));
410 tmp = tmp->b_this_page;
412 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
413 local_irq_restore(flags);
416 * If none of the buffers had errors and they are all
417 * uptodate then we can set the page uptodate.
419 if (page_uptodate && !PageError(page))
420 SetPageUptodate(page);
425 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
426 local_irq_restore(flags);
431 * Completion handler for block_write_full_page() - pages which are unlocked
432 * during I/O, and which have PageWriteback cleared upon I/O completion.
434 static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
436 char b[BDEVNAME_SIZE];
438 struct buffer_head *first;
439 struct buffer_head *tmp;
442 BUG_ON(!buffer_async_write(bh));
446 set_buffer_uptodate(bh);
448 if (printk_ratelimit()) {
450 printk(KERN_WARNING "lost page write due to "
452 bdevname(bh->b_bdev, b));
454 set_bit(AS_EIO, &page->mapping->flags);
455 set_buffer_write_io_error(bh);
456 clear_buffer_uptodate(bh);
460 first = page_buffers(page);
461 local_irq_save(flags);
462 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
464 clear_buffer_async_write(bh);
466 tmp = bh->b_this_page;
468 if (buffer_async_write(tmp)) {
469 BUG_ON(!buffer_locked(tmp));
472 tmp = tmp->b_this_page;
474 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
475 local_irq_restore(flags);
476 end_page_writeback(page);
480 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
481 local_irq_restore(flags);
486 * If a page's buffers are under async readin (end_buffer_async_read
487 * completion) then there is a possibility that another thread of
488 * control could lock one of the buffers after it has completed
489 * but while some of the other buffers have not completed. This
490 * locked buffer would confuse end_buffer_async_read() into not unlocking
491 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
492 * that this buffer is not under async I/O.
494 * The page comes unlocked when it has no locked buffer_async buffers
497 * PageLocked prevents anyone starting new async I/O reads any of
500 * PageWriteback is used to prevent simultaneous writeout of the same
503 * PageLocked prevents anyone from starting writeback of a page which is
504 * under read I/O (PageWriteback is only ever set against a locked page).
506 static void mark_buffer_async_read(struct buffer_head *bh)
508 bh->b_end_io = end_buffer_async_read;
509 set_buffer_async_read(bh);
512 void mark_buffer_async_write(struct buffer_head *bh)
514 bh->b_end_io = end_buffer_async_write;
515 set_buffer_async_write(bh);
517 EXPORT_SYMBOL(mark_buffer_async_write);
521 * fs/buffer.c contains helper functions for buffer-backed address space's
522 * fsync functions. A common requirement for buffer-based filesystems is
523 * that certain data from the backing blockdev needs to be written out for
524 * a successful fsync(). For example, ext2 indirect blocks need to be
525 * written back and waited upon before fsync() returns.
527 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
528 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
529 * management of a list of dependent buffers at ->i_mapping->private_list.
531 * Locking is a little subtle: try_to_free_buffers() will remove buffers
532 * from their controlling inode's queue when they are being freed. But
533 * try_to_free_buffers() will be operating against the *blockdev* mapping
534 * at the time, not against the S_ISREG file which depends on those buffers.
535 * So the locking for private_list is via the private_lock in the address_space
536 * which backs the buffers. Which is different from the address_space
537 * against which the buffers are listed. So for a particular address_space,
538 * mapping->private_lock does *not* protect mapping->private_list! In fact,
539 * mapping->private_list will always be protected by the backing blockdev's
542 * Which introduces a requirement: all buffers on an address_space's
543 * ->private_list must be from the same address_space: the blockdev's.
545 * address_spaces which do not place buffers at ->private_list via these
546 * utility functions are free to use private_lock and private_list for
547 * whatever they want. The only requirement is that list_empty(private_list)
548 * be true at clear_inode() time.
550 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
551 * filesystems should do that. invalidate_inode_buffers() should just go
552 * BUG_ON(!list_empty).
554 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
555 * take an address_space, not an inode. And it should be called
556 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
559 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
560 * list if it is already on a list. Because if the buffer is on a list,
561 * it *must* already be on the right one. If not, the filesystem is being
562 * silly. This will save a ton of locking. But first we have to ensure
563 * that buffers are taken *off* the old inode's list when they are freed
564 * (presumably in truncate). That requires careful auditing of all
565 * filesystems (do it inside bforget()). It could also be done by bringing
570 * The buffer's backing address_space's private_lock must be held
572 static inline void __remove_assoc_queue(struct buffer_head *bh)
574 list_del_init(&bh->b_assoc_buffers);
575 WARN_ON(!bh->b_assoc_map);
576 if (buffer_write_io_error(bh))
577 set_bit(AS_EIO, &bh->b_assoc_map->flags);
578 bh->b_assoc_map = NULL;
581 int inode_has_buffers(struct inode *inode)
583 return !list_empty(&inode->i_data.private_list);
587 * osync is designed to support O_SYNC io. It waits synchronously for
588 * all already-submitted IO to complete, but does not queue any new
589 * writes to the disk.
591 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
592 * you dirty the buffers, and then use osync_inode_buffers to wait for
593 * completion. Any other dirty buffers which are not yet queued for
594 * write will not be flushed to disk by the osync.
596 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
598 struct buffer_head *bh;
604 list_for_each_prev(p, list) {
606 if (buffer_locked(bh)) {
610 if (!buffer_uptodate(bh))
622 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
624 * @mapping: the mapping which wants those buffers written
626 * Starts I/O against the buffers at mapping->private_list, and waits upon
629 * Basically, this is a convenience function for fsync().
630 * @mapping is a file or directory which needs those buffers to be written for
631 * a successful fsync().
633 int sync_mapping_buffers(struct address_space *mapping)
635 struct address_space *buffer_mapping = mapping->assoc_mapping;
637 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
640 return fsync_buffers_list(&buffer_mapping->private_lock,
641 &mapping->private_list);
643 EXPORT_SYMBOL(sync_mapping_buffers);
646 * Called when we've recently written block `bblock', and it is known that
647 * `bblock' was for a buffer_boundary() buffer. This means that the block at
648 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
649 * dirty, schedule it for IO. So that indirects merge nicely with their data.
651 void write_boundary_block(struct block_device *bdev,
652 sector_t bblock, unsigned blocksize)
654 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
656 if (buffer_dirty(bh))
657 ll_rw_block(WRITE, 1, &bh);
662 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
664 struct address_space *mapping = inode->i_mapping;
665 struct address_space *buffer_mapping = bh->b_page->mapping;
667 mark_buffer_dirty(bh);
668 if (!mapping->assoc_mapping) {
669 mapping->assoc_mapping = buffer_mapping;
671 BUG_ON(mapping->assoc_mapping != buffer_mapping);
673 if (list_empty(&bh->b_assoc_buffers)) {
674 spin_lock(&buffer_mapping->private_lock);
675 list_move_tail(&bh->b_assoc_buffers,
676 &mapping->private_list);
677 bh->b_assoc_map = mapping;
678 spin_unlock(&buffer_mapping->private_lock);
681 EXPORT_SYMBOL(mark_buffer_dirty_inode);
684 * Add a page to the dirty page list.
686 * It is a sad fact of life that this function is called from several places
687 * deeply under spinlocking. It may not sleep.
689 * If the page has buffers, the uptodate buffers are set dirty, to preserve
690 * dirty-state coherency between the page and the buffers. It the page does
691 * not have buffers then when they are later attached they will all be set
694 * The buffers are dirtied before the page is dirtied. There's a small race
695 * window in which a writepage caller may see the page cleanness but not the
696 * buffer dirtiness. That's fine. If this code were to set the page dirty
697 * before the buffers, a concurrent writepage caller could clear the page dirty
698 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
699 * page on the dirty page list.
701 * We use private_lock to lock against try_to_free_buffers while using the
702 * page's buffer list. Also use this to protect against clean buffers being
703 * added to the page after it was set dirty.
705 * FIXME: may need to call ->reservepage here as well. That's rather up to the
706 * address_space though.
708 int __set_page_dirty_buffers(struct page *page)
710 struct address_space * const mapping = page_mapping(page);
712 if (unlikely(!mapping))
713 return !TestSetPageDirty(page);
715 spin_lock(&mapping->private_lock);
716 if (page_has_buffers(page)) {
717 struct buffer_head *head = page_buffers(page);
718 struct buffer_head *bh = head;
721 set_buffer_dirty(bh);
722 bh = bh->b_this_page;
723 } while (bh != head);
725 spin_unlock(&mapping->private_lock);
727 if (!TestSetPageDirty(page)) {
728 write_lock_irq(&mapping->tree_lock);
729 if (page->mapping) { /* Race with truncate? */
730 if (mapping_cap_account_dirty(mapping))
731 __inc_zone_page_state(page, NR_FILE_DIRTY);
732 radix_tree_tag_set(&mapping->page_tree,
734 PAGECACHE_TAG_DIRTY);
736 write_unlock_irq(&mapping->tree_lock);
737 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
742 EXPORT_SYMBOL(__set_page_dirty_buffers);
745 * Write out and wait upon a list of buffers.
747 * We have conflicting pressures: we want to make sure that all
748 * initially dirty buffers get waited on, but that any subsequently
749 * dirtied buffers don't. After all, we don't want fsync to last
750 * forever if somebody is actively writing to the file.
752 * Do this in two main stages: first we copy dirty buffers to a
753 * temporary inode list, queueing the writes as we go. Then we clean
754 * up, waiting for those writes to complete.
756 * During this second stage, any subsequent updates to the file may end
757 * up refiling the buffer on the original inode's dirty list again, so
758 * there is a chance we will end up with a buffer queued for write but
759 * not yet completed on that list. So, as a final cleanup we go through
760 * the osync code to catch these locked, dirty buffers without requeuing
761 * any newly dirty buffers for write.
763 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
765 struct buffer_head *bh;
766 struct list_head tmp;
769 INIT_LIST_HEAD(&tmp);
772 while (!list_empty(list)) {
773 bh = BH_ENTRY(list->next);
774 __remove_assoc_queue(bh);
775 if (buffer_dirty(bh) || buffer_locked(bh)) {
776 list_add(&bh->b_assoc_buffers, &tmp);
777 if (buffer_dirty(bh)) {
781 * Ensure any pending I/O completes so that
782 * ll_rw_block() actually writes the current
783 * contents - it is a noop if I/O is still in
784 * flight on potentially older contents.
786 ll_rw_block(SWRITE, 1, &bh);
793 while (!list_empty(&tmp)) {
794 bh = BH_ENTRY(tmp.prev);
795 list_del_init(&bh->b_assoc_buffers);
799 if (!buffer_uptodate(bh))
806 err2 = osync_buffers_list(lock, list);
814 * Invalidate any and all dirty buffers on a given inode. We are
815 * probably unmounting the fs, but that doesn't mean we have already
816 * done a sync(). Just drop the buffers from the inode list.
818 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
819 * assumes that all the buffers are against the blockdev. Not true
822 void invalidate_inode_buffers(struct inode *inode)
824 if (inode_has_buffers(inode)) {
825 struct address_space *mapping = &inode->i_data;
826 struct list_head *list = &mapping->private_list;
827 struct address_space *buffer_mapping = mapping->assoc_mapping;
829 spin_lock(&buffer_mapping->private_lock);
830 while (!list_empty(list))
831 __remove_assoc_queue(BH_ENTRY(list->next));
832 spin_unlock(&buffer_mapping->private_lock);
837 * Remove any clean buffers from the inode's buffer list. This is called
838 * when we're trying to free the inode itself. Those buffers can pin it.
840 * Returns true if all buffers were removed.
842 int remove_inode_buffers(struct inode *inode)
846 if (inode_has_buffers(inode)) {
847 struct address_space *mapping = &inode->i_data;
848 struct list_head *list = &mapping->private_list;
849 struct address_space *buffer_mapping = mapping->assoc_mapping;
851 spin_lock(&buffer_mapping->private_lock);
852 while (!list_empty(list)) {
853 struct buffer_head *bh = BH_ENTRY(list->next);
854 if (buffer_dirty(bh)) {
858 __remove_assoc_queue(bh);
860 spin_unlock(&buffer_mapping->private_lock);
866 * Create the appropriate buffers when given a page for data area and
867 * the size of each buffer.. Use the bh->b_this_page linked list to
868 * follow the buffers created. Return NULL if unable to create more
871 * The retry flag is used to differentiate async IO (paging, swapping)
872 * which may not fail from ordinary buffer allocations.
874 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
877 struct buffer_head *bh, *head;
883 while ((offset -= size) >= 0) {
884 bh = alloc_buffer_head(GFP_NOFS);
889 bh->b_this_page = head;
894 atomic_set(&bh->b_count, 0);
895 bh->b_private = NULL;
898 /* Link the buffer to its page */
899 set_bh_page(bh, page, offset);
901 init_buffer(bh, NULL, NULL);
905 * In case anything failed, we just free everything we got.
911 head = head->b_this_page;
912 free_buffer_head(bh);
917 * Return failure for non-async IO requests. Async IO requests
918 * are not allowed to fail, so we have to wait until buffer heads
919 * become available. But we don't want tasks sleeping with
920 * partially complete buffers, so all were released above.
925 /* We're _really_ low on memory. Now we just
926 * wait for old buffer heads to become free due to
927 * finishing IO. Since this is an async request and
928 * the reserve list is empty, we're sure there are
929 * async buffer heads in use.
934 EXPORT_SYMBOL_GPL(alloc_page_buffers);
937 link_dev_buffers(struct page *page, struct buffer_head *head)
939 struct buffer_head *bh, *tail;
944 bh = bh->b_this_page;
946 tail->b_this_page = head;
947 attach_page_buffers(page, head);
951 * Initialise the state of a blockdev page's buffers.
954 init_page_buffers(struct page *page, struct block_device *bdev,
955 sector_t block, int size)
957 struct buffer_head *head = page_buffers(page);
958 struct buffer_head *bh = head;
959 int uptodate = PageUptodate(page);
962 if (!buffer_mapped(bh)) {
963 init_buffer(bh, NULL, NULL);
965 bh->b_blocknr = block;
967 set_buffer_uptodate(bh);
968 set_buffer_mapped(bh);
971 bh = bh->b_this_page;
972 } while (bh != head);
976 * Create the page-cache page that contains the requested block.
978 * This is user purely for blockdev mappings.
981 grow_dev_page(struct block_device *bdev, sector_t block,
982 pgoff_t index, int size)
984 struct inode *inode = bdev->bd_inode;
986 struct buffer_head *bh;
988 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
992 BUG_ON(!PageLocked(page));
994 if (page_has_buffers(page)) {
995 bh = page_buffers(page);
996 if (bh->b_size == size) {
997 init_page_buffers(page, bdev, block, size);
1000 if (!try_to_free_buffers(page))
1005 * Allocate some buffers for this page
1007 bh = alloc_page_buffers(page, size, 0);
1012 * Link the page to the buffers and initialise them. Take the
1013 * lock to be atomic wrt __find_get_block(), which does not
1014 * run under the page lock.
1016 spin_lock(&inode->i_mapping->private_lock);
1017 link_dev_buffers(page, bh);
1018 init_page_buffers(page, bdev, block, size);
1019 spin_unlock(&inode->i_mapping->private_lock);
1025 page_cache_release(page);
1030 * Create buffers for the specified block device block's page. If
1031 * that page was dirty, the buffers are set dirty also.
1033 * Except that's a bug. Attaching dirty buffers to a dirty
1034 * blockdev's page can result in filesystem corruption, because
1035 * some of those buffers may be aliases of filesystem data.
1036 * grow_dev_page() will go BUG() if this happens.
1039 grow_buffers(struct block_device *bdev, sector_t block, int size)
1048 } while ((size << sizebits) < PAGE_SIZE);
1050 index = block >> sizebits;
1053 * Check for a block which wants to lie outside our maximum possible
1054 * pagecache index. (this comparison is done using sector_t types).
1056 if (unlikely(index != block >> sizebits)) {
1057 char b[BDEVNAME_SIZE];
1059 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1061 __FUNCTION__, (unsigned long long)block,
1065 block = index << sizebits;
1066 /* Create a page with the proper size buffers.. */
1067 page = grow_dev_page(bdev, block, index, size);
1071 page_cache_release(page);
1075 static struct buffer_head *
1076 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1078 /* Size must be multiple of hard sectorsize */
1079 if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1080 (size < 512 || size > PAGE_SIZE))) {
1081 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1083 printk(KERN_ERR "hardsect size: %d\n",
1084 bdev_hardsect_size(bdev));
1091 struct buffer_head * bh;
1094 bh = __find_get_block(bdev, block, size);
1098 ret = grow_buffers(bdev, block, size);
1107 * The relationship between dirty buffers and dirty pages:
1109 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1110 * the page is tagged dirty in its radix tree.
1112 * At all times, the dirtiness of the buffers represents the dirtiness of
1113 * subsections of the page. If the page has buffers, the page dirty bit is
1114 * merely a hint about the true dirty state.
1116 * When a page is set dirty in its entirety, all its buffers are marked dirty
1117 * (if the page has buffers).
1119 * When a buffer is marked dirty, its page is dirtied, but the page's other
1122 * Also. When blockdev buffers are explicitly read with bread(), they
1123 * individually become uptodate. But their backing page remains not
1124 * uptodate - even if all of its buffers are uptodate. A subsequent
1125 * block_read_full_page() against that page will discover all the uptodate
1126 * buffers, will set the page uptodate and will perform no I/O.
1130 * mark_buffer_dirty - mark a buffer_head as needing writeout
1131 * @bh: the buffer_head to mark dirty
1133 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1134 * backing page dirty, then tag the page as dirty in its address_space's radix
1135 * tree and then attach the address_space's inode to its superblock's dirty
1138 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1139 * mapping->tree_lock and the global inode_lock.
1141 void fastcall mark_buffer_dirty(struct buffer_head *bh)
1143 if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1144 __set_page_dirty_nobuffers(bh->b_page);
1148 * Decrement a buffer_head's reference count. If all buffers against a page
1149 * have zero reference count, are clean and unlocked, and if the page is clean
1150 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1151 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1152 * a page but it ends up not being freed, and buffers may later be reattached).
1154 void __brelse(struct buffer_head * buf)
1156 if (atomic_read(&buf->b_count)) {
1160 printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1165 * bforget() is like brelse(), except it discards any
1166 * potentially dirty data.
1168 void __bforget(struct buffer_head *bh)
1170 clear_buffer_dirty(bh);
1171 if (!list_empty(&bh->b_assoc_buffers)) {
1172 struct address_space *buffer_mapping = bh->b_page->mapping;
1174 spin_lock(&buffer_mapping->private_lock);
1175 list_del_init(&bh->b_assoc_buffers);
1176 bh->b_assoc_map = NULL;
1177 spin_unlock(&buffer_mapping->private_lock);
1182 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1185 if (buffer_uptodate(bh)) {
1190 bh->b_end_io = end_buffer_read_sync;
1191 submit_bh(READ, bh);
1193 if (buffer_uptodate(bh))
1201 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1202 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1203 * refcount elevated by one when they're in an LRU. A buffer can only appear
1204 * once in a particular CPU's LRU. A single buffer can be present in multiple
1205 * CPU's LRUs at the same time.
1207 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1208 * sb_find_get_block().
1210 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1211 * a local interrupt disable for that.
1214 #define BH_LRU_SIZE 8
1217 struct buffer_head *bhs[BH_LRU_SIZE];
1220 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1223 #define bh_lru_lock() local_irq_disable()
1224 #define bh_lru_unlock() local_irq_enable()
1226 #define bh_lru_lock() preempt_disable()
1227 #define bh_lru_unlock() preempt_enable()
1230 static inline void check_irqs_on(void)
1232 #ifdef irqs_disabled
1233 BUG_ON(irqs_disabled());
1238 * The LRU management algorithm is dopey-but-simple. Sorry.
1240 static void bh_lru_install(struct buffer_head *bh)
1242 struct buffer_head *evictee = NULL;
1247 lru = &__get_cpu_var(bh_lrus);
1248 if (lru->bhs[0] != bh) {
1249 struct buffer_head *bhs[BH_LRU_SIZE];
1255 for (in = 0; in < BH_LRU_SIZE; in++) {
1256 struct buffer_head *bh2 = lru->bhs[in];
1261 if (out >= BH_LRU_SIZE) {
1262 BUG_ON(evictee != NULL);
1269 while (out < BH_LRU_SIZE)
1271 memcpy(lru->bhs, bhs, sizeof(bhs));
1280 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1282 static struct buffer_head *
1283 lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1285 struct buffer_head *ret = NULL;
1291 lru = &__get_cpu_var(bh_lrus);
1292 for (i = 0; i < BH_LRU_SIZE; i++) {
1293 struct buffer_head *bh = lru->bhs[i];
1295 if (bh && bh->b_bdev == bdev &&
1296 bh->b_blocknr == block && bh->b_size == size) {
1299 lru->bhs[i] = lru->bhs[i - 1];
1314 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1315 * it in the LRU and mark it as accessed. If it is not present then return
1318 struct buffer_head *
1319 __find_get_block(struct block_device *bdev, sector_t block, int size)
1321 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1324 bh = __find_get_block_slow(bdev, block);
1332 EXPORT_SYMBOL(__find_get_block);
1335 * __getblk will locate (and, if necessary, create) the buffer_head
1336 * which corresponds to the passed block_device, block and size. The
1337 * returned buffer has its reference count incremented.
1339 * __getblk() cannot fail - it just keeps trying. If you pass it an
1340 * illegal block number, __getblk() will happily return a buffer_head
1341 * which represents the non-existent block. Very weird.
1343 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1344 * attempt is failing. FIXME, perhaps?
1346 struct buffer_head *
1347 __getblk(struct block_device *bdev, sector_t block, int size)
1349 struct buffer_head *bh = __find_get_block(bdev, block, size);
1353 bh = __getblk_slow(bdev, block, size);
1356 EXPORT_SYMBOL(__getblk);
1359 * Do async read-ahead on a buffer..
1361 void __breadahead(struct block_device *bdev, sector_t block, int size)
1363 struct buffer_head *bh = __getblk(bdev, block, size);
1365 ll_rw_block(READA, 1, &bh);
1369 EXPORT_SYMBOL(__breadahead);
1372 * __bread() - reads a specified block and returns the bh
1373 * @bdev: the block_device to read from
1374 * @block: number of block
1375 * @size: size (in bytes) to read
1377 * Reads a specified block, and returns buffer head that contains it.
1378 * It returns NULL if the block was unreadable.
1380 struct buffer_head *
1381 __bread(struct block_device *bdev, sector_t block, int size)
1383 struct buffer_head *bh = __getblk(bdev, block, size);
1385 if (likely(bh) && !buffer_uptodate(bh))
1386 bh = __bread_slow(bh);
1389 EXPORT_SYMBOL(__bread);
1392 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1393 * This doesn't race because it runs in each cpu either in irq
1394 * or with preempt disabled.
1396 static void invalidate_bh_lru(void *arg)
1398 struct bh_lru *b = &get_cpu_var(bh_lrus);
1401 for (i = 0; i < BH_LRU_SIZE; i++) {
1405 put_cpu_var(bh_lrus);
1408 static void invalidate_bh_lrus(void)
1410 on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1413 void set_bh_page(struct buffer_head *bh,
1414 struct page *page, unsigned long offset)
1417 BUG_ON(offset >= PAGE_SIZE);
1418 if (PageHighMem(page))
1420 * This catches illegal uses and preserves the offset:
1422 bh->b_data = (char *)(0 + offset);
1424 bh->b_data = page_address(page) + offset;
1426 EXPORT_SYMBOL(set_bh_page);
1429 * Called when truncating a buffer on a page completely.
1431 static void discard_buffer(struct buffer_head * bh)
1434 clear_buffer_dirty(bh);
1436 clear_buffer_mapped(bh);
1437 clear_buffer_req(bh);
1438 clear_buffer_new(bh);
1439 clear_buffer_delay(bh);
1444 * block_invalidatepage - invalidate part of all of a buffer-backed page
1446 * @page: the page which is affected
1447 * @offset: the index of the truncation point
1449 * block_invalidatepage() is called when all or part of the page has become
1450 * invalidatedby a truncate operation.
1452 * block_invalidatepage() does not have to release all buffers, but it must
1453 * ensure that no dirty buffer is left outside @offset and that no I/O
1454 * is underway against any of the blocks which are outside the truncation
1455 * point. Because the caller is about to free (and possibly reuse) those
1458 void block_invalidatepage(struct page *page, unsigned long offset)
1460 struct buffer_head *head, *bh, *next;
1461 unsigned int curr_off = 0;
1463 BUG_ON(!PageLocked(page));
1464 if (!page_has_buffers(page))
1467 head = page_buffers(page);
1470 unsigned int next_off = curr_off + bh->b_size;
1471 next = bh->b_this_page;
1474 * is this block fully invalidated?
1476 if (offset <= curr_off)
1478 curr_off = next_off;
1480 } while (bh != head);
1483 * We release buffers only if the entire page is being invalidated.
1484 * The get_block cached value has been unconditionally invalidated,
1485 * so real IO is not possible anymore.
1488 try_to_release_page(page, 0);
1492 EXPORT_SYMBOL(block_invalidatepage);
1495 * We attach and possibly dirty the buffers atomically wrt
1496 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1497 * is already excluded via the page lock.
1499 void create_empty_buffers(struct page *page,
1500 unsigned long blocksize, unsigned long b_state)
1502 struct buffer_head *bh, *head, *tail;
1504 head = alloc_page_buffers(page, blocksize, 1);
1507 bh->b_state |= b_state;
1509 bh = bh->b_this_page;
1511 tail->b_this_page = head;
1513 spin_lock(&page->mapping->private_lock);
1514 if (PageUptodate(page) || PageDirty(page)) {
1517 if (PageDirty(page))
1518 set_buffer_dirty(bh);
1519 if (PageUptodate(page))
1520 set_buffer_uptodate(bh);
1521 bh = bh->b_this_page;
1522 } while (bh != head);
1524 attach_page_buffers(page, head);
1525 spin_unlock(&page->mapping->private_lock);
1527 EXPORT_SYMBOL(create_empty_buffers);
1530 * We are taking a block for data and we don't want any output from any
1531 * buffer-cache aliases starting from return from that function and
1532 * until the moment when something will explicitly mark the buffer
1533 * dirty (hopefully that will not happen until we will free that block ;-)
1534 * We don't even need to mark it not-uptodate - nobody can expect
1535 * anything from a newly allocated buffer anyway. We used to used
1536 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1537 * don't want to mark the alias unmapped, for example - it would confuse
1538 * anyone who might pick it with bread() afterwards...
1540 * Also.. Note that bforget() doesn't lock the buffer. So there can
1541 * be writeout I/O going on against recently-freed buffers. We don't
1542 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1543 * only if we really need to. That happens here.
1545 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1547 struct buffer_head *old_bh;
1551 old_bh = __find_get_block_slow(bdev, block);
1553 clear_buffer_dirty(old_bh);
1554 wait_on_buffer(old_bh);
1555 clear_buffer_req(old_bh);
1559 EXPORT_SYMBOL(unmap_underlying_metadata);
1562 * NOTE! All mapped/uptodate combinations are valid:
1564 * Mapped Uptodate Meaning
1566 * No No "unknown" - must do get_block()
1567 * No Yes "hole" - zero-filled
1568 * Yes No "allocated" - allocated on disk, not read in
1569 * Yes Yes "valid" - allocated and up-to-date in memory.
1571 * "Dirty" is valid only with the last case (mapped+uptodate).
1575 * While block_write_full_page is writing back the dirty buffers under
1576 * the page lock, whoever dirtied the buffers may decide to clean them
1577 * again at any time. We handle that by only looking at the buffer
1578 * state inside lock_buffer().
1580 * If block_write_full_page() is called for regular writeback
1581 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1582 * locked buffer. This only can happen if someone has written the buffer
1583 * directly, with submit_bh(). At the address_space level PageWriteback
1584 * prevents this contention from occurring.
1586 static int __block_write_full_page(struct inode *inode, struct page *page,
1587 get_block_t *get_block, struct writeback_control *wbc)
1591 sector_t last_block;
1592 struct buffer_head *bh, *head;
1593 const unsigned blocksize = 1 << inode->i_blkbits;
1594 int nr_underway = 0;
1596 BUG_ON(!PageLocked(page));
1598 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1600 if (!page_has_buffers(page)) {
1601 create_empty_buffers(page, blocksize,
1602 (1 << BH_Dirty)|(1 << BH_Uptodate));
1606 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1607 * here, and the (potentially unmapped) buffers may become dirty at
1608 * any time. If a buffer becomes dirty here after we've inspected it
1609 * then we just miss that fact, and the page stays dirty.
1611 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1612 * handle that here by just cleaning them.
1615 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1616 head = page_buffers(page);
1620 * Get all the dirty buffers mapped to disk addresses and
1621 * handle any aliases from the underlying blockdev's mapping.
1624 if (block > last_block) {
1626 * mapped buffers outside i_size will occur, because
1627 * this page can be outside i_size when there is a
1628 * truncate in progress.
1631 * The buffer was zeroed by block_write_full_page()
1633 clear_buffer_dirty(bh);
1634 set_buffer_uptodate(bh);
1635 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1636 WARN_ON(bh->b_size != blocksize);
1637 err = get_block(inode, block, bh, 1);
1640 if (buffer_new(bh)) {
1641 /* blockdev mappings never come here */
1642 clear_buffer_new(bh);
1643 unmap_underlying_metadata(bh->b_bdev,
1647 bh = bh->b_this_page;
1649 } while (bh != head);
1652 if (!buffer_mapped(bh))
1655 * If it's a fully non-blocking write attempt and we cannot
1656 * lock the buffer then redirty the page. Note that this can
1657 * potentially cause a busy-wait loop from pdflush and kswapd
1658 * activity, but those code paths have their own higher-level
1661 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1663 } else if (test_set_buffer_locked(bh)) {
1664 redirty_page_for_writepage(wbc, page);
1667 if (test_clear_buffer_dirty(bh)) {
1668 mark_buffer_async_write(bh);
1672 } while ((bh = bh->b_this_page) != head);
1675 * The page and its buffers are protected by PageWriteback(), so we can
1676 * drop the bh refcounts early.
1678 BUG_ON(PageWriteback(page));
1679 set_page_writeback(page);
1682 struct buffer_head *next = bh->b_this_page;
1683 if (buffer_async_write(bh)) {
1684 submit_bh(WRITE, bh);
1688 } while (bh != head);
1693 if (nr_underway == 0) {
1695 * The page was marked dirty, but the buffers were
1696 * clean. Someone wrote them back by hand with
1697 * ll_rw_block/submit_bh. A rare case.
1701 if (!buffer_uptodate(bh)) {
1705 bh = bh->b_this_page;
1706 } while (bh != head);
1708 SetPageUptodate(page);
1709 end_page_writeback(page);
1711 * The page and buffer_heads can be released at any time from
1714 wbc->pages_skipped++; /* We didn't write this page */
1720 * ENOSPC, or some other error. We may already have added some
1721 * blocks to the file, so we need to write these out to avoid
1722 * exposing stale data.
1723 * The page is currently locked and not marked for writeback
1726 /* Recovery: lock and submit the mapped buffers */
1728 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1730 mark_buffer_async_write(bh);
1733 * The buffer may have been set dirty during
1734 * attachment to a dirty page.
1736 clear_buffer_dirty(bh);
1738 } while ((bh = bh->b_this_page) != head);
1740 BUG_ON(PageWriteback(page));
1741 set_page_writeback(page);
1744 struct buffer_head *next = bh->b_this_page;
1745 if (buffer_async_write(bh)) {
1746 clear_buffer_dirty(bh);
1747 submit_bh(WRITE, bh);
1751 } while (bh != head);
1755 static int __block_prepare_write(struct inode *inode, struct page *page,
1756 unsigned from, unsigned to, get_block_t *get_block)
1758 unsigned block_start, block_end;
1761 unsigned blocksize, bbits;
1762 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1764 BUG_ON(!PageLocked(page));
1765 BUG_ON(from > PAGE_CACHE_SIZE);
1766 BUG_ON(to > PAGE_CACHE_SIZE);
1769 blocksize = 1 << inode->i_blkbits;
1770 if (!page_has_buffers(page))
1771 create_empty_buffers(page, blocksize, 0);
1772 head = page_buffers(page);
1774 bbits = inode->i_blkbits;
1775 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1777 for(bh = head, block_start = 0; bh != head || !block_start;
1778 block++, block_start=block_end, bh = bh->b_this_page) {
1779 block_end = block_start + blocksize;
1780 if (block_end <= from || block_start >= to) {
1781 if (PageUptodate(page)) {
1782 if (!buffer_uptodate(bh))
1783 set_buffer_uptodate(bh);
1788 clear_buffer_new(bh);
1789 if (!buffer_mapped(bh)) {
1790 WARN_ON(bh->b_size != blocksize);
1791 err = get_block(inode, block, bh, 1);
1794 if (buffer_new(bh)) {
1795 unmap_underlying_metadata(bh->b_bdev,
1797 if (PageUptodate(page)) {
1798 set_buffer_uptodate(bh);
1801 if (block_end > to || block_start < from) {
1804 kaddr = kmap_atomic(page, KM_USER0);
1808 if (block_start < from)
1809 memset(kaddr+block_start,
1810 0, from-block_start);
1811 flush_dcache_page(page);
1812 kunmap_atomic(kaddr, KM_USER0);
1817 if (PageUptodate(page)) {
1818 if (!buffer_uptodate(bh))
1819 set_buffer_uptodate(bh);
1822 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1823 (block_start < from || block_end > to)) {
1824 ll_rw_block(READ, 1, &bh);
1829 * If we issued read requests - let them complete.
1831 while(wait_bh > wait) {
1832 wait_on_buffer(*--wait_bh);
1833 if (!buffer_uptodate(*wait_bh))
1840 clear_buffer_new(bh);
1841 } while ((bh = bh->b_this_page) != head);
1846 * Zero out any newly allocated blocks to avoid exposing stale
1847 * data. If BH_New is set, we know that the block was newly
1848 * allocated in the above loop.
1853 block_end = block_start+blocksize;
1854 if (block_end <= from)
1856 if (block_start >= to)
1858 if (buffer_new(bh)) {
1861 clear_buffer_new(bh);
1862 kaddr = kmap_atomic(page, KM_USER0);
1863 memset(kaddr+block_start, 0, bh->b_size);
1864 flush_dcache_page(page);
1865 kunmap_atomic(kaddr, KM_USER0);
1866 set_buffer_uptodate(bh);
1867 mark_buffer_dirty(bh);
1870 block_start = block_end;
1871 bh = bh->b_this_page;
1872 } while (bh != head);
1876 static int __block_commit_write(struct inode *inode, struct page *page,
1877 unsigned from, unsigned to)
1879 unsigned block_start, block_end;
1882 struct buffer_head *bh, *head;
1884 blocksize = 1 << inode->i_blkbits;
1886 for(bh = head = page_buffers(page), block_start = 0;
1887 bh != head || !block_start;
1888 block_start=block_end, bh = bh->b_this_page) {
1889 block_end = block_start + blocksize;
1890 if (block_end <= from || block_start >= to) {
1891 if (!buffer_uptodate(bh))
1894 set_buffer_uptodate(bh);
1895 mark_buffer_dirty(bh);
1900 * If this is a partial write which happened to make all buffers
1901 * uptodate then we can optimize away a bogus readpage() for
1902 * the next read(). Here we 'discover' whether the page went
1903 * uptodate as a result of this (potentially partial) write.
1906 SetPageUptodate(page);
1911 * Generic "read page" function for block devices that have the normal
1912 * get_block functionality. This is most of the block device filesystems.
1913 * Reads the page asynchronously --- the unlock_buffer() and
1914 * set/clear_buffer_uptodate() functions propagate buffer state into the
1915 * page struct once IO has completed.
1917 int block_read_full_page(struct page *page, get_block_t *get_block)
1919 struct inode *inode = page->mapping->host;
1920 sector_t iblock, lblock;
1921 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
1922 unsigned int blocksize;
1924 int fully_mapped = 1;
1926 BUG_ON(!PageLocked(page));
1927 blocksize = 1 << inode->i_blkbits;
1928 if (!page_has_buffers(page))
1929 create_empty_buffers(page, blocksize, 0);
1930 head = page_buffers(page);
1932 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1933 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
1939 if (buffer_uptodate(bh))
1942 if (!buffer_mapped(bh)) {
1946 if (iblock < lblock) {
1947 WARN_ON(bh->b_size != blocksize);
1948 err = get_block(inode, iblock, bh, 0);
1952 if (!buffer_mapped(bh)) {
1953 void *kaddr = kmap_atomic(page, KM_USER0);
1954 memset(kaddr + i * blocksize, 0, blocksize);
1955 flush_dcache_page(page);
1956 kunmap_atomic(kaddr, KM_USER0);
1958 set_buffer_uptodate(bh);
1962 * get_block() might have updated the buffer
1965 if (buffer_uptodate(bh))
1969 } while (i++, iblock++, (bh = bh->b_this_page) != head);
1972 SetPageMappedToDisk(page);
1976 * All buffers are uptodate - we can set the page uptodate
1977 * as well. But not if get_block() returned an error.
1979 if (!PageError(page))
1980 SetPageUptodate(page);
1985 /* Stage two: lock the buffers */
1986 for (i = 0; i < nr; i++) {
1989 mark_buffer_async_read(bh);
1993 * Stage 3: start the IO. Check for uptodateness
1994 * inside the buffer lock in case another process reading
1995 * the underlying blockdev brought it uptodate (the sct fix).
1997 for (i = 0; i < nr; i++) {
1999 if (buffer_uptodate(bh))
2000 end_buffer_async_read(bh, 1);
2002 submit_bh(READ, bh);
2007 /* utility function for filesystems that need to do work on expanding
2008 * truncates. Uses prepare/commit_write to allow the filesystem to
2009 * deal with the hole.
2011 static int __generic_cont_expand(struct inode *inode, loff_t size,
2012 pgoff_t index, unsigned int offset)
2014 struct address_space *mapping = inode->i_mapping;
2016 unsigned long limit;
2020 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2021 if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2022 send_sig(SIGXFSZ, current, 0);
2025 if (size > inode->i_sb->s_maxbytes)
2029 page = grab_cache_page(mapping, index);
2032 err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2035 * ->prepare_write() may have instantiated a few blocks
2036 * outside i_size. Trim these off again.
2039 page_cache_release(page);
2040 vmtruncate(inode, inode->i_size);
2044 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2047 page_cache_release(page);
2054 int generic_cont_expand(struct inode *inode, loff_t size)
2057 unsigned int offset;
2059 offset = (size & (PAGE_CACHE_SIZE - 1)); /* Within page */
2061 /* ugh. in prepare/commit_write, if from==to==start of block, we
2062 ** skip the prepare. make sure we never send an offset for the start
2065 if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2066 /* caller must handle this extra byte. */
2069 index = size >> PAGE_CACHE_SHIFT;
2071 return __generic_cont_expand(inode, size, index, offset);
2074 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2076 loff_t pos = size - 1;
2077 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
2078 unsigned int offset = (pos & (PAGE_CACHE_SIZE - 1)) + 1;
2080 /* prepare/commit_write can handle even if from==to==start of block. */
2081 return __generic_cont_expand(inode, size, index, offset);
2085 * For moronic filesystems that do not allow holes in file.
2086 * We may have to extend the file.
2089 int cont_prepare_write(struct page *page, unsigned offset,
2090 unsigned to, get_block_t *get_block, loff_t *bytes)
2092 struct address_space *mapping = page->mapping;
2093 struct inode *inode = mapping->host;
2094 struct page *new_page;
2098 unsigned blocksize = 1 << inode->i_blkbits;
2101 while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2103 new_page = grab_cache_page(mapping, pgpos);
2106 /* we might sleep */
2107 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2108 unlock_page(new_page);
2109 page_cache_release(new_page);
2112 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2113 if (zerofrom & (blocksize-1)) {
2114 *bytes |= (blocksize-1);
2117 status = __block_prepare_write(inode, new_page, zerofrom,
2118 PAGE_CACHE_SIZE, get_block);
2121 kaddr = kmap_atomic(new_page, KM_USER0);
2122 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2123 flush_dcache_page(new_page);
2124 kunmap_atomic(kaddr, KM_USER0);
2125 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2126 unlock_page(new_page);
2127 page_cache_release(new_page);
2130 if (page->index < pgpos) {
2131 /* completely inside the area */
2134 /* page covers the boundary, find the boundary offset */
2135 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2137 /* if we will expand the thing last block will be filled */
2138 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2139 *bytes |= (blocksize-1);
2143 /* starting below the boundary? Nothing to zero out */
2144 if (offset <= zerofrom)
2147 status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2150 if (zerofrom < offset) {
2151 kaddr = kmap_atomic(page, KM_USER0);
2152 memset(kaddr+zerofrom, 0, offset-zerofrom);
2153 flush_dcache_page(page);
2154 kunmap_atomic(kaddr, KM_USER0);
2155 __block_commit_write(inode, page, zerofrom, offset);
2159 ClearPageUptodate(page);
2163 ClearPageUptodate(new_page);
2164 unlock_page(new_page);
2165 page_cache_release(new_page);
2170 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2171 get_block_t *get_block)
2173 struct inode *inode = page->mapping->host;
2174 int err = __block_prepare_write(inode, page, from, to, get_block);
2176 ClearPageUptodate(page);
2180 int block_commit_write(struct page *page, unsigned from, unsigned to)
2182 struct inode *inode = page->mapping->host;
2183 __block_commit_write(inode,page,from,to);
2187 int generic_commit_write(struct file *file, struct page *page,
2188 unsigned from, unsigned to)
2190 struct inode *inode = page->mapping->host;
2191 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2192 __block_commit_write(inode,page,from,to);
2194 * No need to use i_size_read() here, the i_size
2195 * cannot change under us because we hold i_mutex.
2197 if (pos > inode->i_size) {
2198 i_size_write(inode, pos);
2199 mark_inode_dirty(inode);
2206 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2207 * immediately, while under the page lock. So it needs a special end_io
2208 * handler which does not touch the bh after unlocking it.
2210 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2211 * a race there is benign: unlock_buffer() only use the bh's address for
2212 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2215 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2218 set_buffer_uptodate(bh);
2220 /* This happens, due to failed READA attempts. */
2221 clear_buffer_uptodate(bh);
2227 * On entry, the page is fully not uptodate.
2228 * On exit the page is fully uptodate in the areas outside (from,to)
2230 int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2231 get_block_t *get_block)
2233 struct inode *inode = page->mapping->host;
2234 const unsigned blkbits = inode->i_blkbits;
2235 const unsigned blocksize = 1 << blkbits;
2236 struct buffer_head map_bh;
2237 struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2238 unsigned block_in_page;
2239 unsigned block_start;
2240 sector_t block_in_file;
2245 int is_mapped_to_disk = 1;
2248 if (PageMappedToDisk(page))
2251 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2252 map_bh.b_page = page;
2255 * We loop across all blocks in the page, whether or not they are
2256 * part of the affected region. This is so we can discover if the
2257 * page is fully mapped-to-disk.
2259 for (block_start = 0, block_in_page = 0;
2260 block_start < PAGE_CACHE_SIZE;
2261 block_in_page++, block_start += blocksize) {
2262 unsigned block_end = block_start + blocksize;
2267 if (block_start >= to)
2269 map_bh.b_size = blocksize;
2270 ret = get_block(inode, block_in_file + block_in_page,
2274 if (!buffer_mapped(&map_bh))
2275 is_mapped_to_disk = 0;
2276 if (buffer_new(&map_bh))
2277 unmap_underlying_metadata(map_bh.b_bdev,
2279 if (PageUptodate(page))
2281 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2282 kaddr = kmap_atomic(page, KM_USER0);
2283 if (block_start < from) {
2284 memset(kaddr+block_start, 0, from-block_start);
2287 if (block_end > to) {
2288 memset(kaddr + to, 0, block_end - to);
2291 flush_dcache_page(page);
2292 kunmap_atomic(kaddr, KM_USER0);
2295 if (buffer_uptodate(&map_bh))
2296 continue; /* reiserfs does this */
2297 if (block_start < from || block_end > to) {
2298 struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2304 bh->b_state = map_bh.b_state;
2305 atomic_set(&bh->b_count, 0);
2306 bh->b_this_page = NULL;
2308 bh->b_blocknr = map_bh.b_blocknr;
2309 bh->b_size = blocksize;
2310 bh->b_data = (char *)(long)block_start;
2311 bh->b_bdev = map_bh.b_bdev;
2312 bh->b_private = NULL;
2313 read_bh[nr_reads++] = bh;
2318 struct buffer_head *bh;
2321 * The page is locked, so these buffers are protected from
2322 * any VM or truncate activity. Hence we don't need to care
2323 * for the buffer_head refcounts.
2325 for (i = 0; i < nr_reads; i++) {
2328 bh->b_end_io = end_buffer_read_nobh;
2329 submit_bh(READ, bh);
2331 for (i = 0; i < nr_reads; i++) {
2334 if (!buffer_uptodate(bh))
2336 free_buffer_head(bh);
2343 if (is_mapped_to_disk)
2344 SetPageMappedToDisk(page);
2345 SetPageUptodate(page);
2348 * Setting the page dirty here isn't necessary for the prepare_write
2349 * function - commit_write will do that. But if/when this function is
2350 * used within the pagefault handler to ensure that all mmapped pages
2351 * have backing space in the filesystem, we will need to dirty the page
2352 * if its contents were altered.
2355 set_page_dirty(page);
2360 for (i = 0; i < nr_reads; i++) {
2362 free_buffer_head(read_bh[i]);
2366 * Error recovery is pretty slack. Clear the page and mark it dirty
2367 * so we'll later zero out any blocks which _were_ allocated.
2369 kaddr = kmap_atomic(page, KM_USER0);
2370 memset(kaddr, 0, PAGE_CACHE_SIZE);
2371 flush_dcache_page(page);
2372 kunmap_atomic(kaddr, KM_USER0);
2373 SetPageUptodate(page);
2374 set_page_dirty(page);
2377 EXPORT_SYMBOL(nobh_prepare_write);
2379 int nobh_commit_write(struct file *file, struct page *page,
2380 unsigned from, unsigned to)
2382 struct inode *inode = page->mapping->host;
2383 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2385 set_page_dirty(page);
2386 if (pos > inode->i_size) {
2387 i_size_write(inode, pos);
2388 mark_inode_dirty(inode);
2392 EXPORT_SYMBOL(nobh_commit_write);
2395 * nobh_writepage() - based on block_full_write_page() except
2396 * that it tries to operate without attaching bufferheads to
2399 int nobh_writepage(struct page *page, get_block_t *get_block,
2400 struct writeback_control *wbc)
2402 struct inode * const inode = page->mapping->host;
2403 loff_t i_size = i_size_read(inode);
2404 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2409 /* Is the page fully inside i_size? */
2410 if (page->index < end_index)
2413 /* Is the page fully outside i_size? (truncate in progress) */
2414 offset = i_size & (PAGE_CACHE_SIZE-1);
2415 if (page->index >= end_index+1 || !offset) {
2417 * The page may have dirty, unmapped buffers. For example,
2418 * they may have been added in ext3_writepage(). Make them
2419 * freeable here, so the page does not leak.
2422 /* Not really sure about this - do we need this ? */
2423 if (page->mapping->a_ops->invalidatepage)
2424 page->mapping->a_ops->invalidatepage(page, offset);
2427 return 0; /* don't care */
2431 * The page straddles i_size. It must be zeroed out on each and every
2432 * writepage invocation because it may be mmapped. "A file is mapped
2433 * in multiples of the page size. For a file that is not a multiple of
2434 * the page size, the remaining memory is zeroed when mapped, and
2435 * writes to that region are not written out to the file."
2437 kaddr = kmap_atomic(page, KM_USER0);
2438 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2439 flush_dcache_page(page);
2440 kunmap_atomic(kaddr, KM_USER0);
2442 ret = mpage_writepage(page, get_block, wbc);
2444 ret = __block_write_full_page(inode, page, get_block, wbc);
2447 EXPORT_SYMBOL(nobh_writepage);
2450 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2452 int nobh_truncate_page(struct address_space *mapping, loff_t from)
2454 struct inode *inode = mapping->host;
2455 unsigned blocksize = 1 << inode->i_blkbits;
2456 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2457 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2460 const struct address_space_operations *a_ops = mapping->a_ops;
2464 if ((offset & (blocksize - 1)) == 0)
2468 page = grab_cache_page(mapping, index);
2472 to = (offset + blocksize) & ~(blocksize - 1);
2473 ret = a_ops->prepare_write(NULL, page, offset, to);
2475 kaddr = kmap_atomic(page, KM_USER0);
2476 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2477 flush_dcache_page(page);
2478 kunmap_atomic(kaddr, KM_USER0);
2479 set_page_dirty(page);
2482 page_cache_release(page);
2486 EXPORT_SYMBOL(nobh_truncate_page);
2488 int block_truncate_page(struct address_space *mapping,
2489 loff_t from, get_block_t *get_block)
2491 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2492 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2495 unsigned length, pos;
2496 struct inode *inode = mapping->host;
2498 struct buffer_head *bh;
2502 blocksize = 1 << inode->i_blkbits;
2503 length = offset & (blocksize - 1);
2505 /* Block boundary? Nothing to do */
2509 length = blocksize - length;
2510 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2512 page = grab_cache_page(mapping, index);
2517 if (!page_has_buffers(page))
2518 create_empty_buffers(page, blocksize, 0);
2520 /* Find the buffer that contains "offset" */
2521 bh = page_buffers(page);
2523 while (offset >= pos) {
2524 bh = bh->b_this_page;
2530 if (!buffer_mapped(bh)) {
2531 WARN_ON(bh->b_size != blocksize);
2532 err = get_block(inode, iblock, bh, 0);
2535 /* unmapped? It's a hole - nothing to do */
2536 if (!buffer_mapped(bh))
2540 /* Ok, it's mapped. Make sure it's up-to-date */
2541 if (PageUptodate(page))
2542 set_buffer_uptodate(bh);
2544 if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2546 ll_rw_block(READ, 1, &bh);
2548 /* Uhhuh. Read error. Complain and punt. */
2549 if (!buffer_uptodate(bh))
2553 kaddr = kmap_atomic(page, KM_USER0);
2554 memset(kaddr + offset, 0, length);
2555 flush_dcache_page(page);
2556 kunmap_atomic(kaddr, KM_USER0);
2558 mark_buffer_dirty(bh);
2563 page_cache_release(page);
2569 * The generic ->writepage function for buffer-backed address_spaces
2571 int block_write_full_page(struct page *page, get_block_t *get_block,
2572 struct writeback_control *wbc)
2574 struct inode * const inode = page->mapping->host;
2575 loff_t i_size = i_size_read(inode);
2576 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2580 /* Is the page fully inside i_size? */
2581 if (page->index < end_index)
2582 return __block_write_full_page(inode, page, get_block, wbc);
2584 /* Is the page fully outside i_size? (truncate in progress) */
2585 offset = i_size & (PAGE_CACHE_SIZE-1);
2586 if (page->index >= end_index+1 || !offset) {
2588 * The page may have dirty, unmapped buffers. For example,
2589 * they may have been added in ext3_writepage(). Make them
2590 * freeable here, so the page does not leak.
2592 do_invalidatepage(page, 0);
2594 return 0; /* don't care */
2598 * The page straddles i_size. It must be zeroed out on each and every
2599 * writepage invokation because it may be mmapped. "A file is mapped
2600 * in multiples of the page size. For a file that is not a multiple of
2601 * the page size, the remaining memory is zeroed when mapped, and
2602 * writes to that region are not written out to the file."
2604 kaddr = kmap_atomic(page, KM_USER0);
2605 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2606 flush_dcache_page(page);
2607 kunmap_atomic(kaddr, KM_USER0);
2608 return __block_write_full_page(inode, page, get_block, wbc);
2611 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2612 get_block_t *get_block)
2614 struct buffer_head tmp;
2615 struct inode *inode = mapping->host;
2618 tmp.b_size = 1 << inode->i_blkbits;
2619 get_block(inode, block, &tmp, 0);
2620 return tmp.b_blocknr;
2623 static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2625 struct buffer_head *bh = bio->bi_private;
2630 if (err == -EOPNOTSUPP) {
2631 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2632 set_bit(BH_Eopnotsupp, &bh->b_state);
2635 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2640 int submit_bh(int rw, struct buffer_head * bh)
2645 BUG_ON(!buffer_locked(bh));
2646 BUG_ON(!buffer_mapped(bh));
2647 BUG_ON(!bh->b_end_io);
2649 if (buffer_ordered(bh) && (rw == WRITE))
2653 * Only clear out a write error when rewriting, should this
2654 * include WRITE_SYNC as well?
2656 if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2657 clear_buffer_write_io_error(bh);
2660 * from here on down, it's all bio -- do the initial mapping,
2661 * submit_bio -> generic_make_request may further map this bio around
2663 bio = bio_alloc(GFP_NOIO, 1);
2665 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2666 bio->bi_bdev = bh->b_bdev;
2667 bio->bi_io_vec[0].bv_page = bh->b_page;
2668 bio->bi_io_vec[0].bv_len = bh->b_size;
2669 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2673 bio->bi_size = bh->b_size;
2675 bio->bi_end_io = end_bio_bh_io_sync;
2676 bio->bi_private = bh;
2679 submit_bio(rw, bio);
2681 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2689 * ll_rw_block: low-level access to block devices (DEPRECATED)
2690 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
2691 * @nr: number of &struct buffer_heads in the array
2692 * @bhs: array of pointers to &struct buffer_head
2694 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2695 * requests an I/O operation on them, either a %READ or a %WRITE. The third
2696 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2697 * are sent to disk. The fourth %READA option is described in the documentation
2698 * for generic_make_request() which ll_rw_block() calls.
2700 * This function drops any buffer that it cannot get a lock on (with the
2701 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2702 * clean when doing a write request, and any buffer that appears to be
2703 * up-to-date when doing read request. Further it marks as clean buffers that
2704 * are processed for writing (the buffer cache won't assume that they are
2705 * actually clean until the buffer gets unlocked).
2707 * ll_rw_block sets b_end_io to simple completion handler that marks
2708 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2711 * All of the buffers must be for the same device, and must also be a
2712 * multiple of the current approved size for the device.
2714 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2718 for (i = 0; i < nr; i++) {
2719 struct buffer_head *bh = bhs[i];
2723 else if (test_set_buffer_locked(bh))
2726 if (rw == WRITE || rw == SWRITE) {
2727 if (test_clear_buffer_dirty(bh)) {
2728 bh->b_end_io = end_buffer_write_sync;
2730 submit_bh(WRITE, bh);
2734 if (!buffer_uptodate(bh)) {
2735 bh->b_end_io = end_buffer_read_sync;
2746 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2747 * and then start new I/O and then wait upon it. The caller must have a ref on
2750 int sync_dirty_buffer(struct buffer_head *bh)
2754 WARN_ON(atomic_read(&bh->b_count) < 1);
2756 if (test_clear_buffer_dirty(bh)) {
2758 bh->b_end_io = end_buffer_write_sync;
2759 ret = submit_bh(WRITE, bh);
2761 if (buffer_eopnotsupp(bh)) {
2762 clear_buffer_eopnotsupp(bh);
2765 if (!ret && !buffer_uptodate(bh))
2774 * try_to_free_buffers() checks if all the buffers on this particular page
2775 * are unused, and releases them if so.
2777 * Exclusion against try_to_free_buffers may be obtained by either
2778 * locking the page or by holding its mapping's private_lock.
2780 * If the page is dirty but all the buffers are clean then we need to
2781 * be sure to mark the page clean as well. This is because the page
2782 * may be against a block device, and a later reattachment of buffers
2783 * to a dirty page will set *all* buffers dirty. Which would corrupt
2784 * filesystem data on the same device.
2786 * The same applies to regular filesystem pages: if all the buffers are
2787 * clean then we set the page clean and proceed. To do that, we require
2788 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2791 * try_to_free_buffers() is non-blocking.
2793 static inline int buffer_busy(struct buffer_head *bh)
2795 return atomic_read(&bh->b_count) |
2796 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2800 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2802 struct buffer_head *head = page_buffers(page);
2803 struct buffer_head *bh;
2807 if (buffer_write_io_error(bh) && page->mapping)
2808 set_bit(AS_EIO, &page->mapping->flags);
2809 if (buffer_busy(bh))
2811 bh = bh->b_this_page;
2812 } while (bh != head);
2815 struct buffer_head *next = bh->b_this_page;
2817 if (!list_empty(&bh->b_assoc_buffers))
2818 __remove_assoc_queue(bh);
2820 } while (bh != head);
2821 *buffers_to_free = head;
2822 __clear_page_buffers(page);
2828 int try_to_free_buffers(struct page *page)
2830 struct address_space * const mapping = page->mapping;
2831 struct buffer_head *buffers_to_free = NULL;
2834 BUG_ON(!PageLocked(page));
2835 if (PageWriteback(page))
2838 if (mapping == NULL) { /* can this still happen? */
2839 ret = drop_buffers(page, &buffers_to_free);
2843 spin_lock(&mapping->private_lock);
2844 ret = drop_buffers(page, &buffers_to_free);
2845 spin_unlock(&mapping->private_lock);
2848 * If the filesystem writes its buffers by hand (eg ext3)
2849 * then we can have clean buffers against a dirty page. We
2850 * clean the page here; otherwise later reattachment of buffers
2851 * could encounter a non-uptodate page, which is unresolvable.
2852 * This only applies in the rare case where try_to_free_buffers
2853 * succeeds but the page is not freed.
2855 clear_page_dirty(page);
2858 if (buffers_to_free) {
2859 struct buffer_head *bh = buffers_to_free;
2862 struct buffer_head *next = bh->b_this_page;
2863 free_buffer_head(bh);
2865 } while (bh != buffers_to_free);
2869 EXPORT_SYMBOL(try_to_free_buffers);
2871 void block_sync_page(struct page *page)
2873 struct address_space *mapping;
2876 mapping = page_mapping(page);
2878 blk_run_backing_dev(mapping->backing_dev_info, page);
2882 * There are no bdflush tunables left. But distributions are
2883 * still running obsolete flush daemons, so we terminate them here.
2885 * Use of bdflush() is deprecated and will be removed in a future kernel.
2886 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
2888 asmlinkage long sys_bdflush(int func, long data)
2890 static int msg_count;
2892 if (!capable(CAP_SYS_ADMIN))
2895 if (msg_count < 5) {
2898 "warning: process `%s' used the obsolete bdflush"
2899 " system call\n", current->comm);
2900 printk(KERN_INFO "Fix your initscripts?\n");
2909 * Buffer-head allocation
2911 static kmem_cache_t *bh_cachep;
2914 * Once the number of bh's in the machine exceeds this level, we start
2915 * stripping them in writeback.
2917 static int max_buffer_heads;
2919 int buffer_heads_over_limit;
2921 struct bh_accounting {
2922 int nr; /* Number of live bh's */
2923 int ratelimit; /* Limit cacheline bouncing */
2926 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
2928 static void recalc_bh_state(void)
2933 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
2935 __get_cpu_var(bh_accounting).ratelimit = 0;
2936 for_each_online_cpu(i)
2937 tot += per_cpu(bh_accounting, i).nr;
2938 buffer_heads_over_limit = (tot > max_buffer_heads);
2941 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
2943 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
2945 get_cpu_var(bh_accounting).nr++;
2947 put_cpu_var(bh_accounting);
2951 EXPORT_SYMBOL(alloc_buffer_head);
2953 void free_buffer_head(struct buffer_head *bh)
2955 BUG_ON(!list_empty(&bh->b_assoc_buffers));
2956 kmem_cache_free(bh_cachep, bh);
2957 get_cpu_var(bh_accounting).nr--;
2959 put_cpu_var(bh_accounting);
2961 EXPORT_SYMBOL(free_buffer_head);
2964 init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
2966 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
2967 SLAB_CTOR_CONSTRUCTOR) {
2968 struct buffer_head * bh = (struct buffer_head *)data;
2970 memset(bh, 0, sizeof(*bh));
2971 INIT_LIST_HEAD(&bh->b_assoc_buffers);
2975 #ifdef CONFIG_HOTPLUG_CPU
2976 static void buffer_exit_cpu(int cpu)
2979 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
2981 for (i = 0; i < BH_LRU_SIZE; i++) {
2985 get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
2986 per_cpu(bh_accounting, cpu).nr = 0;
2987 put_cpu_var(bh_accounting);
2990 static int buffer_cpu_notify(struct notifier_block *self,
2991 unsigned long action, void *hcpu)
2993 if (action == CPU_DEAD)
2994 buffer_exit_cpu((unsigned long)hcpu);
2997 #endif /* CONFIG_HOTPLUG_CPU */
2999 void __init buffer_init(void)
3003 bh_cachep = kmem_cache_create("buffer_head",
3004 sizeof(struct buffer_head), 0,
3005 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3011 * Limit the bh occupancy to 10% of ZONE_NORMAL
3013 nrpages = (nr_free_buffer_pages() * 10) / 100;
3014 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3015 hotcpu_notifier(buffer_cpu_notify, 0);
3018 EXPORT_SYMBOL(__bforget);
3019 EXPORT_SYMBOL(__brelse);
3020 EXPORT_SYMBOL(__wait_on_buffer);
3021 EXPORT_SYMBOL(block_commit_write);
3022 EXPORT_SYMBOL(block_prepare_write);
3023 EXPORT_SYMBOL(block_read_full_page);
3024 EXPORT_SYMBOL(block_sync_page);
3025 EXPORT_SYMBOL(block_truncate_page);
3026 EXPORT_SYMBOL(block_write_full_page);
3027 EXPORT_SYMBOL(cont_prepare_write);
3028 EXPORT_SYMBOL(end_buffer_read_sync);
3029 EXPORT_SYMBOL(end_buffer_write_sync);
3030 EXPORT_SYMBOL(file_fsync);
3031 EXPORT_SYMBOL(fsync_bdev);
3032 EXPORT_SYMBOL(generic_block_bmap);
3033 EXPORT_SYMBOL(generic_commit_write);
3034 EXPORT_SYMBOL(generic_cont_expand);
3035 EXPORT_SYMBOL(generic_cont_expand_simple);
3036 EXPORT_SYMBOL(init_buffer);
3037 EXPORT_SYMBOL(invalidate_bdev);
3038 EXPORT_SYMBOL(ll_rw_block);
3039 EXPORT_SYMBOL(mark_buffer_dirty);
3040 EXPORT_SYMBOL(submit_bh);
3041 EXPORT_SYMBOL(sync_dirty_buffer);
3042 EXPORT_SYMBOL(unlock_buffer);