4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/capability.h>
28 #include <linux/blkdev.h>
29 #include <linux/file.h>
30 #include <linux/quotaops.h>
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/writeback.h>
34 #include <linux/hash.h>
35 #include <linux/suspend.h>
36 #include <linux/buffer_head.h>
37 #include <linux/task_io_accounting_ops.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
47 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
50 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52 bh->b_end_io = handler;
53 bh->b_private = private;
55 EXPORT_SYMBOL(init_buffer);
57 static int sync_buffer(void *word)
59 struct block_device *bd;
60 struct buffer_head *bh
61 = container_of(word, struct buffer_head, b_state);
66 blk_run_address_space(bd->bd_inode->i_mapping);
71 void __lock_buffer(struct buffer_head *bh)
73 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74 TASK_UNINTERRUPTIBLE);
76 EXPORT_SYMBOL(__lock_buffer);
78 void unlock_buffer(struct buffer_head *bh)
80 clear_bit_unlock(BH_Lock, &bh->b_state);
81 smp_mb__after_clear_bit();
82 wake_up_bit(&bh->b_state, BH_Lock);
84 EXPORT_SYMBOL(unlock_buffer);
87 * Block until a buffer comes unlocked. This doesn't stop it
88 * from becoming locked again - you have to lock it yourself
89 * if you want to preserve its state.
91 void __wait_on_buffer(struct buffer_head * bh)
93 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
95 EXPORT_SYMBOL(__wait_on_buffer);
98 __clear_page_buffers(struct page *page)
100 ClearPagePrivate(page);
101 set_page_private(page, 0);
102 page_cache_release(page);
106 static int quiet_error(struct buffer_head *bh)
108 if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
114 static void buffer_io_error(struct buffer_head *bh)
116 char b[BDEVNAME_SIZE];
117 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
118 bdevname(bh->b_bdev, b),
119 (unsigned long long)bh->b_blocknr);
123 * End-of-IO handler helper function which does not touch the bh after
125 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
126 * a race there is benign: unlock_buffer() only use the bh's address for
127 * hashing after unlocking the buffer, so it doesn't actually touch the bh
130 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
133 set_buffer_uptodate(bh);
135 /* This happens, due to failed READA attempts. */
136 clear_buffer_uptodate(bh);
142 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
143 * unlock the buffer. This is what ll_rw_block uses too.
145 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
147 __end_buffer_read_notouch(bh, uptodate);
150 EXPORT_SYMBOL(end_buffer_read_sync);
152 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
154 char b[BDEVNAME_SIZE];
157 set_buffer_uptodate(bh);
159 if (!buffer_eopnotsupp(bh) && !quiet_error(bh)) {
161 printk(KERN_WARNING "lost page write due to "
163 bdevname(bh->b_bdev, b));
165 set_buffer_write_io_error(bh);
166 clear_buffer_uptodate(bh);
171 EXPORT_SYMBOL(end_buffer_write_sync);
174 * Various filesystems appear to want __find_get_block to be non-blocking.
175 * But it's the page lock which protects the buffers. To get around this,
176 * we get exclusion from try_to_free_buffers with the blockdev mapping's
179 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
180 * may be quite high. This code could TryLock the page, and if that
181 * succeeds, there is no need to take private_lock. (But if
182 * private_lock is contended then so is mapping->tree_lock).
184 static struct buffer_head *
185 __find_get_block_slow(struct block_device *bdev, sector_t block)
187 struct inode *bd_inode = bdev->bd_inode;
188 struct address_space *bd_mapping = bd_inode->i_mapping;
189 struct buffer_head *ret = NULL;
191 struct buffer_head *bh;
192 struct buffer_head *head;
196 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
197 page = find_get_page(bd_mapping, index);
201 spin_lock(&bd_mapping->private_lock);
202 if (!page_has_buffers(page))
204 head = page_buffers(page);
207 if (!buffer_mapped(bh))
209 else if (bh->b_blocknr == block) {
214 bh = bh->b_this_page;
215 } while (bh != head);
217 /* we might be here because some of the buffers on this page are
218 * not mapped. This is due to various races between
219 * file io on the block device and getblk. It gets dealt with
220 * elsewhere, don't buffer_error if we had some unmapped buffers
223 printk("__find_get_block_slow() failed. "
224 "block=%llu, b_blocknr=%llu\n",
225 (unsigned long long)block,
226 (unsigned long long)bh->b_blocknr);
227 printk("b_state=0x%08lx, b_size=%zu\n",
228 bh->b_state, bh->b_size);
229 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
232 spin_unlock(&bd_mapping->private_lock);
233 page_cache_release(page);
238 /* If invalidate_buffers() will trash dirty buffers, it means some kind
239 of fs corruption is going on. Trashing dirty data always imply losing
240 information that was supposed to be just stored on the physical layer
243 Thus invalidate_buffers in general usage is not allwowed to trash
244 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
245 be preserved. These buffers are simply skipped.
247 We also skip buffers which are still in use. For example this can
248 happen if a userspace program is reading the block device.
250 NOTE: In the case where the user removed a removable-media-disk even if
251 there's still dirty data not synced on disk (due a bug in the device driver
252 or due an error of the user), by not destroying the dirty buffers we could
253 generate corruption also on the next media inserted, thus a parameter is
254 necessary to handle this case in the most safe way possible (trying
255 to not corrupt also the new disk inserted with the data belonging to
256 the old now corrupted disk). Also for the ramdisk the natural thing
257 to do in order to release the ramdisk memory is to destroy dirty buffers.
259 These are two special cases. Normal usage imply the device driver
260 to issue a sync on the device (without waiting I/O completion) and
261 then an invalidate_buffers call that doesn't trash dirty buffers.
263 For handling cache coherency with the blkdev pagecache the 'update' case
264 is been introduced. It is needed to re-read from disk any pinned
265 buffer. NOTE: re-reading from disk is destructive so we can do it only
266 when we assume nobody is changing the buffercache under our I/O and when
267 we think the disk contains more recent information than the buffercache.
268 The update == 1 pass marks the buffers we need to update, the update == 2
269 pass does the actual I/O. */
270 void invalidate_bdev(struct block_device *bdev)
272 struct address_space *mapping = bdev->bd_inode->i_mapping;
274 if (mapping->nrpages == 0)
277 invalidate_bh_lrus();
278 invalidate_mapping_pages(mapping, 0, -1);
280 EXPORT_SYMBOL(invalidate_bdev);
283 * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
285 static void free_more_memory(void)
290 wakeup_flusher_threads(1024);
293 for_each_online_node(nid) {
294 (void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
295 gfp_zone(GFP_NOFS), NULL,
298 try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
304 * I/O completion handler for block_read_full_page() - pages
305 * which come unlocked at the end of I/O.
307 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
310 struct buffer_head *first;
311 struct buffer_head *tmp;
313 int page_uptodate = 1;
315 BUG_ON(!buffer_async_read(bh));
319 set_buffer_uptodate(bh);
321 clear_buffer_uptodate(bh);
322 if (!quiet_error(bh))
328 * Be _very_ careful from here on. Bad things can happen if
329 * two buffer heads end IO at almost the same time and both
330 * decide that the page is now completely done.
332 first = page_buffers(page);
333 local_irq_save(flags);
334 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
335 clear_buffer_async_read(bh);
339 if (!buffer_uptodate(tmp))
341 if (buffer_async_read(tmp)) {
342 BUG_ON(!buffer_locked(tmp));
345 tmp = tmp->b_this_page;
347 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
348 local_irq_restore(flags);
351 * If none of the buffers had errors and they are all
352 * uptodate then we can set the page uptodate.
354 if (page_uptodate && !PageError(page))
355 SetPageUptodate(page);
360 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
361 local_irq_restore(flags);
366 * Completion handler for block_write_full_page() - pages which are unlocked
367 * during I/O, and which have PageWriteback cleared upon I/O completion.
369 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
371 char b[BDEVNAME_SIZE];
373 struct buffer_head *first;
374 struct buffer_head *tmp;
377 BUG_ON(!buffer_async_write(bh));
381 set_buffer_uptodate(bh);
383 if (!quiet_error(bh)) {
385 printk(KERN_WARNING "lost page write due to "
387 bdevname(bh->b_bdev, b));
389 set_bit(AS_EIO, &page->mapping->flags);
390 set_buffer_write_io_error(bh);
391 clear_buffer_uptodate(bh);
395 first = page_buffers(page);
396 local_irq_save(flags);
397 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
399 clear_buffer_async_write(bh);
401 tmp = bh->b_this_page;
403 if (buffer_async_write(tmp)) {
404 BUG_ON(!buffer_locked(tmp));
407 tmp = tmp->b_this_page;
409 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
410 local_irq_restore(flags);
411 end_page_writeback(page);
415 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
416 local_irq_restore(flags);
419 EXPORT_SYMBOL(end_buffer_async_write);
422 * If a page's buffers are under async readin (end_buffer_async_read
423 * completion) then there is a possibility that another thread of
424 * control could lock one of the buffers after it has completed
425 * but while some of the other buffers have not completed. This
426 * locked buffer would confuse end_buffer_async_read() into not unlocking
427 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
428 * that this buffer is not under async I/O.
430 * The page comes unlocked when it has no locked buffer_async buffers
433 * PageLocked prevents anyone starting new async I/O reads any of
436 * PageWriteback is used to prevent simultaneous writeout of the same
439 * PageLocked prevents anyone from starting writeback of a page which is
440 * under read I/O (PageWriteback is only ever set against a locked page).
442 static void mark_buffer_async_read(struct buffer_head *bh)
444 bh->b_end_io = end_buffer_async_read;
445 set_buffer_async_read(bh);
448 static void mark_buffer_async_write_endio(struct buffer_head *bh,
449 bh_end_io_t *handler)
451 bh->b_end_io = handler;
452 set_buffer_async_write(bh);
455 void mark_buffer_async_write(struct buffer_head *bh)
457 mark_buffer_async_write_endio(bh, end_buffer_async_write);
459 EXPORT_SYMBOL(mark_buffer_async_write);
463 * fs/buffer.c contains helper functions for buffer-backed address space's
464 * fsync functions. A common requirement for buffer-based filesystems is
465 * that certain data from the backing blockdev needs to be written out for
466 * a successful fsync(). For example, ext2 indirect blocks need to be
467 * written back and waited upon before fsync() returns.
469 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
470 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
471 * management of a list of dependent buffers at ->i_mapping->private_list.
473 * Locking is a little subtle: try_to_free_buffers() will remove buffers
474 * from their controlling inode's queue when they are being freed. But
475 * try_to_free_buffers() will be operating against the *blockdev* mapping
476 * at the time, not against the S_ISREG file which depends on those buffers.
477 * So the locking for private_list is via the private_lock in the address_space
478 * which backs the buffers. Which is different from the address_space
479 * against which the buffers are listed. So for a particular address_space,
480 * mapping->private_lock does *not* protect mapping->private_list! In fact,
481 * mapping->private_list will always be protected by the backing blockdev's
484 * Which introduces a requirement: all buffers on an address_space's
485 * ->private_list must be from the same address_space: the blockdev's.
487 * address_spaces which do not place buffers at ->private_list via these
488 * utility functions are free to use private_lock and private_list for
489 * whatever they want. The only requirement is that list_empty(private_list)
490 * be true at clear_inode() time.
492 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
493 * filesystems should do that. invalidate_inode_buffers() should just go
494 * BUG_ON(!list_empty).
496 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
497 * take an address_space, not an inode. And it should be called
498 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
501 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
502 * list if it is already on a list. Because if the buffer is on a list,
503 * it *must* already be on the right one. If not, the filesystem is being
504 * silly. This will save a ton of locking. But first we have to ensure
505 * that buffers are taken *off* the old inode's list when they are freed
506 * (presumably in truncate). That requires careful auditing of all
507 * filesystems (do it inside bforget()). It could also be done by bringing
512 * The buffer's backing address_space's private_lock must be held
514 static void __remove_assoc_queue(struct buffer_head *bh)
516 list_del_init(&bh->b_assoc_buffers);
517 WARN_ON(!bh->b_assoc_map);
518 if (buffer_write_io_error(bh))
519 set_bit(AS_EIO, &bh->b_assoc_map->flags);
520 bh->b_assoc_map = NULL;
523 int inode_has_buffers(struct inode *inode)
525 return !list_empty(&inode->i_data.private_list);
529 * osync is designed to support O_SYNC io. It waits synchronously for
530 * all already-submitted IO to complete, but does not queue any new
531 * writes to the disk.
533 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
534 * you dirty the buffers, and then use osync_inode_buffers to wait for
535 * completion. Any other dirty buffers which are not yet queued for
536 * write will not be flushed to disk by the osync.
538 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
540 struct buffer_head *bh;
546 list_for_each_prev(p, list) {
548 if (buffer_locked(bh)) {
552 if (!buffer_uptodate(bh))
563 static void do_thaw_all(struct work_struct *work)
565 struct super_block *sb;
566 char b[BDEVNAME_SIZE];
570 list_for_each_entry(sb, &super_blocks, s_list) {
571 if (list_empty(&sb->s_instances))
574 spin_unlock(&sb_lock);
575 down_read(&sb->s_umount);
576 while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
577 printk(KERN_WARNING "Emergency Thaw on %s\n",
578 bdevname(sb->s_bdev, b));
579 up_read(&sb->s_umount);
581 if (__put_super_and_need_restart(sb))
584 spin_unlock(&sb_lock);
586 printk(KERN_WARNING "Emergency Thaw complete\n");
590 * emergency_thaw_all -- forcibly thaw every frozen filesystem
592 * Used for emergency unfreeze of all filesystems via SysRq
594 void emergency_thaw_all(void)
596 struct work_struct *work;
598 work = kmalloc(sizeof(*work), GFP_ATOMIC);
600 INIT_WORK(work, do_thaw_all);
606 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
607 * @mapping: the mapping which wants those buffers written
609 * Starts I/O against the buffers at mapping->private_list, and waits upon
612 * Basically, this is a convenience function for fsync().
613 * @mapping is a file or directory which needs those buffers to be written for
614 * a successful fsync().
616 int sync_mapping_buffers(struct address_space *mapping)
618 struct address_space *buffer_mapping = mapping->assoc_mapping;
620 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
623 return fsync_buffers_list(&buffer_mapping->private_lock,
624 &mapping->private_list);
626 EXPORT_SYMBOL(sync_mapping_buffers);
629 * Called when we've recently written block `bblock', and it is known that
630 * `bblock' was for a buffer_boundary() buffer. This means that the block at
631 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
632 * dirty, schedule it for IO. So that indirects merge nicely with their data.
634 void write_boundary_block(struct block_device *bdev,
635 sector_t bblock, unsigned blocksize)
637 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
639 if (buffer_dirty(bh))
640 ll_rw_block(WRITE, 1, &bh);
645 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
647 struct address_space *mapping = inode->i_mapping;
648 struct address_space *buffer_mapping = bh->b_page->mapping;
650 mark_buffer_dirty(bh);
651 if (!mapping->assoc_mapping) {
652 mapping->assoc_mapping = buffer_mapping;
654 BUG_ON(mapping->assoc_mapping != buffer_mapping);
656 if (!bh->b_assoc_map) {
657 spin_lock(&buffer_mapping->private_lock);
658 list_move_tail(&bh->b_assoc_buffers,
659 &mapping->private_list);
660 bh->b_assoc_map = mapping;
661 spin_unlock(&buffer_mapping->private_lock);
664 EXPORT_SYMBOL(mark_buffer_dirty_inode);
667 * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
670 * If warn is true, then emit a warning if the page is not uptodate and has
671 * not been truncated.
673 static void __set_page_dirty(struct page *page,
674 struct address_space *mapping, int warn)
676 spin_lock_irq(&mapping->tree_lock);
677 if (page->mapping) { /* Race with truncate? */
678 WARN_ON_ONCE(warn && !PageUptodate(page));
679 account_page_dirtied(page, mapping);
680 radix_tree_tag_set(&mapping->page_tree,
681 page_index(page), PAGECACHE_TAG_DIRTY);
683 spin_unlock_irq(&mapping->tree_lock);
684 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
688 * Add a page to the dirty page list.
690 * It is a sad fact of life that this function is called from several places
691 * deeply under spinlocking. It may not sleep.
693 * If the page has buffers, the uptodate buffers are set dirty, to preserve
694 * dirty-state coherency between the page and the buffers. It the page does
695 * not have buffers then when they are later attached they will all be set
698 * The buffers are dirtied before the page is dirtied. There's a small race
699 * window in which a writepage caller may see the page cleanness but not the
700 * buffer dirtiness. That's fine. If this code were to set the page dirty
701 * before the buffers, a concurrent writepage caller could clear the page dirty
702 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
703 * page on the dirty page list.
705 * We use private_lock to lock against try_to_free_buffers while using the
706 * page's buffer list. Also use this to protect against clean buffers being
707 * added to the page after it was set dirty.
709 * FIXME: may need to call ->reservepage here as well. That's rather up to the
710 * address_space though.
712 int __set_page_dirty_buffers(struct page *page)
715 struct address_space *mapping = page_mapping(page);
717 if (unlikely(!mapping))
718 return !TestSetPageDirty(page);
720 spin_lock(&mapping->private_lock);
721 if (page_has_buffers(page)) {
722 struct buffer_head *head = page_buffers(page);
723 struct buffer_head *bh = head;
726 set_buffer_dirty(bh);
727 bh = bh->b_this_page;
728 } while (bh != head);
730 newly_dirty = !TestSetPageDirty(page);
731 spin_unlock(&mapping->private_lock);
734 __set_page_dirty(page, mapping, 1);
737 EXPORT_SYMBOL(__set_page_dirty_buffers);
740 * Write out and wait upon a list of buffers.
742 * We have conflicting pressures: we want to make sure that all
743 * initially dirty buffers get waited on, but that any subsequently
744 * dirtied buffers don't. After all, we don't want fsync to last
745 * forever if somebody is actively writing to the file.
747 * Do this in two main stages: first we copy dirty buffers to a
748 * temporary inode list, queueing the writes as we go. Then we clean
749 * up, waiting for those writes to complete.
751 * During this second stage, any subsequent updates to the file may end
752 * up refiling the buffer on the original inode's dirty list again, so
753 * there is a chance we will end up with a buffer queued for write but
754 * not yet completed on that list. So, as a final cleanup we go through
755 * the osync code to catch these locked, dirty buffers without requeuing
756 * any newly dirty buffers for write.
758 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
760 struct buffer_head *bh;
761 struct list_head tmp;
762 struct address_space *mapping, *prev_mapping = NULL;
765 INIT_LIST_HEAD(&tmp);
768 while (!list_empty(list)) {
769 bh = BH_ENTRY(list->next);
770 mapping = bh->b_assoc_map;
771 __remove_assoc_queue(bh);
772 /* Avoid race with mark_buffer_dirty_inode() which does
773 * a lockless check and we rely on seeing the dirty bit */
775 if (buffer_dirty(bh) || buffer_locked(bh)) {
776 list_add(&bh->b_assoc_buffers, &tmp);
777 bh->b_assoc_map = mapping;
778 if (buffer_dirty(bh)) {
782 * Ensure any pending I/O completes so that
783 * ll_rw_block() actually writes the current
784 * contents - it is a noop if I/O is still in
785 * flight on potentially older contents.
787 ll_rw_block(SWRITE_SYNC_PLUG, 1, &bh);
790 * Kick off IO for the previous mapping. Note
791 * that we will not run the very last mapping,
792 * wait_on_buffer() will do that for us
793 * through sync_buffer().
795 if (prev_mapping && prev_mapping != mapping)
796 blk_run_address_space(prev_mapping);
797 prev_mapping = mapping;
805 while (!list_empty(&tmp)) {
806 bh = BH_ENTRY(tmp.prev);
808 mapping = bh->b_assoc_map;
809 __remove_assoc_queue(bh);
810 /* Avoid race with mark_buffer_dirty_inode() which does
811 * a lockless check and we rely on seeing the dirty bit */
813 if (buffer_dirty(bh)) {
814 list_add(&bh->b_assoc_buffers,
815 &mapping->private_list);
816 bh->b_assoc_map = mapping;
820 if (!buffer_uptodate(bh))
827 err2 = osync_buffers_list(lock, list);
835 * Invalidate any and all dirty buffers on a given inode. We are
836 * probably unmounting the fs, but that doesn't mean we have already
837 * done a sync(). Just drop the buffers from the inode list.
839 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
840 * assumes that all the buffers are against the blockdev. Not true
843 void invalidate_inode_buffers(struct inode *inode)
845 if (inode_has_buffers(inode)) {
846 struct address_space *mapping = &inode->i_data;
847 struct list_head *list = &mapping->private_list;
848 struct address_space *buffer_mapping = mapping->assoc_mapping;
850 spin_lock(&buffer_mapping->private_lock);
851 while (!list_empty(list))
852 __remove_assoc_queue(BH_ENTRY(list->next));
853 spin_unlock(&buffer_mapping->private_lock);
856 EXPORT_SYMBOL(invalidate_inode_buffers);
859 * Remove any clean buffers from the inode's buffer list. This is called
860 * when we're trying to free the inode itself. Those buffers can pin it.
862 * Returns true if all buffers were removed.
864 int remove_inode_buffers(struct inode *inode)
868 if (inode_has_buffers(inode)) {
869 struct address_space *mapping = &inode->i_data;
870 struct list_head *list = &mapping->private_list;
871 struct address_space *buffer_mapping = mapping->assoc_mapping;
873 spin_lock(&buffer_mapping->private_lock);
874 while (!list_empty(list)) {
875 struct buffer_head *bh = BH_ENTRY(list->next);
876 if (buffer_dirty(bh)) {
880 __remove_assoc_queue(bh);
882 spin_unlock(&buffer_mapping->private_lock);
888 * Create the appropriate buffers when given a page for data area and
889 * the size of each buffer.. Use the bh->b_this_page linked list to
890 * follow the buffers created. Return NULL if unable to create more
893 * The retry flag is used to differentiate async IO (paging, swapping)
894 * which may not fail from ordinary buffer allocations.
896 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
899 struct buffer_head *bh, *head;
905 while ((offset -= size) >= 0) {
906 bh = alloc_buffer_head(GFP_NOFS);
911 bh->b_this_page = head;
916 atomic_set(&bh->b_count, 0);
917 bh->b_private = NULL;
920 /* Link the buffer to its page */
921 set_bh_page(bh, page, offset);
923 init_buffer(bh, NULL, NULL);
927 * In case anything failed, we just free everything we got.
933 head = head->b_this_page;
934 free_buffer_head(bh);
939 * Return failure for non-async IO requests. Async IO requests
940 * are not allowed to fail, so we have to wait until buffer heads
941 * become available. But we don't want tasks sleeping with
942 * partially complete buffers, so all were released above.
947 /* We're _really_ low on memory. Now we just
948 * wait for old buffer heads to become free due to
949 * finishing IO. Since this is an async request and
950 * the reserve list is empty, we're sure there are
951 * async buffer heads in use.
956 EXPORT_SYMBOL_GPL(alloc_page_buffers);
959 link_dev_buffers(struct page *page, struct buffer_head *head)
961 struct buffer_head *bh, *tail;
966 bh = bh->b_this_page;
968 tail->b_this_page = head;
969 attach_page_buffers(page, head);
973 * Initialise the state of a blockdev page's buffers.
976 init_page_buffers(struct page *page, struct block_device *bdev,
977 sector_t block, int size)
979 struct buffer_head *head = page_buffers(page);
980 struct buffer_head *bh = head;
981 int uptodate = PageUptodate(page);
984 if (!buffer_mapped(bh)) {
985 init_buffer(bh, NULL, NULL);
987 bh->b_blocknr = block;
989 set_buffer_uptodate(bh);
990 set_buffer_mapped(bh);
993 bh = bh->b_this_page;
994 } while (bh != head);
998 * Create the page-cache page that contains the requested block.
1000 * This is user purely for blockdev mappings.
1002 static struct page *
1003 grow_dev_page(struct block_device *bdev, sector_t block,
1004 pgoff_t index, int size)
1006 struct inode *inode = bdev->bd_inode;
1008 struct buffer_head *bh;
1010 page = find_or_create_page(inode->i_mapping, index,
1011 (mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE);
1015 BUG_ON(!PageLocked(page));
1017 if (page_has_buffers(page)) {
1018 bh = page_buffers(page);
1019 if (bh->b_size == size) {
1020 init_page_buffers(page, bdev, block, size);
1023 if (!try_to_free_buffers(page))
1028 * Allocate some buffers for this page
1030 bh = alloc_page_buffers(page, size, 0);
1035 * Link the page to the buffers and initialise them. Take the
1036 * lock to be atomic wrt __find_get_block(), which does not
1037 * run under the page lock.
1039 spin_lock(&inode->i_mapping->private_lock);
1040 link_dev_buffers(page, bh);
1041 init_page_buffers(page, bdev, block, size);
1042 spin_unlock(&inode->i_mapping->private_lock);
1048 page_cache_release(page);
1053 * Create buffers for the specified block device block's page. If
1054 * that page was dirty, the buffers are set dirty also.
1057 grow_buffers(struct block_device *bdev, sector_t block, int size)
1066 } while ((size << sizebits) < PAGE_SIZE);
1068 index = block >> sizebits;
1071 * Check for a block which wants to lie outside our maximum possible
1072 * pagecache index. (this comparison is done using sector_t types).
1074 if (unlikely(index != block >> sizebits)) {
1075 char b[BDEVNAME_SIZE];
1077 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1079 __func__, (unsigned long long)block,
1083 block = index << sizebits;
1084 /* Create a page with the proper size buffers.. */
1085 page = grow_dev_page(bdev, block, index, size);
1089 page_cache_release(page);
1093 static struct buffer_head *
1094 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1096 /* Size must be multiple of hard sectorsize */
1097 if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1098 (size < 512 || size > PAGE_SIZE))) {
1099 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1101 printk(KERN_ERR "logical block size: %d\n",
1102 bdev_logical_block_size(bdev));
1109 struct buffer_head * bh;
1112 bh = __find_get_block(bdev, block, size);
1116 ret = grow_buffers(bdev, block, size);
1125 * The relationship between dirty buffers and dirty pages:
1127 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1128 * the page is tagged dirty in its radix tree.
1130 * At all times, the dirtiness of the buffers represents the dirtiness of
1131 * subsections of the page. If the page has buffers, the page dirty bit is
1132 * merely a hint about the true dirty state.
1134 * When a page is set dirty in its entirety, all its buffers are marked dirty
1135 * (if the page has buffers).
1137 * When a buffer is marked dirty, its page is dirtied, but the page's other
1140 * Also. When blockdev buffers are explicitly read with bread(), they
1141 * individually become uptodate. But their backing page remains not
1142 * uptodate - even if all of its buffers are uptodate. A subsequent
1143 * block_read_full_page() against that page will discover all the uptodate
1144 * buffers, will set the page uptodate and will perform no I/O.
1148 * mark_buffer_dirty - mark a buffer_head as needing writeout
1149 * @bh: the buffer_head to mark dirty
1151 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1152 * backing page dirty, then tag the page as dirty in its address_space's radix
1153 * tree and then attach the address_space's inode to its superblock's dirty
1156 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1157 * mapping->tree_lock and the global inode_lock.
1159 void mark_buffer_dirty(struct buffer_head *bh)
1161 WARN_ON_ONCE(!buffer_uptodate(bh));
1164 * Very *carefully* optimize the it-is-already-dirty case.
1166 * Don't let the final "is it dirty" escape to before we
1167 * perhaps modified the buffer.
1169 if (buffer_dirty(bh)) {
1171 if (buffer_dirty(bh))
1175 if (!test_set_buffer_dirty(bh)) {
1176 struct page *page = bh->b_page;
1177 if (!TestSetPageDirty(page)) {
1178 struct address_space *mapping = page_mapping(page);
1180 __set_page_dirty(page, mapping, 0);
1184 EXPORT_SYMBOL(mark_buffer_dirty);
1187 * Decrement a buffer_head's reference count. If all buffers against a page
1188 * have zero reference count, are clean and unlocked, and if the page is clean
1189 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1190 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1191 * a page but it ends up not being freed, and buffers may later be reattached).
1193 void __brelse(struct buffer_head * buf)
1195 if (atomic_read(&buf->b_count)) {
1199 WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1201 EXPORT_SYMBOL(__brelse);
1204 * bforget() is like brelse(), except it discards any
1205 * potentially dirty data.
1207 void __bforget(struct buffer_head *bh)
1209 clear_buffer_dirty(bh);
1210 if (bh->b_assoc_map) {
1211 struct address_space *buffer_mapping = bh->b_page->mapping;
1213 spin_lock(&buffer_mapping->private_lock);
1214 list_del_init(&bh->b_assoc_buffers);
1215 bh->b_assoc_map = NULL;
1216 spin_unlock(&buffer_mapping->private_lock);
1220 EXPORT_SYMBOL(__bforget);
1222 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1225 if (buffer_uptodate(bh)) {
1230 bh->b_end_io = end_buffer_read_sync;
1231 submit_bh(READ, bh);
1233 if (buffer_uptodate(bh))
1241 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1242 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1243 * refcount elevated by one when they're in an LRU. A buffer can only appear
1244 * once in a particular CPU's LRU. A single buffer can be present in multiple
1245 * CPU's LRUs at the same time.
1247 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1248 * sb_find_get_block().
1250 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1251 * a local interrupt disable for that.
1254 #define BH_LRU_SIZE 8
1257 struct buffer_head *bhs[BH_LRU_SIZE];
1260 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1263 #define bh_lru_lock() local_irq_disable()
1264 #define bh_lru_unlock() local_irq_enable()
1266 #define bh_lru_lock() preempt_disable()
1267 #define bh_lru_unlock() preempt_enable()
1270 static inline void check_irqs_on(void)
1272 #ifdef irqs_disabled
1273 BUG_ON(irqs_disabled());
1278 * The LRU management algorithm is dopey-but-simple. Sorry.
1280 static void bh_lru_install(struct buffer_head *bh)
1282 struct buffer_head *evictee = NULL;
1287 lru = &__get_cpu_var(bh_lrus);
1288 if (lru->bhs[0] != bh) {
1289 struct buffer_head *bhs[BH_LRU_SIZE];
1295 for (in = 0; in < BH_LRU_SIZE; in++) {
1296 struct buffer_head *bh2 = lru->bhs[in];
1301 if (out >= BH_LRU_SIZE) {
1302 BUG_ON(evictee != NULL);
1309 while (out < BH_LRU_SIZE)
1311 memcpy(lru->bhs, bhs, sizeof(bhs));
1320 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1322 static struct buffer_head *
1323 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1325 struct buffer_head *ret = NULL;
1331 lru = &__get_cpu_var(bh_lrus);
1332 for (i = 0; i < BH_LRU_SIZE; i++) {
1333 struct buffer_head *bh = lru->bhs[i];
1335 if (bh && bh->b_bdev == bdev &&
1336 bh->b_blocknr == block && bh->b_size == size) {
1339 lru->bhs[i] = lru->bhs[i - 1];
1354 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1355 * it in the LRU and mark it as accessed. If it is not present then return
1358 struct buffer_head *
1359 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1361 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1364 bh = __find_get_block_slow(bdev, block);
1372 EXPORT_SYMBOL(__find_get_block);
1375 * __getblk will locate (and, if necessary, create) the buffer_head
1376 * which corresponds to the passed block_device, block and size. The
1377 * returned buffer has its reference count incremented.
1379 * __getblk() cannot fail - it just keeps trying. If you pass it an
1380 * illegal block number, __getblk() will happily return a buffer_head
1381 * which represents the non-existent block. Very weird.
1383 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1384 * attempt is failing. FIXME, perhaps?
1386 struct buffer_head *
1387 __getblk(struct block_device *bdev, sector_t block, unsigned size)
1389 struct buffer_head *bh = __find_get_block(bdev, block, size);
1393 bh = __getblk_slow(bdev, block, size);
1396 EXPORT_SYMBOL(__getblk);
1399 * Do async read-ahead on a buffer..
1401 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1403 struct buffer_head *bh = __getblk(bdev, block, size);
1405 ll_rw_block(READA, 1, &bh);
1409 EXPORT_SYMBOL(__breadahead);
1412 * __bread() - reads a specified block and returns the bh
1413 * @bdev: the block_device to read from
1414 * @block: number of block
1415 * @size: size (in bytes) to read
1417 * Reads a specified block, and returns buffer head that contains it.
1418 * It returns NULL if the block was unreadable.
1420 struct buffer_head *
1421 __bread(struct block_device *bdev, sector_t block, unsigned size)
1423 struct buffer_head *bh = __getblk(bdev, block, size);
1425 if (likely(bh) && !buffer_uptodate(bh))
1426 bh = __bread_slow(bh);
1429 EXPORT_SYMBOL(__bread);
1432 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1433 * This doesn't race because it runs in each cpu either in irq
1434 * or with preempt disabled.
1436 static void invalidate_bh_lru(void *arg)
1438 struct bh_lru *b = &get_cpu_var(bh_lrus);
1441 for (i = 0; i < BH_LRU_SIZE; i++) {
1445 put_cpu_var(bh_lrus);
1448 void invalidate_bh_lrus(void)
1450 on_each_cpu(invalidate_bh_lru, NULL, 1);
1452 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1454 void set_bh_page(struct buffer_head *bh,
1455 struct page *page, unsigned long offset)
1458 BUG_ON(offset >= PAGE_SIZE);
1459 if (PageHighMem(page))
1461 * This catches illegal uses and preserves the offset:
1463 bh->b_data = (char *)(0 + offset);
1465 bh->b_data = page_address(page) + offset;
1467 EXPORT_SYMBOL(set_bh_page);
1470 * Called when truncating a buffer on a page completely.
1472 static void discard_buffer(struct buffer_head * bh)
1475 clear_buffer_dirty(bh);
1477 clear_buffer_mapped(bh);
1478 clear_buffer_req(bh);
1479 clear_buffer_new(bh);
1480 clear_buffer_delay(bh);
1481 clear_buffer_unwritten(bh);
1486 * block_invalidatepage - invalidate part of all of a buffer-backed page
1488 * @page: the page which is affected
1489 * @offset: the index of the truncation point
1491 * block_invalidatepage() is called when all or part of the page has become
1492 * invalidatedby a truncate operation.
1494 * block_invalidatepage() does not have to release all buffers, but it must
1495 * ensure that no dirty buffer is left outside @offset and that no I/O
1496 * is underway against any of the blocks which are outside the truncation
1497 * point. Because the caller is about to free (and possibly reuse) those
1500 void block_invalidatepage(struct page *page, unsigned long offset)
1502 struct buffer_head *head, *bh, *next;
1503 unsigned int curr_off = 0;
1505 BUG_ON(!PageLocked(page));
1506 if (!page_has_buffers(page))
1509 head = page_buffers(page);
1512 unsigned int next_off = curr_off + bh->b_size;
1513 next = bh->b_this_page;
1516 * is this block fully invalidated?
1518 if (offset <= curr_off)
1520 curr_off = next_off;
1522 } while (bh != head);
1525 * We release buffers only if the entire page is being invalidated.
1526 * The get_block cached value has been unconditionally invalidated,
1527 * so real IO is not possible anymore.
1530 try_to_release_page(page, 0);
1534 EXPORT_SYMBOL(block_invalidatepage);
1537 * We attach and possibly dirty the buffers atomically wrt
1538 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1539 * is already excluded via the page lock.
1541 void create_empty_buffers(struct page *page,
1542 unsigned long blocksize, unsigned long b_state)
1544 struct buffer_head *bh, *head, *tail;
1546 head = alloc_page_buffers(page, blocksize, 1);
1549 bh->b_state |= b_state;
1551 bh = bh->b_this_page;
1553 tail->b_this_page = head;
1555 spin_lock(&page->mapping->private_lock);
1556 if (PageUptodate(page) || PageDirty(page)) {
1559 if (PageDirty(page))
1560 set_buffer_dirty(bh);
1561 if (PageUptodate(page))
1562 set_buffer_uptodate(bh);
1563 bh = bh->b_this_page;
1564 } while (bh != head);
1566 attach_page_buffers(page, head);
1567 spin_unlock(&page->mapping->private_lock);
1569 EXPORT_SYMBOL(create_empty_buffers);
1572 * We are taking a block for data and we don't want any output from any
1573 * buffer-cache aliases starting from return from that function and
1574 * until the moment when something will explicitly mark the buffer
1575 * dirty (hopefully that will not happen until we will free that block ;-)
1576 * We don't even need to mark it not-uptodate - nobody can expect
1577 * anything from a newly allocated buffer anyway. We used to used
1578 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1579 * don't want to mark the alias unmapped, for example - it would confuse
1580 * anyone who might pick it with bread() afterwards...
1582 * Also.. Note that bforget() doesn't lock the buffer. So there can
1583 * be writeout I/O going on against recently-freed buffers. We don't
1584 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1585 * only if we really need to. That happens here.
1587 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1589 struct buffer_head *old_bh;
1593 old_bh = __find_get_block_slow(bdev, block);
1595 clear_buffer_dirty(old_bh);
1596 wait_on_buffer(old_bh);
1597 clear_buffer_req(old_bh);
1601 EXPORT_SYMBOL(unmap_underlying_metadata);
1604 * NOTE! All mapped/uptodate combinations are valid:
1606 * Mapped Uptodate Meaning
1608 * No No "unknown" - must do get_block()
1609 * No Yes "hole" - zero-filled
1610 * Yes No "allocated" - allocated on disk, not read in
1611 * Yes Yes "valid" - allocated and up-to-date in memory.
1613 * "Dirty" is valid only with the last case (mapped+uptodate).
1617 * While block_write_full_page is writing back the dirty buffers under
1618 * the page lock, whoever dirtied the buffers may decide to clean them
1619 * again at any time. We handle that by only looking at the buffer
1620 * state inside lock_buffer().
1622 * If block_write_full_page() is called for regular writeback
1623 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1624 * locked buffer. This only can happen if someone has written the buffer
1625 * directly, with submit_bh(). At the address_space level PageWriteback
1626 * prevents this contention from occurring.
1628 * If block_write_full_page() is called with wbc->sync_mode ==
1629 * WB_SYNC_ALL, the writes are posted using WRITE_SYNC_PLUG; this
1630 * causes the writes to be flagged as synchronous writes, but the
1631 * block device queue will NOT be unplugged, since usually many pages
1632 * will be pushed to the out before the higher-level caller actually
1633 * waits for the writes to be completed. The various wait functions,
1634 * such as wait_on_writeback_range() will ultimately call sync_page()
1635 * which will ultimately call blk_run_backing_dev(), which will end up
1636 * unplugging the device queue.
1638 static int __block_write_full_page(struct inode *inode, struct page *page,
1639 get_block_t *get_block, struct writeback_control *wbc,
1640 bh_end_io_t *handler)
1644 sector_t last_block;
1645 struct buffer_head *bh, *head;
1646 const unsigned blocksize = 1 << inode->i_blkbits;
1647 int nr_underway = 0;
1648 int write_op = (wbc->sync_mode == WB_SYNC_ALL ?
1649 WRITE_SYNC_PLUG : WRITE);
1651 BUG_ON(!PageLocked(page));
1653 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1655 if (!page_has_buffers(page)) {
1656 create_empty_buffers(page, blocksize,
1657 (1 << BH_Dirty)|(1 << BH_Uptodate));
1661 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1662 * here, and the (potentially unmapped) buffers may become dirty at
1663 * any time. If a buffer becomes dirty here after we've inspected it
1664 * then we just miss that fact, and the page stays dirty.
1666 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1667 * handle that here by just cleaning them.
1670 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1671 head = page_buffers(page);
1675 * Get all the dirty buffers mapped to disk addresses and
1676 * handle any aliases from the underlying blockdev's mapping.
1679 if (block > last_block) {
1681 * mapped buffers outside i_size will occur, because
1682 * this page can be outside i_size when there is a
1683 * truncate in progress.
1686 * The buffer was zeroed by block_write_full_page()
1688 clear_buffer_dirty(bh);
1689 set_buffer_uptodate(bh);
1690 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1692 WARN_ON(bh->b_size != blocksize);
1693 err = get_block(inode, block, bh, 1);
1696 clear_buffer_delay(bh);
1697 if (buffer_new(bh)) {
1698 /* blockdev mappings never come here */
1699 clear_buffer_new(bh);
1700 unmap_underlying_metadata(bh->b_bdev,
1704 bh = bh->b_this_page;
1706 } while (bh != head);
1709 if (!buffer_mapped(bh))
1712 * If it's a fully non-blocking write attempt and we cannot
1713 * lock the buffer then redirty the page. Note that this can
1714 * potentially cause a busy-wait loop from writeback threads
1715 * and kswapd activity, but those code paths have their own
1716 * higher-level throttling.
1718 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1720 } else if (!trylock_buffer(bh)) {
1721 redirty_page_for_writepage(wbc, page);
1724 if (test_clear_buffer_dirty(bh)) {
1725 mark_buffer_async_write_endio(bh, handler);
1729 } while ((bh = bh->b_this_page) != head);
1732 * The page and its buffers are protected by PageWriteback(), so we can
1733 * drop the bh refcounts early.
1735 BUG_ON(PageWriteback(page));
1736 set_page_writeback(page);
1739 struct buffer_head *next = bh->b_this_page;
1740 if (buffer_async_write(bh)) {
1741 submit_bh(write_op, bh);
1745 } while (bh != head);
1750 if (nr_underway == 0) {
1752 * The page was marked dirty, but the buffers were
1753 * clean. Someone wrote them back by hand with
1754 * ll_rw_block/submit_bh. A rare case.
1756 end_page_writeback(page);
1759 * The page and buffer_heads can be released at any time from
1767 * ENOSPC, or some other error. We may already have added some
1768 * blocks to the file, so we need to write these out to avoid
1769 * exposing stale data.
1770 * The page is currently locked and not marked for writeback
1773 /* Recovery: lock and submit the mapped buffers */
1775 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1776 !buffer_delay(bh)) {
1778 mark_buffer_async_write_endio(bh, handler);
1781 * The buffer may have been set dirty during
1782 * attachment to a dirty page.
1784 clear_buffer_dirty(bh);
1786 } while ((bh = bh->b_this_page) != head);
1788 BUG_ON(PageWriteback(page));
1789 mapping_set_error(page->mapping, err);
1790 set_page_writeback(page);
1792 struct buffer_head *next = bh->b_this_page;
1793 if (buffer_async_write(bh)) {
1794 clear_buffer_dirty(bh);
1795 submit_bh(write_op, bh);
1799 } while (bh != head);
1805 * If a page has any new buffers, zero them out here, and mark them uptodate
1806 * and dirty so they'll be written out (in order to prevent uninitialised
1807 * block data from leaking). And clear the new bit.
1809 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1811 unsigned int block_start, block_end;
1812 struct buffer_head *head, *bh;
1814 BUG_ON(!PageLocked(page));
1815 if (!page_has_buffers(page))
1818 bh = head = page_buffers(page);
1821 block_end = block_start + bh->b_size;
1823 if (buffer_new(bh)) {
1824 if (block_end > from && block_start < to) {
1825 if (!PageUptodate(page)) {
1826 unsigned start, size;
1828 start = max(from, block_start);
1829 size = min(to, block_end) - start;
1831 zero_user(page, start, size);
1832 set_buffer_uptodate(bh);
1835 clear_buffer_new(bh);
1836 mark_buffer_dirty(bh);
1840 block_start = block_end;
1841 bh = bh->b_this_page;
1842 } while (bh != head);
1844 EXPORT_SYMBOL(page_zero_new_buffers);
1846 static int __block_prepare_write(struct inode *inode, struct page *page,
1847 unsigned from, unsigned to, get_block_t *get_block)
1849 unsigned block_start, block_end;
1852 unsigned blocksize, bbits;
1853 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1855 BUG_ON(!PageLocked(page));
1856 BUG_ON(from > PAGE_CACHE_SIZE);
1857 BUG_ON(to > PAGE_CACHE_SIZE);
1860 blocksize = 1 << inode->i_blkbits;
1861 if (!page_has_buffers(page))
1862 create_empty_buffers(page, blocksize, 0);
1863 head = page_buffers(page);
1865 bbits = inode->i_blkbits;
1866 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1868 for(bh = head, block_start = 0; bh != head || !block_start;
1869 block++, block_start=block_end, bh = bh->b_this_page) {
1870 block_end = block_start + blocksize;
1871 if (block_end <= from || block_start >= to) {
1872 if (PageUptodate(page)) {
1873 if (!buffer_uptodate(bh))
1874 set_buffer_uptodate(bh);
1879 clear_buffer_new(bh);
1880 if (!buffer_mapped(bh)) {
1881 WARN_ON(bh->b_size != blocksize);
1882 err = get_block(inode, block, bh, 1);
1885 if (buffer_new(bh)) {
1886 unmap_underlying_metadata(bh->b_bdev,
1888 if (PageUptodate(page)) {
1889 clear_buffer_new(bh);
1890 set_buffer_uptodate(bh);
1891 mark_buffer_dirty(bh);
1894 if (block_end > to || block_start < from)
1895 zero_user_segments(page,
1901 if (PageUptodate(page)) {
1902 if (!buffer_uptodate(bh))
1903 set_buffer_uptodate(bh);
1906 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1907 !buffer_unwritten(bh) &&
1908 (block_start < from || block_end > to)) {
1909 ll_rw_block(READ, 1, &bh);
1914 * If we issued read requests - let them complete.
1916 while(wait_bh > wait) {
1917 wait_on_buffer(*--wait_bh);
1918 if (!buffer_uptodate(*wait_bh))
1922 page_zero_new_buffers(page, from, to);
1926 static int __block_commit_write(struct inode *inode, struct page *page,
1927 unsigned from, unsigned to)
1929 unsigned block_start, block_end;
1932 struct buffer_head *bh, *head;
1934 blocksize = 1 << inode->i_blkbits;
1936 for(bh = head = page_buffers(page), block_start = 0;
1937 bh != head || !block_start;
1938 block_start=block_end, bh = bh->b_this_page) {
1939 block_end = block_start + blocksize;
1940 if (block_end <= from || block_start >= to) {
1941 if (!buffer_uptodate(bh))
1944 set_buffer_uptodate(bh);
1945 mark_buffer_dirty(bh);
1947 clear_buffer_new(bh);
1951 * If this is a partial write which happened to make all buffers
1952 * uptodate then we can optimize away a bogus readpage() for
1953 * the next read(). Here we 'discover' whether the page went
1954 * uptodate as a result of this (potentially partial) write.
1957 SetPageUptodate(page);
1962 * block_write_begin takes care of the basic task of block allocation and
1963 * bringing partial write blocks uptodate first.
1965 * If *pagep is not NULL, then block_write_begin uses the locked page
1966 * at *pagep rather than allocating its own. In this case, the page will
1967 * not be unlocked or deallocated on failure.
1969 int block_write_begin(struct file *file, struct address_space *mapping,
1970 loff_t pos, unsigned len, unsigned flags,
1971 struct page **pagep, void **fsdata,
1972 get_block_t *get_block)
1974 struct inode *inode = mapping->host;
1978 unsigned start, end;
1981 index = pos >> PAGE_CACHE_SHIFT;
1982 start = pos & (PAGE_CACHE_SIZE - 1);
1988 page = grab_cache_page_write_begin(mapping, index, flags);
1995 BUG_ON(!PageLocked(page));
1997 status = __block_prepare_write(inode, page, start, end, get_block);
1998 if (unlikely(status)) {
1999 ClearPageUptodate(page);
2003 page_cache_release(page);
2007 * prepare_write() may have instantiated a few blocks
2008 * outside i_size. Trim these off again. Don't need
2009 * i_size_read because we hold i_mutex.
2011 if (pos + len > inode->i_size)
2012 vmtruncate(inode, inode->i_size);
2019 EXPORT_SYMBOL(block_write_begin);
2021 int block_write_end(struct file *file, struct address_space *mapping,
2022 loff_t pos, unsigned len, unsigned copied,
2023 struct page *page, void *fsdata)
2025 struct inode *inode = mapping->host;
2028 start = pos & (PAGE_CACHE_SIZE - 1);
2030 if (unlikely(copied < len)) {
2032 * The buffers that were written will now be uptodate, so we
2033 * don't have to worry about a readpage reading them and
2034 * overwriting a partial write. However if we have encountered
2035 * a short write and only partially written into a buffer, it
2036 * will not be marked uptodate, so a readpage might come in and
2037 * destroy our partial write.
2039 * Do the simplest thing, and just treat any short write to a
2040 * non uptodate page as a zero-length write, and force the
2041 * caller to redo the whole thing.
2043 if (!PageUptodate(page))
2046 page_zero_new_buffers(page, start+copied, start+len);
2048 flush_dcache_page(page);
2050 /* This could be a short (even 0-length) commit */
2051 __block_commit_write(inode, page, start, start+copied);
2055 EXPORT_SYMBOL(block_write_end);
2057 int generic_write_end(struct file *file, struct address_space *mapping,
2058 loff_t pos, unsigned len, unsigned copied,
2059 struct page *page, void *fsdata)
2061 struct inode *inode = mapping->host;
2062 int i_size_changed = 0;
2064 copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2067 * No need to use i_size_read() here, the i_size
2068 * cannot change under us because we hold i_mutex.
2070 * But it's important to update i_size while still holding page lock:
2071 * page writeout could otherwise come in and zero beyond i_size.
2073 if (pos+copied > inode->i_size) {
2074 i_size_write(inode, pos+copied);
2079 page_cache_release(page);
2082 * Don't mark the inode dirty under page lock. First, it unnecessarily
2083 * makes the holding time of page lock longer. Second, it forces lock
2084 * ordering of page lock and transaction start for journaling
2088 mark_inode_dirty(inode);
2092 EXPORT_SYMBOL(generic_write_end);
2095 * block_is_partially_uptodate checks whether buffers within a page are
2098 * Returns true if all buffers which correspond to a file portion
2099 * we want to read are uptodate.
2101 int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
2104 struct inode *inode = page->mapping->host;
2105 unsigned block_start, block_end, blocksize;
2107 struct buffer_head *bh, *head;
2110 if (!page_has_buffers(page))
2113 blocksize = 1 << inode->i_blkbits;
2114 to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
2116 if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
2119 head = page_buffers(page);
2123 block_end = block_start + blocksize;
2124 if (block_end > from && block_start < to) {
2125 if (!buffer_uptodate(bh)) {
2129 if (block_end >= to)
2132 block_start = block_end;
2133 bh = bh->b_this_page;
2134 } while (bh != head);
2138 EXPORT_SYMBOL(block_is_partially_uptodate);
2141 * Generic "read page" function for block devices that have the normal
2142 * get_block functionality. This is most of the block device filesystems.
2143 * Reads the page asynchronously --- the unlock_buffer() and
2144 * set/clear_buffer_uptodate() functions propagate buffer state into the
2145 * page struct once IO has completed.
2147 int block_read_full_page(struct page *page, get_block_t *get_block)
2149 struct inode *inode = page->mapping->host;
2150 sector_t iblock, lblock;
2151 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2152 unsigned int blocksize;
2154 int fully_mapped = 1;
2156 BUG_ON(!PageLocked(page));
2157 blocksize = 1 << inode->i_blkbits;
2158 if (!page_has_buffers(page))
2159 create_empty_buffers(page, blocksize, 0);
2160 head = page_buffers(page);
2162 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2163 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2169 if (buffer_uptodate(bh))
2172 if (!buffer_mapped(bh)) {
2176 if (iblock < lblock) {
2177 WARN_ON(bh->b_size != blocksize);
2178 err = get_block(inode, iblock, bh, 0);
2182 if (!buffer_mapped(bh)) {
2183 zero_user(page, i * blocksize, blocksize);
2185 set_buffer_uptodate(bh);
2189 * get_block() might have updated the buffer
2192 if (buffer_uptodate(bh))
2196 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2199 SetPageMappedToDisk(page);
2203 * All buffers are uptodate - we can set the page uptodate
2204 * as well. But not if get_block() returned an error.
2206 if (!PageError(page))
2207 SetPageUptodate(page);
2212 /* Stage two: lock the buffers */
2213 for (i = 0; i < nr; i++) {
2216 mark_buffer_async_read(bh);
2220 * Stage 3: start the IO. Check for uptodateness
2221 * inside the buffer lock in case another process reading
2222 * the underlying blockdev brought it uptodate (the sct fix).
2224 for (i = 0; i < nr; i++) {
2226 if (buffer_uptodate(bh))
2227 end_buffer_async_read(bh, 1);
2229 submit_bh(READ, bh);
2233 EXPORT_SYMBOL(block_read_full_page);
2235 /* utility function for filesystems that need to do work on expanding
2236 * truncates. Uses filesystem pagecache writes to allow the filesystem to
2237 * deal with the hole.
2239 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2241 struct address_space *mapping = inode->i_mapping;
2246 err = inode_newsize_ok(inode, size);
2250 err = pagecache_write_begin(NULL, mapping, size, 0,
2251 AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
2256 err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2262 EXPORT_SYMBOL(generic_cont_expand_simple);
2264 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2265 loff_t pos, loff_t *bytes)
2267 struct inode *inode = mapping->host;
2268 unsigned blocksize = 1 << inode->i_blkbits;
2271 pgoff_t index, curidx;
2273 unsigned zerofrom, offset, len;
2276 index = pos >> PAGE_CACHE_SHIFT;
2277 offset = pos & ~PAGE_CACHE_MASK;
2279 while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
2280 zerofrom = curpos & ~PAGE_CACHE_MASK;
2281 if (zerofrom & (blocksize-1)) {
2282 *bytes |= (blocksize-1);
2285 len = PAGE_CACHE_SIZE - zerofrom;
2287 err = pagecache_write_begin(file, mapping, curpos, len,
2288 AOP_FLAG_UNINTERRUPTIBLE,
2292 zero_user(page, zerofrom, len);
2293 err = pagecache_write_end(file, mapping, curpos, len, len,
2300 balance_dirty_pages_ratelimited(mapping);
2303 /* page covers the boundary, find the boundary offset */
2304 if (index == curidx) {
2305 zerofrom = curpos & ~PAGE_CACHE_MASK;
2306 /* if we will expand the thing last block will be filled */
2307 if (offset <= zerofrom) {
2310 if (zerofrom & (blocksize-1)) {
2311 *bytes |= (blocksize-1);
2314 len = offset - zerofrom;
2316 err = pagecache_write_begin(file, mapping, curpos, len,
2317 AOP_FLAG_UNINTERRUPTIBLE,
2321 zero_user(page, zerofrom, len);
2322 err = pagecache_write_end(file, mapping, curpos, len, len,
2334 * For moronic filesystems that do not allow holes in file.
2335 * We may have to extend the file.
2337 int cont_write_begin(struct file *file, struct address_space *mapping,
2338 loff_t pos, unsigned len, unsigned flags,
2339 struct page **pagep, void **fsdata,
2340 get_block_t *get_block, loff_t *bytes)
2342 struct inode *inode = mapping->host;
2343 unsigned blocksize = 1 << inode->i_blkbits;
2347 err = cont_expand_zero(file, mapping, pos, bytes);
2351 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2352 if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2353 *bytes |= (blocksize-1);
2358 err = block_write_begin(file, mapping, pos, len,
2359 flags, pagep, fsdata, get_block);
2363 EXPORT_SYMBOL(cont_write_begin);
2365 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2366 get_block_t *get_block)
2368 struct inode *inode = page->mapping->host;
2369 int err = __block_prepare_write(inode, page, from, to, get_block);
2371 ClearPageUptodate(page);
2374 EXPORT_SYMBOL(block_prepare_write);
2376 int block_commit_write(struct page *page, unsigned from, unsigned to)
2378 struct inode *inode = page->mapping->host;
2379 __block_commit_write(inode,page,from,to);
2382 EXPORT_SYMBOL(block_commit_write);
2385 * block_page_mkwrite() is not allowed to change the file size as it gets
2386 * called from a page fault handler when a page is first dirtied. Hence we must
2387 * be careful to check for EOF conditions here. We set the page up correctly
2388 * for a written page which means we get ENOSPC checking when writing into
2389 * holes and correct delalloc and unwritten extent mapping on filesystems that
2390 * support these features.
2392 * We are not allowed to take the i_mutex here so we have to play games to
2393 * protect against truncate races as the page could now be beyond EOF. Because
2394 * vmtruncate() writes the inode size before removing pages, once we have the
2395 * page lock we can determine safely if the page is beyond EOF. If it is not
2396 * beyond EOF, then the page is guaranteed safe against truncation until we
2400 block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2401 get_block_t get_block)
2403 struct page *page = vmf->page;
2404 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
2407 int ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
2410 size = i_size_read(inode);
2411 if ((page->mapping != inode->i_mapping) ||
2412 (page_offset(page) > size)) {
2413 /* page got truncated out from underneath us */
2418 /* page is wholly or partially inside EOF */
2419 if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
2420 end = size & ~PAGE_CACHE_MASK;
2422 end = PAGE_CACHE_SIZE;
2424 ret = block_prepare_write(page, 0, end, get_block);
2426 ret = block_commit_write(page, 0, end);
2428 if (unlikely(ret)) {
2432 else /* -ENOSPC, -EIO, etc */
2433 ret = VM_FAULT_SIGBUS;
2435 ret = VM_FAULT_LOCKED;
2440 EXPORT_SYMBOL(block_page_mkwrite);
2443 * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2444 * immediately, while under the page lock. So it needs a special end_io
2445 * handler which does not touch the bh after unlocking it.
2447 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2449 __end_buffer_read_notouch(bh, uptodate);
2453 * Attach the singly-linked list of buffers created by nobh_write_begin, to
2454 * the page (converting it to circular linked list and taking care of page
2457 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2459 struct buffer_head *bh;
2461 BUG_ON(!PageLocked(page));
2463 spin_lock(&page->mapping->private_lock);
2466 if (PageDirty(page))
2467 set_buffer_dirty(bh);
2468 if (!bh->b_this_page)
2469 bh->b_this_page = head;
2470 bh = bh->b_this_page;
2471 } while (bh != head);
2472 attach_page_buffers(page, head);
2473 spin_unlock(&page->mapping->private_lock);
2477 * On entry, the page is fully not uptodate.
2478 * On exit the page is fully uptodate in the areas outside (from,to)
2480 int nobh_write_begin(struct file *file, struct address_space *mapping,
2481 loff_t pos, unsigned len, unsigned flags,
2482 struct page **pagep, void **fsdata,
2483 get_block_t *get_block)
2485 struct inode *inode = mapping->host;
2486 const unsigned blkbits = inode->i_blkbits;
2487 const unsigned blocksize = 1 << blkbits;
2488 struct buffer_head *head, *bh;
2492 unsigned block_in_page;
2493 unsigned block_start, block_end;
2494 sector_t block_in_file;
2497 int is_mapped_to_disk = 1;
2499 index = pos >> PAGE_CACHE_SHIFT;
2500 from = pos & (PAGE_CACHE_SIZE - 1);
2503 page = grab_cache_page_write_begin(mapping, index, flags);
2509 if (page_has_buffers(page)) {
2511 page_cache_release(page);
2513 return block_write_begin(file, mapping, pos, len, flags, pagep,
2517 if (PageMappedToDisk(page))
2521 * Allocate buffers so that we can keep track of state, and potentially
2522 * attach them to the page if an error occurs. In the common case of
2523 * no error, they will just be freed again without ever being attached
2524 * to the page (which is all OK, because we're under the page lock).
2526 * Be careful: the buffer linked list is a NULL terminated one, rather
2527 * than the circular one we're used to.
2529 head = alloc_page_buffers(page, blocksize, 0);
2535 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2538 * We loop across all blocks in the page, whether or not they are
2539 * part of the affected region. This is so we can discover if the
2540 * page is fully mapped-to-disk.
2542 for (block_start = 0, block_in_page = 0, bh = head;
2543 block_start < PAGE_CACHE_SIZE;
2544 block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2547 block_end = block_start + blocksize;
2550 if (block_start >= to)
2552 ret = get_block(inode, block_in_file + block_in_page,
2556 if (!buffer_mapped(bh))
2557 is_mapped_to_disk = 0;
2559 unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
2560 if (PageUptodate(page)) {
2561 set_buffer_uptodate(bh);
2564 if (buffer_new(bh) || !buffer_mapped(bh)) {
2565 zero_user_segments(page, block_start, from,
2569 if (buffer_uptodate(bh))
2570 continue; /* reiserfs does this */
2571 if (block_start < from || block_end > to) {
2573 bh->b_end_io = end_buffer_read_nobh;
2574 submit_bh(READ, bh);
2581 * The page is locked, so these buffers are protected from
2582 * any VM or truncate activity. Hence we don't need to care
2583 * for the buffer_head refcounts.
2585 for (bh = head; bh; bh = bh->b_this_page) {
2587 if (!buffer_uptodate(bh))
2594 if (is_mapped_to_disk)
2595 SetPageMappedToDisk(page);
2597 *fsdata = head; /* to be released by nobh_write_end */
2604 * Error recovery is a bit difficult. We need to zero out blocks that
2605 * were newly allocated, and dirty them to ensure they get written out.
2606 * Buffers need to be attached to the page at this point, otherwise
2607 * the handling of potential IO errors during writeout would be hard
2608 * (could try doing synchronous writeout, but what if that fails too?)
2610 attach_nobh_buffers(page, head);
2611 page_zero_new_buffers(page, from, to);
2615 page_cache_release(page);
2618 if (pos + len > inode->i_size)
2619 vmtruncate(inode, inode->i_size);
2623 EXPORT_SYMBOL(nobh_write_begin);
2625 int nobh_write_end(struct file *file, struct address_space *mapping,
2626 loff_t pos, unsigned len, unsigned copied,
2627 struct page *page, void *fsdata)
2629 struct inode *inode = page->mapping->host;
2630 struct buffer_head *head = fsdata;
2631 struct buffer_head *bh;
2632 BUG_ON(fsdata != NULL && page_has_buffers(page));
2634 if (unlikely(copied < len) && head)
2635 attach_nobh_buffers(page, head);
2636 if (page_has_buffers(page))
2637 return generic_write_end(file, mapping, pos, len,
2638 copied, page, fsdata);
2640 SetPageUptodate(page);
2641 set_page_dirty(page);
2642 if (pos+copied > inode->i_size) {
2643 i_size_write(inode, pos+copied);
2644 mark_inode_dirty(inode);
2648 page_cache_release(page);
2652 head = head->b_this_page;
2653 free_buffer_head(bh);
2658 EXPORT_SYMBOL(nobh_write_end);
2661 * nobh_writepage() - based on block_full_write_page() except
2662 * that it tries to operate without attaching bufferheads to
2665 int nobh_writepage(struct page *page, get_block_t *get_block,
2666 struct writeback_control *wbc)
2668 struct inode * const inode = page->mapping->host;
2669 loff_t i_size = i_size_read(inode);
2670 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2674 /* Is the page fully inside i_size? */
2675 if (page->index < end_index)
2678 /* Is the page fully outside i_size? (truncate in progress) */
2679 offset = i_size & (PAGE_CACHE_SIZE-1);
2680 if (page->index >= end_index+1 || !offset) {
2682 * The page may have dirty, unmapped buffers. For example,
2683 * they may have been added in ext3_writepage(). Make them
2684 * freeable here, so the page does not leak.
2687 /* Not really sure about this - do we need this ? */
2688 if (page->mapping->a_ops->invalidatepage)
2689 page->mapping->a_ops->invalidatepage(page, offset);
2692 return 0; /* don't care */
2696 * The page straddles i_size. It must be zeroed out on each and every
2697 * writepage invocation because it may be mmapped. "A file is mapped
2698 * in multiples of the page size. For a file that is not a multiple of
2699 * the page size, the remaining memory is zeroed when mapped, and
2700 * writes to that region are not written out to the file."
2702 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2704 ret = mpage_writepage(page, get_block, wbc);
2706 ret = __block_write_full_page(inode, page, get_block, wbc,
2707 end_buffer_async_write);
2710 EXPORT_SYMBOL(nobh_writepage);
2712 int nobh_truncate_page(struct address_space *mapping,
2713 loff_t from, get_block_t *get_block)
2715 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2716 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2719 unsigned length, pos;
2720 struct inode *inode = mapping->host;
2722 struct buffer_head map_bh;
2725 blocksize = 1 << inode->i_blkbits;
2726 length = offset & (blocksize - 1);
2728 /* Block boundary? Nothing to do */
2732 length = blocksize - length;
2733 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2735 page = grab_cache_page(mapping, index);
2740 if (page_has_buffers(page)) {
2743 page_cache_release(page);
2744 return block_truncate_page(mapping, from, get_block);
2747 /* Find the buffer that contains "offset" */
2749 while (offset >= pos) {
2754 map_bh.b_size = blocksize;
2756 err = get_block(inode, iblock, &map_bh, 0);
2759 /* unmapped? It's a hole - nothing to do */
2760 if (!buffer_mapped(&map_bh))
2763 /* Ok, it's mapped. Make sure it's up-to-date */
2764 if (!PageUptodate(page)) {
2765 err = mapping->a_ops->readpage(NULL, page);
2767 page_cache_release(page);
2771 if (!PageUptodate(page)) {
2775 if (page_has_buffers(page))
2778 zero_user(page, offset, length);
2779 set_page_dirty(page);
2784 page_cache_release(page);
2788 EXPORT_SYMBOL(nobh_truncate_page);
2790 int block_truncate_page(struct address_space *mapping,
2791 loff_t from, get_block_t *get_block)
2793 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2794 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2797 unsigned length, pos;
2798 struct inode *inode = mapping->host;
2800 struct buffer_head *bh;
2803 blocksize = 1 << inode->i_blkbits;
2804 length = offset & (blocksize - 1);
2806 /* Block boundary? Nothing to do */
2810 length = blocksize - length;
2811 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2813 page = grab_cache_page(mapping, index);
2818 if (!page_has_buffers(page))
2819 create_empty_buffers(page, blocksize, 0);
2821 /* Find the buffer that contains "offset" */
2822 bh = page_buffers(page);
2824 while (offset >= pos) {
2825 bh = bh->b_this_page;
2831 if (!buffer_mapped(bh)) {
2832 WARN_ON(bh->b_size != blocksize);
2833 err = get_block(inode, iblock, bh, 0);
2836 /* unmapped? It's a hole - nothing to do */
2837 if (!buffer_mapped(bh))
2841 /* Ok, it's mapped. Make sure it's up-to-date */
2842 if (PageUptodate(page))
2843 set_buffer_uptodate(bh);
2845 if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2847 ll_rw_block(READ, 1, &bh);
2849 /* Uhhuh. Read error. Complain and punt. */
2850 if (!buffer_uptodate(bh))
2854 zero_user(page, offset, length);
2855 mark_buffer_dirty(bh);
2860 page_cache_release(page);
2864 EXPORT_SYMBOL(block_truncate_page);
2867 * The generic ->writepage function for buffer-backed address_spaces
2868 * this form passes in the end_io handler used to finish the IO.
2870 int block_write_full_page_endio(struct page *page, get_block_t *get_block,
2871 struct writeback_control *wbc, bh_end_io_t *handler)
2873 struct inode * const inode = page->mapping->host;
2874 loff_t i_size = i_size_read(inode);
2875 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2878 /* Is the page fully inside i_size? */
2879 if (page->index < end_index)
2880 return __block_write_full_page(inode, page, get_block, wbc,
2883 /* Is the page fully outside i_size? (truncate in progress) */
2884 offset = i_size & (PAGE_CACHE_SIZE-1);
2885 if (page->index >= end_index+1 || !offset) {
2887 * The page may have dirty, unmapped buffers. For example,
2888 * they may have been added in ext3_writepage(). Make them
2889 * freeable here, so the page does not leak.
2891 do_invalidatepage(page, 0);
2893 return 0; /* don't care */
2897 * The page straddles i_size. It must be zeroed out on each and every
2898 * writepage invocation because it may be mmapped. "A file is mapped
2899 * in multiples of the page size. For a file that is not a multiple of
2900 * the page size, the remaining memory is zeroed when mapped, and
2901 * writes to that region are not written out to the file."
2903 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2904 return __block_write_full_page(inode, page, get_block, wbc, handler);
2906 EXPORT_SYMBOL(block_write_full_page_endio);
2909 * The generic ->writepage function for buffer-backed address_spaces
2911 int block_write_full_page(struct page *page, get_block_t *get_block,
2912 struct writeback_control *wbc)
2914 return block_write_full_page_endio(page, get_block, wbc,
2915 end_buffer_async_write);
2917 EXPORT_SYMBOL(block_write_full_page);
2919 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2920 get_block_t *get_block)
2922 struct buffer_head tmp;
2923 struct inode *inode = mapping->host;
2926 tmp.b_size = 1 << inode->i_blkbits;
2927 get_block(inode, block, &tmp, 0);
2928 return tmp.b_blocknr;
2930 EXPORT_SYMBOL(generic_block_bmap);
2932 static void end_bio_bh_io_sync(struct bio *bio, int err)
2934 struct buffer_head *bh = bio->bi_private;
2936 if (err == -EOPNOTSUPP) {
2937 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2938 set_bit(BH_Eopnotsupp, &bh->b_state);
2941 if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
2942 set_bit(BH_Quiet, &bh->b_state);
2944 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2948 int submit_bh(int rw, struct buffer_head * bh)
2953 BUG_ON(!buffer_locked(bh));
2954 BUG_ON(!buffer_mapped(bh));
2955 BUG_ON(!bh->b_end_io);
2956 BUG_ON(buffer_delay(bh));
2957 BUG_ON(buffer_unwritten(bh));
2960 * Mask in barrier bit for a write (could be either a WRITE or a
2963 if (buffer_ordered(bh) && (rw & WRITE))
2964 rw |= WRITE_BARRIER;
2967 * Only clear out a write error when rewriting
2969 if (test_set_buffer_req(bh) && (rw & WRITE))
2970 clear_buffer_write_io_error(bh);
2973 * from here on down, it's all bio -- do the initial mapping,
2974 * submit_bio -> generic_make_request may further map this bio around
2976 bio = bio_alloc(GFP_NOIO, 1);
2978 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2979 bio->bi_bdev = bh->b_bdev;
2980 bio->bi_io_vec[0].bv_page = bh->b_page;
2981 bio->bi_io_vec[0].bv_len = bh->b_size;
2982 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2986 bio->bi_size = bh->b_size;
2988 bio->bi_end_io = end_bio_bh_io_sync;
2989 bio->bi_private = bh;
2992 submit_bio(rw, bio);
2994 if (bio_flagged(bio, BIO_EOPNOTSUPP))
3000 EXPORT_SYMBOL(submit_bh);
3003 * ll_rw_block: low-level access to block devices (DEPRECATED)
3004 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
3005 * @nr: number of &struct buffer_heads in the array
3006 * @bhs: array of pointers to &struct buffer_head
3008 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3009 * requests an I/O operation on them, either a %READ or a %WRITE. The third
3010 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
3011 * are sent to disk. The fourth %READA option is described in the documentation
3012 * for generic_make_request() which ll_rw_block() calls.
3014 * This function drops any buffer that it cannot get a lock on (with the
3015 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
3016 * clean when doing a write request, and any buffer that appears to be
3017 * up-to-date when doing read request. Further it marks as clean buffers that
3018 * are processed for writing (the buffer cache won't assume that they are
3019 * actually clean until the buffer gets unlocked).
3021 * ll_rw_block sets b_end_io to simple completion handler that marks
3022 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
3025 * All of the buffers must be for the same device, and must also be a
3026 * multiple of the current approved size for the device.
3028 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
3032 for (i = 0; i < nr; i++) {
3033 struct buffer_head *bh = bhs[i];
3035 if (rw == SWRITE || rw == SWRITE_SYNC || rw == SWRITE_SYNC_PLUG)
3037 else if (!trylock_buffer(bh))
3040 if (rw == WRITE || rw == SWRITE || rw == SWRITE_SYNC ||
3041 rw == SWRITE_SYNC_PLUG) {
3042 if (test_clear_buffer_dirty(bh)) {
3043 bh->b_end_io = end_buffer_write_sync;
3045 if (rw == SWRITE_SYNC)
3046 submit_bh(WRITE_SYNC, bh);
3048 submit_bh(WRITE, bh);
3052 if (!buffer_uptodate(bh)) {
3053 bh->b_end_io = end_buffer_read_sync;
3062 EXPORT_SYMBOL(ll_rw_block);
3065 * For a data-integrity writeout, we need to wait upon any in-progress I/O
3066 * and then start new I/O and then wait upon it. The caller must have a ref on
3069 int sync_dirty_buffer(struct buffer_head *bh)
3073 WARN_ON(atomic_read(&bh->b_count) < 1);
3075 if (test_clear_buffer_dirty(bh)) {
3077 bh->b_end_io = end_buffer_write_sync;
3078 ret = submit_bh(WRITE_SYNC, bh);
3080 if (buffer_eopnotsupp(bh)) {
3081 clear_buffer_eopnotsupp(bh);
3084 if (!ret && !buffer_uptodate(bh))
3091 EXPORT_SYMBOL(sync_dirty_buffer);
3094 * try_to_free_buffers() checks if all the buffers on this particular page
3095 * are unused, and releases them if so.
3097 * Exclusion against try_to_free_buffers may be obtained by either
3098 * locking the page or by holding its mapping's private_lock.
3100 * If the page is dirty but all the buffers are clean then we need to
3101 * be sure to mark the page clean as well. This is because the page
3102 * may be against a block device, and a later reattachment of buffers
3103 * to a dirty page will set *all* buffers dirty. Which would corrupt
3104 * filesystem data on the same device.
3106 * The same applies to regular filesystem pages: if all the buffers are
3107 * clean then we set the page clean and proceed. To do that, we require
3108 * total exclusion from __set_page_dirty_buffers(). That is obtained with
3111 * try_to_free_buffers() is non-blocking.
3113 static inline int buffer_busy(struct buffer_head *bh)
3115 return atomic_read(&bh->b_count) |
3116 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3120 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3122 struct buffer_head *head = page_buffers(page);
3123 struct buffer_head *bh;
3127 if (buffer_write_io_error(bh) && page->mapping)
3128 set_bit(AS_EIO, &page->mapping->flags);
3129 if (buffer_busy(bh))
3131 bh = bh->b_this_page;
3132 } while (bh != head);
3135 struct buffer_head *next = bh->b_this_page;
3137 if (bh->b_assoc_map)
3138 __remove_assoc_queue(bh);
3140 } while (bh != head);
3141 *buffers_to_free = head;
3142 __clear_page_buffers(page);
3148 int try_to_free_buffers(struct page *page)
3150 struct address_space * const mapping = page->mapping;
3151 struct buffer_head *buffers_to_free = NULL;
3154 BUG_ON(!PageLocked(page));
3155 if (PageWriteback(page))
3158 if (mapping == NULL) { /* can this still happen? */
3159 ret = drop_buffers(page, &buffers_to_free);
3163 spin_lock(&mapping->private_lock);
3164 ret = drop_buffers(page, &buffers_to_free);
3167 * If the filesystem writes its buffers by hand (eg ext3)
3168 * then we can have clean buffers against a dirty page. We
3169 * clean the page here; otherwise the VM will never notice
3170 * that the filesystem did any IO at all.
3172 * Also, during truncate, discard_buffer will have marked all
3173 * the page's buffers clean. We discover that here and clean
3176 * private_lock must be held over this entire operation in order
3177 * to synchronise against __set_page_dirty_buffers and prevent the
3178 * dirty bit from being lost.
3181 cancel_dirty_page(page, PAGE_CACHE_SIZE);
3182 spin_unlock(&mapping->private_lock);
3184 if (buffers_to_free) {
3185 struct buffer_head *bh = buffers_to_free;
3188 struct buffer_head *next = bh->b_this_page;
3189 free_buffer_head(bh);
3191 } while (bh != buffers_to_free);
3195 EXPORT_SYMBOL(try_to_free_buffers);
3197 void block_sync_page(struct page *page)
3199 struct address_space *mapping;
3202 mapping = page_mapping(page);
3204 blk_run_backing_dev(mapping->backing_dev_info, page);
3206 EXPORT_SYMBOL(block_sync_page);
3209 * There are no bdflush tunables left. But distributions are
3210 * still running obsolete flush daemons, so we terminate them here.
3212 * Use of bdflush() is deprecated and will be removed in a future kernel.
3213 * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3215 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3217 static int msg_count;
3219 if (!capable(CAP_SYS_ADMIN))
3222 if (msg_count < 5) {
3225 "warning: process `%s' used the obsolete bdflush"
3226 " system call\n", current->comm);
3227 printk(KERN_INFO "Fix your initscripts?\n");
3236 * Buffer-head allocation
3238 static struct kmem_cache *bh_cachep;
3241 * Once the number of bh's in the machine exceeds this level, we start
3242 * stripping them in writeback.
3244 static int max_buffer_heads;
3246 int buffer_heads_over_limit;
3248 struct bh_accounting {
3249 int nr; /* Number of live bh's */
3250 int ratelimit; /* Limit cacheline bouncing */
3253 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3255 static void recalc_bh_state(void)
3260 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3262 __get_cpu_var(bh_accounting).ratelimit = 0;
3263 for_each_online_cpu(i)
3264 tot += per_cpu(bh_accounting, i).nr;
3265 buffer_heads_over_limit = (tot > max_buffer_heads);
3268 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3270 struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3272 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3273 get_cpu_var(bh_accounting).nr++;
3275 put_cpu_var(bh_accounting);
3279 EXPORT_SYMBOL(alloc_buffer_head);
3281 void free_buffer_head(struct buffer_head *bh)
3283 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3284 kmem_cache_free(bh_cachep, bh);
3285 get_cpu_var(bh_accounting).nr--;
3287 put_cpu_var(bh_accounting);
3289 EXPORT_SYMBOL(free_buffer_head);
3291 static void buffer_exit_cpu(int cpu)
3294 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3296 for (i = 0; i < BH_LRU_SIZE; i++) {
3300 get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
3301 per_cpu(bh_accounting, cpu).nr = 0;
3302 put_cpu_var(bh_accounting);
3305 static int buffer_cpu_notify(struct notifier_block *self,
3306 unsigned long action, void *hcpu)
3308 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
3309 buffer_exit_cpu((unsigned long)hcpu);
3314 * bh_uptodate_or_lock - Test whether the buffer is uptodate
3315 * @bh: struct buffer_head
3317 * Return true if the buffer is up-to-date and false,
3318 * with the buffer locked, if not.
3320 int bh_uptodate_or_lock(struct buffer_head *bh)
3322 if (!buffer_uptodate(bh)) {
3324 if (!buffer_uptodate(bh))
3330 EXPORT_SYMBOL(bh_uptodate_or_lock);
3333 * bh_submit_read - Submit a locked buffer for reading
3334 * @bh: struct buffer_head
3336 * Returns zero on success and -EIO on error.
3338 int bh_submit_read(struct buffer_head *bh)
3340 BUG_ON(!buffer_locked(bh));
3342 if (buffer_uptodate(bh)) {
3348 bh->b_end_io = end_buffer_read_sync;
3349 submit_bh(READ, bh);
3351 if (buffer_uptodate(bh))
3355 EXPORT_SYMBOL(bh_submit_read);
3357 void __init buffer_init(void)
3361 bh_cachep = kmem_cache_create("buffer_head",
3362 sizeof(struct buffer_head), 0,
3363 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3368 * Limit the bh occupancy to 10% of ZONE_NORMAL
3370 nrpages = (nr_free_buffer_pages() * 10) / 100;
3371 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3372 hotcpu_notifier(buffer_cpu_notify, 0);