Index: linux-2.6.11/fs/ext3/inode.c =================================================================== --- linux-2.6.11.orig/fs/ext3/inode.c 2005-03-05 19:37:01.000000000 +0300 +++ linux-2.6.11/fs/ext3/inode.c 2005-03-05 19:37:02.000000000 +0300 @@ -1585,9 +1585,34 @@ .releasepage = ext3_releasepage, }; +static int ext3_wb_set_page_dirty(struct page *page) +{ + return __set_page_dirty_nobuffers(page); +} + +static struct address_space_operations ext3_writeback_da_aops = { + .readpage = ext3_readpage, + .readpages = ext3_readpages, + .writepage = ext3_wb_writepage, + .writepages = ext3_wb_writepages, + .sync_page = block_sync_page, + .prepare_write = ext3_wb_prepare_write, + .commit_write = ext3_wb_commit_write, + .bmap = ext3_bmap, + .invalidatepage = ext3_wb_invalidatepage, + .releasepage = ext3_wb_releasepage, + .set_page_dirty = ext3_wb_set_page_dirty, + .direct_IO = ext3_direct_IO, +}; + void ext3_set_aops(struct inode *inode) { - if (ext3_should_order_data(inode)) + if (S_ISREG(inode->i_mode) && + (EXT3_I(inode)->i_flags & EXT3_EXTENTS_FL) && + test_opt(inode->i_sb, EXTENTS) && + test_opt(inode->i_sb, DELAYED_ALLOC)) + inode->i_mapping->a_ops = &ext3_writeback_da_aops; + else if (ext3_should_order_data(inode)) inode->i_mapping->a_ops = &ext3_ordered_aops; else if (ext3_should_writeback_data(inode)) inode->i_mapping->a_ops = &ext3_writeback_aops; @@ -1612,6 +1637,11 @@ int err; void *kaddr; + if ((EXT3_I(inode)->i_flags & EXT3_EXTENTS_FL) && + test_opt(inode->i_sb, EXTENTS) && + test_opt(inode->i_sb, DELAYED_ALLOC)) + return ext3_wb_block_truncate_page(handle, page, mapping, from); + blocksize = inode->i_sb->s_blocksize; length = blocksize - (offset & (blocksize - 1)); iblock = index << (PAGE_CACHE_SHIFT - inode->i_sb->s_blocksize_bits); Index: linux-2.6.11/fs/ext3/writeback.c =================================================================== --- linux-2.6.11.orig/fs/ext3/writeback.c 2005-03-02 22:42:20.659360368 +0300 +++ linux-2.6.11/fs/ext3/writeback.c 2005-03-06 15:40:33.000000000 +0300 @@ -0,0 +1,1137 @@ +/* + * Copyright (c) 2003, Cluster File Systems, Inc, info@clusterfs.com + * Written by Alex Tomas + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public Licens + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- + */ + +/* + * TODO: + * MUST: + * - flush dirty pages in -ENOSPC case in order to free reserved blocks + * - direct I/O support + * - blocksize != PAGE_CACHE_SIZE support + * TODO: + * - should ext3_wb_writepage() try to flush neighbours? + * - ext3_wb_block_truncate_page() must flush partial truncated pages + * - reservation can be done per write-request in ext3_file_write() + * rather than per-page in ext3_wb_commit_write() + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * If EXT3_WB_STATS is defined, then some stats are collected. + * It will be showed upont umount time. + */ +#define EXT3_WB_STATS + + +/* + * With EXT3_WB_SKIP_SMALL defined the patch will try to avoid + * small I/Os ignoring ->writepages() if mapping hasn't enoug + * contig. dirty pages + */ +#define EXT3_WB_SKIP_SMALL__ + +#define WB_ASSERT(__x__) if (!(__x__)) BUG(); + +#define WB_DEBUG__ +#ifdef WB_DEBUG +#define wb_debug(fmt,a...) printk(fmt, ##a); +#else +#define wb_debug(fmt,a...) +#endif + +#define WB_MAX_PAGES_PER_EXTENT 32000 + +#define WB_PAGES_PER_ARRAY 60 + +struct ext3_wb_pages { + struct list_head list; + struct page *pages[WB_PAGES_PER_ARRAY]; + unsigned short num, start; +}; + +struct ext3_wb_control { + pgoff_t start; + int len, extents; + int blocks_to_release; + struct ext3_wb_pages *pages; + struct list_head list; + struct ext3_extents_tree tree; + struct address_space *mapping; +}; + + +int ext3_wb_invalidatepage(struct page *, unsigned long); + + +static struct page * ext3_wb_pull_page(struct ext3_wb_control *wc) +{ + struct ext3_wb_pages *wp = wc->pages; + + BUG_ON(wp == NULL); + BUG_ON(list_empty(&wc->list)); + BUG_ON(list_empty(&wp->list)); + if (wp->start == wp->num) { + list_del(&wp->list); + kfree(wp); + if (list_empty(&wc->list)) + return NULL; + wp = list_entry(wc->list.next, struct ext3_wb_pages, list); + wc->pages = wp; + } + BUG_ON(list_empty(&wp->list)); + return wp->pages[wp->start++]; +} + +static struct bio * ext3_wb_bio_alloc(struct inode *inode, + sector_t first_block, int nr_vecs) +{ + int gfp_flags = GFP_NOFS | __GFP_HIGH; + struct bio *bio; + int maxreq; + + maxreq = bio_get_nr_vecs(inode->i_sb->s_bdev); + if (maxreq < nr_vecs) + nr_vecs = maxreq; + + bio = bio_alloc(gfp_flags, nr_vecs); + + if (bio == NULL && (current->flags & PF_MEMALLOC)) { + while (!bio && (nr_vecs /= 2)) + bio = bio_alloc(gfp_flags, nr_vecs); + } + + if (bio) { + bio->bi_bdev = inode->i_sb->s_bdev; + bio->bi_sector = first_block << (inode->i_blkbits - 9); + } + return bio; +} + +static int ext3_wb_end_io(struct bio *bio, unsigned int bytes, int err) +{ + const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); + struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; + + if (bio->bi_size) + return 1; + + do { + struct page *page = bvec->bv_page; + + if (--bvec >= bio->bi_io_vec) + prefetchw(&bvec->bv_page->flags); + + if (!uptodate) + SetPageError(page); + end_page_writeback(page); + } while (bvec >= bio->bi_io_vec); + bio_put(bio); + return 0; +} + +static struct bio *ext3_wb_bio_submit(struct bio *bio, handle_t *handle) +{ + bio->bi_end_io = ext3_wb_end_io; + submit_bio(WRITE, bio); + return NULL; +} + +int inline ext3_wb_reserve_space_page(struct page *page, int blocks) +{ + struct inode *inode = page->mapping->host; + int total, mdb, err; + + wb_debug("reserve %d blocks for page %lu from inode %lu\n", + blocks, page->index, inode->i_ino); + + /* user wants us to reserve blocks for his file. reserving space + * for his (data) blocks isn't enough because adding block may + * involve allocation index/leaf blocks for tree/blockmap. + * so, we need to calculate numbers of needed metadata for worst + * case: block per extent */ + + spin_lock(&EXT3_I(inode)->i_wb_reserved_lock); + total = EXT3_I(inode)->i_blocks_reserved + blocks; + mdb = ext3_ext_calc_blockmap_metadata(inode, total); + + /* if blockmap needs more metadata, we have to reserve difference */ + BUG_ON(mdb < EXT3_I(inode)->i_md_reserved); + mdb = mdb - EXT3_I(inode)->i_md_reserved; + + err = ext3_mb_reserve_blocks(inode->i_sb, mdb + blocks); + if (err) { + /* blocks are exhausted? */ + spin_unlock(&EXT3_I(inode)->i_wb_reserved_lock); + return err; + } + + /* blocks have been reserved, account this. I believe + * inode's fields are protected by inode->i_sem */ + EXT3_I(inode)->i_blocks_reserved += blocks; + EXT3_I(inode)->i_md_reserved += mdb; + spin_unlock(&EXT3_I(inode)->i_wb_reserved_lock); + + /* mark page to one that has reserved space. ->writepages() + * or ->invalidatepage() must release this space */ + SetPagePrivate(page); + + return 0; +} + +int inline ext3_wb_release_space(struct inode *inode, int blocks) +{ + int total, mdb; + + wb_debug("release %d blocks reserved from inode %lu\n", + blocks, inode->i_ino); + + spin_lock(&EXT3_I(inode)->i_wb_reserved_lock); + total = EXT3_I(inode)->i_blocks_reserved - blocks; + mdb = ext3_ext_calc_blockmap_metadata(inode, total); + + /* if blockmap needs lesser metadata, we may release ifference */ + BUG_ON(mdb > EXT3_I(inode)->i_md_reserved); + mdb = EXT3_I(inode)->i_md_reserved - mdb; + + ext3_mb_release_blocks(inode->i_sb, mdb + blocks); + + /* block have been released, account this. I believe + * inode's fields are protected by inode->i_sem */ + EXT3_I(inode)->i_blocks_reserved -= blocks; + EXT3_I(inode)->i_md_reserved -= mdb; + spin_unlock(&EXT3_I(inode)->i_wb_reserved_lock); + + return 0; +} + +static inline int ext3_wb_drop_reservation(struct page *page, int blocks) +{ + /* we just allocated blocks for this page. those blocks (and + * probably metadata for them) were reserved before. now we + * should drop reservation mark from the page. if we didn't + * do that then ->invalidatepage() may think page still holds + * reserved blocks. we could release reserved blocks right + * now, but I'd prefer to make this once per several blocks */ + wb_debug("release %d blocks reserved for page %lu from inode %lu\n", + blocks, page->index, inode->i_ino); + ClearPagePrivate(page); + return 0; +} + +static int ext3_wb_submit_extent(struct ext3_wb_control *wc, handle_t *handle, + struct ext3_extent *ex, int new) +{ + struct inode *inode = wc->mapping->host; + int blkbits = inode->i_blkbits; + struct page *page; + unsigned long blk, off, len, remain; + unsigned long pstart, plen, prev; + struct bio *bio = NULL; + int nr_pages; + + /* + * we have list of pages in wc and block numbers in ex + * let's cook bios from them and start real I/O + */ + + BUG_ON(PAGE_CACHE_SHIFT < blkbits); + BUG_ON(list_empty(&wc->list)); + + wb_debug("cook and submit bios for %u/%u/%u for %lu/%u\n", + ex->ee_block, ex->ee_len, ex->ee_start, wc->start, wc->len); + + blk = ex->ee_block; + remain = ex->ee_len; + wc->extents++; + + while (remain) { + page = ext3_wb_pull_page(wc); + if (page == NULL) + break; + + pstart = page->index << (PAGE_CACHE_SHIFT - blkbits); + plen = PAGE_SIZE >> blkbits; + if (pstart > blk) { + /* probably extent covers long space and page + * to be written in the middle of it */ + BUG_ON(pstart - blk >= remain); + remain -= pstart - blk; + blk = pstart; + } + BUG_ON(blk < pstart || blk >= pstart + plen); + + BUG_ON(!PageUptodate(page)); + BUG_ON(!new && PagePrivate(page)); + BUG_ON(new && PageMappedToDisk(page)); + BUG_ON(!new && !PageMappedToDisk(page)); + SetPageMappedToDisk(page); + if (new && PagePrivate(page)) { + /* space is just allocated and it was reserved in + * ->commit_write(). time to release reservation. + * space may not be reserved if page gets dirty + * via mmap. should we reserve it in ->mmap() ? */ + prev = min(plen, remain); + ext3_wb_drop_reservation(page, prev); + wc->blocks_to_release += prev; + } + +alloc_new_bio: + if (bio == NULL) { + /* +2 because head/tail may belong to different pages */ + nr_pages = (ex->ee_len - (blk - ex->ee_block)); + nr_pages = (nr_pages >> (PAGE_CACHE_SHIFT - blkbits)); + off = ex->ee_start + (blk - ex->ee_block); + bio = ext3_wb_bio_alloc(inode, off, nr_pages + 2); + if (bio == NULL) + return -ENOMEM; + } + + off = (blk - pstart) << blkbits; + prev = min(plen, remain); + len = prev << blkbits; + if (bio_add_page(bio, page, len, off) < len) { + bio = ext3_wb_bio_submit(bio, handle); + goto alloc_new_bio; + } + remain -= prev; + blk += prev; + if (blk < pstart + plen) { + /* extent covers part of the page only. + * it's possible that next extent covers + * the tail. so, we leave page */ + printk("blk %lu pstart %lu plen %lu remain %lu prev %lu\n", + blk, pstart, plen, remain, prev); + wc->pages->start--; + BUG_ON(remain != 0); + } + } + if (bio) + ext3_wb_bio_submit(bio, handle); + BUG_ON(new && remain != 0); + return 0; +} + +static int ext3_wb_find_goal(struct inode *inode, struct ext3_ext_path *path, + unsigned long block, int *aflags) +{ + struct ext3_inode_info *ei = EXT3_I(inode); + unsigned long bg_start; + unsigned long colour; + int depth; + + if (path) { + struct ext3_extent *ex; + depth = path->p_depth; + + /* try to predict block placement */ + if ((ex = path[depth].p_ext)) { + if (ex->ee_block + ex->ee_len == block) + *aflags |= EXT3_MB_HINT_MERGE; + return ex->ee_start + (block - ex->ee_block); + } + + /* it looks index is empty + * try to find starting from index itself */ + if (path[depth].p_bh) + return path[depth].p_bh->b_blocknr; + } + + /* OK. use inode's group */ + bg_start = (ei->i_block_group * EXT3_BLOCKS_PER_GROUP(inode->i_sb)) + + le32_to_cpu(EXT3_SB(inode->i_sb)->s_es->s_first_data_block); + colour = (current->pid % 16) * + (EXT3_BLOCKS_PER_GROUP(inode->i_sb) / 16); + return bg_start + colour + block; +} + +static int ext3_wb_handle_extent(struct ext3_extents_tree *tree, + struct ext3_ext_path *path, + struct ext3_ext_cache *ec) +{ + struct ext3_wb_control *wc = tree->private; + struct inode *inode = tree->inode; + struct super_block *sb = inode->i_sb; + int i, count, err, aflags = 0; + struct ext3_extent nex; + sector_t pblock; + unsigned long tgen; + loff_t new_i_size; + handle_t *handle; + + if (ec->ec_type == EXT3_EXT_CACHE_EXTENT) { + /* + * The extent is already allocated. The only thing + * we have to do is to flush correspondend pages. + */ + wb_debug("extent %u/%u/%u exist\n", + ec->ec_block, ec->ec_len, ec->ec_start); + nex.ee_start = ec->ec_start; + nex.ee_block = ec->ec_block; + nex.ee_len = ec->ec_len; + err = ext3_wb_submit_extent(wc, NULL, &nex, 0); + return err; + } + + wb_debug("extent %u/%u DOES NOT exist\n", ec->ec_block, ec->ec_len); + + /* space for some pages we want to flush hasn't allocated + * yet. so, it's time to allocate space */ + tgen = EXT_GENERATION(tree); + count = ext3_ext_calc_credits_for_insert(tree, path); + up(&EXT3_I(inode)->truncate_sem); + + handle = ext3_journal_start(inode, count + EXT3_ALLOC_NEEDED + 1); + if (IS_ERR(handle)) { + down(&EXT3_I(inode)->truncate_sem); + return PTR_ERR(handle); + } + + /* FIXME: we could analyze current path and advice allocator + * to find additional blocks if goal can't be allocated + * this is for better interaction between extents and mballoc + * plus this should improve overall performance */ + + down(&EXT3_I(inode)->truncate_sem); + if (tgen != EXT_GENERATION(tree)) { + /* the tree has changed. so path can be invalid at moment */ + ext3_journal_stop(handle); + return EXT_REPEAT; + } + + pblock = ext3_wb_find_goal(inode, path, ec->ec_block, &aflags); + count = ec->ec_len; + aflags |= EXT3_MB_HINT_RESERVED; /* block have been already reserved */ + pblock = ext3_mb_new_blocks(handle, inode, pblock, &count, aflags, &err); + if (!pblock) + goto out; + + BUG_ON(count > ec->ec_len); + BUG_ON(count == 0); + wb_debug("allocated %lu/%u for %lu (asked %u)\n", + (unsigned long) pblock, count, inode->i_ino, ec->ec_len); + + /* insert new extent */ + nex.ee_start = pblock; + nex.ee_start_hi = 0; + nex.ee_len = count; + nex.ee_block = ec->ec_block; + err = ext3_ext_insert_extent(handle, tree, path, &nex); + if (err) + goto out; + + /* + * Putting len of the actual extent we just inserted, + * we are asking ext3_ext_walk_space() to continue + * scaning after that block + */ + ec->ec_len = nex.ee_len; + BUG_ON(nex.ee_len == 0); + +#ifdef EXT3_WB_STATS + { + struct super_block *sb = inode->i_sb; + spin_lock(&EXT3_SB(sb)->s_wb_lock); + EXT3_SB(sb)->s_wb_allocated += nex.ee_len; + spin_unlock(&EXT3_SB(sb)->s_wb_lock); + } +#endif + + wb_debug("inserted %lu/%lu/%lu for %lu (asked %u)\n", + (unsigned long) nex.ee_block, (unsigned long) nex.ee_len, + (unsigned long) nex.ee_start, inode->i_ino, ec->ec_len); + + /* + * Important! The nex can change after insert. So do not + * use ec for following + */ + + /* block have been allocated for data, so time to drop dirty + * in correspondend buffer_heads to prevent corruptions */ + for (i = 0; i < nex.ee_len; i++) + unmap_underlying_metadata(sb->s_bdev, nex.ee_start + i); + + /* correct on-disk inode size */ + if (nex.ee_len > 0) { + new_i_size = (loff_t) nex.ee_block + nex.ee_len; + new_i_size = new_i_size << inode->i_blkbits; + if (new_i_size > i_size_read(inode)) + new_i_size = i_size_read(inode); + if (new_i_size > EXT3_I(inode)->i_disksize) { + EXT3_I(inode)->i_disksize = new_i_size; + err = ext3_mark_inode_dirty(handle, inode); + } + } + + if (ext3_should_order_data(inode)) + err = ext3_wb_submit_extent(wc, handle, &nex, 1); + else + err = ext3_wb_submit_extent(wc, NULL, &nex, 1); + + /* we don't want to recalculate needed reservation for + * each page. we may do this for each new extent */ + ext3_wb_release_space(inode, wc->blocks_to_release); + wc->blocks_to_release = 0; + +out: + ext3_journal_stop(handle); + if (err) + printk("EXT3-fs: writeback error = %d\n", err); + return err; +} + +static int ext3_wb_flush(struct ext3_wb_control *wc) +{ + struct list_head *cur, *tmp; + struct inode *inode; + int err, num = 0; + + if (wc->len == 0) + return 0; + + inode = wc->mapping->host; + wb_debug("start flushing %lu/%u from inode %lu\n", + wc->start, wc->len, inode->i_ino); + + wc->pages = list_entry(wc->list.next, struct ext3_wb_pages, list); + wc->extents = 0; + + down(&EXT3_I(inode)->truncate_sem); + /* FIXME: handle blocksize < PAGE_CACHE_SIZE! */ + /* FIXME: last page may be partial */ + err = ext3_ext_walk_space(&wc->tree, wc->start, wc->len, + ext3_wb_handle_extent); + up(&EXT3_I(inode)->truncate_sem); + + list_for_each_safe(cur, tmp, &wc->list) { + struct ext3_wb_pages *wp; + wp = list_entry(cur, struct ext3_wb_pages, list); + if (err) { + while (wp->start < wp->num) { + struct page *page = wp->pages[wp->start]; + BUG_ON(!PageWriteback(page)); + end_page_writeback(page); + __set_page_dirty_nobuffers(page); + wp->start++; + } + } else { + BUG_ON(num != 0); + BUG_ON(wp->start != wp->num - 1 && + wp->start != wp->num); + } + list_del(&wp->list); + kfree(wp); + num++; + } + wc->pages = NULL; + wc->len = 0; + wc->extents = 0; + + return err; +} + +static int ext3_wb_add_page(struct ext3_wb_control *wc, struct page *page) +{ + struct ext3_wb_pages * wp = wc->pages; + + if (wp == NULL || wp->num == WB_PAGES_PER_ARRAY) { + wp = kmalloc(sizeof(struct ext3_wb_pages), GFP_NOFS); + if (wp == NULL) { + printk("no mem for ext3_wb_pages!\n"); + return -ENOMEM; + } + wp->num = 0; + wp->start = 0; + list_add_tail(&wp->list, &wc->list); + wc->pages = wp; + } + + wp->pages[wp->num] = page; + wp->num++; + + return 0; +} + +static inline void +ext3_wb_init_control(struct ext3_wb_control *wc, struct address_space *mapping) +{ + wc->tree.private = wc; + wc->mapping = mapping; + wc->len = 0; + wc->blocks_to_release = 0; + INIT_LIST_HEAD(&wc->list); + wc->pages = NULL; + ext3_init_tree_desc(&wc->tree, mapping->host); +} + +static inline int +ext3_wb_can_merge(struct ext3_wb_control *wc, unsigned long next) +{ + if (wc->start + wc->len == next && + wc->len <= WB_MAX_PAGES_PER_EXTENT) + return 1; + return 0; +} + +int ext3_wb_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct backing_dev_info *bdi = mapping->backing_dev_info; + struct inode *inode = mapping->host; + int nr_pages, i, err = 0, done = 0; + struct ext3_wb_control wc; + struct pagevec pvec; + pgoff_t index = 0; + int written = 0; + int extents = 0; + pgoff_t pindex = 0; + + wb_debug("->writepages on inode %lu (%u reserved)\n", + inode->i_ino, EXT3_I(inode)->i_blocks_reserved); +#ifdef EXT3_WB_SKIP_SMALL + if (wbc->nr_to_write <= 64 && wbc->sync_mode == WB_SYNC_NONE) + return 0; +#endif + atomic_inc(&EXT3_I(inode)->i_wb_writers); +#ifdef EXT3_WB_STATS + { + struct super_block *sb = inode->i_sb; + spin_lock(&EXT3_SB(sb)->s_wb_lock); + EXT3_SB(sb)->s_wb_reqs++; + EXT3_SB(sb)->s_wb_nr_to_write += wbc->nr_to_write; + if (atomic_read(&EXT3_I(inode)->i_wb_writers) != 1) + EXT3_SB(sb)->s_wb_collisions++; + spin_unlock(&EXT3_SB(sb)->s_wb_lock); + } +#endif + +#if 0 + if (wbc->nr_to_write <= 64) { + printk("short ->writepages(#%lu, %ld), sync mode %d\n", + (unsigned long) inode->i_ino, wbc->nr_to_write, + (int) wbc->sync_mode); + } +#endif + ext3_wb_init_control(&wc, mapping); + + pagevec_init(&pvec, 0); + while (!done && (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, + PAGECACHE_TAG_DIRTY, PAGEVEC_SIZE))) { + for (i = 0; i < nr_pages; i++) { + struct page *page = pvec.pages[i]; + + lock_page(page); + + if (wbc->sync_mode != WB_SYNC_NONE) + wait_on_page_writeback(page); + + if (page->mapping != mapping) { + unlock_page(page); + continue; + } + if (PageWriteback(page)) { + unlock_page(page); + continue; + } + + if (wc.len && ext3_wb_can_merge(&wc, page->index) && + wbc->nr_to_write <= 0) { + /* + * If we already exhausted blocks we got + * to write and new extent starts, stop + * writeback + */ + unlock_page(page); + done = 1; + break; + + } + + if (!test_clear_page_dirty(page)) { + unlock_page(page); + continue; + } + + set_page_writeback(page); + unlock_page(page); + + if (wc.len == 0) { + wc.start = page->index; + wc.len = 1; + extents++; + } else if (ext3_wb_can_merge(&wc, page->index)) { + wc.len++; + } else { + /* end of current extent: flush it ... */ +#if 0 + if (wc.len < 64 && wc.len > 0) { + printk("#%u: wow! short extent %d for flush on #%lu\n", + (unsigned) current->pid, wc.len, inode->i_ino); + printk("#%u: done = %d, nr_to_write %ld, sync = %d\n", + (unsigned) current->pid, done, wbc->nr_to_write, + wbc->sync_mode); + printk("#%u: written %d, extents %d\n", + (unsigned) current->pid, written, extents); + printk("#%u: cur %lu, prev %lu\n", + (unsigned) current->pid, + (unsigned long) page->index, + (unsigned long) pindex); + } +#endif + err = ext3_wb_flush(&wc); + if (err) { + done = 1; + end_page_writeback(page); + break; + } + + /* ... and start new one */ + BUG_ON(!PageWriteback(page)); + wc.start = page->index; + wc.len = 1; + extents++; + } + + pindex = page->index; + err = ext3_wb_add_page(&wc, page); + if (err) { + done = 1; + end_page_writeback(page); + break; + } + written++; + + wbc->nr_to_write--; +#if 0 + if ((--(wbc->nr_to_write) <= 0)) + done = 1; +#endif + if (wbc->nonblocking && bdi_write_congested(bdi)) { +#ifdef EXT3_WB_STATS + struct super_block *sb = inode->i_sb; + spin_lock(&EXT3_SB(sb)->s_wb_lock); + EXT3_SB(sb)->s_wb_congested++; + spin_unlock(&EXT3_SB(sb)->s_wb_lock); +#endif + wbc->encountered_congestion = 1; + done = 1; + } + } + pagevec_release(&pvec); + } + if (!err) { +#ifdef EXT3_WB_SKIP_SMALL + if (wc.len > 0 && wc.len < 64 && wbc->sync_mode == WB_SYNC_NONE) { + struct list_head *cur, *tmp; +#if 0 + printk("#%u: wow! short extent %d for flush on #%lu (2)\n", + (unsigned) current->pid, wc.len, inode->i_ino); + printk("#%u: done = %d, nr_to_write %ld, sync = %d\n", + (unsigned) current->pid, done, wbc->nr_to_write, + wbc->sync_mode); + printk("#%u: written %d, extents %d\n", + (unsigned) current->pid, written, extents); +#endif + list_for_each_safe(cur, tmp, &wc.list) { + struct ext3_wb_pages *wp; + wp = list_entry(cur, struct ext3_wb_pages, list); + for (i = wp->start; i < wp->num; i++) { + struct page *page = wp->pages[i]; + BUG_ON(!PageWriteback(page)); + end_page_writeback(page); + __set_page_dirty_nobuffers(page); + } + wbc->nr_to_write += i; + list_del(&wp->list); + kfree(wp); + } + } else +#endif + ext3_wb_flush(&wc); + } + + atomic_dec(&EXT3_I(inode)->i_wb_writers); + +#ifdef EXT3_WB_STATS + { + struct inode *inode = mapping->host; + struct super_block *sb = inode->i_sb; + spin_lock(&EXT3_SB(sb)->s_wb_lock); + EXT3_SB(sb)->s_wb_blocks += written; + EXT3_SB(sb)->s_wb_extents += extents; + spin_unlock(&EXT3_SB(sb)->s_wb_lock); + } +#endif + return 0; +} + +static void ext3_wb_clear_page(struct page *page, int from, int to) +{ + void *kaddr; + + if (to < PAGE_CACHE_SIZE || from > 0) { + kaddr = kmap_atomic(page, KM_USER0); + if (PAGE_CACHE_SIZE > to) + memset(kaddr + to, 0, PAGE_CACHE_SIZE - to); + if (0 < from) + memset(kaddr, 0, from); + flush_dcache_page(page); + kunmap_atomic(kaddr, KM_USER0); + } +} + +int ext3_wb_prepare_write(struct file *file, struct page *page, + unsigned from, unsigned to) +{ + struct inode *inode = page->mapping->host; + struct buffer_head bh, *bhw = &bh; + int err = 0; + + wb_debug("prepare page %lu (%u-%u) for inode %lu\n", + page->index, from, to, page->mapping->host->i_ino); + + /* if page is uptodate this means that ->prepare_write() has + * been called on page before and page is mapped to disk or + * we did reservation. page is protected and nobody can + * access it. hence, it safe to use page->private to pass + * flag that ->commit_write() has to reserve blocks. because + * an error may occur after ->prepare_write() we should not + * reserve block here. it's better to do in ->commit_write() + * when we're sure page is to be written */ + page->private = 0; + if (!PageUptodate(page)) { + /* first write to this page */ + bh.b_state = 0; + err = ext3_ext_get_block(NULL, inode, page->index, bhw, 0, 0); + if (err) + return err; + if (!buffer_mapped(bhw)) { + /* this block isn't allocated yet, reserve space */ + wb_debug("reserve space for new block\n"); + page->private = 1; + ext3_wb_clear_page(page, from, to); + } else { + /* block is already mapped, so no need to reserve */ + BUG_ON(PagePrivate(page)); + if (to - from < PAGE_CACHE_SIZE) { + wb_debug("read block %lu\n", bhw->b_blocknr); + set_bh_page(bhw, page, 0); + bhw->b_this_page = 0; + bhw->b_size = 1 << inode->i_blkbits; + ll_rw_block(READ, 1, &bhw); + wait_on_buffer(bhw); + if (!buffer_uptodate(bhw)) + return -EIO; + } + SetPageMappedToDisk(page); + } + } else if (!PageMappedToDisk(page) && !PagePrivate(page)) { + /* this page was a hole at time of mmap() calling + * now someone wants to modify it by sys_write() */ + wb_debug("reserve block for hole\n"); + page->private = 1; + } + + return 0; +} + +int ext3_wb_commit_write(struct file *file, struct page *page, + unsigned from, unsigned to) +{ + loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to; + struct inode *inode = page->mapping->host; + int err = 0; + + wb_debug("commit page %lu (%u-%u) for inode %lu\n", + page->index, from, to, inode->i_ino); + + if (page->private) { + /* ->prepare_write() observed that block for this + * page hasn't been allocated yet. there fore it + * asked to reserve block for later allocation */ + page->private = 0; + err = ext3_wb_reserve_space_page(page, 1); + if (err) + return err; + } + + /* ok. block for this page is allocated already or it has + * been reserved succesfully. so, user may use it */ + __set_page_dirty_nobuffers(page); + SetPageUptodate(page); + + /* correct in-core size, not on-disk on-disk size will + * be corrected upon allocation */ + if (pos > inode->i_size) + i_size_write(inode, pos); + + return err; +} + +int ext3_wb_write_single_page(struct page *page, + struct writeback_control *wbc) +{ + struct inode *inode = page->mapping->host; + struct ext3_wb_control wc; + int err; + + atomic_inc(&EXT3_I(inode)->i_wb_writers); + +#ifdef EXT3_WB_STATS + { + spin_lock(&EXT3_SB(inode->i_sb)->s_wb_lock); + EXT3_SB(inode->i_sb)->s_wb_single_pages++; + if (atomic_read(&EXT3_I(inode)->i_wb_writers) != 1) + EXT3_SB(inode->i_sb)->s_wb_collisions_sp++; + spin_unlock(&EXT3_SB(inode->i_sb)->s_wb_lock); + } +#endif + + ext3_wb_init_control(&wc, page->mapping); + + BUG_ON(PageWriteback(page)); + set_page_writeback(page); + unlock_page(page); + + wc.start = page->index; + wc.len = 1; + + err = ext3_wb_add_page(&wc, page); + if (err) { + printk(KERN_ERR "EXT3-fs: cant add page at %s:%d - %d\n", + __FILE__, __LINE__, err); + end_page_writeback(page); + return err; + } + err = ext3_wb_flush(&wc); + atomic_dec(&EXT3_I(inode)->i_wb_writers); + + return err; +} + +int ext3_wb_writepage(struct page *page, struct writeback_control *wbc) +{ + struct inode *inode = page->mapping->host; + loff_t i_size = i_size_read(inode); + pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT; + unsigned offset; + void *kaddr; + + wb_debug("writepage %lu from inode %lu\n", page->index, inode->i_ino); + + /* + * FIXME: just to play ... + * If another thread is writing inode's data and the page + * hasn't space on a disk yet, leave it for that thread + */ +#if 1 + if (atomic_read(&EXT3_I(page->mapping->host)->i_wb_writers) + && !PageMappedToDisk(page)) { + __set_page_dirty_nobuffers(page); + unlock_page(page); + return 0; + } +#endif + + /* we give up here if we're reentered, because + * it might be for a different filesystem */ + if (ext3_journal_current_handle()) { + __set_page_dirty_nobuffers(page); + unlock_page(page); + return 0; + } + + /* Is the page fully inside i_size? */ + if (page->index < end_index) + return ext3_wb_write_single_page(page, wbc); + + /* Is the page fully outside i_size? (truncate in progress) */ + offset = i_size & (PAGE_CACHE_SIZE-1); + if (page->index >= end_index + 1 || !offset) { + /* + * The page may have dirty, unmapped buffers. For example, + * they may have been added in ext3_writepage(). Make them + * freeable here, so the page does not leak. + */ + ext3_wb_invalidatepage(page, 0); + unlock_page(page); + return 0; /* don't care */ + } + + /* + * The page straddles i_size. It must be zeroed out on each and every + * writepage invocation because it may be mmapped. "A file is mapped + * in multiples of the page size. For a file that is not a multiple of + * the page size, the remaining memory is zeroed when mapped, and + * writes to that region are not written out to the file." + */ + kaddr = kmap_atomic(page, KM_USER0); + memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset); + flush_dcache_page(page); + kunmap_atomic(kaddr, KM_USER0); + return ext3_wb_write_single_page(page, wbc); +} + +int ext3_wb_releasepage(struct page *page, int wait) +{ + wb_debug("release page %lu from inode %lu (wait %d)\n", + page->index, page->mapping->host->i_ino, wait); + + if (PageWriteback(page)) + return 0; + + if (PagePrivate(page)) { + BUG_ON(1); + return 1; + } + return 0; +} + +int ext3_wb_invalidatepage(struct page *page, unsigned long offset) +{ + struct inode *inode = page->mapping->host; + int ret = 0; + + /* ->invalidatepage() is called when page is marked Private. + * for our page being Private mean that space has been + * reserved for this page and it is being truncated. so, + * it's time to drop reservation */ + wb_debug("invalidate page %lu from inode %lu (offset %lu)\n", + page->index, inode->i_ino, offset); + if (offset == 0) { + ext3_wb_release_space(inode, 1); + ext3_wb_drop_reservation(page, 1); + ret = try_to_release_page(page, 0); + } + return ret; +} + +int ext3_wb_block_truncate_page(handle_t *handle, struct page *page, + struct address_space *mapping, loff_t from) +{ + unsigned offset = from & (PAGE_CACHE_SIZE-1); + struct inode *inode = mapping->host; + struct buffer_head bh, *bhw = &bh; + unsigned blocksize, length; + void *kaddr; + int err = 0; + + wb_debug("partial truncate from %lu on page %lu from inode %lu\n", + (unsigned long) from, page->index, inode->i_ino); + + blocksize = inode->i_sb->s_blocksize; + length = blocksize - (offset & (blocksize - 1)); + + /* if page isn't uptodate we have to check has it assigned block + * if it has then that block is to be read before memset() */ + if (!PageUptodate(page)) { + BUG_ON(PageMappedToDisk(page)); + bh.b_state = 0; + err = ext3_ext_get_block(NULL, inode, page->index, bhw, 0, 0); + if (err) + goto err_out; + BUG_ON(buffer_new(bhw)); + if (buffer_mapped(bhw)) { + /* time to retrieve data from a disk */ + wb_debug("read block %lu for part.trunc on %lu\n", + bhw->b_blocknr, page->index); + set_bh_page(bhw, page, 0); + bhw->b_this_page = 0; + bhw->b_size = 1 << inode->i_blkbits; + ll_rw_block(READ, 1, &bhw); + wait_on_buffer(bhw); + err = -EIO; + if (!buffer_uptodate(bhw)) + goto err_out; + SetPageMappedToDisk(page); + } else { + wb_debug("zero page %lu (part.trunc)\n", page->index); + offset = 0; + length = blocksize; + } + } + + kaddr = kmap_atomic(page, KM_USER0); + memset(kaddr + offset, 0, length); + flush_dcache_page(page); + kunmap_atomic(kaddr, KM_USER0); + SetPageUptodate(page); + __set_page_dirty_nobuffers(page); + +err_out: + unlock_page(page); + page_cache_release(page); + return err; +} + +void ext3_wb_init(struct super_block *sb) +{ + if (!test_opt(sb, DELAYED_ALLOC)) + return; + + if (PAGE_CACHE_SHIFT != sb->s_blocksize_bits) { + printk(KERN_ERR "EXT3-fs: delayed allocation isn't" + "supported for PAGE_CACHE_SIZE != blocksize yet\n"); + clear_opt (EXT3_SB(sb)->s_mount_opt, DELAYED_ALLOC); + return; + } + printk("EXT3-fs: delayed allocation enabled\n"); +#ifdef EXT3_WB_STATS + spin_lock_init(&EXT3_SB(sb)->s_wb_lock); +#endif +} + +void ext3_wb_release(struct super_block *sb) +{ + struct ext3_sb_info *sbi = EXT3_SB(sb); + + if (!test_opt(sb, DELAYED_ALLOC)) + return; + + if (sbi->s_wb_reqs == 0) + return; +#ifdef EXT3_WB_STATS + printk("EXT3-fs: writeback: %lu blocks %lu extents in %lu reqs (%lu ave)\n", + sbi->s_wb_blocks, sbi->s_wb_extents, sbi->s_wb_reqs, + sbi->s_wb_blocks / sbi->s_wb_reqs); + printk("EXT3-fs: writeback: %lu nr_to_write, %lu congestions, %lu singles\n", + sbi->s_wb_nr_to_write, sbi->s_wb_congested, + sbi->s_wb_single_pages); + printk("EXT3-fs: writeback: %lu collisions, %lu single-page collisions\n", + sbi->s_wb_collisions, sbi->s_wb_collisions_sp); + printk("EXT3-fs: writeback: %lu allocated\n", + sbi->s_wb_allocated); +#endif +} + Index: linux-2.6.11/fs/ext3/super.c =================================================================== --- linux-2.6.11.orig/fs/ext3/super.c 2005-03-05 19:37:01.000000000 +0300 +++ linux-2.6.11/fs/ext3/super.c 2005-03-05 19:37:02.000000000 +0300 @@ -377,6 +377,7 @@ struct ext3_super_block *es = sbi->s_es; int i; + ext3_wb_release(sb); ext3_mb_release(sb); ext3_ext_release(sb); ext3_xattr_put_super(sb); @@ -446,6 +447,13 @@ ei->i_rsv_window.rsv_end = EXT3_RESERVE_WINDOW_NOT_ALLOCATED; ei->vfs_inode.i_version = 1; memset(&ei->i_cached_extent, 0, sizeof(struct ext3_ext_cache)); + + /* FIXME: these wb-related fields could be initialized once */ + ei->i_blocks_reserved = 0; + ei->i_md_reserved = 0; + atomic_set(&ei->i_wb_writers, 0); + spin_lock_init(&ei->i_wb_reserved_lock); + return &ei->vfs_inode; } @@ -584,6 +592,7 @@ Opt_abort, Opt_data_journal, Opt_data_ordered, Opt_data_writeback, Opt_usrjquota, Opt_grpjquota, Opt_offusrjquota, Opt_offgrpjquota, Opt_jqfmt_vfsold, Opt_jqfmt_vfsv0, Opt_mballoc, Opt_mbfactor, + Opt_delayed_alloc, Opt_ignore, Opt_barrier, Opt_err, Opt_resize, Opt_extents, Opt_extdebug, }; @@ -636,6 +645,7 @@ {Opt_extdebug, "extdebug"}, {Opt_mballoc, "mballoc"}, {Opt_mballoc, "mbfactor=%u"}, + {Opt_delayed_alloc, "delalloc"}, {Opt_err, NULL}, {Opt_resize, "resize"}, }; @@ -920,6 +930,9 @@ else clear_opt(sbi->s_mount_opt, BARRIER); break; + case Opt_delayed_alloc: + set_opt(sbi->s_mount_opt, DELAYED_ALLOC); + break; case Opt_ignore: break; case Opt_resize: @@ -1634,6 +1647,7 @@ ext3_ext_init(sb); ext3_mb_init(sb, needs_recovery); + ext3_wb_init(sb); lock_kernel(); return 0; Index: linux-2.6.11/fs/ext3/Makefile =================================================================== --- linux-2.6.11.orig/fs/ext3/Makefile 2005-03-05 19:37:01.000000000 +0300 +++ linux-2.6.11/fs/ext3/Makefile 2005-03-05 19:37:02.000000000 +0300 @@ -6,7 +6,7 @@ ext3-y := balloc.o bitmap.o dir.o file.o fsync.o ialloc.o inode.o \ ioctl.o namei.o super.o symlink.o hash.o resize.o extents.o \ - mballoc.o + mballoc.o writeback.o ext3-$(CONFIG_EXT3_FS_XATTR) += xattr.o xattr_user.o xattr_trusted.o ext3-$(CONFIG_EXT3_FS_POSIX_ACL) += acl.o Index: linux-2.6.11/include/linux/ext3_fs.h =================================================================== --- linux-2.6.11.orig/include/linux/ext3_fs.h 2005-03-05 19:37:01.000000000 +0300 +++ linux-2.6.11/include/linux/ext3_fs.h 2005-03-05 19:37:02.000000000 +0300 @@ -372,6 +372,7 @@ #define EXT3_MOUNT_EXTENTS 0x40000 /* Extents support */ #define EXT3_MOUNT_EXTDEBUG 0x80000 /* Extents debug */ #define EXT3_MOUNT_MBALLOC 0x100000/* Buddy allocation support */ +#define EXT3_MOUNT_DELAYED_ALLOC 0x200000/* Delayed allocation support */ /* Compatibility, for having both ext2_fs.h and ext3_fs.h included at once */ #ifndef _LINUX_EXT2_FS_H @@ -883,6 +884,18 @@ extern void ext3_wb_init(struct super_block *); extern void ext3_wb_release(struct super_block *); +/* writeback.c */ +extern int ext3_wb_writepages(struct address_space *, struct writeback_control *); +extern int ext3_wb_prepare_write(struct file *file, struct page *page, + unsigned from, unsigned to); +extern int ext3_wb_commit_write(struct file *, struct page *, unsigned, unsigned); +extern int ext3_wb_writepage(struct page *, struct writeback_control *); +extern int ext3_wb_invalidatepage(struct page *, unsigned long); +extern int ext3_wb_releasepage(struct page *, int); +extern int ext3_wb_block_truncate_page(handle_t *, struct page *, struct address_space *, loff_t); +extern void ext3_wb_init(struct super_block *); +extern void ext3_wb_release(struct super_block *); + #endif /* __KERNEL__ */ #endif /* _LINUX_EXT3_FS_H */ Index: linux-2.6.11/include/linux/ext3_fs_i.h =================================================================== --- linux-2.6.11.orig/include/linux/ext3_fs_i.h 2005-03-05 19:37:00.000000000 +0300 +++ linux-2.6.11/include/linux/ext3_fs_i.h 2005-03-05 19:37:02.000000000 +0300 @@ -131,6 +131,11 @@ struct inode vfs_inode; struct ext3_ext_cache i_cached_extent; + + __u32 i_blocks_reserved; + __u32 i_md_reserved; + spinlock_t i_wb_reserved_lock; /* to protect i_md_reserved */ + atomic_t i_wb_writers; }; #endif /* _LINUX_EXT3_FS_I */ Index: linux-2.6.11/include/linux/ext3_fs_sb.h =================================================================== --- linux-2.6.11.orig/include/linux/ext3_fs_sb.h 2005-03-05 19:37:01.000000000 +0300 +++ linux-2.6.11/include/linux/ext3_fs_sb.h 2005-03-05 19:37:02.000000000 +0300 @@ -127,6 +127,17 @@ unsigned long s_bal_ex_scanned; /* total extents scanned */ unsigned long s_bal_goals; /* goal hits */ unsigned long s_bal_breaks; /* too long searches */ + + unsigned long s_wb_reqs; + unsigned long s_wb_nr_to_write; + unsigned long s_wb_blocks; + unsigned long s_wb_extents; + unsigned long s_wb_congested; + unsigned long s_wb_single_pages; + unsigned long s_wb_collisions; + unsigned long s_wb_collisions_sp; + unsigned long s_wb_allocated; + spinlock_t s_wb_lock; }; #endif /* _LINUX_EXT3_FS_SB */