On Fri, Jun 27, 2025 at 09:02:39AM +0200, Christoph Hellwig wrote:
> Now that the writeback code has the proper abstractions, all the ioend
> code can be self-contained in ioend.c.
>
> Signed-off-by: Christoph Hellwig <hch@xxxxxx>
> Reviewed-by: Joanne Koong <joannelkoong@xxxxxxxxx>
Looks ok,
Reviewed-by: "Darrick J. Wong" <djwong@xxxxxxxxxx>
--D
> ---
> fs/iomap/buffered-io.c | 215 ----------------------------------------
> fs/iomap/internal.h | 1 -
> fs/iomap/ioend.c | 220 ++++++++++++++++++++++++++++++++++++++++-
> 3 files changed, 219 insertions(+), 217 deletions(-)
>
> diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
> index d152456d41a8..3e0ce6f42df5 100644
> --- a/fs/iomap/buffered-io.c
> +++ b/fs/iomap/buffered-io.c
> @@ -1559,221 +1559,6 @@ void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
> }
> EXPORT_SYMBOL_GPL(iomap_finish_folio_write);
>
> -/*
> - * We're now finished for good with this ioend structure. Update the page
> - * state, release holds on bios, and finally free up memory. Do not use the
> - * ioend after this.
> - */
> -u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
> -{
> - struct inode *inode = ioend->io_inode;
> - struct bio *bio = &ioend->io_bio;
> - struct folio_iter fi;
> - u32 folio_count = 0;
> -
> - if (ioend->io_error) {
> - mapping_set_error(inode->i_mapping, ioend->io_error);
> - if (!bio_flagged(bio, BIO_QUIET)) {
> - pr_err_ratelimited(
> -"%s: writeback error on inode %lu, offset %lld, sector %llu",
> - inode->i_sb->s_id, inode->i_ino,
> - ioend->io_offset, ioend->io_sector);
> - }
> - }
> -
> - /* walk all folios in bio, ending page IO on them */
> - bio_for_each_folio_all(fi, bio) {
> - iomap_finish_folio_write(inode, fi.folio, fi.length);
> - folio_count++;
> - }
> -
> - bio_put(bio); /* frees the ioend */
> - return folio_count;
> -}
> -
> -static void ioend_writeback_end_bio(struct bio *bio)
> -{
> - struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
> -
> - ioend->io_error = blk_status_to_errno(bio->bi_status);
> - iomap_finish_ioend_buffered(ioend);
> -}
> -
> -/*
> - * We cannot cancel the ioend directly in case of an error, so call the bio end
> - * I/O handler with the error status here to run the normal I/O completion
> - * handler.
> - */
> -int ioend_writeback_submit(struct iomap_writeback_ctx *wpc, int error)
> -{
> - struct iomap_ioend *ioend = wpc->wb_ctx;
> -
> - if (!ioend->io_bio.bi_end_io)
> - ioend->io_bio.bi_end_io = ioend_writeback_end_bio;
> -
> - if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE))
> - error = -EIO;
> -
> - if (error) {
> - ioend->io_bio.bi_status = errno_to_blk_status(error);
> - bio_endio(&ioend->io_bio);
> - return error;
> - }
> -
> - submit_bio(&ioend->io_bio);
> - return 0;
> -}
> -EXPORT_SYMBOL_GPL(ioend_writeback_submit);
> -
> -static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writeback_ctx *wpc,
> - loff_t pos, u16 ioend_flags)
> -{
> - struct bio *bio;
> -
> - bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
> - REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc),
> - GFP_NOFS, &iomap_ioend_bioset);
> - bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
> - bio->bi_write_hint = wpc->inode->i_write_hint;
> - wbc_init_bio(wpc->wbc, bio);
> - wpc->nr_folios = 0;
> - return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags);
> -}
> -
> -static bool iomap_can_add_to_ioend(struct iomap_writeback_ctx *wpc, loff_t pos,
> - u16 ioend_flags)
> -{
> - struct iomap_ioend *ioend = wpc->wb_ctx;
> -
> - if (ioend_flags & IOMAP_IOEND_BOUNDARY)
> - return false;
> - if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
> - (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
> - return false;
> - if (pos != ioend->io_offset + ioend->io_size)
> - return false;
> - if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) &&
> - iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio))
> - return false;
> - /*
> - * Limit ioend bio chain lengths to minimise IO completion latency. This
> - * also prevents long tight loops ending page writeback on all the
> - * folios in the ioend.
> - */
> - if (wpc->nr_folios >= IOEND_BATCH_SIZE)
> - return false;
> - return true;
> -}
> -
> -/*
> - * Test to see if we have an existing ioend structure that we could append to
> - * first; otherwise finish off the current ioend and start another.
> - *
> - * If a new ioend is created and cached, the old ioend is submitted to the block
> - * layer instantly. Batching optimisations are provided by higher level block
> - * plugging.
> - *
> - * At the end of a writeback pass, there will be a cached ioend remaining on the
> - * writepage context that the caller will need to submit.
> - */
> -ssize_t iomap_add_to_ioend(struct iomap_writeback_ctx *wpc, struct folio *folio,
> - loff_t pos, loff_t end_pos, unsigned int dirty_len)
> -{
> - struct iomap_ioend *ioend = wpc->wb_ctx;
> - size_t poff = offset_in_folio(folio, pos);
> - unsigned int ioend_flags = 0;
> - unsigned int map_len = min_t(u64, dirty_len,
> - wpc->iomap.offset + wpc->iomap.length - pos);
> - int error;
> -
> - trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap);
> -
> - WARN_ON_ONCE(!folio->private && map_len < dirty_len);
> -
> - switch (wpc->iomap.type) {
> - case IOMAP_INLINE:
> - WARN_ON_ONCE(1);
> - return -EIO;
> - case IOMAP_HOLE:
> - return map_len;
> - default:
> - break;
> - }
> -
> - if (wpc->iomap.type == IOMAP_UNWRITTEN)
> - ioend_flags |= IOMAP_IOEND_UNWRITTEN;
> - if (wpc->iomap.flags & IOMAP_F_SHARED)
> - ioend_flags |= IOMAP_IOEND_SHARED;
> - if (folio_test_dropbehind(folio))
> - ioend_flags |= IOMAP_IOEND_DONTCACHE;
> - if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
> - ioend_flags |= IOMAP_IOEND_BOUNDARY;
> -
> - if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) {
> -new_ioend:
> - if (ioend) {
> - error = wpc->ops->writeback_submit(wpc, 0);
> - if (error)
> - return error;
> - }
> - wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags);
> - }
> -
> - if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff))
> - goto new_ioend;
> -
> - iomap_start_folio_write(wpc->inode, folio, map_len);
> -
> - /*
> - * Clamp io_offset and io_size to the incore EOF so that ondisk
> - * file size updates in the ioend completion are byte-accurate.
> - * This avoids recovering files with zeroed tail regions when
> - * writeback races with appending writes:
> - *
> - * Thread 1: Thread 2:
> - * ------------ -----------
> - * write [A, A+B]
> - * update inode size to A+B
> - * submit I/O [A, A+BS]
> - * write [A+B, A+B+C]
> - * update inode size to A+B+C
> - * <I/O completes, updates disk size to min(A+B+C, A+BS)>
> - * <power failure>
> - *
> - * After reboot:
> - * 1) with A+B+C < A+BS, the file has zero padding in range
> - * [A+B, A+B+C]
> - *
> - * |< Block Size (BS) >|
> - * |DDDDDDDDDDDD0000000000000|
> - * ^ ^ ^
> - * A A+B A+B+C
> - * (EOF)
> - *
> - * 2) with A+B+C > A+BS, the file has zero padding in range
> - * [A+B, A+BS]
> - *
> - * |< Block Size (BS) >|< Block Size (BS) >|
> - * |DDDDDDDDDDDD0000000000000|00000000000000000000000000|
> - * ^ ^ ^ ^
> - * A A+B A+BS A+B+C
> - * (EOF)
> - *
> - * D = Valid Data
> - * 0 = Zero Padding
> - *
> - * Note that this defeats the ability to chain the ioends of
> - * appending writes.
> - */
> - ioend->io_size += map_len;
> - if (ioend->io_offset + ioend->io_size > end_pos)
> - ioend->io_size = end_pos - ioend->io_offset;
> -
> - wbc_account_cgroup_owner(wpc->wbc, folio, map_len);
> - return map_len;
> -}
> -EXPORT_SYMBOL_GPL(iomap_add_to_ioend);
> -
> static int iomap_writeback_range(struct iomap_writeback_ctx *wpc,
> struct folio *folio, u64 pos, u32 rlen, u64 end_pos,
> bool *wb_pending)
> diff --git a/fs/iomap/internal.h b/fs/iomap/internal.h
> index f6992a3bf66a..d05cb3aed96e 100644
> --- a/fs/iomap/internal.h
> +++ b/fs/iomap/internal.h
> @@ -4,7 +4,6 @@
>
> #define IOEND_BATCH_SIZE 4096
>
> -u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend);
> u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend);
>
> #endif /* _IOMAP_INTERNAL_H */
> diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c
> index 18894ebba6db..ce0a4c13d008 100644
> --- a/fs/iomap/ioend.c
> +++ b/fs/iomap/ioend.c
> @@ -1,10 +1,13 @@
> // SPDX-License-Identifier: GPL-2.0
> /*
> - * Copyright (c) 2024-2025 Christoph Hellwig.
> + * Copyright (c) 2016-2025 Christoph Hellwig.
> */
> #include <linux/iomap.h>
> #include <linux/list_sort.h>
> +#include <linux/pagemap.h>
> +#include <linux/writeback.h>
> #include "internal.h"
> +#include "trace.h"
>
> struct bio_set iomap_ioend_bioset;
> EXPORT_SYMBOL_GPL(iomap_ioend_bioset);
> @@ -28,6 +31,221 @@ struct iomap_ioend *iomap_init_ioend(struct inode *inode,
> }
> EXPORT_SYMBOL_GPL(iomap_init_ioend);
>
> +/*
> + * We're now finished for good with this ioend structure. Update the folio
> + * state, release holds on bios, and finally free up memory. Do not use the
> + * ioend after this.
> + */
> +static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
> +{
> + struct inode *inode = ioend->io_inode;
> + struct bio *bio = &ioend->io_bio;
> + struct folio_iter fi;
> + u32 folio_count = 0;
> +
> + if (ioend->io_error) {
> + mapping_set_error(inode->i_mapping, ioend->io_error);
> + if (!bio_flagged(bio, BIO_QUIET)) {
> + pr_err_ratelimited(
> +"%s: writeback error on inode %lu, offset %lld, sector %llu",
> + inode->i_sb->s_id, inode->i_ino,
> + ioend->io_offset, ioend->io_sector);
> + }
> + }
> +
> + /* walk all folios in bio, ending page IO on them */
> + bio_for_each_folio_all(fi, bio) {
> + iomap_finish_folio_write(inode, fi.folio, fi.length);
> + folio_count++;
> + }
> +
> + bio_put(bio); /* frees the ioend */
> + return folio_count;
> +}
> +
> +static void ioend_writeback_end_bio(struct bio *bio)
> +{
> + struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
> +
> + ioend->io_error = blk_status_to_errno(bio->bi_status);
> + iomap_finish_ioend_buffered(ioend);
> +}
> +
> +/*
> + * We cannot cancel the ioend directly in case of an error, so call the bio end
> + * I/O handler with the error status here to run the normal I/O completion
> + * handler.
> + */
> +int ioend_writeback_submit(struct iomap_writeback_ctx *wpc, int error)
> +{
> + struct iomap_ioend *ioend = wpc->wb_ctx;
> +
> + if (!ioend->io_bio.bi_end_io)
> + ioend->io_bio.bi_end_io = ioend_writeback_end_bio;
> +
> + if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE))
> + error = -EIO;
> +
> + if (error) {
> + ioend->io_bio.bi_status = errno_to_blk_status(error);
> + bio_endio(&ioend->io_bio);
> + return error;
> + }
> +
> + submit_bio(&ioend->io_bio);
> + return 0;
> +}
> +EXPORT_SYMBOL_GPL(ioend_writeback_submit);
> +
> +static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writeback_ctx *wpc,
> + loff_t pos, u16 ioend_flags)
> +{
> + struct bio *bio;
> +
> + bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
> + REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc),
> + GFP_NOFS, &iomap_ioend_bioset);
> + bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
> + bio->bi_write_hint = wpc->inode->i_write_hint;
> + wbc_init_bio(wpc->wbc, bio);
> + wpc->nr_folios = 0;
> + return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags);
> +}
> +
> +static bool iomap_can_add_to_ioend(struct iomap_writeback_ctx *wpc, loff_t pos,
> + u16 ioend_flags)
> +{
> + struct iomap_ioend *ioend = wpc->wb_ctx;
> +
> + if (ioend_flags & IOMAP_IOEND_BOUNDARY)
> + return false;
> + if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
> + (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
> + return false;
> + if (pos != ioend->io_offset + ioend->io_size)
> + return false;
> + if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) &&
> + iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio))
> + return false;
> + /*
> + * Limit ioend bio chain lengths to minimise IO completion latency. This
> + * also prevents long tight loops ending page writeback on all the
> + * folios in the ioend.
> + */
> + if (wpc->nr_folios >= IOEND_BATCH_SIZE)
> + return false;
> + return true;
> +}
> +
> +/*
> + * Test to see if we have an existing ioend structure that we could append to
> + * first; otherwise finish off the current ioend and start another.
> + *
> + * If a new ioend is created and cached, the old ioend is submitted to the block
> + * layer instantly. Batching optimisations are provided by higher level block
> + * plugging.
> + *
> + * At the end of a writeback pass, there will be a cached ioend remaining on the
> + * writepage context that the caller will need to submit.
> + */
> +ssize_t iomap_add_to_ioend(struct iomap_writeback_ctx *wpc, struct folio *folio,
> + loff_t pos, loff_t end_pos, unsigned int dirty_len)
> +{
> + struct iomap_ioend *ioend = wpc->wb_ctx;
> + size_t poff = offset_in_folio(folio, pos);
> + unsigned int ioend_flags = 0;
> + unsigned int map_len = min_t(u64, dirty_len,
> + wpc->iomap.offset + wpc->iomap.length - pos);
> + int error;
> +
> + trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap);
> +
> + WARN_ON_ONCE(!folio->private && map_len < dirty_len);
> +
> + switch (wpc->iomap.type) {
> + case IOMAP_INLINE:
> + WARN_ON_ONCE(1);
> + return -EIO;
> + case IOMAP_HOLE:
> + return map_len;
> + default:
> + break;
> + }
> +
> + if (wpc->iomap.type == IOMAP_UNWRITTEN)
> + ioend_flags |= IOMAP_IOEND_UNWRITTEN;
> + if (wpc->iomap.flags & IOMAP_F_SHARED)
> + ioend_flags |= IOMAP_IOEND_SHARED;
> + if (folio_test_dropbehind(folio))
> + ioend_flags |= IOMAP_IOEND_DONTCACHE;
> + if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
> + ioend_flags |= IOMAP_IOEND_BOUNDARY;
> +
> + if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) {
> +new_ioend:
> + if (ioend) {
> + error = wpc->ops->writeback_submit(wpc, 0);
> + if (error)
> + return error;
> + }
> + wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags);
> + }
> +
> + if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff))
> + goto new_ioend;
> +
> + iomap_start_folio_write(wpc->inode, folio, map_len);
> +
> + /*
> + * Clamp io_offset and io_size to the incore EOF so that ondisk
> + * file size updates in the ioend completion are byte-accurate.
> + * This avoids recovering files with zeroed tail regions when
> + * writeback races with appending writes:
> + *
> + * Thread 1: Thread 2:
> + * ------------ -----------
> + * write [A, A+B]
> + * update inode size to A+B
> + * submit I/O [A, A+BS]
> + * write [A+B, A+B+C]
> + * update inode size to A+B+C
> + * <I/O completes, updates disk size to min(A+B+C, A+BS)>
> + * <power failure>
> + *
> + * After reboot:
> + * 1) with A+B+C < A+BS, the file has zero padding in range
> + * [A+B, A+B+C]
> + *
> + * |< Block Size (BS) >|
> + * |DDDDDDDDDDDD0000000000000|
> + * ^ ^ ^
> + * A A+B A+B+C
> + * (EOF)
> + *
> + * 2) with A+B+C > A+BS, the file has zero padding in range
> + * [A+B, A+BS]
> + *
> + * |< Block Size (BS) >|< Block Size (BS) >|
> + * |DDDDDDDDDDDD0000000000000|00000000000000000000000000|
> + * ^ ^ ^ ^
> + * A A+B A+BS A+B+C
> + * (EOF)
> + *
> + * D = Valid Data
> + * 0 = Zero Padding
> + *
> + * Note that this defeats the ability to chain the ioends of
> + * appending writes.
> + */
> + ioend->io_size += map_len;
> + if (ioend->io_offset + ioend->io_size > end_pos)
> + ioend->io_size = end_pos - ioend->io_offset;
> +
> + wbc_account_cgroup_owner(wpc->wbc, folio, map_len);
> + return map_len;
> +}
> +EXPORT_SYMBOL_GPL(iomap_add_to_ioend);
> +
> static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
> {
> if (ioend->io_parent) {
> --
> 2.47.2
>
>
