[PATCH 4/4] erofs: use buffered I/O for file-backed mounts by default

[Gao Xiang]

For many use cases (e.g. container images are just fetched from remote),
performance will be impacted if underlay page cache is up-to-date but
direct i/o will flush dirty pages first.

Instead, let's use buffered I/O by default and add a (re)mount option to
explicitly enable direct I/O if underlay files are supported too.

The startup time for containerd is improved as below:
                                     unpack        1st run  non-1st runs
EROFS snapshotter buffered I/O file  4.586404265s  0.308s   0.198s
EROFS snapshotter direct I/O file    4.581742849s  2.238s   0.222s
EROFS snapshotter loop               4.596023152s  0.346s   0.201s
Overlayfs snapshotter                5.382851037s  0.206s   0.214s

Fixes: fb176750266a ("erofs: add file-backed mount support")
Cc: Derek McGowan <de...@mcg.dev>
Signed-off-by: Gao Xiang <hsiang...@linux.alibaba.com>
---
 fs/erofs/fileio.c   |  7 +++++--
 fs/erofs/internal.h |  1 +
 fs/erofs/super.c    | 21 +++++++++++++--------
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c
index a61b8faec651..33f8539dda4a 100644
--- a/fs/erofs/fileio.c
+++ b/fs/erofs/fileio.c
@@ -9,6 +9,7 @@ struct erofs_fileio_rq {
        struct bio_vec bvecs[BIO_MAX_VECS];
        struct bio bio;
        struct kiocb iocb;
+       struct super_block *sb;
 };
 
 struct erofs_fileio {
@@ -52,8 +53,9 @@ static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq)
        rq->iocb.ki_pos = rq->bio.bi_iter.bi_sector << SECTOR_SHIFT;
        rq->iocb.ki_ioprio = get_current_ioprio();
        rq->iocb.ki_complete = erofs_fileio_ki_complete;
-       rq->iocb.ki_flags = (rq->iocb.ki_filp->f_mode & FMODE_CAN_ODIRECT) ?
-                               IOCB_DIRECT : 0;
+       if (test_opt(&EROFS_SB(rq->sb)->opt, DIRECT_IO) &&
+           rq->iocb.ki_filp->f_mode & FMODE_CAN_ODIRECT)
+               rq->iocb.ki_flags = IOCB_DIRECT;
        iov_iter_bvec(&iter, ITER_DEST, rq->bvecs, rq->bio.bi_vcnt,
                      rq->bio.bi_iter.bi_size);
        ret = vfs_iocb_iter_read(rq->iocb.ki_filp, &rq->iocb, &iter);
@@ -68,6 +70,7 @@ static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct 
erofs_map_dev *mdev)
 
        bio_init(&rq->bio, NULL, rq->bvecs, BIO_MAX_VECS, REQ_OP_READ);
        rq->iocb.ki_filp = mdev->m_dif->file;
+       rq->sb = mdev->m_sb;
        return rq;
 }
 
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 59e7739b7c3b..bc46000ccc6b 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -177,6 +177,7 @@ struct erofs_sb_info {
 #define EROFS_MOUNT_POSIX_ACL          0x00000020
 #define EROFS_MOUNT_DAX_ALWAYS         0x00000040
 #define EROFS_MOUNT_DAX_NEVER          0x00000080
+#define EROFS_MOUNT_DIRECT_IO          0x00000100
 
 #define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option)
 #define set_opt(opt, option)   ((opt)->mount_opt |= EROFS_MOUNT_##option)
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 934fb1513095..cbb9566cc6dc 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -364,14 +364,8 @@ static void erofs_default_options(struct erofs_sb_info 
*sbi)
 }
 
 enum {
-       Opt_user_xattr,
-       Opt_acl,
-       Opt_cache_strategy,
-       Opt_dax,
-       Opt_dax_enum,
-       Opt_device,
-       Opt_fsid,
-       Opt_domain_id,
+       Opt_user_xattr, Opt_acl, Opt_cache_strategy, Opt_dax, Opt_dax_enum,
+       Opt_device, Opt_fsid, Opt_domain_id, Opt_directio,
        Opt_err
 };
 
@@ -398,6 +392,7 @@ static const struct fs_parameter_spec erofs_fs_parameters[] 
= {
        fsparam_string("device",        Opt_device),
        fsparam_string("fsid",          Opt_fsid),
        fsparam_string("domain_id",     Opt_domain_id),
+       fsparam_flag_no("directio",     Opt_directio),
        {}
 };
 
@@ -511,6 +506,16 @@ static int erofs_fc_parse_param(struct fs_context *fc,
                errorfc(fc, "%s option not supported", 
erofs_fs_parameters[opt].name);
                break;
 #endif
+       case Opt_directio:
+#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE
+               if (result.boolean)
+                       set_opt(&sbi->opt, DIRECT_IO);
+               else
+                       clear_opt(&sbi->opt, DIRECT_IO);
+#else
+               errorfc(fc, "%s option not supported", 
erofs_fs_parameters[opt].name);
+#endif
+               break;
        default:
                return -ENOPARAM;
        }
-- 
2.43.5