Currently, although fragment compression is already multi-threaded,
the data parts of inodes prior to their own fragments are still
single-threaded if fragment deduplication is on. This can greatly
slow down `-Eall-fragments` image building at least for the current
mkfs codebase.
Let's add an extended option `-E^fragdedupe` to explicitly disable it.
After this commit, the Fedora Kiwi builds I'm testing can be reduced
from 1148s (3,096,842,240 bytes, 2.9G) to 137s (2,969,956,352 bytes,
2.8G) with `-Eall-fragments,^fragdedupe -C524288 -z lzma,level=6,
dictsize=524288` on Intel(R) Xeon(R) Platinum 8269CY CPU @ 2.50GHz with
32 cores.
Cc: Neal Gompa <ngomp...@gmail.com>
Signed-off-by: Gao Xiang <hsiang...@linux.alibaba.com>
---
Anyway, it's just a quick fix for faster image builds, but I think
`-E^fragdedupe` is also useful in the future, since it may produce
smaller builds if `-Ededupe` is off.
include/erofs/config.h | 1 +
lib/compress.c | 3 ++-
mkfs/main.c | 10 ++++++++++
3 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index bb03e70..47e4d00 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -53,6 +53,7 @@ struct erofs_configure {
bool c_fragments;
bool c_all_fragments;
bool c_dedupe;
+ bool c_nofragdedupe;
bool c_ignore_mtime;
bool c_showprogress;
bool c_extra_ea_name_prefixes;
diff --git a/lib/compress.c b/lib/compress.c
index 917916e..6ac9c75 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1527,7 +1527,8 @@ void *erofs_begin_compressed_file(struct erofs_inode
*inode, int fd, u64 fpos)
* Handle tails in advance to avoid writing duplicated
* parts into the packed inode.
*/
- if (cfg.c_fragments && !erofs_is_packed_inode(inode)) {
+ if (cfg.c_fragments && !erofs_is_packed_inode(inode) &&
+ !cfg.c_nofragdedupe) {
ret = z_erofs_fragments_dedupe(inode, fd, &ictx->tof_chksum);
if (ret < 0)
goto err_free_ictx;
diff --git a/mkfs/main.c b/mkfs/main.c
index 1f4b7c6..7d559d9 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -306,6 +306,15 @@ static int erofs_mkfs_feat_set_dedupe(bool en, const char
*val,
return 0;
}
+static int erofs_mkfs_feat_set_fragdedupe(bool en, const char *val,
+ unsigned int vallen)
+{
+ if (vallen)
+ return -EINVAL;
+ cfg.c_nofragdedupe = !en;
+ return 0;
+}
+
static struct {
char *feat;
int (*set)(bool en, const char *val, unsigned int len);
@@ -315,6 +324,7 @@ static struct {
{"fragments", erofs_mkfs_feat_set_fragments},
{"all-fragments", erofs_mkfs_feat_set_all_fragments},
{"dedupe", erofs_mkfs_feat_set_dedupe},
+ {"fragdedupe", erofs_mkfs_feat_set_fragdedupe},
{NULL, NULL},
};
--
2.43.5
