From: zhaoyifan <zhaoyifa...@huawei.com> This patch introduces configuration options for the upcoming experimental S3 support, including configuration parsing and passwd_file reading logic.
User could specify the following options: - S3 service endpoint (Compulsory) - S3 credentials file, in the format of "$ak:%sk" (Optional) - S3 API calling style (Optional) - S3 API signature version, only sigV2 supported yet (Optional) Signed-off-by: Yifan Zhao <zhaoyifa...@huawei.com> --- change since v1: - rename: include/erofs/s3.h => lib/liberofs_s3.h - add liberofs_s3.h in this patch rather than previous one lib/liberofs_s3.h | 40 +++++++++ lib/remotes/s3.c | 3 +- mkfs/main.c | 220 ++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 233 insertions(+), 30 deletions(-) create mode 100644 lib/liberofs_s3.h diff --git a/lib/liberofs_s3.h b/lib/liberofs_s3.h new file mode 100644 index 0000000..16a06c9 --- /dev/null +++ b/lib/liberofs_s3.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +/* + * Copyright (C) 2025 HUAWEI, Inc. + * http://www.huawei.com/ + * Created by Yifan Zhao <zhaoyifa...@huawei.com> + */ +#ifndef __EROFS_S3_H +#define __EROFS_S3_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum s3erofs_url_style { + S3EROFS_URL_STYLE_PATH, // Path style: https://s3.amazonaws.com/bucket/object + S3EROFS_URL_STYLE_VIRTUAL_HOST, // Virtual host style: https://bucket.s3.amazonaws.com/object +}; + +enum s3erofs_signature_version { + S3EROFS_SIGNATURE_VERSION_2, + S3EROFS_SIGNATURE_VERSION_4, +}; + +#define S3_ACCESS_KEY_LEN 256 +#define S3_SECRET_KEY_LEN 256 + +struct erofs_s3 { + const char *endpoint, *bucket; + char access_key[S3_ACCESS_KEY_LEN + 1]; + char secret_key[S3_SECRET_KEY_LEN + 1]; + + enum s3erofs_url_style url_style; + enum s3erofs_signature_version sig; +}; + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/lib/remotes/s3.c b/lib/remotes/s3.c index ed2b023..358ee91 100644 --- a/lib/remotes/s3.c +++ b/lib/remotes/s3.c @@ -3,4 +3,5 @@ * Copyright (C) 2025 HUAWEI, Inc. * http://www.huawei.com/ * Created by Yifan Zhao <zhaoyifa...@huawei.com> - */ \ No newline at end of file + */ +#include "liberofs_s3.h" \ No newline at end of file diff --git a/mkfs/main.c b/mkfs/main.c index 3aa1421..f524f45 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -31,6 +31,7 @@ #include "../lib/liberofs_private.h" #include "../lib/liberofs_uuid.h" #include "../lib/liberofs_metabox.h" +#include "../lib/liberofs_s3.h" #include "../lib/compressor.h" static struct option long_options[] = { @@ -59,6 +60,9 @@ static struct option long_options[] = { {"gid-offset", required_argument, NULL, 17}, {"tar", optional_argument, NULL, 20}, {"aufs", no_argument, NULL, 21}, +#ifdef HAVE_S3 + {"s3", required_argument, NULL, 22}, +#endif {"mount-point", required_argument, NULL, 512}, {"xattr-prefix", required_argument, NULL, 19}, #ifdef WITH_ANDROID @@ -197,6 +201,12 @@ static void usage(int argc, char **argv) " --root-xattr-isize=# ensure the inline xattr size of the root directory is # bytes at least\n" " --aufs replace aufs special files with overlayfs metadata\n" " --sort=<path,none> data sorting order for tarballs as input (default: path)\n" +#ifdef HAVE_S3 + " --s3=X generate an index-only image from s3-compatible object store backend\n" + " [,passwd_file=Y] X=endpoint, Y=s3 credentials file\n" + " [,style=Z] S3 API calling style (Z = vhost|path) (default: vhost)\n" + " [,sig=<2,4>] S3 API signature version (default: 2)\n" +#endif " --tar=X generate a full or index-only image from a tarball(-ish) source\n" " (X = f|i|headerball; f=full mode, i=index mode,\n" " headerball=file data is omited in the source stream)\n" @@ -247,6 +257,10 @@ static struct erofs_tarfile erofstar = { static bool incremental_mode; static u8 metabox_algorithmid; +#ifdef HAVE_S3 +static struct erofs_s3 s3cfg; +#endif + enum { EROFS_MKFS_DATA_IMPORT_DEFAULT, EROFS_MKFS_DATA_IMPORT_FULLDATA, @@ -257,6 +271,9 @@ enum { enum { EROFS_MKFS_SOURCE_DEFAULT, EROFS_MKFS_SOURCE_TAR, +#ifdef HAVE_S3 + EROFS_MKFS_SOURCE_S3, +#endif EROFS_MKFS_SOURCE_REBUILD, } source_mode; @@ -522,6 +539,139 @@ static void mkfs_parse_tar_cfg(char *cfg) erofstar.index_mode = true; } +#ifdef HAVE_S3 +static int mkfs_parse_s3_cfg_passwd(const char *filepath, char *ak, char *sk) +{ + struct stat file_stat; + int fd, n, ret = 0; + char buf[S3_ACCESS_KEY_LEN + S3_SECRET_KEY_LEN + 3]; + char *colon; + + fd = open(filepath, O_RDONLY); + if (fd < 0) { + erofs_err("failed to open passwd_file %s", filepath); + return -errno; + } + + if (fstat(fd, &file_stat) != 0) { + ret = -errno; + goto err; + } + + if (!S_ISREG(file_stat.st_mode)) { + erofs_err("%s is not a regular file", filepath); + ret = -EINVAL; + goto err; + } + + if ((file_stat.st_mode & (S_IXUSR | S_IROTH | S_IWOTH | S_IXOTH | S_IRGRP | + S_IWGRP | S_IXGRP)) != 0) { + erofs_err("%s should not have others/group permission", filepath); + ret = -EPERM; + goto err; + } + + if (file_stat.st_size > S3_ACCESS_KEY_LEN + S3_SECRET_KEY_LEN + 3) { + erofs_err("passwd_file %s with size %lu is too big", filepath, + file_stat.st_size); + ret = -EINVAL; + goto err; + } + + n = read(fd, buf, file_stat.st_size); + if (n < 0) { + ret = -errno; + goto err; + } + buf[n] = '\0'; + + while (n > 0 && (buf[n - 1] == '\n' || buf[n - 1] == '\r')) + buf[--n] = '\0'; + + colon = strchr(buf, ':'); + if (!colon) { + ret = -EINVAL; + goto err; + } + *colon = '\0'; + + strcpy(ak, buf); + strcpy(sk, colon + 1); + +err: + close(fd); + return ret; +} + +static int mkfs_parse_s3_cfg(char *cfg_str) +{ + char *p, *q, *opt; + int ret = 0; + + if (source_mode != EROFS_MKFS_SOURCE_DEFAULT) { + erofs_warn("generation image from s3 conflicting with other " + "options, ignoring it"); + return 0; + } + source_mode = EROFS_MKFS_SOURCE_S3; + + if (!cfg_str) { + erofs_err("s3: missing parameter"); + return -EINVAL; + } + + s3cfg.url_style = S3EROFS_URL_STYLE_VIRTUAL_HOST; + s3cfg.sig = S3EROFS_SIGNATURE_VERSION_2; + + p = strchr(cfg_str, ','); + if (p) { + s3cfg.endpoint = strndup(cfg_str, p - cfg_str); + } else { + s3cfg.endpoint = strdup(cfg_str); + return 0; + } + + opt = p + 1; + while (opt) { + q = strchr(opt, ','); + if (q) + *q = '\0'; + + if ((p = strstr(opt, "passwd_file="))) { + p += strlen("passwd_file="); + ret = mkfs_parse_s3_cfg_passwd(p, s3cfg.access_key, + s3cfg.secret_key); + if (ret) + return ret; + } else if ((p = strstr(opt, "style="))) { + p += strlen("style="); + if (strncmp(p, "vhost", 5) == 0) { + s3cfg.url_style = S3EROFS_URL_STYLE_VIRTUAL_HOST; + } else if (strncmp(p, "path", 4) == 0) { + s3cfg.url_style = S3EROFS_URL_STYLE_PATH; + } else { + erofs_err("invalid s3 style %s", p); + return -EINVAL; + } + } else if ((p = strstr(opt, "sig="))) { + p += strlen("sig="); + if (strncmp(p, "4", 1) == 0) { + erofs_warn("AWS Signature Version 4 is not supported yet, using Version 2"); + } else if (strncmp(p, "2", 1) == 0) { + s3cfg.sig = S3EROFS_SIGNATURE_VERSION_2; + } else { + erofs_err("invalid s3 sig %s", p); + return -EINVAL; + } + } + + opt = q ? q + 1 : NULL; + } + + return 0; +} +#endif + static int mkfs_parse_one_compress_alg(char *alg, struct erofs_compr_opts *copts) { @@ -670,6 +820,13 @@ static int mkfs_parse_sources(int argc, char *argv[], int optind) erofstar.ios.dumpfd = fd; } break; +#ifdef HAVE_S3 + case EROFS_MKFS_SOURCE_S3: + s3cfg.bucket = strdup(argv[optind++]); + if (!s3cfg.bucket) + return -ENOMEM; + break; +#endif case EROFS_MKFS_SOURCE_REBUILD: default: erofs_err("unexpected source_mode: %d", source_mode); @@ -944,6 +1101,13 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) case 21: erofstar.aufs = true; break; +#ifdef HAVE_S3 + case 22: + err = mkfs_parse_s3_cfg(optarg); + if (err) + return err; + break; +#endif case 516: if (!optarg || !strcmp(optarg, "1")) cfg.c_ovlfs_strip = true; @@ -1539,35 +1703,7 @@ int main(int argc, char **argv) erofs_inode_manager_init(); - if (source_mode == EROFS_MKFS_SOURCE_TAR) { - root = erofs_rebuild_make_root(&g_sbi); - if (IS_ERR(root)) { - err = PTR_ERR(root); - goto exit; - } - - while (!(err = tarerofs_parse_tar(root, &erofstar))); - - if (err < 0) - goto exit; - - err = erofs_rebuild_dump_tree(root, incremental_mode); - if (err < 0) - goto exit; - } else if (source_mode == EROFS_MKFS_SOURCE_REBUILD) { - root = erofs_rebuild_make_root(&g_sbi); - if (IS_ERR(root)) { - err = PTR_ERR(root); - goto exit; - } - - err = erofs_mkfs_rebuild_load_trees(root); - if (err) - goto exit; - err = erofs_rebuild_dump_tree(root, incremental_mode); - if (err) - goto exit; - } else { + if (source_mode == EROFS_MKFS_SOURCE_DEFAULT) { err = erofs_build_shared_xattrs_from_path(&g_sbi, cfg.c_src_path); if (err) { erofs_err("failed to build shared xattrs: %s", @@ -1584,6 +1720,32 @@ int main(int argc, char **argv) root = NULL; goto exit; } + } else { + root = erofs_rebuild_make_root(&g_sbi); + if (IS_ERR(root)) { + err = PTR_ERR(root); + goto exit; + } + + if (source_mode == EROFS_MKFS_SOURCE_TAR) { + while (!(err = tarerofs_parse_tar(root, &erofstar))) + ; + if (err < 0) + goto exit; + } else if (source_mode == EROFS_MKFS_SOURCE_REBUILD) { + err = erofs_mkfs_rebuild_load_trees(root); + if (err) + goto exit; +#ifdef HAVE_S3 + } else if (source_mode == EROFS_MKFS_SOURCE_S3) { + err = -EOPNOTSUPP; + goto exit; +#endif + } + + err = erofs_rebuild_dump_tree(root, incremental_mode); + if (err) + goto exit; } if (tar_index_512b) { -- 2.46.0
