On 2025/8/1 15:30, Yifan Zhao wrote:
From: zhaoyifan <zhaoyifa...@huawei.com>
This patch introduces configuration options for the upcoming experimental S3
support, including configuration parsing and passwd_file reading logic.
User could specify the following options:
- S3 service endpoint (Compulsory)
- S3 credentials file, in the format of "$ak:%sk" (Optional)
- S3 API calling style (Optional)
- S3 API signature version, only sigV2 supported yet (Optional)
Signed-off-by: Yifan Zhao <zhaoyifa...@huawei.com>
---
change since v1:
- rename: include/erofs/s3.h => lib/liberofs_s3.h
- add liberofs_s3.h in this patch rather than previous one
lib/liberofs_s3.h | 40 +++++++++
lib/remotes/s3.c | 3 +-
mkfs/main.c | 220 ++++++++++++++++++++++++++++++++++++++++------
3 files changed, 233 insertions(+), 30 deletions(-)
create mode 100644 lib/liberofs_s3.h
diff --git a/lib/liberofs_s3.h b/lib/liberofs_s3.h
new file mode 100644
index 0000000..16a06c9
--- /dev/null
+++ b/lib/liberofs_s3.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+/*
+ * Copyright (C) 2025 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Yifan Zhao <zhaoyifa...@huawei.com>
+ */
+#ifndef __EROFS_S3_H
+#define __EROFS_S3_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum s3erofs_url_style {
+ S3EROFS_URL_STYLE_PATH, // Path style:
https://s3.amazonaws.com/bucket/object
+ S3EROFS_URL_STYLE_VIRTUAL_HOST, // Virtual host style:
https://bucket.s3.amazonaws.com/object
+};
+
+enum s3erofs_signature_version {
+ S3EROFS_SIGNATURE_VERSION_2,
+ S3EROFS_SIGNATURE_VERSION_4,
+};
+
+#define S3_ACCESS_KEY_LEN 256
+#define S3_SECRET_KEY_LEN 256
+
+struct erofs_s3 {
+ const char *endpoint, *bucket;
+ char access_key[S3_ACCESS_KEY_LEN + 1];
+ char secret_key[S3_SECRET_KEY_LEN + 1];
+
+ enum s3erofs_url_style url_style;
+ enum s3erofs_signature_version sig;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/lib/remotes/s3.c b/lib/remotes/s3.c
index ed2b023..358ee91 100644
--- a/lib/remotes/s3.c
+++ b/lib/remotes/s3.c
@@ -3,4 +3,5 @@
* Copyright (C) 2025 HUAWEI, Inc.
* http://www.huawei.com/
* Created by Yifan Zhao <zhaoyifa...@huawei.com>
- */
\ No newline at end of file
+ */
+#include "liberofs_s3.h"
\ No newline at end of file
diff --git a/mkfs/main.c b/mkfs/main.c
index 3aa1421..f524f45 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -31,6 +31,7 @@
#include "../lib/liberofs_private.h"
#include "../lib/liberofs_uuid.h"
#include "../lib/liberofs_metabox.h"
+#include "../lib/liberofs_s3.h"
#include "../lib/compressor.h"
static struct option long_options[] = {
@@ -59,6 +60,9 @@ static struct option long_options[] = {
{"gid-offset", required_argument, NULL, 17},
{"tar", optional_argument, NULL, 20},
{"aufs", no_argument, NULL, 21},
+#ifdef HAVE_S3
+ {"s3", required_argument, NULL, 22},
+#endif
{"mount-point", required_argument, NULL, 512},
{"xattr-prefix", required_argument, NULL, 19},
#ifdef WITH_ANDROID
@@ -197,6 +201,12 @@ static void usage(int argc, char **argv)
" --root-xattr-isize=# ensure the inline xattr size of the root
directory is # bytes at least\n"
" --aufs replace aufs special files with overlayfs
metadata\n"
" --sort=<path,none> data sorting order for tarballs as input
(default: path)\n"
+#ifdef HAVE_S3
HAVE_S3 is a bit odd, how about using
S3_ENABLED (like LZ4_ENABLED?)
+ " --s3=X generate an index-only image from
s3-compatible object store backend\n"
+ " [,passwd_file=Y] X=endpoint, Y=s3 credentials file\n"
What's s3 credentials file? Is it documented
somewhere? Why is it named as passwd_file?
Can we have an option to pass in accesskey
too?
+ " [,style=Z] S3 API calling style (Z = vhost|path)
(default: vhost)\n"
+ " [,sig=<2,4>] S3 API signature version (default: 2)\n"
+#endif
" --tar=X generate a full or index-only image from a
tarball(-ish) source\n"
" (X = f|i|headerball; f=full mode, i=index
mode,\n"
" headerball=file data is
omited in the source stream)\n"
@@ -247,6 +257,10 @@ static struct erofs_tarfile erofstar = {
static bool incremental_mode;
static u8 metabox_algorithmid;
+#ifdef HAVE_S3
+static struct erofs_s3 s3cfg;
+#endif
+
enum {
EROFS_MKFS_DATA_IMPORT_DEFAULT,
EROFS_MKFS_DATA_IMPORT_FULLDATA,
@@ -257,6 +271,9 @@ enum {
enum {
EROFS_MKFS_SOURCE_DEFAULT,
EROFS_MKFS_SOURCE_LOCALDIR,
Thanks,
Gao Xiang
