This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d72635e919b [chore](tde) Add scoped KMS credentials to regression
clusters (#64564)
d72635e919b is described below
commit d72635e919b495627812e152c617828fc85d82af
Author: Jamie <[email protected]>
AuthorDate: Tue Jul 7 01:50:48 2026 +0800
[chore](tde) Add scoped KMS credentials to regression clusters (#64564)
- Add provider-scoped TDE credential options for AWS and Aliyun in
doris-compose.
- Pass scoped TDE credentials into FE container environment variables.
- Extend regression cluster options/configuration for scoped
credentials.
- Mask TDE credential values in regression logs and log only set/unset
status for configured secrets.
---
docker/runtime/doris-compose/cluster.py | 25 ++++++++++-
docker/runtime/doris-compose/command.py | 25 +++++++++++
docker/runtime/doris-compose/resource/init_fe.sh | 4 ++
.../org/apache/doris/regression/Config.groovy | 32 +++++++++++++-
.../apache/doris/regression/ConfigOptions.groovy | 32 +++++++++++++-
.../doris/regression/suite/SuiteCluster.groovy | 51 +++++++++++++++++++++-
6 files changed, 163 insertions(+), 6 deletions(-)
diff --git a/docker/runtime/doris-compose/cluster.py
b/docker/runtime/doris-compose/cluster.py
index 769a1af4be0..965c4057b9d 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -371,6 +371,18 @@ class Node(object):
def get_tde_sk(self):
return self.cluster.tde_sk
+ def get_tde_aws_ak(self):
+ return getattr(self.cluster, "tde_aws_ak", "")
+
+ def get_tde_aws_sk(self):
+ return getattr(self.cluster, "tde_aws_sk", "")
+
+ def get_tde_aliyun_ak(self):
+ return getattr(self.cluster, "tde_aliyun_ak", "")
+
+ def get_tde_aliyun_sk(self):
+ return getattr(self.cluster, "tde_aliyun_sk", "")
+
def get_default_named_ports(self):
# port_name : default_port
# the port_name come from fe.conf, be.conf, cloud.conf, etc
@@ -414,6 +426,10 @@ class Node(object):
"ENABLE_STORAGE_VAULT": 1 if getattr(self.cluster,
"enable_storage_vault", False) else 0,
"TDE_AK": self.get_tde_ak(),
"TDE_SK": self.get_tde_sk(),
+ "TDE_AWS_AK": self.get_tde_aws_ak(),
+ "TDE_AWS_SK": self.get_tde_aws_sk(),
+ "TDE_ALIYUN_AK": self.get_tde_aliyun_ak(),
+ "TDE_ALIYUN_SK": self.get_tde_aliyun_sk(),
}
if self.cluster.is_cloud:
@@ -933,6 +949,7 @@ class Cluster(object):
local_network_ip, fe_follower, be_disks, be_cluster, reg_be,
extra_hosts, env, coverage_dir, cloud_store_config,
sql_mode_node_mgr, be_metaservice_endpoint, be_cluster_id,
tde_ak, tde_sk,
+ tde_aws_ak, tde_aws_sk, tde_aliyun_ak, tde_aliyun_sk,
external_ms_cluster, instance_id, cluster_snapshot="",
enable_storage_vault=False):
self.name = name
@@ -974,6 +991,10 @@ class Cluster(object):
self.be_cluster_id = be_cluster_id
self.tde_ak = tde_ak
self.tde_sk = tde_sk
+ self.tde_aws_ak = tde_aws_ak
+ self.tde_aws_sk = tde_aws_sk
+ self.tde_aliyun_ak = tde_aliyun_ak
+ self.tde_aliyun_sk = tde_aliyun_sk
@staticmethod
def new(name, image, is_cloud, is_root_user, fe_config, be_config,
@@ -981,6 +1002,7 @@ class Cluster(object):
fe_follower, be_disks, be_cluster, reg_be, extra_hosts, env,
coverage_dir, cloud_store_config, sql_mode_node_mgr,
be_metaservice_endpoint, be_cluster_id, tde_ak, tde_sk,
+ tde_aws_ak, tde_aws_sk, tde_aliyun_ak, tde_aliyun_sk,
external_ms_cluster, instance_id, cluster_snapshot="",
enable_storage_vault=False):
if not os.path.exists(LOCAL_DORIS_PATH):
@@ -997,7 +1019,8 @@ class Cluster(object):
be_disks, be_cluster, reg_be, extra_hosts, env,
coverage_dir, cloud_store_config,
sql_mode_node_mgr, be_metaservice_endpoint,
- be_cluster_id, tde_ak, tde_sk,
external_ms_cluster,
+ be_cluster_id, tde_ak, tde_sk, tde_aws_ak,
tde_aws_sk,
+ tde_aliyun_ak, tde_aliyun_sk,
external_ms_cluster,
instance_id, cluster_snapshot,
enable_storage_vault)
os.makedirs(cluster.get_path(), exist_ok=True)
os.makedirs(get_status_path(name), exist_ok=True)
diff --git a/docker/runtime/doris-compose/command.py
b/docker/runtime/doris-compose/command.py
index 11151ca7f1b..d3cb0821b4a 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -554,6 +554,30 @@ class UpCommand(Command):
default="",
help="tde sk")
+ parser.add_argument(
+ "--tde-aws-ak",
+ type=str,
+ default="",
+ help="tde aws ak")
+
+ parser.add_argument(
+ "--tde-aws-sk",
+ type=str,
+ default="",
+ help="tde aws sk")
+
+ parser.add_argument(
+ "--tde-aliyun-ak",
+ type=str,
+ default="",
+ help="tde aliyun ak")
+
+ parser.add_argument(
+ "--tde-aliyun-sk",
+ type=str,
+ default="",
+ help="tde aliyun sk")
+
# if default==True, use this style to parser, like --detach
if self._support_boolean_action():
parser.add_argument(
@@ -684,6 +708,7 @@ class UpCommand(Command):
args.be_disks if args.be_disks is not None else ["HDD=1"],
args.be_cluster, args.reg_be, args.extra_hosts, args.env,
args.coverage_dir, cloud_store_config, args.sql_mode_node_mgr,
args.be_metaservice_endpoint, args.be_cluster_id, args.tde_ak,
args.tde_sk,
+ args.tde_aws_ak, args.tde_aws_sk, args.tde_aliyun_ak,
args.tde_aliyun_sk,
external_ms_cluster, instance_id, cluster_snapshot,
enable_storage_vault)
LOG.info("Create new cluster {} succ, cluster path is {}".format(
diff --git a/docker/runtime/doris-compose/resource/init_fe.sh
b/docker/runtime/doris-compose/resource/init_fe.sh
index 4e3a3ffffe7..380d791e362 100755
--- a/docker/runtime/doris-compose/resource/init_fe.sh
+++ b/docker/runtime/doris-compose/resource/init_fe.sh
@@ -92,6 +92,10 @@ fe_daemon() {
run_fe() {
export DORIS_TDE_AK=${TDE_AK}
export DORIS_TDE_SK=${TDE_SK}
+ export DORIS_TDE_AWS_AK=${TDE_AWS_AK}
+ export DORIS_TDE_AWS_SK=${TDE_AWS_SK}
+ export DORIS_TDE_ALIYUN_AK=${TDE_ALIYUN_AK}
+ export DORIS_TDE_ALIYUN_SK=${TDE_ALIYUN_SK}
health_log "run start_fe.sh"
# Add cluster_snapshot parameter for first startup only (
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index c666af4162c..3b5e335a96c 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -170,6 +170,10 @@ class Config {
public String tdeAk
public String tdeSk
+ public String tdeAwsAk
+ public String tdeAwsSk
+ public String tdeAliyunAk
+ public String tdeAliyunSk
public String tdeKeyEndpoint
public String tdeKeyRegion
public String tdeKeyProvider
@@ -240,6 +244,10 @@ class Config {
String cloudVersion,
String tdeAk,
String tdeSk,
+ String tdeAwsAk,
+ String tdeAwsSk,
+ String tdeAliyunAk,
+ String tdeAliyunSk,
String tdeKeyEndpoint,
String tdeKeyRegion,
String tdeKeyProvider,
@@ -304,6 +312,10 @@ class Config {
this.cloudVersion = cloudVersion
this.tdeAk = tdeAk
this.tdeSk = tdeSk
+ this.tdeAwsAk = tdeAwsAk
+ this.tdeAwsSk = tdeAwsSk
+ this.tdeAliyunAk = tdeAliyunAk
+ this.tdeAliyunSk = tdeAliyunSk
this.tdeKeyEndpoint = tdeKeyEndpoint
this.tdeKeyRegion = tdeKeyRegion
this.tdeKeyProvider = tdeKeyProvider
@@ -313,6 +325,10 @@ class Config {
this.enableClusterSnapshot = enableClusterSnapshot
}
+ static String secretStatus(String value) {
+ return value == null || value == "" ? "unset" : "set"
+ }
+
static String removeDirectoryPrefix(String str) {
def prefixes = ["./regression-test/suites/", "regression-test/suites/"]
@@ -512,9 +528,17 @@ class Config {
log.info("cloudVersion is ${config.cloudVersion}".toString())
config.tdeAk = cmd.getOptionValue(tdeAkOpt, config.tdeAk)
- log.info("tdeAk is ${config.tdeAk}".toString())
+ log.info("tdeAk is ${secretStatus(config.tdeAk)}".toString())
config.tdeSk = cmd.getOptionValue(tdeSkOpt, config.tdeSk)
- log.info("tdeSk is ${config.tdeSk}".toString())
+ log.info("tdeSk is ${secretStatus(config.tdeSk)}".toString())
+ config.tdeAwsAk = cmd.getOptionValue(tdeAwsAkOpt, config.tdeAwsAk)
+ log.info("tdeAwsAk is ${secretStatus(config.tdeAwsAk)}".toString())
+ config.tdeAwsSk = cmd.getOptionValue(tdeAwsSkOpt, config.tdeAwsSk)
+ log.info("tdeAwsSk is ${secretStatus(config.tdeAwsSk)}".toString())
+ config.tdeAliyunAk = cmd.getOptionValue(tdeAliyunAkOpt,
config.tdeAliyunAk)
+ log.info("tdeAliyunAk is
${secretStatus(config.tdeAliyunAk)}".toString())
+ config.tdeAliyunSk = cmd.getOptionValue(tdeAliyunSkOpt,
config.tdeAliyunSk)
+ log.info("tdeAliyunSk is
${secretStatus(config.tdeAliyunSk)}".toString())
config.tdeKeyEndpoint = cmd.getOptionValue(tdeKeyEndpointOpt,
config.tdeKeyEndpoint)
log.info("tdeKeyEndpoint is ${config.tdeKeyEndpoint}".toString())
config.tdeKeyRegion = cmd.getOptionValue(tdeKeyRegionOpt,
config.tdeKeyRegion)
@@ -657,6 +681,10 @@ class Config {
configToString(obj.cloudVersion),
configToString(obj.tdeAk),
configToString(obj.tdeSk),
+ configToString(obj.tdeAwsAk),
+ configToString(obj.tdeAwsSk),
+ configToString(obj.tdeAliyunAk),
+ configToString(obj.tdeAliyunSk),
configToString(obj.tdeKeyEndpoint),
configToString(obj.tdeKeyRegion),
configToString(obj.tdeKeyProvider),
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
index 0f55f55fabb..c62599a5976 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
@@ -100,6 +100,10 @@ class ConfigOptions {
static Option cloudVersionOpt
static Option tdeAkOpt
static Option tdeSkOpt
+ static Option tdeAwsAkOpt
+ static Option tdeAwsSkOpt
+ static Option tdeAliyunAkOpt
+ static Option tdeAliyunSkOpt
static Option tdeKeyEndpointOpt
static Option tdeKeyRegionOpt
static Option tdeKeyProviderOpt
@@ -611,14 +615,34 @@ class ConfigOptions {
.build()
tdeAkOpt = Option.builder("tdeAk")
.required(false)
- .hasArg(false)
+ .hasArg(true)
.desc("TDE Access Key")
.build();
tdeSkOpt = Option.builder("tdeSk")
.required(false)
- .hasArg(false)
+ .hasArg(true)
.desc("TDE Secret Key")
.build();
+ tdeAwsAkOpt = Option.builder("tdeAwsAk")
+ .required(false)
+ .hasArg(true)
+ .desc("TDE AWS Access Key")
+ .build();
+ tdeAwsSkOpt = Option.builder("tdeAwsSk")
+ .required(false)
+ .hasArg(true)
+ .desc("TDE AWS Secret Key")
+ .build();
+ tdeAliyunAkOpt = Option.builder("tdeAliyunAk")
+ .required(false)
+ .hasArg(true)
+ .desc("TDE Aliyun Access Key")
+ .build();
+ tdeAliyunSkOpt = Option.builder("tdeAliyunSk")
+ .required(false)
+ .hasArg(true)
+ .desc("TDE Aliyun Secret Key")
+ .build();
tdeKeyEndpointOpt = Option.builder("tdeKeyEndpoint")
.required(false)
.hasArg(false)
@@ -714,6 +738,10 @@ class ConfigOptions {
.addOption(cloudVersionOpt)
.addOption(tdeAkOpt)
.addOption(tdeSkOpt)
+ .addOption(tdeAwsAkOpt)
+ .addOption(tdeAwsSkOpt)
+ .addOption(tdeAliyunAkOpt)
+ .addOption(tdeAliyunSkOpt)
.addOption(tdeKeyEndpointOpt)
.addOption(tdeKeyRegionOpt)
.addOption(tdeKeyProviderOpt)
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
index ef5282eb218..899ba118f5e 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
@@ -100,6 +100,10 @@ class ClusterOptions {
String tdeAk = "";
String tdeSk = "";
+ String tdeAwsAk = "";
+ String tdeAwsSk = "";
+ String tdeAliyunAk = "";
+ String tdeAliyunSk = "";
// Use external meta service cluster (shared MS/FDB)
// Specify the cluster name that provides MS/FDB services
@@ -422,6 +426,26 @@ class SuiteCluster {
cmd += options.tdeSk
}
+ if (options.tdeAwsAk != null && options.tdeAwsAk != "") {
+ cmd += ['--tde-aws-ak']
+ cmd += options.tdeAwsAk
+ }
+
+ if (options.tdeAwsSk != null && options.tdeAwsSk != "") {
+ cmd += ['--tde-aws-sk']
+ cmd += options.tdeAwsSk
+ }
+
+ if (options.tdeAliyunAk != null && options.tdeAliyunAk != "") {
+ cmd += ['--tde-aliyun-ak']
+ cmd += options.tdeAliyunAk
+ }
+
+ if (options.tdeAliyunSk != null && options.tdeAliyunSk != "") {
+ cmd += ['--tde-aliyun-sk']
+ cmd += options.tdeAliyunSk
+ }
+
if (options.externalMsCluster != null && options.externalMsCluster !=
"") {
cmd += ['--external-ms', options.externalMsCluster]
}
@@ -914,9 +938,34 @@ class SuiteCluster {
}
// Execute command with proper argument list to avoid shell escaping issues
+ private static List<String> maskSensitiveArgs(List<String> cmdList) {
+ Set<String> sensitiveOptions = [
+ '--tde-ak',
+ '--tde-sk',
+ '--tde-aws-ak',
+ '--tde-aws-sk',
+ '--tde-aliyun-ak',
+ '--tde-aliyun-sk'
+ ] as Set
+ List<String> masked = []
+ boolean maskNext = false
+ for (String arg : cmdList) {
+ if (maskNext) {
+ masked += '***'
+ maskNext = false
+ continue
+ }
+ masked += arg
+ if (sensitiveOptions.contains(arg)) {
+ maskNext = true
+ }
+ }
+ return masked
+ }
+
private Object runCmdList(List<String> cmdList, int timeoutSecond = 60)
throws Exception {
def fullCmdList = ['python', '-W', 'ignore', config.dorisComposePath]
+ cmdList + ['-v', '--output-json']
- logger.info('Run doris compose cmd: {}', fullCmdList.join(' '))
+ logger.info('Run doris compose cmd: {}',
maskSensitiveArgs(fullCmdList).join(' '))
def proc = fullCmdList.execute()
def outBuf = new StringBuilder()
def errBuf = new StringBuilder()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
