xushiyan commented on code in PR #410:
URL: https://github.com/apache/hudi-rs/pull/410#discussion_r2260858066
##########
crates/core/src/table/builder.rs:
##########
@@ -267,6 +280,23 @@ impl OptionResolver {
Ok(())
}
+
+ /// Parse Hudi environment variable format:
HOODIE_ENV_fs_DOT_s3a_DOT_access_DOT_key
+ /// Returns: fs.s3a.access.key
+ fn parse_hudi_env_var(&self, env_key: &str) -> String {
+ env_key
+ .strip_prefix("HOODIE_ENV_")
+ .unwrap_or(env_key)
+ .replace("_DOT_", ".")
+ }
+
+ fn map_hudi_to_storage_key(&self, hudi_key: &str) -> Option<String> {
+ match hudi_key {
+ "fs.s3a.access.key" | "fs.s3.awsAccessKeyId" =>
Some("aws_access_key_id".to_string()),
+ "fs.s3a.secret.key" | "fs.s3.awsSecretAccessKey" =>
Some("aws_secret_access_key".to_string()),
+ _ => None,
Review Comment:
we should parse and get generic keys, instead of looking for specific ones.
Any env var could be applicable if it's set in that `HOODIE_ENV_` type of form
##########
crates/core/src/table/builder.rs:
##########
@@ -202,13 +202,26 @@ impl OptionResolver {
if Storage::CLOUD_STORAGE_PREFIXES
.iter()
.any(|prefix| key.starts_with(prefix))
- &&
!self.storage_options.contains_key(&key.to_ascii_lowercase())
{
- self.storage_options.insert(key.to_ascii_lowercase(), value);
+ if let Some(storage_key) = self.process_cloud_env_var(&key) {
+ if !self.storage_options.contains_key(&storage_key) {
+ self.storage_options.insert(storage_key, value);
+ }
+ }
}
}
}
+ /// Process a cloud environment variable and return the appropriate
storage key
+ fn process_cloud_env_var(&self, env_key: &str) -> Option<String> {
+ if env_key.starts_with("HOODIE_ENV_fs_DOT_s3a_DOT_") ||
env_key.starts_with("HOODIE_ENV_fs_DOT_s3_DOT_") {
Review Comment:
this does not seem right. why are these 2 strings checked in particular?
shouldn't it be just `HOODIE_ENV_` ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
