dwsmith1983 commented on code in PR #5872:
URL: https://github.com/apache/datafusion-comet/pull/5872#discussion_r4174426095
##########
native/core/src/parquet/objectstore/s3.rs:
##########
@@ -1000,11 +1173,393 @@ impl CredentialProviderMetadata {
}
}
+/// The STS region the SDK profile provider takes, on its regional host, when
no region is found.
+const STS_FALLBACK_REGION: &str = "us-east-1";
+
+/// The region whose STS endpoint is the global https://sts.amazonaws.com,
signed for us-east-1,
+/// where the Java SDK sends a role profile's request when no region is found.
+const STS_GLOBAL_REGION: &str = "aws-global";
+
+/// Profile properties that make a profile resolve credentials other than its
static keys.
+const CREDENTIAL_PROPERTIES: [&str; 10] = [
+ "role_arn",
+ "credential_source",
+ "web_identity_token_file",
+ "credential_process",
+ "login_session",
+ "sso_session",
+ "sso_account_id",
+ "sso_region",
+ "sso_role_name",
+ "sso_start_url",
+];
+
+/// A role a profile assumes from its `source_profile`.
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+struct ProfileRole {
+ role_arn: String,
+ external_id: Option<String>,
+ session_name: Option<String>,
+ region: Option<String>,
+}
+
+/// A role profile's chain: the profile whose credentials start it and the
roles assumed from
+/// them, outermost first.
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+struct ProfileRoleChain {
+ base: String,
+ /// Whether the base is a web identity role, the only base that calls STS.
+ base_needs_region: bool,
+ roles: Vec<ProfileRole>,
+}
+
+fn has_only_static_keys(profile: &Profile) -> bool {
+ profile.get("aws_access_key_id").is_some()
+ && !CREDENTIAL_PROPERTIES
+ .iter()
+ .any(|property| profile.get(property).is_some())
+}
+
+/// Follows `role_arn` and `source_profile` from `selected` the way the SDK's
profile provider
+/// does (aws-config's profile/credentials/repr.rs). That provider assumes
every role with one
+/// STS region and offers no per-role endpoint, while Hadoop's Java SDK gives
each role its own,
+/// so the roles are assumed here instead. A chain this does not mirror
returns the reason, to
+/// stay on the SDK provider.
+fn resolve_role_chain(profiles: &ProfileSet, selected: &str) ->
Result<ProfileRoleChain, String> {
+ let mut name = selected;
+ let mut visited = Vec::new();
+ let mut roles = Vec::new();
+ loop {
+ let profile = profiles
+ .get_profile(name)
+ .ok_or_else(|| format!("profile {name} is not defined"))?;
+ if visited.contains(&name) {
+ return Err(format!("profile {name} is in a source_profile cycle"));
+ }
+ visited.push(name);
+ // The SDK takes a source profile's static keys ahead of its other
settings, which the
+ // base provider, reading the profile as its selected one, would not.
+ if visited.len() > 1
+ && profile.get("aws_access_key_id").is_some()
+ && !has_only_static_keys(profile)
+ {
+ return Err(format!(
Review Comment:
> Could we resolve mixed source profiles using Java's precedence instead of
delegating the entire chain, and add a regression test asserting the signing
key?
Changed approach in aa73a0d6c. Instead of matching more of the Java SDK in
Rust, a bucket whose provider list names `ProfileAWSCredentialsProvider` now
gets its credentials from the existing `HadoopS3ACredentialProviderAdapter`,
which builds Hadoop's own provider list on the executor. The mixed source
profile, `credential_source` chains, per-role regions and the global STS
endpoint all resolve in the Java SDK, so native signs with whatever identity
Hadoop picks.
Before switching I ran the real Hadoop 3.4.2 provider over 63 profile
fixtures against the Rust walker; it still differed on 22, including this one,
self-referencing roles, SSO with a role, and the list's error handling. The
walker and the default-credentials forward are gone.
`HadoopProfileCredentialsProviderSuite` reads from MinIO with only the profile
provider set and checks the adapter served it; it passes on Spark 4.1 and fails
with `Unsupported credential provider` when the routing is turned off.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
