steveloughran commented on code in PR #6406:
URL: https://github.com/apache/hadoop/pull/6406#discussion_r1467034744
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java:
##########
@@ -68,6 +68,10 @@ private Constants() {
public static final String AWS_CREDENTIALS_PROVIDER =
"fs.s3a.aws.credentials.provider";
+ // aws credentials providers mapping with key/value pairs
Review Comment:
nit: javadocs with @value
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java:
##########
@@ -206,6 +210,52 @@ public void testFallbackToDefaults() throws Throwable {
assertTrue("empty credentials", credentials.size() > 0);
}
+ @Test
+ public void testAssumedRoleWithRemap() throws Throwable {
+ Configuration conf = new Configuration(false);
+ conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER,
+
"custom.assume.role.key1,custom.assume.role.key2,custom.assume.role.key3");
+ conf.set(AWS_CREDENTIALS_PROVIDER_MAPPING,
+ "custom.assume.role.key1="
+ + CredentialProviderListFactory.ENVIRONMENT_CREDENTIALS_V2
+ + " ,custom.assume.role.key2 ="
+ + CountInvocationsProvider.NAME
+ + ", custom.assume.role.key3= "
+ + CredentialProviderListFactory.PROFILE_CREDENTIALS_V1);
+ final AWSCredentialProviderList credentials =
+ buildAWSProviderList(
+ new URI("s3a://bucket1"),
+ conf,
+ ASSUMED_ROLE_CREDENTIALS_PROVIDER,
+ new ArrayList<>(),
+ new HashSet<>());
+ assertEquals("Credentials not matching", 3, credentials.size());
Review Comment:
assertJ size assert
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java:
##########
@@ -233,6 +236,11 @@ public static AWSCredentialProviderList
buildAWSProviderList(
key, className, mapped);
className = mapped;
}
+ if (awsCredsMappedClasses != null &&
awsCredsMappedClasses.containsKey(className)) {
Review Comment:
make an `else` unless we really want to support remapping of the standard
map to different values. Which I suppose we might...
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java:
##########
@@ -206,6 +210,52 @@ public void testFallbackToDefaults() throws Throwable {
assertTrue("empty credentials", credentials.size() > 0);
}
+ @Test
+ public void testAssumedRoleWithRemap() throws Throwable {
+ Configuration conf = new Configuration(false);
+ conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER,
+
"custom.assume.role.key1,custom.assume.role.key2,custom.assume.role.key3");
+ conf.set(AWS_CREDENTIALS_PROVIDER_MAPPING,
+ "custom.assume.role.key1="
+ + CredentialProviderListFactory.ENVIRONMENT_CREDENTIALS_V2
+ + " ,custom.assume.role.key2 ="
+ + CountInvocationsProvider.NAME
+ + ", custom.assume.role.key3= "
+ + CredentialProviderListFactory.PROFILE_CREDENTIALS_V1);
+ final AWSCredentialProviderList credentials =
+ buildAWSProviderList(
+ new URI("s3a://bucket1"),
+ conf,
+ ASSUMED_ROLE_CREDENTIALS_PROVIDER,
+ new ArrayList<>(),
+ new HashSet<>());
+ assertEquals("Credentials not matching", 3, credentials.size());
+ }
+
+ @Test
+ public void testAwsCredentialProvidersWithRemap() throws Throwable {
+ Configuration conf = new Configuration(false);
+ conf.set(AWS_CREDENTIALS_PROVIDER,
+
"custom.aws.creds.key1,custom.aws.creds.key2,custom.aws.creds.key3,custom.aws.creds.key4");
+ conf.set(AWS_CREDENTIALS_PROVIDER_MAPPING,
+ "custom.aws.creds.key1="
+ + CredentialProviderListFactory.ENVIRONMENT_CREDENTIALS_V2
+ + " ,\ncustom.aws.creds.key2="
+ + CountInvocationsProvider.NAME
+ + "\n, custom.aws.creds.key3="
+ + CredentialProviderListFactory.PROFILE_CREDENTIALS_V1
+ + ",custom.aws.creds.key4 = "
+ + CredentialProviderListFactory.PROFILE_CREDENTIALS_V2);
+ final AWSCredentialProviderList credentials =
+ buildAWSProviderList(
+ new URI("s3a://bucket1"),
+ conf,
+ AWS_CREDENTIALS_PROVIDER,
+ new ArrayList<>(),
+ new HashSet<>());
+ assertEquals("Credentials not matching", 4, credentials.size());
Review Comment:
assertJ. It's predictable I'll expect these, save time by embracing the api
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java:
##########
@@ -233,6 +236,11 @@ public static AWSCredentialProviderList
buildAWSProviderList(
key, className, mapped);
className = mapped;
}
+ if (awsCredsMappedClasses != null &&
awsCredsMappedClasses.containsKey(className)) {
+ final String mapped = awsCredsMappedClasses.get(className);
+ LOG_REMAPPED_ENTRY.info("Credential entry {} is mapped to {}",
className, mapped);
Review Comment:
debug
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
