ahmarsuhail commented on code in PR #7295:
URL: https://github.com/apache/hadoop/pull/7295#discussion_r1925046174
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java:
##########
@@ -230,13 +227,33 @@ public class S3AStoreImpl
@Override
protected void serviceInit(final Configuration conf) throws Exception {
- objectInputStreamFactory = createStreamFactory(conf);
+ if(conf.getEnum(INPUT_STREAM_TYPE, InputStreamType.DEFAULT_STREAM_TYPE) ==
InputStreamType.Analytics) {
+ final S3AsyncClient s3AsyncClient = getOrCreateAsyncCRTClient(conf);
+ objectInputStreamFactory = createStreamFactory(conf, s3AsyncClient);
+ } else {
+ objectInputStreamFactory = createStreamFactory(conf);
+ }
addService(objectInputStreamFactory);
// init all child services
super.serviceInit(conf);
}
+ private S3AsyncClient getOrCreateAsyncCRTClient(final Configuration conf)
throws Exception {
+ final S3AsyncClient s3AsyncClient;
+ boolean analyticsAcceleratorCRTEnabled =
conf.getBoolean(ANALYTICS_ACCELERATOR_CRT_ENABLED,
+ ANALYTICS_ACCELERATOR_CRT_ENABLED_DEFAULT);
+ LOG.info("Using S3SeekableInputStream");
+ if(analyticsAcceleratorCRTEnabled) {
+ LOG.info("Using S3 CRT client for analytics accelerator S3");
Review Comment:
downgrade to level: debug
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java:
##########
@@ -230,13 +227,33 @@ public class S3AStoreImpl
@Override
protected void serviceInit(final Configuration conf) throws Exception {
- objectInputStreamFactory = createStreamFactory(conf);
+ if(conf.getEnum(INPUT_STREAM_TYPE, InputStreamType.DEFAULT_STREAM_TYPE) ==
InputStreamType.Analytics) {
+ final S3AsyncClient s3AsyncClient = getOrCreateAsyncCRTClient(conf);
+ objectInputStreamFactory = createStreamFactory(conf, s3AsyncClient);
+ } else {
+ objectInputStreamFactory = createStreamFactory(conf);
+ }
addService(objectInputStreamFactory);
// init all child services
super.serviceInit(conf);
}
+ private S3AsyncClient getOrCreateAsyncCRTClient(final Configuration conf)
throws Exception {
+ final S3AsyncClient s3AsyncClient;
+ boolean analyticsAcceleratorCRTEnabled =
conf.getBoolean(ANALYTICS_ACCELERATOR_CRT_ENABLED,
+ ANALYTICS_ACCELERATOR_CRT_ENABLED_DEFAULT);
+ LOG.info("Using S3SeekableInputStream");
Review Comment:
nit: let's remove this log as this method could be used by something that is
not using analytics accelerator
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
