[
https://issues.apache.org/jira/browse/BEAM-12435?focusedWorklogId=715906&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-715906
]
ASF GitHub Bot logged work on BEAM-12435:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 26/Jan/22 18:31
Start Date: 26/Jan/22 18:31
Worklog Time Spent: 10m
Work Description: mattrudary commented on a change in pull request #16607:
URL: https://github.com/apache/beam/pull/16607#discussion_r792928048
##########
File path:
sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemConfiguration.java
##########
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.aws2.s3;
+
+import com.google.auto.value.AutoValue;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.annotations.Experimental.Kind;
+import org.apache.beam.sdk.io.aws2.options.S3ClientBuilderFactory;
+import org.apache.beam.sdk.io.aws2.options.S3Options;
+import org.apache.beam.sdk.util.InstanceBuilder;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+
+/**
+ * Object used to configure {@link S3FileSystem}.
+ *
+ * @see S3Options
+ * @see S3FileSystemSchemeRegistrar
+ */
+@AutoValue
+@Experimental(Kind.FILESYSTEM)
+public abstract class S3FileSystemConfiguration {
+ public static final int MINIMUM_UPLOAD_BUFFER_SIZE_BYTES = 5_242_880;
+
+ /** The uri scheme used by resources on this filesystem. */
+ public abstract String getScheme();
+
+ /** The AWS S3 storage class used for creating S3 objects. */
+ public abstract String getS3StorageClass();
+
+ /** Size of S3 upload chnks. */
+ public abstract int getS3UploadBufferSizeBytes();
+
+ /** Thread pool size, limiting the max concurrent S3 operations. */
+ public abstract int getS3ThreadPoolSize();
+
+ /** Algorithm for SSE-S3 encryption, e.g. AES256. */
+ public abstract @Nullable String getSSEAlgorithm();
+
+ /** SSE key for SSE-C encryption, e.g. a base64 encoded key and the
algorithm. */
+ public abstract SSECustomerKey getSSECustomerKey();
+
+ /** KMS key id for SSE-KMS encyrption, e.g. "arn:aws:kms:..." */
+ public abstract @Nullable String getSSEKMSKeyId();
+
+ /** Builder used to create the {@code S3Client}. */
+ public abstract S3ClientBuilder getS3ClientBuilder();
+
+ /** Creates a new uninitialized {@link Builder}. */
+ public static Builder builder() {
+ return new AutoValue_S3FileSystemConfiguration.Builder();
+ }
+
+ /** Creates a new {@link Builder} with values initialized by this instance's
properties. */
+ public abstract Builder toBuilder();
+
+ /**
+ * Creates a new {@link Builder} with values initialized by the properties
of {@code s3Options}.
+ */
+ public static Builder fromS3Options(S3Options s3Options) {
Review comment:
Why not both :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 715906)
Time Spent: 6.5h (was: 6h 20m)
> Generalize S3FileSystem
> -----------------------
>
> Key: BEAM-12435
> URL: https://issues.apache.org/jira/browse/BEAM-12435
> Project: Beam
> Issue Type: Improvement
> Components: io-java-aws
> Reporter: Matt Rudary
> Assignee: Matt Rudary
> Priority: P2
> Labels: aws, aws-s3
> Fix For: 2.32.0
>
> Time Spent: 6.5h
> Remaining Estimate: 0h
>
> I'm working with multiple storage systems that speak the S3 api. I would like
> to support FileIO operations for these storage systems, but S3FileSystem
> hardcodes the s3 scheme (the various systems use different URI schemes) and
> it is in any case impossible to instantiate more than one in the current
> design.
> I'd like to refactor the code in org.apache.beam.sdk.io.aws.s3 (and maybe
> ...aws.options) somewhat to enable this use-case. I haven't worked out the
> details yet, but it will take some thought to make this work in a non-hacky
> way.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
