peterxcli commented on code in PR #8366:
URL: https://github.com/apache/ozone/pull/8366#discussion_r2097762968
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java:
##########
@@ -131,16 +136,70 @@ public static WebApplicationException
wrapOS3Exception(OS3Exception ex) {
.build());
}
- public static boolean hasSignedPayloadHeader(HttpHeaders headers) {
- final String signingAlgorithm =
headers.getHeaderString(X_AMZ_CONTENT_SHA256);
- if (signingAlgorithm == null) {
- return false;
+ public static boolean hasUnsignedPayload(@Nonnull String
amzContentSha256Header) {
+ Objects.requireNonNull(amzContentSha256Header);
+ return amzContentSha256Header.equals(UNSIGNED_PAYLOAD) ||
+ amzContentSha256Header.equals(STREAMING_UNSIGNED_PAYLOAD_TRAILER);
+ }
+
+ public static boolean hasMultiChunksPayload(@Nonnull String
amzContentSha256Header) {
+ Objects.requireNonNull(amzContentSha256Header);
+ // Multiple chunk uploads
+ // - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+ // -
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
+ // Possible values
+ // - STREAMING-UNSIGNED-PAYLOAD-TRAILER
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+ // - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD
+ // - STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER
+ // Currently since all the multi chunks values have x-amz-content-sha256
header value that starts
+ // with STREAMING, we can use this prefix to differentiates between multi
chunks and single chunks upload.
+ // In the future if there are more multi chunks signature algorithms that
has the same prefix,
+ // this function will be able to handle detect it.
+ return amzContentSha256Header.startsWith(MULTI_CHUNKS_UPLOAD_PREFIX);
+ }
+
+ public static void validateMultiChunksUpload(HttpHeaders headers, String
amzDecodedContentLength,
+ String resource) throws
OS3Exception {
+ final String contentEncoding =
headers.getHeaderString(HttpHeaders.CONTENT_ENCODING);
+ // "Content-Encoding : aws-chunked" seems to only be sent for SDK V2, so
ignore if there is no
+ // Content-Encoding header
+ if (contentEncoding != null) {
+ // Amazon S3 supports multiple content encoding values for example
"Content-Encoding : aws-chunked,gzip"
+ // We are only interested on "aws-chunked"
+ boolean containsAwsChunked = Arrays.stream(contentEncoding.split(","))
+ .map(String::trim)
+ .anyMatch(AWS_CHUNKED::equals);
+ if (!containsAwsChunked) {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
resource);
+ ex.setErrorMessage("An error occurred (InvalidArgument) for multi
chunks upload: " +
+ "The " + HttpHeaders.CONTENT_ENCODING + " header does not contain
" + AWS_CHUNKED);
+ throw ex;
+ }
+ }
+
+ if (amzDecodedContentLength == null) {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
resource);
+ ex.setErrorMessage("An error occurred (InvalidArgument) for multi chunks
upload: " +
+ "The " + DECODED_CONTENT_LENGTH_HEADER + " header is not specified");
+ throw ex;
+ }
+ }
+
+ public static String validateSignatureHeader(HttpHeaders headers, String
resource) throws OS3Exception {
+ String xAmzContentSha256Header =
headers.getHeaderString(X_AMZ_CONTENT_SHA256);
+ if (xAmzContentSha256Header == null) {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
resource);
+ ex.setErrorMessage("An error occurred (InvalidArgument): " +
+ "The " + X_AMZ_CONTENT_SHA256 + " header is not specified");
+ throw ex;
}
- // Handles both AWS Signature Version 4 (HMAC-256) and AWS Signature
Version 4A (ECDSA-P256-SHA256)
- return signingAlgorithm.equals(STREAMING_AWS4_HMAC_SHA256_PAYLOAD) ||
- signingAlgorithm.equals(STREAMING_AWS4_HMAC_SHA256_PAYLOAD_TRAILER) ||
- signingAlgorithm.equals(STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD) ||
-
signingAlgorithm.equals(STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD_TRAILER);
+ return xAmzContentSha256Header;
+ }
+
+ public static boolean eol(int prev, int curr) {
Review Comment:
nit:
```suggestion
/**
* Checks if the given pair of bytes represent the end-of-line sequence
(\r\n).
*
* @param prev the previous byte value (should be 13 for '\r')
* @param curr the current byte value (should be 10 for '\n')
* @return true if the pair forms a CRLF sequence, false otherwise
*/
public static boolean eol(int prev, int curr) {
```
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java:
##########
@@ -46,12 +82,21 @@ public class SignedChunksInputStream extends InputStream {
*/
private int remainingData = 0;
+ /**
+ * Every chunked uploads (multiple chunks) contains a final zero-byte final
additional
Review Comment:
nit:
```suggestion
* Every chunked uploads (multiple chunks) contains a final zero-byte
additional
```
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3;
+
+import static org.apache.hadoop.ozone.s3.util.S3Utils.eol;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Objects;
+
+/**
+ * Input stream implementation to read body of an unsigned chunked upload.
+ * <p>
+ * Currently, the only valid value of x-amz-content-sha256 header to indicate
+ * transfer unsigned payload in multiple chunks is
STREAMING-UNSIGNED-PAYLOAD-TRAILER.
+ * Therefore, the input stream should work with chunked payloads with checksum
trailer.
+ * Nevertheless, this input stream also supports chunked upload without
trailer.
Review Comment:
So this would just skip the checksum trailer check? Does
`SignedChunkInputStream` skip too?
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UnsignedChunksInputStream.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3;
+
+import static org.apache.hadoop.ozone.s3.util.S3Utils.eol;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Objects;
+
+/**
+ * Input stream implementation to read body of an unsigned chunked upload.
+ * <p>
+ * Currently, the only valid value of x-amz-content-sha256 header to indicate
+ * transfer unsigned payload in multiple chunks is
STREAMING-UNSIGNED-PAYLOAD-TRAILER.
+ * Therefore, the input stream should work with chunked payloads with checksum
trailer.
+ * Nevertheless, this input stream also supports chunked upload without
trailer.
+ * </p>
+ * <p>
+ * Example chunk data:
+ * <pre>
+ * 10000\r\n
+ * <65536-bytes>\r\n
+ * 0\r\n
+ * x-amz-checksum-crc64nvme:2wstOANdZ/o=\r\n
+ * </pre>
+ * </p>
+ * <p>
+ * The 10000 will be read and decoded from base-16 representation to 65536,
which is the size of
+ * the subsequent chunk payload. Each chunk upload ends with a zero-byte final
additional chunk.
+ * At the end, there will be a trailer checksum payload
+ * </p>
+ *
+ * <p>
+ * The logic is similar to {@link SignedChunksInputStream}, but since it is an
unsigned chunked upload
+ * there is no "chunk-signature" to parse.
+ * </p>
+ *
+ * <p>
+ * Note that there is not actual trailer checksum verification taking place.
The InputStream only
+ * * returns the actual chunk payload from chunked signatures format.
+ * </p>
+ *
+ * Reference:
+ * <ul>
+ * <li>
+ * <a
href="https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html";>
+ * Signature Calculation: Transfer Payload in Multiple Chunks</a>
+ * </li>
+ * <li>
+ * <a
href="https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html";>
+ * Signature Calculation: Including Trailing Headers</a>
+ * </li>
+ * </ul>
+ */
+public class UnsignedChunksInputStream extends InputStream {
+
+ private final InputStream originalStream;
+
+ /**
+ * Size of the chunk payload. If zero, the content length should be parsed to
+ * retrieve the subsequent chunk payload size.
+ */
+ private int remainingData = 0;
+
+ /**
+ * Every chunked uploads (multiple chunks) contains a final zero-byte final
additional
Review Comment:
```suggestion
* Every chunked uploads (multiple chunks) contains a final zero-byte
additional
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
