This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9468227842e [Fix](tvf) Pass through user-defined properties (#35515)
(#35747)
9468227842e is described below
commit 9468227842ed48da4c9f862691fa4a0d82bd7a41
Author: Mingyu Chen <[email protected]>
AuthorDate: Fri May 31 22:50:26 2024 +0800
[Fix](tvf) Pass through user-defined properties (#35515) (#35747)
bp #35515
Co-authored-by: Tiewei Fang <[email protected]>
---
.../tablefunction/HdfsTableValuedFunction.java | 7 ++-
.../doris/tablefunction/S3TableValuedFunction.java | 14 ++---
.../property/PropertyPassThroughTest.java | 68 ++++++++++++++++++++++
3 files changed, 80 insertions(+), 9 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
index dd85ec55a61..80149e3d138 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.analysis.StorageBackend.StorageType;
import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.URI;
import org.apache.doris.thrift.TFileType;
@@ -70,8 +71,10 @@ public class HdfsTableValuedFunction extends
ExternalFileTableValuedFunction {
locationProperties.put(HdfsResource.HADOOP_FS_NAME,
uri.getScheme() + "://" + uri.getAuthority());
}
- // 4. parse file
- parseFile();
+ if (!FeConstants.runningUnitTest) {
+ // 4. parse file
+ parseFile();
+ }
}
// =========== implement abstract methods of
ExternalFileTableValuedFunction =================
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
index 44cbd482263..98b35de7d3e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
@@ -73,9 +73,8 @@ public class S3TableValuedFunction extends
ExternalFileTableValuedFunction {
S3URI s3uri = getS3Uri(uriStr,
Boolean.parseBoolean(usePathStyle.toLowerCase()),
Boolean.parseBoolean(forceParsingByStandardUri.toLowerCase()));
- String endpoint = otherProps.containsKey(S3Properties.ENDPOINT) ?
otherProps.get(S3Properties.ENDPOINT) :
- s3uri.getEndpoint().orElseThrow(() ->
- new AnalysisException(String.format("Properties '%s'
is required.", S3Properties.ENDPOINT)));
+ String endpoint = getOrDefaultAndRemove(otherProps,
S3Properties.ENDPOINT, s3uri.getEndpoint().orElseThrow(() ->
+ new AnalysisException(String.format("Properties '%s' is
required.", S3Properties.ENDPOINT))));
if (!otherProps.containsKey(S3Properties.REGION)) {
String region = s3uri.getRegion().orElseThrow(() ->
new AnalysisException(String.format("Properties '%s' is
required.", S3Properties.REGION)));
@@ -83,16 +82,17 @@ public class S3TableValuedFunction extends
ExternalFileTableValuedFunction {
}
checkNecessaryS3Properties(otherProps);
CloudCredentialWithEndpoint credential = new
CloudCredentialWithEndpoint(endpoint,
- otherProps.get(S3Properties.REGION),
- otherProps.get(S3Properties.ACCESS_KEY),
- otherProps.get(S3Properties.SECRET_KEY));
+ getOrDefaultAndRemove(otherProps, S3Properties.REGION, ""),
+ getOrDefaultAndRemove(otherProps, S3Properties.ACCESS_KEY, ""),
+ getOrDefaultAndRemove(otherProps, S3Properties.SECRET_KEY,
""));
if (otherProps.containsKey(S3Properties.SESSION_TOKEN)) {
-
credential.setSessionToken(otherProps.get(S3Properties.SESSION_TOKEN));
+ credential.setSessionToken(getOrDefaultAndRemove(otherProps,
S3Properties.SESSION_TOKEN, ""));
}
locationProperties = S3Properties.credentialToMap(credential);
locationProperties.put(PropertyConverter.USE_PATH_STYLE, usePathStyle);
locationProperties.putAll(S3ClientBEProperties.getBeFSProperties(locationProperties));
+ locationProperties.putAll(otherProps);
filePath = NAME + S3URI.SCHEME_DELIM + s3uri.getBucket() +
S3URI.PATH_DELIM + s3uri.getKey();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
new file mode 100644
index 00000000000..32a212c5cf7
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property;
+
+import org.apache.doris.analysis.SelectStmt;
+import org.apache.doris.analysis.TableValuedFunctionRef;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.tablefunction.HdfsTableValuedFunction;
+import org.apache.doris.tablefunction.S3TableValuedFunction;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class PropertyPassThroughTest extends TestWithFeService {
+ @Test
+ public void testS3TVFPropertiesPassThrough() throws Exception {
+ FeConstants.runningUnitTest = true;
+ String queryOld = "select * from s3(\n"
+ + " 'uri' =
'http://s3.us-east-1.amazonaws.com/my-bucket/test.parquet',\n"
+ + " 'access_key' = 'akk',\n"
+ + " 'secret_key' = 'skk',\n"
+ + " 'region' = 'us-east-1',\n"
+ + " 'format' = 'parquet',\n"
+ + " 'fs.s3a.list.version' = '1',\n"
+ + " 'test_property' = 'test',\n"
+ + " 'use_path_style' = 'true'\n"
+ + ") limit 10;";
+ SelectStmt analyzedStmt = createStmt(queryOld);
+ Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1);
+ TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef)
analyzedStmt.getTableRefs().get(0);
+ S3TableValuedFunction s3Tvf = (S3TableValuedFunction)
oldFuncTable.getTableFunction();
+
Assertions.assertTrue(s3Tvf.getBrokerDesc().getProperties().containsKey("fs.s3a.list.version"));
+
Assertions.assertTrue(s3Tvf.getBrokerDesc().getProperties().containsKey("test_property"));
+ }
+
+ @Test
+ public void testHdfsTVFPropertiesPassThrough() throws Exception {
+ FeConstants.runningUnitTest = true;
+ String queryOld = "select * from hdfs(\n"
+ + " 'uri' =
'hdfs://HDFS11111871/path/example_table/country=USA/city=NewYork/000000_0',\n"
+ + " 'hadoop.username' = 'hadoop',\n"
+ + " 'path_partition_keys' = 'country,city',\n"
+ + " 'format' = 'orc',\n"
+ + " 'test_property' = 'test'\n"
+ + ") limit 10;";
+ SelectStmt analyzedStmt = createStmt(queryOld);
+ Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1);
+ TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef)
analyzedStmt.getTableRefs().get(0);
+ HdfsTableValuedFunction hdfsTvf = (HdfsTableValuedFunction)
oldFuncTable.getTableFunction();
+
Assertions.assertTrue(hdfsTvf.getBrokerDesc().getProperties().containsKey("test_property"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
