This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from 96a4125a94 [feature wip](multi catalog)Support iceberg schema
evolution. (#15836)
new 6b2c6460fb [Feature](ES): catalog support mapping es _id #15943
new c9e68b7481 [Feature](multi-catalog)Add support for JuiceFS (#15969)
new 1db3e975cd [fix](DOE) only return first batch data in ES 8.x (#16025)
new a553bc6d7b [Fix][regression-test] Fix test_hdfs_tvf.groovy by update
HDFS conf URI to uri and better error msg handling. (#16029)
new 641c49d925 [fix](jdbc) fix jdbc driver bug and external datasource p2
test case issue (#16033)
new e10346ba53 [fix](multi-catalog)switching catalogs after dropping will
get NPE. (#16067)
new 88096e0993 [fix](multi-catalog)Make ES catalog and resource compatible
(#16096)
new e35ef74f72 [fix](oracle catalog) oracle catalog support `TIMESTAMP`
dateType of oracle (#16113)
new c7d1535dba [feature](jdbc catalog) support SQLServer jdbc catalog
(#16093)
new c32a372b3c [Fix](Oracle External Table) fix that oracle external table
can not insert batch values (#16117)
new bfc414ee53 [refactor](fe) refactor and upgrade dependency tree of FE
and support AWS glue catalog (#16046)
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/es/es_scan_reader.cpp | 5 +-
be/src/exec/table_connector.cpp | 41 +-
be/src/exec/table_connector.h | 9 +
be/src/util/s3_uri.cpp | 7 +
be/src/util/s3_uri.h | 1 +
be/src/vec/exec/scan/new_jdbc_scanner.cpp | 15 +-
be/src/vec/exec/scan/new_jdbc_scanner.h | 2 +
be/src/vec/exec/vjdbc_connector.cpp | 25 +-
be/src/vec/exec/vjdbc_connector.h | 16 +-
bin/start_fe.sh | 9 +
.../docker-compose/oracle/init/03-create-table.sql | 10 +
.../docker-compose/oracle/init/04-insert.sql | 8 +
.../sqlserver/init/01-drop-database.sql | 18 +
.../sqlserver/init/02-create-database.sql | 18 +
.../sqlserver/init/03-create-table.sql | 89 +
.../docker-compose/sqlserver/init/04-insert.sql | 52 +
.../docker-compose/sqlserver/sqlserver.env | 19 +
.../docker-compose/sqlserver/sqlserver.yaml | 47 +
docker/thirdparties/start-thirdparties-docker.sh | 7 +
docker/thirdparties/stop-thirdparties-docker.sh | 3 +
docs/en/docs/advanced/broker.md | 15 +-
.../Create/CREATE-CATALOG.md | 24 +
docs/zh-CN/docs/advanced/broker.md | 13 +
.../Create/CREATE-CATALOG.md | 24 +
fe/check/checkstyle/suppressions.xml | 5 +
.../java/org/apache/doris/sparkdpp}/DppResult.java | 4 +-
.../org/apache/doris/sparkdpp}/EtlJobConfig.java | 2 +-
fe/fe-core/pom.xml | 62 +-
.../metastore/hive2/ProxyMetaStoreClient.java | 2193 ++++++++++++
.../converters/BaseCatalogToHiveConverter.java | 541 +++
.../catalog/converters/CatalogToHiveConverter.java | 58 +
.../converters/CatalogToHiveConverterFactory.java | 54 +
.../glue/catalog/converters/ConverterUtils.java | 49 +
.../catalog/converters/GlueInputConverter.java | 116 +
.../converters/Hive3CatalogToHiveConverter.java | 70 +
.../catalog/converters/HiveToCatalogConverter.java | 372 ++
.../catalog/converters/PartitionNameParser.java | 143 +
.../ConfigurationAWSCredentialsProvider.java | 60 +
...onfigurationAWSCredentialsProviderFactory.java} | 14 +-
.../exceptions/InvalidPartitionNameException.java} | 18 +-
.../exceptions/LakeFormationException.java} | 18 +-
.../metastore/AWSCatalogMetastoreClient.java | 2481 ++++++++++++++
.../metastore/AWSCredentialsProviderFactory.java} | 16 +-
.../catalog/metastore/AWSGlueClientFactory.java | 157 +
.../catalog/metastore/AWSGlueDecoratorBase.java | 1153 +++++++
.../glue/catalog/metastore/AWSGlueMetastore.java | 133 +
.../metastore/AWSGlueMetastoreBaseDecorator.java | 198 ++
.../metastore/AWSGlueMetastoreCacheDecorator.java | 185 +
.../catalog/metastore/AWSGlueMetastoreFactory.java | 47 +
.../metastore/AWSGlueMultipleCatalogDecorator.java | 370 ++
.../DefaultAWSCredentialsProviderFactory.java} | 22 +-
.../catalog/metastore/DefaultAWSGlueMetastore.java | 662 ++++
.../metastore/DefaultExecutorServiceFactory.java | 43 +
.../catalog/metastore/ExecutorServiceFactory.java} | 18 +-
.../glue/catalog/metastore/GlueClientFactory.java} | 19 +-
.../metastore/GlueMetastoreClientDelegate.java | 1843 ++++++++++
.../SessionCredentialsProviderFactory.java | 56 +
.../amazonaws/glue/catalog/util/AWSGlueConfig.java | 64 +
.../catalog/util/BatchCreatePartitionsHelper.java | 153 +
.../catalog/util/BatchDeletePartitionsHelper.java | 147 +
.../glue/catalog/util/ExpressionHelper.java | 242 ++
.../glue/catalog/util/HiveTableValidator.java | 86 +
.../amazonaws/glue/catalog/util/LoggingHelper.java | 57 +
.../glue/catalog/util/MetastoreClientUtils.java | 141 +
.../amazonaws/glue/catalog/util/PartitionKey.java | 60 +
.../glue/catalog/util/PartitionUtils.java | 57 +
.../java/org/apache/doris/analysis/BrokerDesc.java | 1 +
.../java/org/apache/doris/analysis/ExportStmt.java | 5 +-
.../org/apache/doris/analysis/StorageBackend.java | 3 +
.../java/org/apache/doris/backup/BlobStorage.java | 6 +-
.../main/java/org/apache/doris/catalog/Env.java | 5 +-
.../java/org/apache/doris/catalog/EsResource.java | 20 +-
.../java/org/apache/doris/catalog/EsTable.java | 2 +-
.../java/org/apache/doris/catalog/HMSResource.java | 54 +-
.../doris/catalog/HiveMetaStoreClientHelper.java | 3 +-
.../doris/catalog/InternalSchemaInitializer.java | 16 +-
.../org/apache/doris/catalog/JdbcResource.java | 10 +-
.../java/org/apache/doris/catalog/OdbcTable.java | 12 +-
.../java/org/apache/doris/catalog/S3Resource.java | 12 +-
.../doris/catalog/external/HMSExternalTable.java | 4 -
.../java/org/apache/doris/common/FeConstants.java | 1 +
.../apache/doris/datasource/CatalogProperty.java | 2 +
.../apache/doris/datasource/EsExternalCatalog.java | 13 +-
.../doris/datasource/HMSExternalCatalog.java | 3 +-
.../apache/doris/datasource/InternalCatalog.java | 5 +-
.../doris/datasource/hive/HiveMetaStoreClient.java | 2758 ---------------
.../datasource/hive/PooledHiveMetaStoreClient.java | 5 +
.../doris/external/elasticsearch/EsRestClient.java | 2 +
.../doris/external/elasticsearch/EsUtil.java | 14 +-
.../org/apache/doris/external/jdbc/JdbcClient.java | 96 +-
.../main/java/org/apache/doris/load/EtlStatus.java | 2 +-
.../doris/load/loadv2/SparkEtlJobHandler.java | 9 +-
.../org/apache/doris/load/loadv2/SparkLoadJob.java | 4 +-
.../doris/load/loadv2/SparkLoadPendingTask.java | 22 +-
.../org/apache/doris/planner/BrokerScanNode.java | 8 +-
.../org/apache/doris/planner/HiveScanNode.java | 7 +-
.../org/apache/doris/planner/HudiScanNode.java | 344 --
.../apache/doris/planner/PredicatePushDown.java | 4 +-
.../apache/doris/planner/SingleNodePlanner.java | 5 +-
.../doris/planner/external/HiveScanProvider.java | 6 +-
.../tablefunction/HdfsTableValuedFunction.java | 3 +
.../doris/tablefunction/S3TableValuedFunction.java | 13 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 3552 ++++++++++++++++++++
.../catalog/HiveMetaStoreClientHelperTest.java | 275 --
.../doris/load/loadv2/SparkEtlJobHandlerTest.java | 2 +-
.../apache/doris/load/loadv2/SparkLoadJobTest.java | 2 +-
.../load/loadv2/SparkLoadPendingTaskTest.java | 12 +-
fe/hive-udf/pom.xml | 24 -
fe/java-udf/pom.xml | 6 +
.../java/org/apache/doris/udf/JdbcExecutor.java | 20 +-
fe/pom.xml | 97 +-
fe/spark-dpp/pom.xml | 1 -
.../apache/doris/load/loadv2/dpp/ColumnParser.java | 2 +-
.../load/loadv2/dpp/DorisRangePartitioner.java | 2 +-
.../org/apache/doris/load/loadv2/dpp/DppUtils.java | 2 +-
.../dpp/MinimumCoverageRollupTreeBuilder.java | 2 +-
.../doris/load/loadv2/dpp/RollupTreeBuilder.java | 2 +-
.../doris/load/loadv2/dpp/RollupTreeNode.java | 2 +-
.../org/apache/doris/load/loadv2/dpp/SparkDpp.java | 3 +-
.../doris/load/loadv2/dpp/SparkRDDAggregator.java | 2 +-
.../apache/doris/load/loadv2/etl/SparkEtlJob.java | 11 +-
.../doris/load/loadv2/dpp/ColumnParserTest.java | 2 +-
.../load/loadv2/dpp/DorisRangePartitionerTest.java | 2 +-
.../apache/doris/load/loadv2/dpp/DppUtilsTest.java | 2 +-
.../dpp/MinimumCoverageRollupTreeBuilderTest.java | 2 +-
.../apache/doris/load/loadv2/dpp/SparkDppTest.java | 2 +-
.../doris/load/loadv2/etl/SparkEtlJobTest.java | 17 +-
.../doris/broker/hdfs/FileSystemManager.java | 132 +-
gensrc/thrift/Types.thrift | 1 +
regression-test/conf/regression-conf.groovy | 1 +
.../jdbc_catalog_p0/test_oracle_jdbc_catalog.out | 12 +-
.../test_sqlserver_jdbc_catalog.out | 43 +
.../table_valued_function/test_hdfs_tvf.groovy | 4 +-
.../test_oracle_jdbc_catalog.groovy | 1 +
.../jdbc_catalog_p0/test_pg_jdbc_catalog.groovy | 10 +-
...g.groovy => test_sqlserver_jdbc_catalog.groovy} | 58 +-
136 files changed, 16719 insertions(+), 3734 deletions(-)
create mode 100644
docker/thirdparties/docker-compose/sqlserver/init/01-drop-database.sql
create mode 100644
docker/thirdparties/docker-compose/sqlserver/init/02-create-database.sql
create mode 100644
docker/thirdparties/docker-compose/sqlserver/init/03-create-table.sql
create mode 100644
docker/thirdparties/docker-compose/sqlserver/init/04-insert.sql
create mode 100644 docker/thirdparties/docker-compose/sqlserver/sqlserver.env
create mode 100644 docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml
rename fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp =>
fe-common/src/main/java/org/apache/doris/sparkdpp}/DppResult.java (96%)
rename fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl =>
fe-common/src/main/java/org/apache/doris/sparkdpp}/EtlJobConfig.java (99%)
create mode 100644
fe/fe-core/src/main/java/com/aliyun/datalake/metastore/hive2/ProxyMetaStoreClient.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/converters/BaseCatalogToHiveConverter.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/converters/CatalogToHiveConverter.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/converters/CatalogToHiveConverterFactory.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/converters/ConverterUtils.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/converters/GlueInputConverter.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/converters/Hive3CatalogToHiveConverter.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/converters/HiveToCatalogConverter.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/converters/PartitionNameParser.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java
copy
fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/RollupTreeBuilder.java
=>
fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProviderFactory.java}
(62%)
copy
fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/RollupTreeBuilder.java
=>
fe-core/src/main/java/com/amazonaws/glue/catalog/exceptions/InvalidPartitionNameException.java}
(65%)
copy
fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/RollupTreeBuilder.java
=>
fe-core/src/main/java/com/amazonaws/glue/catalog/exceptions/LakeFormationException.java}
(66%)
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSCatalogMetastoreClient.java
copy
fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/RollupTreeBuilder.java
=>
fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSCredentialsProviderFactory.java}
(67%)
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueClientFactory.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueDecoratorBase.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueMetastore.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueMetastoreBaseDecorator.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueMetastoreCacheDecorator.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueMetastoreFactory.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueMultipleCatalogDecorator.java
copy
fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/RollupTreeBuilder.java
=>
fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/DefaultAWSCredentialsProviderFactory.java}
(57%)
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/DefaultAWSGlueMetastore.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/DefaultExecutorServiceFactory.java
copy
fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/RollupTreeBuilder.java
=>
fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/ExecutorServiceFactory.java}
(65%)
copy
fe/{spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/RollupTreeBuilder.java
=>
fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/GlueClientFactory.java}
(66%)
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/SessionCredentialsProviderFactory.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/BatchCreatePartitionsHelper.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/BatchDeletePartitionsHelper.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/ExpressionHelper.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/HiveTableValidator.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/LoggingHelper.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/MetastoreClientUtils.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/PartitionKey.java
create mode 100644
fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/PartitionUtils.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClient.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/planner/HudiScanNode.java
create mode 100644
fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
delete mode 100644
fe/fe-core/src/test/java/org/apache/doris/catalog/HiveMetaStoreClientHelperTest.java
create mode 100644
regression-test/data/jdbc_catalog_p0/test_sqlserver_jdbc_catalog.out
copy regression-test/suites/jdbc_catalog_p0/{test_oracle_jdbc_catalog.groovy
=> test_sqlserver_jdbc_catalog.groovy} (51%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
