This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from a41c5cbfd IMPALA-14416: JniFrontend.getDbs() should handle
InconsistentMetadataFetchException
new 826c8cf9b IMPALA-14081: Support create/drop paimon table for impala
new 7fabd2709 IMPALA-14411: enable_workload_mgmt should work with V2
profile
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/service/query-options-test.cc | 2 +-
be/src/service/query-state-record.cc | 67 +-
bin/impala-config.sh | 3 +-
bin/rat_exclude_files.txt | 1 +
common/thrift/CatalogObjects.thrift | 28 +-
common/thrift/Descriptors.thrift | 1 +
docs/impala.ditamap | 1 +
docs/impala_keydefs.ditamap | 4 +
docs/topics/impala_paimon.xml | 352 +++++++++
docs/topics/impala_reserved_words.xml | 7 +
fe/pom.xml | 16 +
fe/src/main/cup/sql-parser.cup | 8 +-
.../org/apache/impala/analysis/AnalysisUtils.java | 13 +-
.../java/org/apache/impala/analysis/Analyzer.java | 4 +-
.../apache/impala/analysis/CreateTableStmt.java | 16 +-
.../org/apache/impala/analysis/ShowFilesStmt.java | 15 +-
.../org/apache/impala/analysis/ShowStatsStmt.java | 4 +
.../org/apache/impala/analysis/ToSqlUtils.java | 59 +-
.../impala/analysis/paimon/PaimonAnalyzer.java | 290 +++++++
.../org/apache/impala/catalog/HdfsFileFormat.java | 14 +-
.../impala/catalog/HdfsStorageDescriptor.java | 3 +-
.../main/java/org/apache/impala/catalog/Table.java | 6 +-
.../org/apache/impala/catalog/VirtualColumn.java | 10 +
.../impala/catalog/local/LocalPaimonTable.java | 77 ++
.../apache/impala/catalog/local/LocalTable.java | 6 +-
.../impala/catalog/paimon/FePaimonTable.java | 132 ++++
.../catalog/paimon/FeShowFileStmtSupport.java | 29 +-
.../impala/catalog/paimon/ImpalaTypeUtils.java | 283 +++++++
.../catalog/paimon/PaimonCatalogOpExecutor.java | 212 +++++
.../apache/impala/catalog/paimon/PaimonTable.java | 266 +++++++
.../paimon/PaimonTableLoadingException.java | 11 +-
.../apache/impala/catalog/paimon/PaimonUtil.java | 861 +++++++++++++++++++++
.../apache/impala/service/CatalogOpExecutor.java | 142 +++-
.../java/org/apache/impala/service/Frontend.java | 8 +-
.../java/org/apache/impala/util/DebugUtils.java | 3 +
fe/src/main/jflex/sql-scanner.flex | 1 +
.../org/apache/impala/analysis/AnalyzeDDLTest.java | 62 +-
.../org/apache/impala/analysis/ParserTest.java | 4 +-
.../java/org/apache/impala/analysis/ToSqlTest.java | 42 +
.../impala/catalog/paimon/ImpalaTypeUtilsTest.java | 161 ++++
java/pom.xml | 1 +
...-0ca8873e-4b8c-4a80-ac31-3cd5bd95ae01-0.parquet | Bin 0 -> 1315 bytes
...-488765b8-d0d7-4091-ad1a-2753f9fb0b63-0.parquet | Bin 0 -> 1115 bytes
...-5ab52dc4-cd44-4de4-9b81-fcd30baeee0d-0.parquet | Bin 0 -> 1671 bytes
...-6684e5ef-26fa-4f13-a1bd-9c2a5bc9557c-0.parquet | Bin 0 -> 1936 bytes
...-ad75a7e8-4ae0-4803-8f49-92d58fd4a222-0.parquet | Bin 0 -> 1315 bytes
...manifest-1eecc41b-3dcd-4388-afdb-54527b51be2a-0 | Bin 0 -> 748 bytes
...manifest-3505f2c0-e77f-42ff-9a54-368038f7eb3e-0 | Bin 0 -> 2225 bytes
...manifest-671ffa6b-2665-4d30-b039-d21a5565d698-0 | Bin 0 -> 2220 bytes
...manifest-9e6bcbde-1da9-4b06-a17c-c71579de18a5-0 | Bin 0 -> 2227 bytes
...manifest-b4b02924-241f-4abf-8ad8-d7a2ac5536e2-0 | Bin 0 -> 1963 bytes
...manifest-c32aa2fd-3d98-4f09-9067-8a269d8115eb-0 | Bin 0 -> 1964 bytes
...est-list-025dfcce-8e34-42c0-a8f7-def795d489df-0 | Bin 0 -> 1050 bytes
...est-list-025dfcce-8e34-42c0-a8f7-def795d489df-1 | Bin 0 -> 989 bytes
...est-list-5cd761f7-4c95-4181-841e-aa8b405e71be-0 | Bin 0 -> 884 bytes
...est-list-5cd761f7-4c95-4181-841e-aa8b405e71be-1 | Bin 0 -> 884 bytes
...est-list-6e11688a-4b7c-4bfd-9118-5e49eac61016-0 | Bin 0 -> 884 bytes
...est-list-6e11688a-4b7c-4bfd-9118-5e49eac61016-1 | Bin 0 -> 989 bytes
...est-list-d86f1254-05df-4560-8885-693e94b46cbf-0 | Bin 0 -> 989 bytes
...est-list-d86f1254-05df-4560-8885-693e94b46cbf-1 | Bin 0 -> 884 bytes
.../paimon_non_partitioned/schema/schema-0 | 28 +
.../paimon_non_partitioned/schema/schema-1 | 29 +
.../paimon_non_partitioned/snapshot/EARLIEST | 1 +
.../paimon_non_partitioned/snapshot/LATEST | 1 +
.../paimon_non_partitioned/snapshot/snapshot-10 | 19 +
.../paimon_non_partitioned/snapshot/snapshot-11 | 19 +
.../paimon_non_partitioned/snapshot/snapshot-12 | 20 +
.../paimon_non_partitioned/snapshot/snapshot-9 | 19 +
.../stat-eaea66dc-9657-4d89-823a-98315aa27329-0 | 44 ++
...manifest-1db1d7b8-9b21-4241-a0c6-8189c368a0bc-0 | Bin 0 -> 2243 bytes
...manifest-5a2ea60f-e1b6-4126-aee4-088ff0b82f28-0 | Bin 0 -> 2245 bytes
...manifest-9216adfd-451c-486d-a6db-b66884314880-0 | Bin 0 -> 2244 bytes
...est-list-22716ac0-0c0e-4f03-a48c-31c807cbf8d9-0 | Bin 0 -> 1064 bytes
...est-list-22716ac0-0c0e-4f03-a48c-31c807cbf8d9-1 | Bin 0 -> 884 bytes
...est-list-fececc90-17e5-4cde-bc69-84a98da91b79-0 | Bin 0 -> 1028 bytes
...est-list-fececc90-17e5-4cde-bc69-84a98da91b79-1 | Bin 0 -> 992 bytes
.../paimon_partitioned/schema/schema-0 | 28 +
.../paimon_partitioned/schema/schema-1 | 29 +
.../paimon_partitioned/schema/schema-2 | 31 +
.../paimon_partitioned/schema/schema-3 | 30 +
.../paimon_partitioned/schema/schema-4 | 29 +
.../paimon_partitioned/schema/schema-5 | 28 +
.../paimon_partitioned/snapshot/EARLIEST} | 0
.../paimon_partitioned/snapshot/LATEST} | 0
.../paimon_partitioned/snapshot/snapshot-5 | 18 +
.../paimon_partitioned/snapshot/snapshot-6 | 19 +
.../stat-7061d17d-5a4d-4532-b98f-714d66103721-0 | 44 ++
...-d7e0b3b7-a5e7-407f-b591-033cdf2da158-0.parquet | Bin 0 -> 1301 bytes
...-010c93fa-86ab-4389-aa03-4d76b7ece890-0.parquet | Bin 0 -> 1657 bytes
...-4decfe67-c7e1-43f9-96ee-ae5188779a39-0.parquet | Bin 0 -> 1301 bytes
...-ee1eb48c-19ba-4275-9a73-ca788d55c57d-0.parquet | Bin 0 -> 1101 bytes
...-86902019-80d0-4395-8bd4-666e1ffe2422-0.parquet | Bin 0 -> 1922 bytes
.../functional/functional_schema_template.sql | 25 +
.../datasets/functional/schema_constraints.csv | 4 +
.../queries/QueryTest/paimon-ddl-negative.test | 24 +
.../queries/QueryTest/paimon-ddl.test | 211 +++++
.../queries/QueryTest/paimon-show-stats.test | 49 ++
.../functional-query/queries/QueryTest/set.test | 2 +-
.../QueryTest/show-create-table-paimon.test | 201 +++++
tests/custom_cluster/test_query_log.py | 36 +
tests/metadata/test_show_create_table.py | 8 +-
.../test_paimon.py} | 40 +-
102 files changed, 4171 insertions(+), 90 deletions(-)
create mode 100644 docs/topics/impala_paimon.xml
mode change 100755 => 100644
fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
create mode 100644
fe/src/main/java/org/apache/impala/analysis/paimon/PaimonAnalyzer.java
create mode 100644
fe/src/main/java/org/apache/impala/catalog/local/LocalPaimonTable.java
create mode 100644
fe/src/main/java/org/apache/impala/catalog/paimon/FePaimonTable.java
copy
java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
=>
fe/src/main/java/org/apache/impala/catalog/paimon/FeShowFileStmtSupport.java
(57%)
create mode 100644
fe/src/main/java/org/apache/impala/catalog/paimon/ImpalaTypeUtils.java
create mode 100644
fe/src/main/java/org/apache/impala/catalog/paimon/PaimonCatalogOpExecutor.java
create mode 100644
fe/src/main/java/org/apache/impala/catalog/paimon/PaimonTable.java
copy
java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java
=>
fe/src/main/java/org/apache/impala/catalog/paimon/PaimonTableLoadingException.java
(76%)
create mode 100644
fe/src/main/java/org/apache/impala/catalog/paimon/PaimonUtil.java
mode change 100755 => 100644
fe/src/test/java/org/apache/impala/analysis/ParserTest.java
create mode 100644
fe/src/test/java/org/apache/impala/catalog/paimon/ImpalaTypeUtilsTest.java
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/bucket-0/data-0ca8873e-4b8c-4a80-ac31-3cd5bd95ae01-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/bucket-0/data-488765b8-d0d7-4091-ad1a-2753f9fb0b63-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/bucket-0/data-5ab52dc4-cd44-4de4-9b81-fcd30baeee0d-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/bucket-0/data-6684e5ef-26fa-4f13-a1bd-9c2a5bc9557c-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/bucket-0/data-ad75a7e8-4ae0-4803-8f49-92d58fd4a222-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/index-manifest-1eecc41b-3dcd-4388-afdb-54527b51be2a-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-3505f2c0-e77f-42ff-9a54-368038f7eb3e-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-671ffa6b-2665-4d30-b039-d21a5565d698-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-9e6bcbde-1da9-4b06-a17c-c71579de18a5-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-b4b02924-241f-4abf-8ad8-d7a2ac5536e2-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-c32aa2fd-3d98-4f09-9067-8a269d8115eb-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-list-025dfcce-8e34-42c0-a8f7-def795d489df-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-list-025dfcce-8e34-42c0-a8f7-def795d489df-1
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-list-5cd761f7-4c95-4181-841e-aa8b405e71be-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-list-5cd761f7-4c95-4181-841e-aa8b405e71be-1
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-list-6e11688a-4b7c-4bfd-9118-5e49eac61016-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-list-6e11688a-4b7c-4bfd-9118-5e49eac61016-1
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-list-d86f1254-05df-4560-8885-693e94b46cbf-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/manifest/manifest-list-d86f1254-05df-4560-8885-693e94b46cbf-1
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/schema/schema-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/schema/schema-1
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/snapshot/EARLIEST
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/snapshot/LATEST
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/snapshot/snapshot-10
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/snapshot/snapshot-11
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/snapshot/snapshot-12
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/snapshot/snapshot-9
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_non_partitioned/statistics/stat-eaea66dc-9657-4d89-823a-98315aa27329-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/manifest/manifest-1db1d7b8-9b21-4241-a0c6-8189c368a0bc-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/manifest/manifest-5a2ea60f-e1b6-4126-aee4-088ff0b82f28-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/manifest/manifest-9216adfd-451c-486d-a6db-b66884314880-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/manifest/manifest-list-22716ac0-0c0e-4f03-a48c-31c807cbf8d9-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/manifest/manifest-list-22716ac0-0c0e-4f03-a48c-31c807cbf8d9-1
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/manifest/manifest-list-fececc90-17e5-4cde-bc69-84a98da91b79-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/manifest/manifest-list-fececc90-17e5-4cde-bc69-84a98da91b79-1
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/schema/schema-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/schema/schema-1
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/schema/schema-2
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/schema/schema-3
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/schema/schema-4
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/schema/schema-5
copy
testdata/data/{iceberg_test/hadoop_catalog/ice/iceberg_spark_compaction_with_dangling_delete/metadata/version-hint.text
=>
paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/snapshot/EARLIEST}
(100%)
copy
testdata/data/{iceberg_test/hadoop_catalog/ice/iceberg_multiple_storage_locations/metadata/version-hint.text
=>
paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/snapshot/LATEST}
(100%)
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/snapshot/snapshot-5
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/snapshot/snapshot-6
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/statistics/stat-7061d17d-5a4d-4532-b98f-714d66103721-0
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/userId=1/bucket-0/data-d7e0b3b7-a5e7-407f-b591-033cdf2da158-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/userId=2/bucket-0/data-010c93fa-86ab-4389-aa03-4d76b7ece890-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/userId=3/bucket-0/data-4decfe67-c7e1-43f9-96ee-ae5188779a39-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/userId=4/bucket-0/data-ee1eb48c-19ba-4275-9a73-ca788d55c57d-0.parquet
create mode 100644
testdata/data/paimon_test/paimon_catalog/warehouse/functional.db/paimon_partitioned/userId=5/bucket-0/data-86902019-80d0-4395-8bd4-666e1ffe2422-0.parquet
create mode 100644
testdata/workloads/functional-query/queries/QueryTest/paimon-ddl-negative.test
create mode 100644
testdata/workloads/functional-query/queries/QueryTest/paimon-ddl.test
create mode 100644
testdata/workloads/functional-query/queries/QueryTest/paimon-show-stats.test
create mode 100644
testdata/workloads/functional-query/queries/QueryTest/show-create-table-paimon.test
copy tests/{experiments/test_targeted_perf.py => query_test/test_paimon.py}
(51%)
