This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a change to branch branch-4.1.1
in repository https://gitbox.apache.org/repos/asf/impala.git
from 44dc157a2 IMPALA-11344: Missing slots in all cases should be allowed
to be read
new 1a0b39442 IMPALA-11034: Resolve schema of old data files in migrated
Iceberg tables
new b5c3c91ea IMPALA-10865: Fix initialize SelectStmt's groupingExprs_ in
analyzeGroupingExprs
new 15fd47ed1 IMPALA-11457 Fix regression with unknown disk id
new 39af39f09 IMPALA-11464: Skip listing staging dirs to avoid failures on
them
new 14b2d414f IMPALA-11401,IMPALA-10794: Add logs and thread names for
catalogd RPCs
new 5cae46a4b IMPALA-11281: Load table metadata for ResetMetadataStmt
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/orc-metadata-utils.cc | 63 ++++-
be/src/exec/orc-metadata-utils.h | 9 +-
be/src/exec/parquet/parquet-metadata-utils.cc | 69 ++++-
be/src/exec/parquet/parquet-metadata-utils.h | 27 +-
common/fbs/CatalogObjects.fbs | 4 +-
.../org/apache/impala/analysis/ColumnName.java | 6 +
.../org/apache/impala/analysis/FunctionName.java | 4 +
.../apache/impala/analysis/ResetMetadataStmt.java | 5 +
.../org/apache/impala/analysis/SelectStmt.java | 15 +-
.../apache/impala/analysis/StmtMetadataLoader.java | 12 +-
.../java/org/apache/impala/analysis/TableName.java | 4 +
.../impala/catalog/local/CatalogdMetaProvider.java | 4 +-
.../org/apache/impala/common/FileSystemUtil.java | 21 +-
.../ExtractCompoundVerticalBarExprRule.java | 6 +-
.../apache/impala/service/CatalogOpExecutor.java | 11 +-
.../java/org/apache/impala/service/JniCatalog.java | 146 ++++++++---
.../java/org/apache/impala/util/CatalogOpUtil.java | 148 +++++++++++
.../java/org/apache/impala/util/DebugUtils.java | 4 +
.../org/apache/impala/util/CatalogOpUtilTest.java | 221 ++++++++++++++++
testdata/data/README | 28 +++
.../iceberg_migrated_alter_test/000000_0 | Bin 0 -> 817 bytes
.../c9f83a82-60f4-443b-9ca4-359cad16fe12-m0.avro} | Bin 3034 -> 3182 bytes
...396-1-c9f83a82-60f4-443b-9ca4-359cad16fe12.avro | Bin 0 -> 1986 bytes
.../metadata/v1.metadata.json | 59 +++--
.../metadata/v2.metadata.json | 105 ++++++++
.../metadata/version-hint.text | 0
.../iceberg_migrated_alter_test_orc/000000_0 | Bin 0 -> 418 bytes
.../340a3b82-71e3-4f50-b030-aecb5a5ea730-m0.avro} | Bin 3034 -> 3182 bytes
...038-1-340a3b82-71e3-4f50-b030-aecb5a5ea730.avro | Bin 0 -> 1990 bytes
.../metadata/v1.metadata.json | 59 +++--
.../metadata/v2.metadata.json | 105 ++++++++
.../metadata/version-hint.text | 0
.../iceberg_migrated_complex_test/000000_0 | Bin 0 -> 3006 bytes
.../152e384f-2851-44b7-9ada-1bfbec74e9fc-m0.avro} | Bin 3034 -> 3218 bytes
...148-1-152e384f-2851-44b7-9ada-1bfbec74e9fc.avro | Bin 0 -> 1988 bytes
.../metadata/v1.metadata.json | 255 +++++++++++++++++++
.../metadata/v2.metadata.json | 279 +++++++++++++++++++++
.../metadata/version-hint.text | 0
.../iceberg_migrated_complex_test_orc/000000_0 | Bin 0 -> 1217 bytes
.../8588fd4b-13c1-4451-80ad-5cf71a959b94-m0.avro} | Bin 3034 -> 3232 bytes
...504-1-8588fd4b-13c1-4451-80ad-5cf71a959b94.avro | Bin 0 -> 1992 bytes
.../metadata/v1.metadata.json | 255 +++++++++++++++++++
.../metadata/v2.metadata.json | 279 +++++++++++++++++++++
.../metadata/version-hint.text | 0
.../queries/PlannerTest/aggregation.test | 23 +-
...iceberg-migrated-table-field-id-resolution.test | 208 +++++++++++++++
tests/authorization/test_ranger.py | 29 +++
tests/common/file_utils.py | 34 +++
tests/metadata/test_recursive_listing.py | 116 ++++++++-
tests/query_test/test_iceberg.py | 13 +
tests/util/filesystem_base.py | 7 +
tests/util/hdfs_util.py | 12 +
52 files changed, 2506 insertions(+), 139 deletions(-)
create mode 100644 fe/src/main/java/org/apache/impala/util/CatalogOpUtil.java
create mode 100644
fe/src/test/java/org/apache/impala/util/CatalogOpUtilTest.java
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_alter_test/000000_0
copy
testdata/data/iceberg_test/{hadoop_catalog/ice/airports_orc/metadata/1ebf435e-7da7-41e7-bebf-eb3ebf1b1002-m0.avro
=>
iceberg_migrated_alter_test/metadata/c9f83a82-60f4-443b-9ca4-359cad16fe12-m0.avro}
(86%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_alter_test/metadata/snap-2941076094076108396-1-c9f83a82-60f4-443b-9ca4-359cad16fe12.avro
copy testdata/data/iceberg_test/{hadoop_catalog/ice/airports_parquet =>
iceberg_migrated_alter_test}/metadata/v1.metadata.json (52%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_alter_test/metadata/v2.metadata.json
copy
testdata/data/iceberg_test/{hadoop_catalog/ice/complextypestbl_iceberg_orc =>
iceberg_migrated_alter_test}/metadata/version-hint.text (100%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_alter_test_orc/000000_0
copy
testdata/data/iceberg_test/{hadoop_catalog/ice/airports_orc/metadata/1ebf435e-7da7-41e7-bebf-eb3ebf1b1002-m0.avro
=>
iceberg_migrated_alter_test_orc/metadata/340a3b82-71e3-4f50-b030-aecb5a5ea730-m0.avro}
(86%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_alter_test_orc/metadata/snap-2205107170480729038-1-340a3b82-71e3-4f50-b030-aecb5a5ea730.avro
copy testdata/data/iceberg_test/{hadoop_catalog/ice/airports_orc =>
iceberg_migrated_alter_test_orc}/metadata/v1.metadata.json (51%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_alter_test_orc/metadata/v2.metadata.json
copy
testdata/data/iceberg_test/{hadoop_catalog/ice/complextypestbl_iceberg_orc =>
iceberg_migrated_alter_test_orc}/metadata/version-hint.text (100%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_complex_test/000000_0
copy
testdata/data/iceberg_test/{hadoop_catalog/ice/airports_orc/metadata/1ebf435e-7da7-41e7-bebf-eb3ebf1b1002-m0.avro
=>
iceberg_migrated_complex_test/metadata/152e384f-2851-44b7-9ada-1bfbec74e9fc-m0.avro}
(85%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_complex_test/metadata/snap-3911840040574896148-1-152e384f-2851-44b7-9ada-1bfbec74e9fc.avro
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_complex_test/metadata/v1.metadata.json
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_complex_test/metadata/v2.metadata.json
copy
testdata/data/iceberg_test/{hadoop_catalog/ice/complextypestbl_iceberg_orc =>
iceberg_migrated_complex_test}/metadata/version-hint.text (100%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_complex_test_orc/000000_0
copy
testdata/data/iceberg_test/{hadoop_catalog/ice/airports_orc/metadata/1ebf435e-7da7-41e7-bebf-eb3ebf1b1002-m0.avro
=>
iceberg_migrated_complex_test_orc/metadata/8588fd4b-13c1-4451-80ad-5cf71a959b94-m0.avro}
(85%)
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_complex_test_orc/metadata/snap-3622599918649152504-1-8588fd4b-13c1-4451-80ad-5cf71a959b94.avro
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_complex_test_orc/metadata/v1.metadata.json
create mode 100644
testdata/data/iceberg_test/iceberg_migrated_complex_test_orc/metadata/v2.metadata.json
copy
testdata/data/iceberg_test/{hadoop_catalog/ice/complextypestbl_iceberg_orc =>
iceberg_migrated_complex_test_orc}/metadata/version-hint.text (100%)
create mode 100644
testdata/workloads/functional-query/queries/QueryTest/iceberg-migrated-table-field-id-resolution.test
