This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 37f44a58f IMPALA-10918: Allow map type in SELECT list
new bc92661bd IMPALA-11490: Add more metrics for event processor
new f598b2ad6 IMPALA-10610: Support multiple file formats in a single
Iceberg Table
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/hdfs-scan-node-base.cc | 93 +++++++----
be/src/util/event-metrics.cc | 98 ++++++++++-
be/src/util/event-metrics.h | 50 +++++-
common/thrift/JniCatalog.thrift | 41 +++--
common/thrift/metrics.json | 114 ++++++++++++-
.../impala/catalog/events/MetastoreEvents.java | 2 +
.../catalog/events/MetastoreEventsProcessor.java | 100 ++++++++++--
.../org/apache/impala/planner/HdfsScanNode.java | 4 +-
.../org/apache/impala/planner/IcebergScanNode.java | 28 +++-
testdata/data/README | 12 ++
...0b6136a-job_16619542960420_0002-1-00001.parquet | Bin 0 -> 872 bytes
...e500a19c1d1-job_16619542960420_0003-1-00001.orc | Bin 0 -> 437 bytes
...80faff0-job_16619542960420_0004-1-00001.parquet | Bin 0 -> 872 bytes
...19c5500ed04-job_16619542960420_0004-1-00001.orc | Bin 0 -> 435 bytes
.../055baf62-de6d-4583-bf21-f187f9482343-m0.avro} | Bin 3218 -> 3297 bytes
.../871d1473-8566-46c0-a530-a2256b3f396f-m0.avro} | Bin 3218 -> 3297 bytes
.../a0332d05-4c9c-4640-8656-69ba61b7859a-m0.avro} | Bin 3218 -> 3304 bytes
.../d43cc1ea-096f-4594-9583-b1b27f8f0230-m0.avro} | Bin 3218 -> 3305 bytes
...35-1-a0332d05-4c9c-4640-8656-69ba61b7859a.avro} | Bin 1986 -> 1989 bytes
...59-1-055baf62-de6d-4583-bf21-f187f9482343.avro} | Bin 1992 -> 2379 bytes
...54-1-d43cc1ea-096f-4594-9583-b1b27f8f0230.avro} | Bin 1992 -> 2249 bytes
...72-1-871d1473-8566-46c0-a530-a2256b3f396f.avro} | Bin 1992 -> 2119 bytes
.../metadata/v1.metadata.json | 35 ++--
.../metadata/v2.metadata.json | 94 +++++++++++
.../metadata/v3.metadata.json} | 63 ++++---
.../metadata/v4.metadata.json | 125 ++++++++++++++
.../metadata/v5.metadata.json | 128 +++++++++++++++
.../metadata/v6.metadata.json | 153 +++++++++++++++++
.../metadata/v7.metadata.json | 156 ++++++++++++++++++
.../metadata/v8.metadata.json | 181 +++++++++++++++++++++
.../metadata/version-hint.txt | 1 +
.../QueryTest/iceberg-mixed-file-format.test | 11 ++
tests/query_test/test_iceberg.py | 5 +
33 files changed, 1369 insertions(+), 125 deletions(-)
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/data/00000-0-data-gfurnstahl_20220906113044_157fc172-f5d3-4c70-8653-fff150b6136a-job_16619542960420_0002-1-00001.parquet
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/data/00000-0-data-gfurnstahl_20220906113255_8d49367d-e338-4996-ade5-ee500a19c1d1-job_16619542960420_0003-1-00001.orc
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/data/00000-0-data-gfurnstahl_20220906114830_907f72c7-36ac-4135-8315-27ff880faff0-job_16619542960420_0004-1-00001.parquet
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/data/00000-0-data-gfurnstahl_20220906114900_9c1b7b46-5643-428f-a007-519c5500ed04-job_16619542960420_0004-1-00001.orc
copy
testdata/data/iceberg_test/{iceberg_migrated_complex_test/metadata/152e384f-2851-44b7-9ada-1bfbec74e9fc-m0.avro
=>
iceberg_mixed_file_format_test/metadata/055baf62-de6d-4583-bf21-f187f9482343-m0.avro}
(89%)
copy
testdata/data/iceberg_test/{iceberg_migrated_complex_test/metadata/152e384f-2851-44b7-9ada-1bfbec74e9fc-m0.avro
=>
iceberg_mixed_file_format_test/metadata/871d1473-8566-46c0-a530-a2256b3f396f-m0.avro}
(89%)
copy
testdata/data/iceberg_test/{iceberg_migrated_complex_test/metadata/152e384f-2851-44b7-9ada-1bfbec74e9fc-m0.avro
=>
iceberg_mixed_file_format_test/metadata/a0332d05-4c9c-4640-8656-69ba61b7859a-m0.avro}
(89%)
copy
testdata/data/iceberg_test/{iceberg_migrated_complex_test/metadata/152e384f-2851-44b7-9ada-1bfbec74e9fc-m0.avro
=>
iceberg_mixed_file_format_test/metadata/d43cc1ea-096f-4594-9583-b1b27f8f0230-m0.avro}
(89%)
copy
testdata/data/iceberg_test/{iceberg_migrated_alter_test/metadata/snap-2941076094076108396-1-c9f83a82-60f4-443b-9ca4-359cad16fe12.avro
=>
iceberg_mixed_file_format_test/metadata/snap-1847986799759674135-1-a0332d05-4c9c-4640-8656-69ba61b7859a.avro}
(90%)
copy
testdata/data/iceberg_test/{iceberg_migrated_complex_test_orc/metadata/snap-3622599918649152504-1-8588fd4b-13c1-4451-80ad-5cf71a959b94.avro
=>
iceberg_mixed_file_format_test/metadata/snap-5574591442446832859-1-055baf62-de6d-4583-bf21-f187f9482343.avro}
(75%)
copy
testdata/data/iceberg_test/{iceberg_migrated_complex_test_orc/metadata/snap-3622599918649152504-1-8588fd4b-13c1-4451-80ad-5cf71a959b94.avro
=>
iceberg_mixed_file_format_test/metadata/snap-5589333083859779254-1-d43cc1ea-096f-4594-9583-b1b27f8f0230.avro}
(79%)
copy
testdata/data/iceberg_test/{iceberg_migrated_complex_test_orc/metadata/snap-3622599918649152504-1-8588fd4b-13c1-4451-80ad-5cf71a959b94.avro
=>
iceberg_mixed_file_format_test/metadata/snap-6603964444137547572-1-871d1473-8566-46c0-a530-a2256b3f396f.avro}
(84%)
copy testdata/data/iceberg_test/{iceberg_migrated_alter_test =>
iceberg_mixed_file_format_test}/metadata/v1.metadata.json (57%)
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/metadata/v2.metadata.json
copy
testdata/data/iceberg_test/{iceberg_migrated_alter_test_orc/metadata/v2.metadata.json
=> iceberg_mixed_file_format_test/metadata/v3.metadata.json} (51%)
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/metadata/v4.metadata.json
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/metadata/v5.metadata.json
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/metadata/v6.metadata.json
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/metadata/v7.metadata.json
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/metadata/v8.metadata.json
create mode 100644
testdata/data/iceberg_test/iceberg_mixed_file_format_test/metadata/version-hint.txt
create mode 100644
testdata/workloads/functional-query/queries/QueryTest/iceberg-mixed-file-format.test
