This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 28062ed3b IMPALA-14738: Add basic testing for Iceberg V3 tables
28062ed3b is described below
commit 28062ed3b26e96a86d523dece3d5993b2949f83b
Author: Zoltan Borok-Nagy <[email protected]>
AuthorDate: Thu Feb 12 21:15:44 2026 +0100
IMPALA-14738: Add basic testing for Iceberg V3 tables
With the Iceberg 1.10 upgrade now Impala is able to create Iceberg V3
tables. Impala has still limited functionalities on such tables, so
this patch adds basic testing against V3 tables.
This patch also adds Iceberg V3 tables with deletion vectors and
default values (written by Spark), these tables can be easily
loaded for testing.
Tables with V3 data types (VARIANT, UNKNOWN, GEOMETRY, GEOGRAPHY), and
multi-argument partition transforms are still not supported by the
latest Iceberg/Spark, so these will come later.
Testing
=======
* Iceberg V3 tables can be created
* INSERT works
* ALTER TABLE statements work
* Time-travel queries work
Negative tests
==============
* Impala raises errors for the following operations:
- DELETE
- UPDATE
- MERGE
- OPTIMIZE
* Impala raises error when a table contains
- Deletion Vectors
- Columns with default values
Change-Id: Ic1b90f1af539731d4653e83b848d55517a3acb58
Reviewed-on: http://gerrit.cloudera.org:8080/23982
Reviewed-by: Peter Rozsa <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
common/thrift/CatalogObjects.thrift | 7 ++
.../apache/impala/analysis/IcebergDeleteImpl.java | 6 ++
.../apache/impala/analysis/IcebergMergeImpl.java | 5 +
.../apache/impala/analysis/IcebergUpdateImpl.java | 5 +
.../org/apache/impala/analysis/OptimizeStmt.java | 6 ++
.../org/apache/impala/analysis/UpdateStmt.java | 4 -
.../java/org/apache/impala/catalog/Column.java | 10 +-
.../org/apache/impala/catalog/IcebergColumn.java | 11 +++
.../impala/catalog/IcebergContentFileStore.java | 18 ++++
.../catalog/iceberg/GroupedContentFiles.java | 20 +++-
.../apache/impala/planner/IcebergScanPlanner.java | 47 ++++++++-
.../apache/impala/util/IcebergSchemaConverter.java | 5 -
.../java/org/apache/impala/util/IcebergUtil.java | 17 +++-
testdata/data/README | 20 ++++
...1b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet | Bin 0 -> 392 bytes
...0eb-5773-431a-b440-5e9caa4ac641-0-00001.parquet | Bin 0 -> 603 bytes
.../0e51a6a0-b006-45a2-aae3-5604ed104518-m0.avro | Bin 0 -> 7793 bytes
.../ed3d9f93-7a24-464c-9e74-f3d910f45451-m0.avro | Bin 0 -> 7848 bytes
...452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro | Bin 0 -> 4825 bytes
...855-1-0e51a6a0-b006-45a2-aae3-5604ed104518.avro | Bin 0 -> 4754 bytes
.../metadata/v1.metadata.json | 1 +
.../metadata/v2.metadata.json | 1 +
.../metadata/v3.metadata.json | 1 +
.../metadata/v4.metadata.json | 1 +
.../metadata/version-hint.text | 1 +
...627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet | Bin 0 -> 392 bytes
...5b6-4635-8b34-94b49abc87d9-00001-deletes.puffin | Bin 0 -> 824 bytes
...627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet | Bin 0 -> 392 bytes
...627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet | Bin 0 -> 392 bytes
...627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet | Bin 0 -> 392 bytes
...627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet | Bin 0 -> 392 bytes
.../5e791f63-c675-4124-8390-6bbce9e61755-m0.avro | Bin 0 -> 7847 bytes
.../e9787b6c-e745-4040-b78f-e4012aa178ab-m0.avro | Bin 0 -> 7840 bytes
...397-1-5e791f63-c675-4124-8390-6bbce9e61755.avro | Bin 0 -> 4762 bytes
...685-1-e9787b6c-e745-4040-b78f-e4012aa178ab.avro | Bin 0 -> 4840 bytes
.../metadata/v1.metadata.json | 1 +
.../metadata/v2.metadata.json | 1 +
.../metadata/v3.metadata.json | 1 +
.../metadata/version-hint.text | 0
.../queries/QueryTest/iceberg-v3-basic.test | 31 ++++++
.../queries/QueryTest/iceberg-v3-negative.test | 106 +++++++++++++++++++++
tests/query_test/test_iceberg.py | 24 +++++
tests/util/iceberg_metadata_util.py | 7 +-
43 files changed, 333 insertions(+), 24 deletions(-)
diff --git a/common/thrift/CatalogObjects.thrift
b/common/thrift/CatalogObjects.thrift
index 60009e523..5e97cef72 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -664,6 +664,12 @@ struct THash128 {
2: required i64 low
}
+struct TIcebergDeletionVector {
+ 1: required string path;
+ 2: required i64 content_offset;
+ 3: required i64 content_size_in_bytes;
+}
+
// Contains maps from 128-bit XXH128 hash of file path to its file descriptor
struct TIcebergContentFileStore {
1: optional map<THash128, THdfsFileDesc>
path_hash_to_data_file_without_deletes
@@ -675,6 +681,7 @@ struct TIcebergContentFileStore {
7: optional bool has_parquet
8: optional list<string> missing_files
9: optional list<TIcebergPartition> partitions
+ 10: optional map<THash128, TIcebergDeletionVector> data_path_hash_to_dv
}
// Represents a drop partition request for Iceberg tables
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergDeleteImpl.java
b/fe/src/main/java/org/apache/impala/analysis/IcebergDeleteImpl.java
index 35192150e..7eea62c0a 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IcebergDeleteImpl.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergDeleteImpl.java
@@ -40,6 +40,12 @@ public class IcebergDeleteImpl extends IcebergModifyImpl {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
+ if (originalTargetTable_.getFormatVersion() > 2) {
+ throw new AnalysisException(String.format(
+ "Impala does not support DELETE statements on Iceberg tables with
format " +
+ "version %d", originalTargetTable_.getFormatVersion()));
+ }
+
// Make the virtual position delete table the new target table.
modifyStmt_.setTargetTable(icePosDelTable_);
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergMergeImpl.java
b/fe/src/main/java/org/apache/impala/analysis/IcebergMergeImpl.java
index 2a0100a72..cfce65458 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IcebergMergeImpl.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergMergeImpl.java
@@ -119,6 +119,11 @@ public class IcebergMergeImpl implements MergeImpl {
Preconditions.checkState(table instanceof FeIcebergTable);
icebergTable_ = (FeIcebergTable) table;
IcebergUtil.validateIcebergTableForInsert(icebergTable_);
+ if (icebergTable_.getFormatVersion() > 2) {
+ throw new AnalysisException(String.format(
+ "Impala does not support MERGE statements on Iceberg tables with
format " +
+ "version %d", icebergTable_.getFormatVersion()));
+ }
String modifyWriteMode = icebergTable_.getIcebergApiTable().properties()
.get(TableProperties.MERGE_MODE);
if (modifyWriteMode != null && !Objects.equals(modifyWriteMode,
"merge-on-read")
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergUpdateImpl.java
b/fe/src/main/java/org/apache/impala/analysis/IcebergUpdateImpl.java
index d04eb2452..473d95df0 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IcebergUpdateImpl.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergUpdateImpl.java
@@ -63,6 +63,11 @@ public class IcebergUpdateImpl extends IcebergModifyImpl {
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
+ if (originalTargetTable_.getFormatVersion() > 2) {
+ throw new AnalysisException(String.format(
+ "Impala does not support UPDATE statements on Iceberg tables with
format " +
+ "version %d", originalTargetTable_.getFormatVersion()));
+ }
deleteTableId_ = analyzer.getDescTbl().addTargetTable(icePosDelTable_);
IcebergUtil.validateIcebergTableForInsert(originalTargetTable_);
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/OptimizeStmt.java
b/fe/src/main/java/org/apache/impala/analysis/OptimizeStmt.java
index 543229f5d..02361aacf 100644
--- a/fe/src/main/java/org/apache/impala/analysis/OptimizeStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/OptimizeStmt.java
@@ -169,6 +169,12 @@ public class OptimizeStmt extends DmlStatementBase {
FeIcebergTable iceTable = (FeIcebergTable) table_;
IcebergUtil.validateIcebergTableForInsert(iceTable);
+ if (iceTable.getFormatVersion() > 2) {
+ throw new AnalysisException(String.format(
+ "Impala does not support OPTIMIZE statements on Iceberg tables with
format " +
+ "version %d", iceTable.getFormatVersion()));
+ }
+
selectFiles(iceTable);
prepareExpressions(analyzer);
diff --git a/fe/src/main/java/org/apache/impala/analysis/UpdateStmt.java
b/fe/src/main/java/org/apache/impala/analysis/UpdateStmt.java
index fd5349467..693d4d45c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/UpdateStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/UpdateStmt.java
@@ -24,13 +24,9 @@ import java.util.List;
import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.FeKuduTable;
-import org.apache.impala.common.AnalysisException;
-import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.common.Pair;
import org.apache.impala.planner.DataSink;
-import com.google.common.base.Preconditions;
-
/**
* Representation of an Update statement.
*
diff --git a/fe/src/main/java/org/apache/impala/catalog/Column.java
b/fe/src/main/java/org/apache/impala/catalog/Column.java
index ca85b6546..a376ee801 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Column.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Column.java
@@ -83,14 +83,18 @@ public class Column {
stats_.update(type_, statsData);
}
- @Override
- public String toString() {
+ protected MoreObjects.ToStringHelper toStringHelper() {
return MoreObjects.toStringHelper(this.getClass())
.add("name_", name_)
.add("type_", type_)
.add("comment_", comment_)
.add("stats", stats_)
- .add("position_", position_).toString();
+ .add("position_", position_);
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper().toString();
}
public static Column fromThrift(TColumn columnDesc) throws
ImpalaRuntimeException {
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergColumn.java
b/fe/src/main/java/org/apache/impala/catalog/IcebergColumn.java
index a5483d1b8..5cbd7f29c 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergColumn.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergColumn.java
@@ -20,6 +20,8 @@ package org.apache.impala.catalog;
import org.apache.impala.thrift.TColumn;
import org.apache.impala.thrift.TColumnDescriptor;
+import com.google.common.base.MoreObjects;
+
/**
* Represents an Iceberg column.
*
@@ -80,4 +82,13 @@ public class IcebergColumn extends Column {
desc.setIcebergFieldMapValueId(fieldMapValueId_);
return desc;
}
+
+ @Override
+ protected MoreObjects.ToStringHelper toStringHelper() {
+ return super.toStringHelper()
+ .add("fieldId_", fieldId_)
+ .add("fieldMapKeyId_", fieldMapKeyId_)
+ .add("fieldMapValueId_", fieldMapValueId_)
+ .add("isNullable_", isNullable_);
+ }
}
diff --git
a/fe/src/main/java/org/apache/impala/catalog/IcebergContentFileStore.java
b/fe/src/main/java/org/apache/impala/catalog/IcebergContentFileStore.java
index 95434cea2..12fdc2a6b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergContentFileStore.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergContentFileStore.java
@@ -47,6 +47,7 @@ import org.apache.impala.fb.FbIcebergMetadata;
import org.apache.impala.thrift.THash128;
import org.apache.impala.thrift.THdfsFileDesc;
import org.apache.impala.thrift.TIcebergContentFileStore;
+import org.apache.impala.thrift.TIcebergDeletionVector;
import org.apache.impala.thrift.TIcebergPartition;
import org.apache.impala.thrift.TNetworkAddress;
import org.apache.impala.util.Hash128;
@@ -157,6 +158,7 @@ public class IcebergContentFileStore {
private MapListContainer dataFilesWithDeletes_ = new MapListContainer();
private MapListContainer positionDeleteFiles_ = new MapListContainer();
private MapListContainer equalityDeleteFiles_ = new MapListContainer();
+ private Map<Hash128, TIcebergDeletionVector> dataFileToDV_ = new HashMap<>();
private Set<String> missingFiles_ = new HashSet<>();
// Partitions with their corresponding ids that are used to refer to the
partition info
// from the IcebergFileDescriptors.
@@ -204,6 +206,7 @@ public class IcebergContentFileStore {
for (DeleteFile deleteFile : icebergFiles.equalityDeleteFiles) {
storeFile(deleteFile, fileDescMap, equalityDeleteFiles_);
}
+ dataFileToDV_ = icebergFiles.dataFileToDV;
}
private void storeFile(ContentFile<?> contentFile,
@@ -278,6 +281,10 @@ public class IcebergContentFileStore {
return missingFiles_;
}
+ public Map<Hash128, TIcebergDeletionVector> getDataFileToDV() {
+ return dataFileToDV_;
+ }
+
public long getNumFiles() {
return dataFilesWithoutDeletes_.getNumFiles() +
dataFilesWithDeletes_.getNumFiles() +
@@ -372,6 +379,11 @@ public class IcebergContentFileStore {
ret.setHas_parquet(hasParquet_);
ret.setMissing_files(new ArrayList<>(missingFiles_));
ret.setPartitions(convertPartitionMapToList(partitions_));
+ Map<THash128, TIcebergDeletionVector> tdeletion_vectors = new HashMap<>();
+ for (Map.Entry<Hash128, TIcebergDeletionVector> entry :
dataFileToDV_.entrySet()) {
+ tdeletion_vectors.put(entry.getKey().toThrift(), entry.getValue());
+ }
+ ret.setData_path_hash_to_dv(tdeletion_vectors);
return ret;
}
@@ -399,6 +411,12 @@ public class IcebergContentFileStore {
tFileStore.getPath_hash_to_equality_delete_file(),
networkAddresses, hostIndex);
}
+ if (tFileStore.isSetData_path_hash_to_dv()) {
+ for (Map.Entry<THash128, TIcebergDeletionVector> entry :
+ tFileStore.getData_path_hash_to_dv().entrySet()) {
+ ret.dataFileToDV_.put(Hash128.fromThrift(entry.getKey()),
entry.getValue());
+ }
+ }
ret.hasAvro_ = tFileStore.isSetHas_avro() ? tFileStore.isHas_avro() :
false;
ret.hasOrc_ = tFileStore.isSetHas_orc() ? tFileStore.isHas_orc() : false;
ret.hasParquet_ = tFileStore.isSetHas_parquet() ?
tFileStore.isHas_parquet() : false;
diff --git
a/fe/src/main/java/org/apache/impala/catalog/iceberg/GroupedContentFiles.java
b/fe/src/main/java/org/apache/impala/catalog/iceberg/GroupedContentFiles.java
index 67f03a6f6..74a37f240 100644
---
a/fe/src/main/java/org/apache/impala/catalog/iceberg/GroupedContentFiles.java
+++
b/fe/src/main/java/org/apache/impala/catalog/iceberg/GroupedContentFiles.java
@@ -18,19 +18,26 @@
package org.apache.impala.catalog.iceberg;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.FileContent;
+import org.apache.iceberg.FileFormat;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.io.CloseableIterable;
+import org.apache.impala.thrift.TIcebergDeletionVector;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
+import org.apache.impala.util.Hash128;
+import org.apache.impala.util.IcebergUtil;
/**
* Struct-like object to group different Iceberg content files:
@@ -47,6 +54,7 @@ public class GroupedContentFiles {
public List<DataFile> dataFilesWithDeletes = new ArrayList<>();
public Set<DeleteFile> positionDeleteFiles = new HashSet<>();
public Set<DeleteFile> equalityDeleteFiles = new HashSet<>();
+ public Map<Hash128, TIcebergDeletionVector> dataFileToDV = new HashMap<>();
public GroupedContentFiles() { }
@@ -55,10 +63,18 @@ public class GroupedContentFiles {
if (scanTask.deletes().isEmpty()) {
dataFilesWithoutDeletes.add(scanTask.file());
} else {
- dataFilesWithDeletes.add(scanTask.file());
+ DataFile dataFile = scanTask.file();
+ dataFilesWithDeletes.add(dataFile);
for (DeleteFile delFile : scanTask.deletes()) {
if (delFile.content() == FileContent.POSITION_DELETES) {
- positionDeleteFiles.add(delFile);
+ if (delFile.format() == FileFormat.PUFFIN) {
+ Preconditions.checkState(scanTask.deletes().size() == 1);
+ dataFileToDV.put(
+ IcebergUtil.getFilePathHash(dataFile),
+ IcebergUtil.createTIcebergDeletionVector(delFile));
+ } else {
+ positionDeleteFiles.add(delFile);
+ }
} else {
Preconditions.checkState(delFile.content() ==
FileContent.EQUALITY_DELETES);
equalityDeleteFiles.add(delFile);
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
index c470029ef..d790250d4 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
@@ -35,8 +35,10 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.iceberg.ContentFile;
+import org.apache.iceberg.DataFile;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.FileContent;
+import org.apache.iceberg.FileFormat;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.ExpressionUtil;
@@ -47,6 +49,7 @@ import org.apache.iceberg.metrics.MetricsReport;
import org.apache.iceberg.metrics.MetricsReporter;
import org.apache.iceberg.metrics.ScanMetricsResult;
import org.apache.iceberg.metrics.ScanReport;
+import org.apache.iceberg.types.Types.NestedField;
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.BinaryPredicate;
import org.apache.impala.analysis.BinaryPredicate.Operator;
@@ -85,6 +88,7 @@ import org.apache.impala.fb.FbIcebergMetadata;
import org.apache.impala.planner.JoinNode.DistributionMode;
import org.apache.impala.service.Frontend;
import org.apache.impala.thrift.TColumnStats;
+import org.apache.impala.thrift.TIcebergDeletionVector;
import org.apache.impala.thrift.TIcebergPartition;
import org.apache.impala.thrift.TIcebergPartitionTransformType;
import org.apache.impala.thrift.TQueryOptions;
@@ -146,6 +150,7 @@ public class IcebergScanPlanner {
private List<IcebergFileDescriptor> dataFilesWithoutDeletes_ = new
ArrayList<>();
private List<IcebergFileDescriptor> dataFilesWithDeletes_ = new
ArrayList<>();
private Set<IcebergFileDescriptor> positionDeleteFiles_ = new HashSet<>();
+ public Map<Hash128, TIcebergDeletionVector> dataFileToDV_ = new HashMap<>();
// Holds all the equalityFieldIds from the equality delete file descriptors
involved in
// this query.
@@ -238,16 +243,38 @@ public class IcebergScanPlanner {
}
dataFilesWithDeletes_ = fileStore.getDataFilesWithDeletes();
positionDeleteFiles_ = new HashSet<>(fileStore.getPositionDeleteFiles());
+ dataFileToDV_ = fileStore.getDataFileToDV();
initEqualityIds(fileStore.getEqualityDeleteFiles());
updateDeleteStatistics();
}
private boolean noDeleteFiles() {
- return positionDeleteFiles_.isEmpty() &&
equalityIdsToDeleteFiles_.isEmpty();
+ return positionDeleteFiles_.isEmpty() &&
+ equalityIdsToDeleteFiles_.isEmpty() &&
+ dataFileToDV_.isEmpty();
+ }
+
+ private void validateSlotRefs() throws ImpalaException {
+ if (getIceTable().getFormatVersion() < 3) return;
+ for (SlotDescriptor slotDesc : tblRef_.getDesc().getSlots()) {
+ if (!slotDesc.isMaterialized()) continue;
+ Column column = slotDesc.getColumn();
+ if (column == null) continue;
+ int fieldId = ((IcebergColumn) column).getFieldId();
+ NestedField field =
getIceTable().getIcebergApiTable().schema().findField(fieldId);
+ if (field.initialDefaultLiteral() != null) {
+ throw new ImpalaRuntimeException(String.format(
+ "Iceberg columns with default values not supported yet. " +
+ "Table: %s Column: %s Default value: %s",
+ getIceTable().getFullName(), column.getName(),
+ field.initialDefaultLiteral().toString()));
+ }
+ }
}
private PlanNode createIcebergScanPlanImpl() throws ImpalaException {
+ validateSlotRefs();
boolean isPartitionKeyScan = IsPartitionKeyScan();
if (noDeleteFiles()) {
Preconditions.checkState(!tblRef_.optimizeCountStarForIcebergV2());
@@ -261,6 +288,11 @@ public class IcebergScanPlanner {
ret.init(analyzer_);
return ret;
}
+ if (!dataFileToDV_.isEmpty()) {
+ throw new ImpalaRuntimeException(
+ "Iceberg tables with Deletion Vectors are not supported yet: " +
+ getIceTable().getFullName());
+ }
PlanNode joinNode = null;
if (!positionDeleteFiles_.isEmpty()) joinNode = createPositionJoinNode();
@@ -690,12 +722,13 @@ public class IcebergScanPlanner {
metricsReporter_)) {
long dataFilesCacheMisses = 0;
for (FileScanTask fileScanTask : fileScanTasks) {
+ DataFile dataFile = fileScanTask.file();
Expression residualExpr = fileScanTask.residual();
if (residualExpr != null && !(residualExpr instanceof True)) {
residualExpressions_.add(residualExpr);
}
Pair<IcebergFileDescriptor, Boolean> fileDesc =
- getFileDescriptor(fileScanTask.file(), fileStore);
+ getFileDescriptor(dataFile, fileStore);
if (!fileDesc.second) ++dataFilesCacheMisses;
if (fileScanTask.deletes().isEmpty()) {
dataFilesWithoutDeletes_.add(fileDesc.first);
@@ -708,8 +741,14 @@ public class IcebergScanPlanner {
if (delFile.content() == FileContent.EQUALITY_DELETES) {
addEqualityDeletesAndIds(delFileDesc.first);
} else {
- Preconditions.checkState(delFile.content() ==
FileContent.POSITION_DELETES);
- positionDeleteFiles_.add(delFileDesc.first);
+ if (delFile.format() == FileFormat.PUFFIN) {
+ Preconditions.checkState(fileScanTask.deletes().size() == 1);
+ dataFileToDV_.put(
+ IcebergUtil.getFilePathHash(dataFile.location()),
+ IcebergUtil.createTIcebergDeletionVector(delFile));
+ } else {
+ positionDeleteFiles_.add(delFileDesc.first);
+ }
}
}
}
diff --git
a/fe/src/main/java/org/apache/impala/util/IcebergSchemaConverter.java
b/fe/src/main/java/org/apache/impala/util/IcebergSchemaConverter.java
index 8b68e36b7..b60319e67 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergSchemaConverter.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergSchemaConverter.java
@@ -26,16 +26,12 @@ import java.util.Set;
import com.google.common.collect.Lists;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.hive.HiveSchemaUtil;
import org.apache.iceberg.types.Types;
-import org.apache.impala.analysis.IcebergPartitionField;
-import org.apache.impala.analysis.IcebergPartitionSpec;
-import org.apache.impala.analysis.IcebergPartitionTransform;
import org.apache.impala.catalog.ArrayType;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.IcebergColumn;
@@ -48,7 +44,6 @@ import org.apache.impala.catalog.Type;
import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.thrift.TColumn;
import org.apache.impala.thrift.TColumnType;
-import org.apache.impala.thrift.TIcebergPartitionTransformType;
/**
* Utility class for converting between Iceberg and Impala schemas and types.
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index 38f10a20d..20738d88b 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -20,8 +20,6 @@ package org.apache.impala.util;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import com.google.flatbuffers.FlatBufferBuilder;
@@ -42,13 +40,14 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.ContentFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileContent;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.PartitionField;
@@ -67,9 +66,7 @@ import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.expressions.Term;
import org.apache.iceberg.hadoop.HadoopFileIO;
import org.apache.iceberg.io.CloseableIterable;
-import org.apache.iceberg.metrics.LoggingMetricsReporter;
import org.apache.iceberg.metrics.MetricsReporter;
-import org.apache.iceberg.metrics.MetricsReporters;
import org.apache.iceberg.mr.Catalogs;
import org.apache.iceberg.transforms.PartitionSpecVisitor;
import org.apache.iceberg.transforms.Transforms;
@@ -118,6 +115,7 @@ import org.apache.impala.thrift.TCompressionCodec;
import org.apache.impala.thrift.THdfsCompression;
import org.apache.impala.thrift.THdfsFileFormat;
import org.apache.impala.thrift.TIcebergCatalog;
+import org.apache.impala.thrift.TIcebergDeletionVector;
import org.apache.impala.thrift.TIcebergFileFormat;
import org.apache.impala.thrift.TIcebergPartition;
import org.apache.impala.thrift.TIcebergPartitionField;
@@ -1465,6 +1463,15 @@ public class IcebergUtil {
return "";
}
+ public static TIcebergDeletionVector createTIcebergDeletionVector(
+ DeleteFile delFile) {
+ Preconditions.checkState(delFile.content() ==
FileContent.POSITION_DELETES);
+ Preconditions.checkState(delFile.format() == FileFormat.PUFFIN);
+ return new TIcebergDeletionVector(
+ delFile.location(),
+ delFile.contentOffset(), delFile.contentSizeInBytes());
+ }
+
/**
* This is a helper class for converting the value of the
* 'impala.computeStatsSnapshotIds' table property to a Map representation
and back.
diff --git a/testdata/data/README b/testdata/data/README
index 6db023867..831be8871 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -1199,6 +1199,26 @@ iceberg_v2_null_delete_record:
* manually edited the metadata JSON, and the manifest and manifest list
files to
register the delete files in the table
+testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors
+Iceberg V3 table created by Spark 4.0.2.
+Built Spark with the following:
+> ./build/mvn -Phive -Phive-thriftserver -DskipTests clean package
+Copy Impala's core-site.xml, hdfs-site.xml, hive-site.xml into Spark's conf/
directory.
+Run spark-shell with the following command:
+> bin/spark-shell --jars iceberg-spark-runtime-4.0_2.13-1.10.1.jar \
+ --conf spark.sql.catalogImplementation=hive \
+ --conf spark.sql.catalog.spark_catalog.type=hive \
+ --conf
spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog
+Then Spark SQL commands were used to create the tables:
+sql("create table iceberg_v3_deletion_vectors (i int) using iceberg
options('format-version'='3', 'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read')").collect()
+...
+
+testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value
+Iceberg V3 table created by Spark 4.0.2, see above
(iceberg_v3_deletion_vectors).
+Since Spark still doesn't support default values, the metadata.json file was
edited
+manually to set the default value for column 'j':
+{"id":2,"name":"j","required":false,"type":"int","initial-default":-1,"write-default":-1}
+
arrays_big.parq:
Generated with RandomNestedDataGenerator.java from the following schema:
{
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/data/00000-0-4bcf51b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/data/00000-0-4bcf51b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet
new file mode 100644
index 000000000..c1a3e60cc
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/data/00000-0-4bcf51b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/data/00000-1-18a720eb-5773-431a-b440-5e9caa4ac641-0-00001.parquet
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/data/00000-1-18a720eb-5773-431a-b440-5e9caa4ac641-0-00001.parquet
new file mode 100644
index 000000000..c43edc2a5
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/data/00000-1-18a720eb-5773-431a-b440-5e9caa4ac641-0-00001.parquet
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/0e51a6a0-b006-45a2-aae3-5604ed104518-m0.avro
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/0e51a6a0-b006-45a2-aae3-5604ed104518-m0.avro
new file mode 100644
index 000000000..8d42e5d07
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/0e51a6a0-b006-45a2-aae3-5604ed104518-m0.avro
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/ed3d9f93-7a24-464c-9e74-f3d910f45451-m0.avro
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/ed3d9f93-7a24-464c-9e74-f3d910f45451-m0.avro
new file mode 100644
index 000000000..8b13928ff
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/ed3d9f93-7a24-464c-9e74-f3d910f45451-m0.avro
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/snap-5466875730470528452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/snap-5466875730470528452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro
new file mode 100644
index 000000000..7c4a62f14
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/snap-5466875730470528452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/snap-7752776210164022855-1-0e51a6a0-b006-45a2-aae3-5604ed104518.avro
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/snap-7752776210164022855-1-0e51a6a0-b006-45a2-aae3-5604ed104518.avro
new file mode 100644
index 000000000..8b1117a06
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/snap-7752776210164022855-1-0e51a6a0-b006-45a2-aae3-5604ed104518.avro
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v1.metadata.json
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v1.metadata.json
new file mode 100644
index 000000000..a80ba9346
--- /dev/null
+++
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v1.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"a0ffb3b6-f99a-4df8-8ddc-ffb49ecb56e2","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value","last-sequence-number":0,"last-updated-ms":1770927959140,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id
[...]
\ No newline at end of file
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v2.metadata.json
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v2.metadata.json
new file mode 100644
index 000000000..69b0ebadd
--- /dev/null
+++
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v2.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"a0ffb3b6-f99a-4df8-8ddc-ffb49ecb56e2","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value","last-sequence-number":1,"last-updated-ms":1770927988103,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id
[...]
\ No newline at end of file
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v3.metadata.json
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v3.metadata.json
new file mode 100644
index 000000000..8f2888241
--- /dev/null
+++
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v3.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"a0ffb3b6-f99a-4df8-8ddc-ffb49ecb56e2","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value","last-sequence-number":1,"last-updated-ms":1770928057357,"last-column-id":2,"current-schema-id":1,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]},{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"j","required":false,"type":"int","i
[...]
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v4.metadata.json
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v4.metadata.json
new file mode 100644
index 000000000..690b20bc7
--- /dev/null
+++
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v4.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"a0ffb3b6-f99a-4df8-8ddc-ffb49ecb56e2","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value","last-sequence-number":2,"last-updated-ms":1770928072044,"last-column-id":2,"current-schema-id":1,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]},{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"j","required":false,"type":"int","i
[...]
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/version-hint.text
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/version-hint.text
new file mode 100644
index 000000000..b8626c4cf
--- /dev/null
+++
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/version-hint.text
@@ -0,0 +1 @@
+4
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00000-0-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00000-0-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
new file mode 100644
index 000000000..c1a3e60cc
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00000-0-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00000-9-08e88179-85b6-4635-8b34-94b49abc87d9-00001-deletes.puffin
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00000-9-08e88179-85b6-4635-8b34-94b49abc87d9-00001-deletes.puffin
new file mode 100644
index 000000000..7958acd09
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00000-9-08e88179-85b6-4635-8b34-94b49abc87d9-00001-deletes.puffin
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00001-1-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00001-1-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
new file mode 100644
index 000000000..c62edd6fa
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00001-1-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00002-2-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00002-2-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
new file mode 100644
index 000000000..c0dbeadee
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00002-2-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00003-3-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00003-3-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
new file mode 100644
index 000000000..1f900399e
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00003-3-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00004-4-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00004-4-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
new file mode 100644
index 000000000..917dce90f
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/data/00004-4-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/5e791f63-c675-4124-8390-6bbce9e61755-m0.avro
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/5e791f63-c675-4124-8390-6bbce9e61755-m0.avro
new file mode 100644
index 000000000..0d561ae1c
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/5e791f63-c675-4124-8390-6bbce9e61755-m0.avro
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/e9787b6c-e745-4040-b78f-e4012aa178ab-m0.avro
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/e9787b6c-e745-4040-b78f-e4012aa178ab-m0.avro
new file mode 100644
index 000000000..ac74f455e
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/e9787b6c-e745-4040-b78f-e4012aa178ab-m0.avro
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/snap-2700858159721908397-1-5e791f63-c675-4124-8390-6bbce9e61755.avro
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/snap-2700858159721908397-1-5e791f63-c675-4124-8390-6bbce9e61755.avro
new file mode 100644
index 000000000..0e3dc73d0
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/snap-2700858159721908397-1-5e791f63-c675-4124-8390-6bbce9e61755.avro
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/snap-3685752423136077685-1-e9787b6c-e745-4040-b78f-e4012aa178ab.avro
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/snap-3685752423136077685-1-e9787b6c-e745-4040-b78f-e4012aa178ab.avro
new file mode 100644
index 000000000..1750ee097
Binary files /dev/null and
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/snap-3685752423136077685-1-e9787b6c-e745-4040-b78f-e4012aa178ab.avro
differ
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v1.metadata.json
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v1.metadata.json
new file mode 100644
index 000000000..f5316a6d1
--- /dev/null
+++
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v1.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"d1e3085d-73f3-4c65-9704-6b1f6436ba40","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_deletion_vectors","last-sequence-number":0,"last-updated-ms":1770903202273,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order
[...]
\ No newline at end of file
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v2.metadata.json
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v2.metadata.json
new file mode 100644
index 000000000..e27c0b966
--- /dev/null
+++
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v2.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"d1e3085d-73f3-4c65-9704-6b1f6436ba40","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_deletion_vectors","last-sequence-number":1,"last-updated-ms":1770903464282,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order
[...]
\ No newline at end of file
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v3.metadata.json
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v3.metadata.json
new file mode 100644
index 000000000..94c3cb527
--- /dev/null
+++
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/v3.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"d1e3085d-73f3-4c65-9704-6b1f6436ba40","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_deletion_vectors","last-sequence-number":2,"last-updated-ms":1770903499349,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order
[...]
diff --git
a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/version-hint.text
b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_deletion_vectors/metadata/version-hint.text
new file mode 100644
index 000000000..e69de29bb
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-basic.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-basic.test
new file mode 100644
index 000000000..5cf5bcb13
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-basic.test
@@ -0,0 +1,31 @@
+====
+---- QUERY
+CREATE TABLE ice_v3 (i int, s string)
+STORED BY ICEBERG
+TBLPROPERTIES('format-version'='3');
+
+INSERT INTO ice_v3 VALUES (1, 'impala');
+INSERT INTO ice_v3 VALUES (2, 'iceberg');
+INSERT INTO ice_v3 VALUES (3, 'spark');
+====
+---- QUERY
+SELECT * FROM ice_v3;
+---- RESULTS
+1,'impala'
+2,'iceberg'
+3,'spark'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+ALTER TABLE ice_v3 ADD COLUMN new_col STRING;
+INSERT INTO ice_v3 VALUES (4, 'hive', 'new_value');
+SELECT * FROM ice_v3;
+---- RESULTS
+1,'impala','NULL'
+2,'iceberg','NULL'
+3,'spark','NULL'
+4,'hive','new_value'
+---- TYPES
+INT,STRING,STRING
+====
\ No newline at end of file
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
new file mode 100644
index 000000000..664b342cb
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
@@ -0,0 +1,106 @@
+====
+---- QUERY
+CREATE TABLE ice_v3 (i int, s string)
+STORED BY ICEBERG
+TBLPROPERTIES('format-version'='3');
+
+INSERT INTO ice_v3 VALUES (1, 'impala');
+INSERT INTO ice_v3 VALUES (2, 'iceberg');
+INSERT INTO ice_v3 VALUES (3, 'spark');
+====
+---- QUERY
+DELETE FROM ice_v3 WHERE i = 1;
+---- CATCH
+Impala does not support DELETE statements on Iceberg tables with format
version 3
+====
+---- QUERY
+UPDATE ice_v3 SET s = 'updated' WHERE i = 1;
+---- CATCH
+Impala does not support UPDATE statements on Iceberg tables with format
version 3
+====
+---- QUERY
+MERGE INTO ice_v3 AS target
+USING (SELECT 1 AS i, 'updated' AS s) AS source ON target.i = source.i
+WHEN MATCHED THEN UPDATE SET target.s = source.s;
+---- CATCH
+Impala does not support MERGE statements on Iceberg tables with format version
3
+====
+---- QUERY
+OPTIMIZE TABLE ice_v3;
+---- CATCH
+Impala does not support OPTIMIZE statements on Iceberg tables with format
version 3
+====
+---- QUERY
+SELECT * FROM iceberg_v3_deletion_vectors;
+---- CATCH
+Iceberg tables with Deletion Vectors are not supported yet
+====
+---- QUERY
+# The first snapshot has no deletion vectors.
+SELECT *
+FROM iceberg_v3_deletion_vectors FOR SYSTEM_VERSION AS OF 2700858159721908397;
+---- RESULTS
+1
+2
+3
+4
+5
+---- TYPES
+INT
+====
+---- QUERY
+SELECT * FROM iceberg_v3_default_value;
+---- CATCH
+Iceberg columns with default values not supported yet.
+====
+---- QUERY
+# Column 'i' does not have a default value.
+SELECT i FROM iceberg_v3_default_value;
+---- RESULTS
+1
+2
+---- TYPES
+INT
+====
+---- QUERY
+# The first snapshot only has column 'i' which does not have a default value.
+SELECT * FROM iceberg_v3_default_value FOR SYSTEM_VERSION AS OF
7752776210164022855;
+---- RESULTS
+1
+---- TYPES
+INT
+====
+---- QUERY
+# 'j' is not materialized.
+select i from (select * from iceberg_v3_default_value) v;
+---- RESULTS
+1
+2
+---- TYPES
+INT
+====
+---- QUERY
+# 'j' is not materialized.
+select i from (select i, j from iceberg_v3_default_value) v;
+---- RESULTS
+1
+2
+---- TYPES
+INT
+====
+---- QUERY
+select i, file__position, input__file__name from iceberg_v3_default_value;
+---- RESULTS
+1,0,regex:'.*00000-0-4bcf51b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet'
+2,0,regex:'.*00000-1-18a720eb-5773-431a-b440-5e9caa4ac641-0-00001.parquet'
+---- TYPES
+INT,BIGINT,STRING
+====
+---- QUERY
+# 'j' is not materialized.
+select count(*) from (select j from iceberg_v3_default_value) v;
+---- RESULTS
+2
+---- TYPES
+BIGINT
+====
\ No newline at end of file
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index be3eae8bb..63f834ae1 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -2311,6 +2311,30 @@ class TestIcebergV2Table(IcebergTestSuite):
assert len(result.data) == 3
+class TestIcebergV3Table(IcebergTestSuite):
+ """Tests related to Iceberg V3 tables."""
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestIcebergV3Table, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_constraint(
+ lambda v: v.get_value('table_format').file_format == 'parquet')
+
+ def test_v3_basic(self, vector, unique_database):
+ self.run_test_case('QueryTest/iceberg-v3-basic', vector, unique_database)
+
+ def test_v3_negative(self, vector, unique_database):
+ create_iceberg_table_from_directory(self.client, unique_database,
+ "iceberg_v3_deletion_vectors", "parquet",
+ table_location="${IMPALA_HOME}/testdata/data/iceberg_test/iceberg_v3",
+ warehouse_prefix=os.getenv("FILESYSTEM_PREFIX"))
+ create_iceberg_table_from_directory(self.client, unique_database,
+ "iceberg_v3_default_value", "parquet",
+ table_location="${IMPALA_HOME}/testdata/data/iceberg_test/iceberg_v3",
+ warehouse_prefix=os.getenv("FILESYSTEM_PREFIX"))
+ self.run_test_case('QueryTest/iceberg-v3-negative', vector,
unique_database)
+
+
# Tests to exercise the DIRECTED distribution mode for V2 Iceberg tables.
Note, that most
# of the test coverage is in TestIcebergV2Table.test_read_position_deletes but
since it
# runs also with the V2 optimizations setting turned off, some tests were
moved here.
diff --git a/tests/util/iceberg_metadata_util.py
b/tests/util/iceberg_metadata_util.py
index 0f8e9b830..988f97005 100644
--- a/tests/util/iceberg_metadata_util.py
+++ b/tests/util/iceberg_metadata_util.py
@@ -37,7 +37,7 @@ def rewrite_metadata(prefix, unique_database, metadata_dir):
continue
version = metadata['format-version']
- if version < 1 or version > 2:
+ if version < 1 or version > 3:
print("WARN: skipping {}, unknown version {}".format(f, version))
continue
@@ -163,6 +163,11 @@ def add_prefix_to_snapshot_entry(table_params, entry):
if 'data_file' in entry:
entry['data_file']['file_path'] = generate_new_path(
table_params, entry['data_file']['file_path'])
+ if ('referenced_data_file' in entry['data_file']
+ and entry['data_file']['referenced_data_file'] is not None):
+ entry['data_file']['referenced_data_file'] = \
+ generate_new_path(
+ table_params, entry['data_file']['referenced_data_file'])
return entry
