This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 13cff7ff484 [feature](fe) Support read-only Iceberg unknown columns
(#70112)
13cff7ff484 is described below
commit 13cff7ff484596f29aa1381750e906db3c8bb704
Author: Socrates <[email protected]>
AuthorDate: Mon Nov 16 18:40:24 2026 +0800
[feature](fe) Support read-only Iceberg unknown columns (#70112)
Problem Summary:
Iceberg V3 `unknown` fields represent NULL-only values. Doris currently
exposes them as `UNSUPPORTED`, so users cannot project those columns or
read the complete table with `SELECT *`.
This PR adds read-only support:
- Preserve the logical `unknown` type for schema display and
serialization, while using the existing SQL NULL execution type.
- Emit NULL carriers in scan schemas and synthesize NULL before matching
physical fields or partition values. Retain parent null maps, collection
offsets, and map keys for nested values.
- Keep predicates on unknown columns out of physical-statistics
pushdown. Compare current unknown equality-delete keys as typed NULLs,
while preserving physical-type resolution for dropped historical keys
whose retained schema only records unknown.
- Reject data mutations against a target schema containing unknown,
including omitted/nested columns, metadata-only DELETE, and empty-table
rewrites. Keep unknown column creation unsupported.
- Add frontend/backend unit coverage and runtime-generated Parquet/ORC
regression fixtures for absent/present physical fields, nested values,
metadata display, NULL predicates and aggregates, schema promotion,
historical snapshots, and write rejection.
Other unsupported Iceberg types retain their existing behavior. No
Thrift/Protobuf definitions or storage-file formats change. The new
unknown scan semantics require the accompanying BE changes;
mixed-version execution has not been validated.
### Release note
Support reading Iceberg V3 unknown columns as NULL, including nested
columns. Tables containing unknown columns remain read-only.
---
be/src/format_v2/column_mapper.cpp | 33 ++++
be/src/format_v2/table/iceberg_reader.cpp | 11 ++
be/src/format_v2/table_reader.h | 15 ++
be/test/format_v2/column_mapper_test.cpp | 60 ++++++
be/test/format_v2/table/iceberg_reader_test.cpp | 23 +++
be/test/format_v2/table_reader_test.cpp | 68 +++++++
.../scripts/java/CreateIcebergUnknownFixtures.java | 188 ++++++++++++++++++
.../iceberg/IcebergConnectorMetadata.java | 2 +
.../iceberg/IcebergConnectorTransaction.java | 2 +
.../iceberg/IcebergPredicateConverter.java | 5 +
.../connector/iceberg/IcebergProcedureOps.java | 10 +-
.../connector/iceberg/IcebergScanPlanProvider.java | 2 +-
.../connector/iceberg/IcebergSchemaUtils.java | 49 ++++-
.../connector/iceberg/IcebergTypeMapping.java | 8 +-
.../iceberg/IcebergWriteSchemaContext.java | 1 +
.../iceberg/IcebergMetadataDeleteTest.java | 12 ++
.../iceberg/IcebergPredicateConverterTest.java | 12 ++
.../connector/iceberg/IcebergProcedureOpsTest.java | 16 ++
.../connector/iceberg/IcebergSchemaUtilsTest.java | 72 ++++++-
.../iceberg/IcebergTypeMappingReadTest.java | 11 +-
.../IcebergWriteTransactionCoverageTest.java | 18 ++
.../converter/ConnectorColumnConverter.java | 6 +-
.../connector/converter/ConnectorUnknownType.java | 38 ++++
.../org/apache/doris/persist/gson/GsonUtils.java | 2 +
.../converter/ConnectorColumnConverterTest.java | 45 +++++
.../iceberg/test_iceberg_unknown_read.out | 219 +++++++++++++++++++++
.../iceberg/test_iceberg_unknown_read.groovy | 198 +++++++++++++++++++
27 files changed, 1106 insertions(+), 20 deletions(-)
diff --git a/be/src/format_v2/column_mapper.cpp
b/be/src/format_v2/column_mapper.cpp
index e8e275612d4..f1eb3d07b93 100644
--- a/be/src/format_v2/column_mapper.cpp
+++ b/be/src/format_v2/column_mapper.cpp
@@ -1882,6 +1882,16 @@ static bool build_variant_projection(const
ColumnMapping& mapping, LocalColumnIn
static Status build_complex_projection(const ColumnMapping& mapping,
LocalColumnIndex* projection,
bool enable_variant_leaf_projection =
false);
+static bool needs_full_map_shape_for_null_value(const DataTypePtr& file_type,
+ const
std::vector<ColumnMapping>& children) {
+ // A physical MAP must retain its value stream to preserve a valid
key/value shape, even
+ // when its table value is UNKNOWN. Materialization replaces that stream
with NULLs.
+ return remove_nullable(file_type)->get_primitive_type() == TYPE_MAP &&
+ std::ranges::any_of(children, [](const ColumnMapping& child) {
+ return !child.file_local_id.has_value() &&
child.table_type->is_null_literal();
+ });
+}
+
// Build the projected file children/type according to the pruned complex
projection. For example,
// if we have a struct column `s` with children `id` and `name`, and the
projection only keeps
// `s.name`, then the file reader should expose `STRUCT<name ...>`.
@@ -1892,6 +1902,11 @@ static Status rebuild_projected_file_children_and_type(
DORIS_CHECK(file_type != nullptr);
DORIS_CHECK(projected_file_children != nullptr);
DORIS_CHECK(projected_type != nullptr);
+ if (needs_full_map_shape_for_null_value(file_type, child_mappings)) {
+ *projected_file_children = original_file_children;
+ *projected_type = file_type;
+ return Status::OK();
+ }
ColumnDefinition field;
field.type = file_type;
field.children = original_file_children;
@@ -1931,6 +1946,10 @@ static Status build_complex_projection(const
ColumnMapping& mapping, LocalColumn
projection->timestamp_is_adjusted_to_utc =
mapping.timestamp_is_adjusted_to_utc;
projection->project_all_children = mapping.child_mappings.empty();
projection->children.clear();
+ if (needs_full_map_shape_for_null_value(mapping.file_type,
mapping.child_mappings)) {
+ projection->project_all_children = true;
+ return Status::OK();
+ }
const auto present_children =
present_child_mappings_in_file_order(mapping.child_mappings);
if (!projection->project_all_children && present_children.empty()) {
// All requested table children under this complex node are
missing/default-only. The file
@@ -2345,6 +2364,14 @@ Status
TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab
mapping->table_column_name = table_column.name;
mapping->table_type = table_column.type;
mapping->variant_access_paths = table_column.variant_access_paths;
+ if (table_column.type->is_null_literal()) {
+ // Iceberg V3 unknown is NULL even if a physical field with the same
ID/name exists.
+ // Resolve it before partition/physical matching so file values and
statistics cannot leak in.
+ DORIS_CHECK(table_column.type->is_nullable());
+ _set_constant_mapping(mapping,
VExprContext::create_shared(VLiteral::create_shared(
+ table_column.type, Field {})));
+ return Status::OK();
+ }
const auto iceberg_metadata_type = [&] {
if (!_options.enable_iceberg_metadata_virtual_columns ||
!table_column.is_synthesized) {
return TableVirtualColumnType::INVALID;
@@ -3014,6 +3041,12 @@ Status TableColumnMapper::_create_direct_mapping(const
ColumnDefinition& table_c
synthesized_used_file_child_ids.push_back(file_child->file_local_id());
}
}
+ if (table_child.type->is_null_literal()) {
+ // An unknown child is always synthesized. Keep the physical
parent for its null
+ // map and ARRAY/MAP offsets, but never decode this child's
value into the result.
+ DORIS_CHECK(table_child.type->is_nullable());
+ file_child = nullptr;
+ }
if (file_child == nullptr) {
if (table_child.default_expr == nullptr &&
table_child.initial_default_value.has_value()) {
diff --git a/be/src/format_v2/table/iceberg_reader.cpp
b/be/src/format_v2/table/iceberg_reader.cpp
index e27d9847709..6e5dc48d5fb 100644
--- a/be/src/format_v2/table/iceberg_reader.cpp
+++ b/be/src/format_v2/table/iceberg_reader.cpp
@@ -2092,6 +2092,17 @@ Status
IcebergTableReader::_append_equality_delete_predicates(format::FileScanRe
DataTypes comparison_types;
bool has_missing_key = false;
for (size_t idx = 0; idx < filter.field_ids.size(); ++idx) {
+ if (filter.key_types[idx]->is_null_literal()) {
+ // Current Iceberg UNKNOWN is NULL even when a same-ID
physical field exists.
+ // Compare a typed NULL against the tombstones, never the
ignored physical key.
+ // Dropped historical UNKNOWN keys retain a STRING carrier for
physical resolution.
+ const auto comparison_type =
remove_nullable(filter.delete_key_types[idx]);
+ key_exprs.push_back(
+
VLiteral::create_shared(make_nullable(comparison_type), Field {}));
+ comparison_types.push_back(comparison_type);
+ has_missing_key = true;
+ continue;
+ }
EqualityDeleteColumnPath data_path;
bool complete_path = false;
RETURN_IF_ERROR(
diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h
index bdd0a434c79..4c388835ef0 100644
--- a/be/src/format_v2/table_reader.h
+++ b/be/src/format_v2/table_reader.h
@@ -1438,6 +1438,12 @@ protected:
DORIS_CHECK(column != nullptr);
DORIS_CHECK(mapping.file_type != nullptr);
DORIS_CHECK(mapping.table_type != nullptr);
+ if (mapping.table_type->is_null_literal()) {
+ // ARRAY/MAP retain a physical child stream to recover offsets.
UNKNOWN values must
+ // still be NULL, rather than cast from that stream (for example
STRING to UInt8).
+ return _materialize_default_or_missing_column(mapping, nullptr,
rows, column,
+
nullable_parent_null_map);
+ }
*column = file_column;
if (!mapping.is_trivial) {
if (!mapping.child_mappings.empty()) {
@@ -2023,6 +2029,15 @@ protected:
const ColumnMapping* value_mapping = nullptr;
for (const auto& child_mapping : mapping.child_mappings) {
if (!child_mapping.file_local_id.has_value()) {
+ if (child_mapping.table_type->is_null_literal()) {
+ // Iceberg map keys are required, so only the value can be
UNKNOWN. Its
+ // mapping deliberately has no file ID even though the
parent has offsets.
+ DORIS_CHECK(
+ assert_cast<const
DataTypeMap&>(*remove_nullable(mapping.table_type))
+ .get_value_type()
+ ->is_null_literal());
+ value_mapping = &child_mapping;
+ }
continue;
}
if (*child_mapping.file_local_id == 0) {
diff --git a/be/test/format_v2/column_mapper_test.cpp
b/be/test/format_v2/column_mapper_test.cpp
index 82f20aa169c..aa130783a79 100644
--- a/be/test/format_v2/column_mapper_test.cpp
+++ b/be/test/format_v2/column_mapper_test.cpp
@@ -253,6 +253,66 @@ TEST(ColumnMapperDebugTest,
CoversDebugStringEnumAndNestedBranches) {
}
}
+TEST(ColumnMapperConstantTest,
NullTypeIgnoresPresentPhysicalFieldsAndPartitionValues) {
+ auto nested = std::make_shared<DataTypeUInt8>();
+ nested->set_null_literal(true);
+ const auto null_type = make_nullable(nested);
+ auto column = field_id_col("u", 1, null_type);
+ column.is_partition_key = true;
+ for (const auto mode : {TableColumnMappingMode::BY_FIELD_ID,
TableColumnMappingMode::BY_NAME,
+ TableColumnMappingMode::BY_INDEX}) {
+ TableColumnMapper mapper({.mode = mode});
+ const std::map<std::string, Field> partitions = {
+ {"u", Field::create_field<TYPE_STRING>("must be ignored")}};
+ ASSERT_TRUE(
+ mapper.create_mapping({column}, partitions, {field_id_col("u",
1, str(), 0)}).ok());
+ const auto& mapping = mapper.mappings().front();
+ EXPECT_FALSE(mapping.file_local_id.has_value());
+ EXPECT_TRUE(mapping.constant_index.has_value());
+ EXPECT_EQ(mapping.filter_conversion, FilterConversionType::CONSTANT);
+ const auto& literal = assert_cast<const
VLiteral&>(*mapping.default_expr->root());
+ EXPECT_TRUE((*literal.get_column_ptr())[0].is_null());
+ FileScanRequest request;
+ ASSERT_TRUE(mapper.create_scan_request({}, {column}, &request).ok());
+ EXPECT_TRUE(request.predicate_columns.empty());
+ EXPECT_TRUE(request.non_predicate_columns.empty());
+ }
+}
+
+TEST(ColumnMapperCreateMappingTest,
NullChildrenIgnorePhysicalValuesUnderStructArrayAndMap) {
+ auto nested = std::make_shared<DataTypeUInt8>();
+ nested->set_null_literal(true);
+ const auto null_type = make_nullable(nested);
+ const auto table_struct = struct_col("s", 1, {field_id_col("u", 2,
null_type)});
+ const auto file_struct = struct_col("s", 1, {field_id_col("u", 2, str(),
0)}, 0);
+ const auto table_array = array_col("a", 3, field_id_col("element", 4,
null_type));
+ const auto file_array = array_col("a", 3, field_id_col("element", 4,
str(), 0), 1);
+ const auto table_map =
+ map_col("m", 5, {field_id_col("key", 6, str()),
field_id_col("value", 7, null_type)},
+ str(), null_type);
+ const auto file_map =
+ map_col("m", 5, {field_id_col("key", 6, str(), 0),
field_id_col("value", 7, str(), 1)},
+ str(), str(), 2);
+ TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID});
+ ASSERT_TRUE(mapper.create_mapping({table_struct, table_array, table_map},
{},
+ {file_struct, file_array, file_map})
+ .ok());
+ for (size_t index = 0; index < 3; ++index) {
+ const auto& mapping = mapper.mappings()[index];
+ EXPECT_TRUE(mapping.file_local_id.has_value());
+ EXPECT_FALSE(mapping.is_trivial);
+ const auto& child = mapping.child_mappings[index == 2 ? 1 : 0];
+ EXPECT_FALSE(child.file_local_id.has_value());
+ EXPECT_EQ(child.filter_conversion,
FilterConversionType::FINALIZE_ONLY);
+ EXPECT_TRUE(child.table_type->is_null_literal());
+ }
+ FileScanRequest request;
+ ASSERT_TRUE(
+ mapper.create_scan_request({}, {table_struct, table_array,
table_map}, &request).ok());
+ ASSERT_EQ(request.non_predicate_columns.size(), 3);
+ EXPECT_TRUE(request.non_predicate_columns[2].project_all_children);
+}
+
TEST(ColumnMapperTest, ParquetRetainsIdlessComplexWrapperWithNestedFieldId) {
auto table_child = field_id_col("a", 1, i32());
auto table_struct = struct_col("s", 10, {table_child});
diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp
b/be/test/format_v2/table/iceberg_reader_test.cpp
index 65df6d3f57c..930a8a21e74 100644
--- a/be/test/format_v2/table/iceberg_reader_test.cpp
+++ b/be/test/format_v2/table/iceberg_reader_test.cpp
@@ -5495,6 +5495,29 @@ Status read_ids_after_key_equality_delete(const
std::string& test_name,
return Status::OK();
}
+TEST(IcebergV2ReaderTest, UnknownEqualityKeyIgnoresPresentPhysicalValues) {
+ for (const bool delete_null : {false, true}) {
+ std::vector<std::vector<int32_t>> ids;
+ const auto status = read_ids_after_key_equality_delete(
+ delete_null ? "doris_unknown_eq_null" :
"doris_unknown_eq_value",
+ {{.ids = {1, 2}, .keys = build_nullable_int64_array({10, 20})},
+ {.ids = {3}, .keys = nullptr}},
+ build_nullable_int64_array(
+ delete_null ? std::vector<std::optional<int64_t>>
{std::nullopt}
+ : std::vector<std::optional<int64_t>>
{10}),
+ external_schema_field("k", 1, {}, std::nullopt,
+
external_primitive_type(TPrimitiveType::NULL_TYPE), false,
+ true),
+ &ids);
+ ASSERT_TRUE(status.ok()) << status;
+ if (delete_null) {
+ EXPECT_EQ(ids, (std::vector<std::vector<int32_t>> {{}, {}}));
+ } else {
+ EXPECT_EQ(ids, (std::vector<std::vector<int32_t>> {{1, 2}, {3}}));
+ }
+ }
+}
+
constexpr int64_t MICROS_PER_DAY = 86400LL * 1000 * 1000;
constexpr int32_t DAYS_1000_01_01 = -354285;
constexpr int32_t DAYS_1600_01_01 = -135140;
diff --git a/be/test/format_v2/table_reader_test.cpp
b/be/test/format_v2/table_reader_test.cpp
index 0d71071a4b5..4ea240f3cc2 100644
--- a/be/test/format_v2/table_reader_test.cpp
+++ b/be/test/format_v2/table_reader_test.cpp
@@ -491,9 +491,77 @@ private:
class TableReaderMaterializeTestHelper final : public TableReader {
public:
using TableReader::_materialize_mapping_column;
+ using TableReader::_materialize_complex_mapping_column;
using TableReader::_materialize_map_mapping_column;
};
+TEST(TableReaderTest, UnknownCollectionChildrenPreserveOffsetsAndParentNulls) {
+ auto nested = std::make_shared<DataTypeUInt8>();
+ nested->set_null_literal(true);
+ const auto null_type = make_nullable(nested);
+ const auto string_type = std::make_shared<DataTypeString>();
+ ColumnMapping unknown;
+ unknown.table_column_name = "value";
+ unknown.table_type = null_type;
+ unknown.file_type = null_type;
+
+ for (const bool is_map : {false, true}) {
+ auto values = ColumnString::create();
+ values->insert_value("ignore-one");
+ values->insert_value("ignore-two");
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ for (const auto offset : {2, 2, 2}) {
+ offsets->insert_value(offset);
+ }
+ ColumnMapping mapping;
+ mapping.child_mappings = {unknown};
+ ColumnPtr source;
+ if (is_map) {
+ auto keys = ColumnString::create();
+ keys->insert_value("a");
+ keys->insert_value("b");
+ source = ColumnMap::create(std::move(keys), std::move(values),
std::move(offsets));
+ mapping.table_type =
+ make_nullable(std::make_shared<DataTypeMap>(string_type,
null_type));
+ mapping.file_type =
+ make_nullable(std::make_shared<DataTypeMap>(string_type,
string_type));
+ } else {
+ source = ColumnArray::create(std::move(values),
std::move(offsets));
+ mapping.table_type =
make_nullable(std::make_shared<DataTypeArray>(null_type));
+ mapping.file_type =
make_nullable(std::make_shared<DataTypeArray>(string_type));
+ }
+ auto nulls = ColumnUInt8::create();
+ nulls->insert_value(0);
+ nulls->insert_value(1);
+ nulls->insert_value(0);
+ source = ColumnNullable::create(IColumn::mutate(std::move(source)),
std::move(nulls));
+ TableReaderMaterializeTestHelper reader;
+ ColumnPtr result;
+ ASSERT_TRUE(reader._materialize_complex_mapping_column(mapping,
source, 3, &result).ok());
+ const auto& nullable = assert_cast<const ColumnNullable&>(*result);
+ EXPECT_EQ(nullable.get_null_map_data()[0], 0);
+ EXPECT_EQ(nullable.get_null_map_data()[1], 1);
+ EXPECT_EQ(nullable.get_null_map_data()[2], 0);
+ const auto& parent = nullable.get_nested_column();
+ if (is_map) {
+ const auto& keys = assert_cast<const
ColumnMap&>(parent).get_keys();
+ EXPECT_EQ(keys.get_data_at(0).to_string(), "a");
+ EXPECT_EQ(keys.get_data_at(1).to_string(), "b");
+ }
+ const auto& result_values = is_map ? assert_cast<const
ColumnMap&>(parent).get_values()
+ : assert_cast<const
ColumnArray&>(parent).get_data();
+ const auto& result_offsets = is_map ? assert_cast<const
ColumnMap&>(parent).get_offsets()
+ : assert_cast<const
ColumnArray&>(parent).get_offsets();
+ ASSERT_EQ(result_offsets.size(), 3);
+ EXPECT_EQ(result_offsets[0], 2);
+ EXPECT_EQ(result_offsets[1], 2);
+ EXPECT_EQ(result_offsets[2], 2);
+ ASSERT_EQ(result_values.size(), 2);
+ EXPECT_TRUE(result_values[0].is_null());
+ EXPECT_TRUE(result_values[1].is_null());
+ }
+}
+
TEST(TableReaderTest, LastProjectionDetachesNestedMapWithoutCopyingStrings) {
auto keys = ColumnString::create();
keys->insert_data(std::string(1UL << 20, 'k').data(), 1UL << 20);
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/java/CreateIcebergUnknownFixtures.java
b/docker/thirdparties/docker-compose/iceberg/scripts/java/CreateIcebergUnknownFixtures.java
new file mode 100644
index 00000000000..77086d4b318
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/java/CreateIcebergUnknownFixtures.java
@@ -0,0 +1,188 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.data.GenericAppenderFactory;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptionKeyMetadata;
+import org.apache.iceberg.io.DataWriter;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.UUID;
+
+/** Generates unknown-column fixtures; physical schemas deliberately retain
the same field IDs. */
+public final class CreateIcebergUnknownFixtures {
+ private CreateIcebergUnknownFixtures() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ throw new IllegalArgumentException("Usage:
CreateIcebergUnknownFixtures <create|evolve> <namespace>");
+ }
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "rest");
+ properties.put("uri", "http://rest:8181");
+ properties.put("warehouse", "s3://warehouse/wh/");
+ properties.put("io-impl", "org.apache.iceberg.aws.s3.S3FileIO");
+ properties.put("s3.endpoint", "http://minio:9000");
+ properties.put("s3.path-style-access", "true");
+ properties.put("s3.region", "us-east-1");
+ Catalog catalog = CatalogUtil.buildIcebergCatalog("demo", properties,
null);
+ Namespace namespace = Namespace.of(args[1]);
+ if (!((SupportsNamespaces) catalog).namespaceExists(namespace)) {
+ ((SupportsNamespaces) catalog).createNamespace(namespace);
+ }
+ for (FileFormat format : Arrays.asList(FileFormat.PARQUET,
FileFormat.ORC)) {
+ String suffix = format.name().toLowerCase(Locale.ROOT);
+ TableIdentifier valid = TableIdentifier.of(namespace,
"unknown_read_" + suffix);
+ if ("create".equals(args[0])) {
+ create(catalog, valid, format, false);
+ create(catalog, TableIdentifier.of(namespace,
"unknown_physical_" + suffix), format, true);
+ TableIdentifier empty = TableIdentifier.of(namespace,
"unknown_empty_" + suffix);
+ catalog.dropTable(empty);
+ catalog.buildTable(empty, schema(true,
false)).withProperty("format-version", "3").create();
+ } else if ("evolve".equals(args[0])) {
+ Table table = catalog.loadTable(valid);
+ // The SDK update API does not implement UNKNOWN promotion on
all supported versions.
+ // Publish the spec-permitted schema evolution directly,
retaining all field IDs.
+ TableOperations ops = ((BaseTable) table).operations();
+ TableMetadata base = ops.current();
+ Schema promoted = TypeUtil.reassignIds(schema(false, false),
table.schema());
+ Schema evolved = new Schema(table.schema().schemaId() + 1,
promoted.columns());
+ ops.commit(base,
TableMetadata.buildFrom(base).setCurrentSchema(evolved,
base.lastColumnId()).build());
+ table.refresh();
+ Schema physical = table.schema().select("id", "u");
+ GenericRecord row = GenericRecord.create(physical);
+ row.setField("id", 4);
+ row.setField("u", "now known");
+ append(table, physical, format,
Collections.singletonList(row));
+ } else {
+ throw new IllegalArgumentException("Unknown action: " +
args[0]);
+ }
+ }
+ }
+
+ private static Schema schema(boolean unknown, boolean omitUnknown) {
+ org.apache.iceberg.types.Type leaf = unknown ? Types.UnknownType.get()
: Types.StringType.get();
+ Types.StructType struct = omitUnknown
+ ? Types.StructType.of(Types.NestedField.optional(4, "known",
Types.IntegerType.get()))
+ : Types.StructType.of(Types.NestedField.optional(4, "known",
Types.IntegerType.get()),
+ Types.NestedField.optional(5, "u", leaf));
+ Types.StructType element = omitUnknown
+ ? Types.StructType.of(Types.NestedField.optional(8, "known",
Types.IntegerType.get()))
+ : Types.StructType.of(Types.NestedField.optional(8, "known",
Types.IntegerType.get()),
+ Types.NestedField.optional(9, "u", leaf));
+ Types.StructType value = omitUnknown
+ ? Types.StructType.of(Types.NestedField.optional(13, "known",
Types.IntegerType.get()))
+ : Types.StructType.of(Types.NestedField.optional(13, "known",
Types.IntegerType.get()),
+ Types.NestedField.optional(14, "u", leaf));
+ java.util.List<Types.NestedField> fields = new java.util.ArrayList<>();
+ fields.add(Types.NestedField.required(1, "id",
Types.IntegerType.get()));
+ if (!omitUnknown) {
+ fields.add(Types.NestedField.optional(2, "u", leaf));
+ }
+ fields.add(Types.NestedField.optional(3, "s", struct));
+ fields.add(Types.NestedField.optional(6, "a",
Types.ListType.ofOptional(7, element)));
+ fields.add(Types.NestedField.optional(10, "m",
Types.MapType.ofOptional(11, 12, Types.StringType.get(), value)));
+ if (!omitUnknown) {
+ fields.add(Types.NestedField.optional(15, "null_array",
Types.ListType.ofOptional(16, leaf)));
+ fields.add(Types.NestedField.optional(17, "null_map",
+ Types.MapType.ofOptional(18, 19, Types.StringType.get(),
leaf)));
+ }
+ return new Schema(fields);
+ }
+
+ private static void create(Catalog catalog, TableIdentifier id, FileFormat
format, boolean present)
+ throws Exception {
+ catalog.dropTable(id);
+ // Use MOR so UPDATE/DELETE/MERGE reach the UNKNOWN write rejection.
+ Table table = catalog.buildTable(id, schema(true, false))
+ .withProperty("format-version", "3")
+ .withProperty("write.delete.mode", "merge-on-read")
+ .withProperty("write.update.mode", "merge-on-read")
+ .withProperty("write.merge.mode", "merge-on-read")
+ .withProperty("write.format.default",
format.name().toLowerCase(Locale.ROOT)).create();
+ // Valid files omit UNKNOWN. The companion fixture intentionally
stores non-NULL STRING
+ // fields under those IDs to prove readers ignore the values AND their
pruning statistics.
+ Schema physical = TypeUtil.reassignIds(schema(false, !present),
table.schema());
+ GenericRecord nullParent = GenericRecord.create(physical);
+ nullParent.setField("id", 1);
+ GenericRecord populated = GenericRecord.create(physical);
+ populated.setField("id", 2);
+ populated.setField("s", child(physical.findType("s").asStructType(),
present));
+ populated.setField("a",
Arrays.asList(child(physical.findType("a").asListType()
+ .elementType().asStructType(), present), null));
+ populated.setField("m", Collections.singletonMap("key",
+
child(physical.findType("m").asMapType().valueType().asStructType(), present)));
+ GenericRecord empty = GenericRecord.create(physical);
+ empty.setField("id", 3);
+ empty.setField("a", Collections.emptyList());
+ empty.setField("m", Collections.emptyMap());
+ if (present) {
+ nullParent.setField("u", "must be ignored");
+ populated.setField("u", "must be ignored");
+ empty.setField("u", "must be ignored");
+ populated.setField("null_array", Arrays.asList("ignored", null));
+ populated.setField("null_map", Collections.singletonMap("key",
"ignored"));
+ empty.setField("null_array", Collections.emptyList());
+ empty.setField("null_map", Collections.emptyMap());
+ }
+ append(table, physical, format, Arrays.asList(nullParent, populated,
empty));
+ }
+
+ private static Record child(Types.StructType type, boolean present) {
+ GenericRecord record = GenericRecord.create(type);
+ record.setField("known", 20);
+ if (present) {
+ record.setField("u", "must be ignored");
+ }
+ return record;
+ }
+
+ private static void append(Table table, Schema physical, FileFormat format,
+ java.util.List<? extends Record> records) throws Exception {
+ String path = table.location() + "/data/" + UUID.randomUUID() + "."
+ + format.name().toLowerCase(Locale.ROOT);
+ DataWriter<Record> writer = new GenericAppenderFactory(physical,
table.spec()).newDataWriter(
+ EncryptedFiles.encryptedOutput(table.io().newOutputFile(path),
EncryptionKeyMetadata.EMPTY),
+ format, null);
+ try (DataWriter<Record> closeable = writer) {
+ for (Record record : records) {
+ closeable.write(record);
+ }
+ }
+ table.newAppend().appendFile(writer.toDataFile()).commit();
+ }
+}
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
index ac669d74d13..2e937d1d48e 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
@@ -2141,6 +2141,7 @@ public class IcebergConnectorMetadata implements
ConnectorMetadata {
return OptionalLong.empty();
}
validateBoundWriteGeneration(table, boundWriteMetadataIdentity);
+ IcebergSchemaUtils.validateNoUnknownForWrite(table.schema());
Expression deleteExpr = predicate.isPresent()
? wholeFileDeleteExpression(table, session, predicate.get())
: Expressions.alwaysTrue();
@@ -2182,6 +2183,7 @@ public class IcebergConnectorMetadata implements
ConnectorMetadata {
// metadata delete (the row-level fallback then reports an exact
affected-row count).
Transaction transaction = IcebergConnectorTransaction.openTransaction(
table, context, null, false, boundWriteMetadataIdentity,
refreshedSnapshotId);
+
IcebergSchemaUtils.validateNoUnknownForWrite(transaction.table().schema());
DeleteFiles deleteFiles =
transaction.newDelete().deleteFromRowFilter(deleteExpr);
if (isCancelled.getAsBoolean()) {
return OptionalLong.empty();
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java
index 9db7005897d..b3068188fad 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java
@@ -282,6 +282,7 @@ public class IcebergConnectorTransaction implements
ConnectorTransaction, Rewrit
this.table = loaded;
try {
validateBoundWriteGeneration(ctx, loaded);
+
IcebergSchemaUtils.validateNoUnknownForWrite(loaded.schema());
if (writeSchemaContext != null) {
writeSchemaContext.validateCurrentSchema(loaded,
ctx.isOverwrite());
}
@@ -290,6 +291,7 @@ public class IcebergConnectorTransaction implements
ConnectorTransaction, Rewrit
// BaseTable.newTransaction refreshes metadata. Fence that
second load too, closing the
// drop/recreate race between the initial generation check
and transaction construction.
validateBoundWriteGeneration(ctx, loaded);
+
IcebergSchemaUtils.validateNoUnknownForWrite(loaded.schema());
if (writeSchemaContext != null) {
writeSchemaContext.validateCurrentSchema(loaded,
ctx.isOverwrite());
}
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java
index 3f15bc80809..1e5701502c6 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPredicateConverter.java
@@ -523,6 +523,11 @@ public class IcebergPredicateConverter {
return null;
}
Types.NestedField field = schema.caseInsensitiveFindField(colName);
+ if (field != null && field.type().typeId() == TypeID.UNKNOWN) {
+ // UNKNOWN ignores any physical value, including its
null-count/bounds statistics.
+ // For example, an old same-ID STRING field must not prune
`unknown_col <=> NULL`.
+ return null;
+ }
// Iceberg 1.11 widens retained four-byte DATE bounds to unscaled
longs. Pruning those
// epoch-day bounds with nanosecond predicates can discard matching
files in every planner.
return field != null && unsafeNanosFields.contains(field.fieldId()) ?
null : field;
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergProcedureOps.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergProcedureOps.java
index 1b77d762060..295b552fc2b 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergProcedureOps.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergProcedureOps.java
@@ -237,16 +237,20 @@ public class IcebergProcedureOps implements
ConnectorProcedureOps {
// Resolve the per-request ops before the auth scope (see
runInAuthScope): a session=user fail-closed
// surfaces verbatim.
IcebergCatalogOps ops = catalogOpsResolver.apply(session);
+ Function<Table, List<RewriteDataGroup>> planTasks = table -> {
+ IcebergSchemaUtils.validateNoUnknownForWrite(table.schema());
+ return
planner.planAndOrganizeTasks(IcebergScanPlanning.requiresServerSideScanPlanning(table)
+ ? table : wrapTableForProcedure(table));
+ };
List<RewriteDataGroup> groups;
if (context == null) {
- groups = ops.withTable(handle.getDbName(), handle.getTableName(),
planner::planAndOrganizeTasks);
+ groups = ops.withTable(handle.getDbName(), handle.getTableName(),
planTasks);
} else {
try {
// A server-planned table (RESTTable) keeps its own newScan(),
which asks the REST server for the
// file scan tasks; rebuilding it as a plain BaseTable would
plan from local manifests instead.
groups = context.executeAuthenticated(() ->
ops.withTable(handle.getDbName(), handle.getTableName(),
- table ->
planner.planAndOrganizeTasks(IcebergScanPlanning.requiresServerSideScanPlanning(table)
- ? table : wrapTableForProcedure(table))));
+ planTasks));
} catch (DorisConnectorException e) {
throw e;
} catch (Exception e) {
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java
index ac4649fa36e..a87ab31ddc9 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java
@@ -2319,7 +2319,7 @@ public class IcebergScanPlanProvider implements
ConnectorScanPlanProvider {
List<NestedField> equalityFields =
schemaForPotentialEqualityDeletes(
table, exactScan.snapshot(), scanSchema,
applicableEqualityDeleteFieldIds);
dict = IcebergSchemaUtils.encodeEqualitySchemaEvolutionProp(
- table, equalityFields, appendRowLineage,
+ table, scanSchema, equalityFields, appendRowLineage,
enableVarbinary, enableTimestampTz);
} else if (iceHandle.hasSnapshotPin()) {
dict = IcebergSchemaUtils.encodeSchemaEvolutionProp(
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java
index 02e0604c8ac..20d0db4e979 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java
@@ -41,6 +41,7 @@ import org.apache.iceberg.mapping.NameMappingParser;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Type.TypeID;
+import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -56,6 +57,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
/**
* Builds the native-reader schema dictionary ({@code current_schema_id} +
{@code history_schema_info}) so BE
@@ -173,11 +175,12 @@ public final class IcebergSchemaUtils {
* drop/re-add operations may legitimately contribute two fields with the
same name and different IDs;
* Iceberg validates names when a Schema is constructed even though BE
consumes this carrier strictly by ID.
*/
- static String encodeEqualitySchemaEvolutionProp(Table table,
List<Types.NestedField> fields,
+ static String encodeEqualitySchemaEvolutionProp(Table table, Schema
scanSchema, List<Types.NestedField> fields,
boolean appendRowLineage, boolean enableVarbinary, boolean
enableTimestampTz) {
Optional<Map<Integer, List<String>>> nameMapping =
extractNameMapping(table);
TSchema current = buildCurrentSchema(fields,
nameMapping.orElse(Collections.emptyMap()),
- nameMapping.isPresent(), enableVarbinary, enableTimestampTz);
+ nameMapping.isPresent(), enableVarbinary, enableTimestampTz,
+ TypeUtil.indexById(scanSchema.asStruct()).keySet());
if (appendRowLineage) {
appendRowLineageFields(current.getRootField());
}
@@ -324,13 +327,13 @@ public final class IcebergSchemaUtils {
private static TSchema buildCurrentSchema(List<Types.NestedField> fields,
Map<Integer, List<String>> nameMapping, boolean hasNameMapping,
boolean enableVarbinary,
- boolean enableTimestampTz) {
+ boolean enableTimestampTz, Set<Integer> currentFieldIds) {
TSchema tSchema = new TSchema();
tSchema.setSchemaId(CURRENT_SCHEMA_ID);
TStructField root = new TStructField();
for (Types.NestedField field : fields) {
addField(root, buildField(field, field.name(), nameMapping,
- hasNameMapping, enableVarbinary, enableTimestampTz));
+ hasNameMapping, enableVarbinary, enableTimestampTz,
currentFieldIds));
}
tSchema.setRootField(root);
return tSchema;
@@ -354,6 +357,13 @@ public final class IcebergSchemaUtils {
private static TField buildField(Types.NestedField field, String
nameOverride,
Map<Integer, List<String>> nameMapping, boolean hasNameMapping,
boolean enableVarbinary,
boolean enableTimestampTz) {
+ return buildField(field, nameOverride, nameMapping, hasNameMapping,
+ enableVarbinary, enableTimestampTz, null);
+ }
+
+ private static TField buildField(Types.NestedField field, String
nameOverride,
+ Map<Integer, List<String>> nameMapping, boolean hasNameMapping,
boolean enableVarbinary,
+ boolean enableTimestampTz, Set<Integer> currentFieldIds) {
TField tField = new TField();
tField.setId(field.fieldId());
tField.setName(nameOverride != null ? nameOverride :
field.name().toLowerCase(Locale.ROOT));
@@ -394,7 +404,15 @@ public final class IcebergSchemaUtils {
Type type = field.type();
TColumnType columnType = new TColumnType();
if (type.isPrimitiveType()) {
- tField.setType(buildPrimitiveColumnType(type, enableVarbinary,
enableTimestampTz));
+ TColumnType primitive = buildPrimitiveColumnType(type,
enableVarbinary, enableTimestampTz);
+ if (type.typeId() == TypeID.UNKNOWN && currentFieldIds != null
+ && !currentFieldIds.contains(field.fieldId())) {
+ // A dropped equality key may retain only its old UNKNOWN
schema after promotion
+ // schemas expire. It is not a current unknown column: BE must
still resolve the
+ // physical key's concrete type, as it did with the
unsupported STRING placeholder.
+ primitive.setType(TPrimitiveType.STRING);
+ }
+ tField.setType(primitive);
if (type.typeId() == TypeID.TIMESTAMP && ((Types.TimestampType)
type).shouldAdjustToUTC()) {
// Iceberg decodes a Parquet timestamp by the table type, not
by the file annotation. Spark and Hive
// write timestamptz as INT96, which has no annotation, so
without this marker BE reads such a
@@ -421,7 +439,7 @@ public final class IcebergSchemaUtils {
TArrayField arrayField = new TArrayField();
arrayField.setItemField(fieldPtr(
buildField(listType.fields().get(0), null,
nameMapping, hasNameMapping,
- enableVarbinary, enableTimestampTz)));
+ enableVarbinary, enableTimestampTz,
currentFieldIds)));
nestedField.setArrayField(arrayField);
break;
}
@@ -432,10 +450,10 @@ public final class IcebergSchemaUtils {
TMapField mapField = new TMapField();
mapField.setKeyField(fieldPtr(
buildField(kv.get(0), null, nameMapping,
hasNameMapping, enableVarbinary,
- enableTimestampTz)));
+ enableTimestampTz, currentFieldIds)));
mapField.setValueField(fieldPtr(
buildField(kv.get(1), null, nameMapping,
hasNameMapping, enableVarbinary,
- enableTimestampTz)));
+ enableTimestampTz, currentFieldIds)));
nestedField.setMapField(mapField);
break;
}
@@ -445,7 +463,7 @@ public final class IcebergSchemaUtils {
TStructField structField = new TStructField();
for (Types.NestedField child : structType.fields()) {
addField(structField, buildField(child, null, nameMapping,
hasNameMapping,
- enableVarbinary, enableTimestampTz));
+ enableVarbinary, enableTimestampTz,
currentFieldIds));
}
nestedField.setStructField(structField);
break;
@@ -462,10 +480,23 @@ public final class IcebergSchemaUtils {
return tField;
}
+ /** Reject every data mutation, including writes that project only
ordinary columns. */
+ static void validateNoUnknownForWrite(Schema schema) {
+ // Once unknown columns become queryable, projection-based write
checks can miss a nested
+ // unknown or an omitted target column. Validate the authoritative
target schema instead.
+ if (TypeUtil.indexById(schema.asStruct()).values().stream()
+ .anyMatch(field -> field.type().typeId() == TypeID.UNKNOWN)) {
+ throw new DorisConnectorException("Writing to Iceberg tables with
unknown type columns is not supported");
+ }
+ }
+
private static TColumnType buildPrimitiveColumnType(
Type type, boolean enableVarbinary, boolean enableTimestampTz) {
TColumnType columnType = new TColumnType();
switch (type.typeId()) {
+ case UNKNOWN:
+ columnType.setType(TPrimitiveType.NULL_TYPE);
+ break;
case BOOLEAN:
columnType.setType(TPrimitiveType.BOOLEAN);
break;
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java
index 39d6bb63f11..5b7be13946a 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java
@@ -154,13 +154,16 @@ public final class IcebergTypeMapping {
// TIMESTAMP_NS is a wall-clock type; zoned nanoseconds need a
distinct carrier.
return ConnectorType.of(((Types.TimestampNanoType)
primitive).shouldAdjustToUTC()
? "UNSUPPORTED" : "TIMESTAMP_NS");
+ case UNKNOWN:
+ // V3 unknown has only NULL values; retain its logical
identity for schema display and writes.
+ return ConnectorType.of("UNKNOWN");
case TIME:
// iceberg TIME has no Doris analogue -> UNSUPPORTED
(explicit, byte-parity with legacy
// IcebergUtils which also mapped TIME to Type.UNSUPPORTED).
return ConnectorType.of("UNSUPPORTED");
default:
// Any primitive iceberg type Doris cannot represent — notably
the v3 types
- // GEOMETRY / GEOGRAPHY / UNKNOWN — degrades to UNSUPPORTED:
the table still LOADS and only this
+ // GEOMETRY / GEOGRAPHY — degrades to UNSUPPORTED: the table
still LOADS and only this
// column is present-but-unqueryable. This DIVERGES from
legacy (IcebergUtils.icebergPrimitiveType-
// ToDorisType threw IllegalArgumentException "Cannot
transform unknown type", failing the whole
// table). Graceful degradation is INTENTIONAL (user decision
2026-07-13: map every unrepresentable
@@ -197,6 +200,9 @@ public final class IcebergTypeMapping {
static Type toIcebergPrimitive(ConnectorType type) {
String name = type.getTypeName().toUpperCase(Locale.ROOT);
switch (name) {
+ case "UNKNOWN":
+ case "NULL_TYPE":
+ throw new DorisConnectorException("Creating Iceberg unknown
columns is not supported");
case "UUID":
return Types.UUIDType.get();
case "BOOLEAN":
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java
index 6d994856a31..3491c3fdef9 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteSchemaContext.java
@@ -100,6 +100,7 @@ final class IcebergWriteSchemaContext {
// A branch selects only the parent snapshot / ref. Iceberg validates
a branch write against the
// table-current schema and stamps that schema on the new snapshot,
and Spark binds it the same way.
Schema schema = table.schema();
+ IcebergSchemaUtils.validateNoUnknownForWrite(schema);
int formatVersion = IcebergWriterHelper.getFormatVersion(table);
TableIdentity identity = pinTableIdentity(table, formatVersion);
return new IcebergWriteSchemaContext(
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergMetadataDeleteTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergMetadataDeleteTest.java
index c6b08bdcdf0..8c17780e934 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergMetadataDeleteTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergMetadataDeleteTest.java
@@ -69,6 +69,18 @@ public class IcebergMetadataDeleteTest {
private static final ConnectorTableHandle HANDLE = new
IcebergTableHandle("db1", "t1");
private static final ConnectorSession SESSION = new FakeSession("UTC");
+ @Test
+ public void metadataDeleteRejectsUnknownEvenWithoutAColumnPredicate() {
+ InMemoryCatalog catalog = freshCatalog();
+ Schema schema = new Schema(Types.NestedField.optional(1, "u",
Types.UnknownType.get()));
+ catalog.createTable(TABLE_ID, schema, PartitionSpec.unpartitioned(),
props("format-version", "3"));
+ org.apache.doris.connector.spi.DorisConnectorException error =
Assertions.assertThrows(
+ org.apache.doris.connector.spi.DorisConnectorException.class,
+ () -> metadata(catalog).deleteUsingMetadata(SESSION, HANDLE,
Optional.empty(), null, () -> false));
+ Assertions.assertTrue(error.getMessage().contains("unknown type
columns"));
+ Assertions.assertNull(reload(catalog).currentSnapshot());
+ }
+
@Test
public void wholePartitionDeleteDropsMatchingFilesOnly() {
InMemoryCatalog catalog = freshCatalog();
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPredicateConverterTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPredicateConverterTest.java
index 2add3b9186b..b020c152cc9 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPredicateConverterTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPredicateConverterTest.java
@@ -54,6 +54,18 @@ import java.util.List;
*/
public class IcebergPredicateConverterTest {
+ @Test
+ public void unknownNullPredicatesNeverUsePhysicalStatistics() {
+ Schema schema = new Schema(Types.NestedField.optional(1, "u",
Types.UnknownType.get()));
+ for (IcebergPredicateConverter.Mode mode :
IcebergPredicateConverter.Mode.values()) {
+ IcebergPredicateConverter converter = new
IcebergPredicateConverter(schema, ZoneOffset.UTC, mode);
+ ConnectorColumnRef unknown = new ConnectorColumnRef("u",
ConnectorType.of("UNKNOWN"));
+ ConnectorComparison predicate = new
ConnectorComparison(ConnectorComparison.Operator.EQ_FOR_NULL,
+ unknown, new
ConnectorLiteral(ConnectorType.of("NULL_TYPE"), null));
+ Assertions.assertTrue(converter.convert(predicate).isEmpty());
+ }
+ }
+
// Same schema (ids/types) as fe-core IcebergPredicateTest.
private static final Schema SCHEMA = new Schema(
Types.NestedField.required(1, "c_int", Types.IntegerType.get()),
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProcedureOpsTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProcedureOpsTest.java
index 4370ccbb367..bb887ec7bf2 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProcedureOpsTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProcedureOpsTest.java
@@ -76,6 +76,22 @@ public class IcebergProcedureOpsTest {
return new IcebergProcedureOps(Collections.emptyMap(),
(IcebergCatalogOps) null, null);
}
+ @Test
+ public void rewriteRejectsUnknownBeforeEmptyTableShortcut() {
+ InMemoryCatalog catalog = new InMemoryCatalog();
+ catalog.initialize("test", Collections.emptyMap());
+ catalog.createNamespace(Namespace.of("db1"));
+ catalog.createTable(TableIdentifier.of("db1", "t"),
+ new Schema(Types.NestedField.optional(1, "u",
Types.UnknownType.get())),
+ PartitionSpec.unpartitioned(),
Collections.singletonMap("format-version", "3"));
+ RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
+ ops.table = catalog.loadTable(TableIdentifier.of("db1", "t"));
+ IcebergProcedureOps procedures = new
IcebergProcedureOps(Collections.emptyMap(), ops, null);
+ Assertions.assertThrows(DorisConnectorException.class,
+ () -> procedures.planRewrite(SESSION, new
IcebergTableHandle("db1", "t"),
+ "rewrite_data_files", Collections.emptyMap(), null,
Collections.emptyList()));
+ }
+
@Test
public void getSupportedProceduresExportsFactoryNamesInLegacyOrder() {
Assertions.assertEquals(
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java
index d45eaccf68f..503efb27aef 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java
@@ -215,7 +215,7 @@ public class IcebergSchemaUtilsTest {
Table table = createTable("mixed_equality", schema);
TFileScanRangeParams params =
decode(IcebergSchemaUtils.encodeEqualitySchemaEvolutionProp(
- table, schema.columns(), false, false, false));
+ table, schema, schema.columns(), false, false, false));
List<TFieldPtr> fields =
params.getHistorySchemaInfo().get(0).getRootField().getFields();
Assertions.assertEquals("ID", fields.get(0).getFieldPtr().getName());
@@ -474,7 +474,7 @@ public class IcebergSchemaUtilsTest {
Table table = createTable("carrier", SCHEMA);
TStructField root =
decode(IcebergSchemaUtils.encodeEqualitySchemaEvolutionProp(
- table, fields, false, false,
false)).getHistorySchemaInfo().get(0).getRootField();
+ table, table.schema(), fields, false, false,
false)).getHistorySchemaInfo().get(0).getRootField();
Assertions.assertEquals(Arrays.asList(10, 20, 30),
root.getFields().stream()
.map(field ->
field.getFieldPtr().getId()).collect(java.util.stream.Collectors.toList()));
@@ -616,6 +616,74 @@ public class IcebergSchemaUtilsTest {
mappedFields.get("binary_col").getType().getType());
}
+ @Test
+ public void unknownUsesNullCarrierAndConcreteEvolutionRebinds() {
+ Schema unknown = new Schema(Types.NestedField.optional(1, "u",
Types.UnknownType.get()));
+ Schema concrete = new Schema(Types.NestedField.optional(1, "u",
Types.LongType.get()));
+ TField oldField =
topFields(IcebergSchemaUtils.buildCurrentSchema(unknown,
+ Collections.singletonList("u"),
Collections.emptyMap())).get("u");
+ TField newField =
topFields(IcebergSchemaUtils.buildCurrentSchema(concrete,
+ Collections.singletonList("u"),
Collections.emptyMap())).get("u");
+ Assertions.assertEquals(TPrimitiveType.NULL_TYPE,
oldField.getType().getType());
+ Assertions.assertEquals(TPrimitiveType.BIGINT,
newField.getType().getType());
+ Assertions.assertEquals(oldField.getId(), newField.getId());
+ Assertions.assertEquals(TPrimitiveType.NULL_TYPE,
oldField.getType().getType(),
+ "rebinding must not mutate an already bound schema");
+ }
+
+ @Test
+ public void nestedUnknownLeavesUseNullCarriersWithoutLosingFieldIds() {
+ Schema schema = new Schema(
+ Types.NestedField.optional(1, "s", Types.StructType.of(
+ Types.NestedField.optional(2, "u",
Types.UnknownType.get()))),
+ Types.NestedField.optional(3, "a",
Types.ListType.ofOptional(4, Types.UnknownType.get())),
+ Types.NestedField.optional(5, "m", Types.MapType.ofOptional(6,
7,
+ Types.StringType.get(), Types.UnknownType.get())));
+ Map<String, TField> fields =
topFields(IcebergSchemaUtils.buildCurrentSchema(schema,
+ Arrays.asList("s", "a", "m"), Collections.emptyMap()));
+ TField structChild = childByName(fields.get("s"), "u");
+ TField arrayChild =
fields.get("a").getNestedField().getArrayField().getItemField().getFieldPtr();
+ TField mapChild =
fields.get("m").getNestedField().getMapField().getValueField().getFieldPtr();
+ Assertions.assertEquals(2, structChild.getId());
+ Assertions.assertEquals(4, arrayChild.getId());
+ Assertions.assertEquals(7, mapChild.getId());
+ for (TField field : Arrays.asList(structChild, arrayChild, mapChild)) {
+ Assertions.assertEquals(TPrimitiveType.NULL_TYPE,
field.getType().getType());
+ Assertions.assertTrue(field.isIsOptional());
+ }
+ }
+
+ @Test
+ public void droppedUnknownEqualityCarrierKeepsPhysicalTypeResolution()
throws Exception {
+ Schema current = new Schema(Types.NestedField.optional(1, "u",
Types.UnknownType.get()));
+ Table table = createTable("unknown_equality_carrier", current,
+ Collections.singletonMap("format-version", "3"));
+ List<Types.NestedField> fields =
Arrays.asList(current.columns().get(0),
+ Types.NestedField.optional(2, "dropped_u",
Types.UnknownType.get()));
+ TSchema schema =
decode(IcebergSchemaUtils.encodeEqualitySchemaEvolutionProp(
+ table, current, fields, false, false,
false)).getHistorySchemaInfo().get(0);
+ Assertions.assertEquals(TPrimitiveType.NULL_TYPE,
topFields(schema).get("u").getType().getType());
+ Assertions.assertEquals(TPrimitiveType.STRING,
topFields(schema).get("dropped_u").getType().getType());
+ }
+
+ @Test
+ public void unknownWriteGuardVisitsNestedLeavesAndCreateRejectsUnknown() {
+ for (org.apache.iceberg.types.Type type : Arrays.asList(
+ Types.UnknownType.get(),
+ Types.StructType.of(Types.NestedField.optional(2, "u",
Types.UnknownType.get())),
+ Types.ListType.ofOptional(3, Types.UnknownType.get()),
+ Types.MapType.ofOptional(4, 5, Types.StringType.get(),
Types.UnknownType.get()))) {
+ Schema schema = new Schema(Types.NestedField.optional(1,
"payload", type));
+
Assertions.assertThrows(org.apache.doris.connector.spi.DorisConnectorException.class,
+ () ->
IcebergSchemaUtils.validateNoUnknownForWrite(schema));
+ }
+ Assertions.assertDoesNotThrow(() ->
IcebergSchemaUtils.validateNoUnknownForWrite(
+ new Schema(Types.NestedField.optional(1, "id",
Types.IntegerType.get()))));
+ Assertions.assertThrows(DorisConnectorException.class,
+ () -> IcebergTypeMapping.toIcebergPrimitive(
+
org.apache.doris.connector.spi.ConnectorType.of("UNKNOWN")));
+ }
+
@Test
public void unsupportedPrimitiveKeepsSafeScalarCarrier() {
Schema schema = new Schema(
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTypeMappingReadTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTypeMappingReadTest.java
index 75b8ced07c3..206e23e4cea 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTypeMappingReadTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTypeMappingReadTest.java
@@ -86,9 +86,9 @@ public class IcebergTypeMappingReadTest {
}
@Test
- public void unknownAndV3TypesDegradeToUnsupportedByDesign() {
+ public void unsupportedV3TypesKeepGracefulDegradation() {
// WHY (user decision 2026-07-13, DV-051): iceberg primitive types
Doris cannot represent —
- // Zoned TIMESTAMP_NANO / GEOMETRY / GEOGRAPHY / UNKNOWN — must map to
UNSUPPORTED WITHOUT throwing,
+ // Zoned TIMESTAMP_NANO / GEOMETRY / GEOGRAPHY — must map to
UNSUPPORTED WITHOUT throwing,
// so the table still loads and only the exotic column is
// present-but-unqueryable. This deliberately DIVERGES from legacy
fe-core, which threw
// IllegalArgumentException("Cannot transform unknown type") at
schema-load and failed the whole table.
@@ -99,12 +99,17 @@ public class IcebergTypeMappingReadTest {
Assertions.assertEquals("UNSUPPORTED",
mapOff(Types.TimestampNanoType.withZone()).getTypeName());
Assertions.assertEquals("UNSUPPORTED",
mapOff(Types.GeometryType.crs84()).getTypeName());
Assertions.assertEquals("UNSUPPORTED",
mapOff(Types.GeographyType.crs84()).getTypeName());
- Assertions.assertEquals("UNSUPPORTED",
mapOff(Types.UnknownType.get()).getTypeName());
Assertions.assertEquals("VARIANT_COMPUTE_V2",
mapOff(Types.VariantType.get()).getTypeName());
// The mapping flags do not rescue an unrepresentable type (both arms
are flag-independent).
Assertions.assertEquals("UNSUPPORTED",
mapOn(Types.GeometryType.crs84()).getTypeName());
}
+ @Test
+ public void unknownRetainsItsReadOnlyIdentityRegardlessOfMappingFlags() {
+ Assertions.assertEquals("UNKNOWN",
mapOff(Types.UnknownType.get()).getTypeName());
+ Assertions.assertEquals("UNKNOWN",
mapOn(Types.UnknownType.get()).getTypeName());
+ }
+
@Test
public void decimalCarriesPrecisionAndScale() {
// WHY: Iceberg DECIMAL(p,s) maps to Doris DECIMALV3(p,s) carrying
both p and s verbatim; legacy
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWriteTransactionCoverageTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWriteTransactionCoverageTest.java
index b29e8e3848d..1f51dee3241 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWriteTransactionCoverageTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWriteTransactionCoverageTest.java
@@ -62,6 +62,24 @@ public class IcebergWriteTransactionCoverageTest {
// above lives under it.
private static final String COMMIT_LOCATION = "s3://bucket/events";
+ @Test
+ public void
everyWriteOperationRejectsAnUnknownTargetBeforeOpeningTransaction() {
+ InMemoryCatalog catalog = catalog();
+ Schema schema = new Schema(
+ Types.NestedField.optional(1, "id", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "payload", Types.StructType.of(
+ Types.NestedField.optional(3, "u",
Types.UnknownType.get()))));
+ Table table = catalog.createTable(IDENTIFIER, schema,
PartitionSpec.unpartitioned(),
+ Collections.singletonMap("format-version", "3"));
+ for (WriteOperation operation : WriteOperation.values()) {
+ IcebergConnectorTransaction transaction = transaction(table);
+ DorisConnectorException error =
Assertions.assertThrows(DorisConnectorException.class,
+ () -> transaction.beginWrite(session(null), "db",
"events", context(operation)));
+ Assertions.assertTrue(error.getMessage().contains("unknown type
columns"), error.getMessage());
+ Assertions.assertNull(table.currentSnapshot());
+ }
+ }
+
@Test
public void appendV1PreservesHistoricalSnapshot() throws IOException {
assertAppendPreservesHistory(1);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java
index 68d88357e51..516264a3057 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java
@@ -213,7 +213,9 @@ public final class ConnectorColumnConverter {
* This is the inverse of {@link #convertType(ConnectorType)}.
*/
public static ConnectorType toConnectorType(Type dorisType) {
- if (dorisType instanceof ConnectorComputeVariantType) {
+ if (dorisType instanceof ConnectorUnknownType) {
+ return ConnectorType.of("UNKNOWN");
+ } else if (dorisType instanceof ConnectorComputeVariantType) {
// Preserve the execution carrier on the reverse write-schema
path; reporting ordinary
// VARIANT would make an unchanged external schema look
concurrently modified.
return ConnectorType.of("VARIANT_COMPUTE_V2");
@@ -388,6 +390,8 @@ public final class ConnectorColumnConverter {
// This carrier is execution-only: connector schemas use it
for native external
// Variant encodings, while persisted Doris table metadata
keeps regular Variant rules.
return new ConnectorComputeVariantType();
+ case "UNKNOWN":
+ return new ConnectorUnknownType();
case "UNSUPPORTED":
return Type.UNSUPPORTED;
default:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorUnknownType.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorUnknownType.java
new file mode 100644
index 00000000000..0d46bebe8f8
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorUnknownType.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.connector.converter;
+
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+
+/** Read-only external unknown type: retain its schema identity while
executing as SQL NULL. */
+public final class ConnectorUnknownType extends ScalarType {
+ public ConnectorUnknownType() {
+ super(PrimitiveType.NULL_TYPE);
+ }
+
+ @Override
+ public String toSql(int depth) {
+ return "unknown";
+ }
+
+ @Override
+ public String toString() {
+ return "unknown";
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index ad05a6e0f32..1a67966d3ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -144,6 +144,7 @@ import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.datasource.connector.converter.ConnectorUnknownType;
import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog;
import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase;
import org.apache.doris.datasource.infoschema.ExternalInfoSchemaTable;
@@ -240,6 +241,7 @@ public class GsonUtils {
.of(org.apache.doris.catalog.Type.class, "clazz")
// TODO: register other sub type after Doris support more types.
.registerSubtype(ScalarType.class,
ScalarType.class.getSimpleName())
+ .registerSubtype(ConnectorUnknownType.class,
ConnectorUnknownType.class.getSimpleName())
.registerSubtype(ArrayType.class, ArrayType.class.getSimpleName())
.registerSubtype(MapType.class, MapType.class.getSimpleName())
.registerSubtype(StructType.class,
StructType.class.getSimpleName())
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverterTest.java
index 0f5c3700bf6..02c23000c7b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverterTest.java
@@ -34,10 +34,55 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Locale;
class ConnectorColumnConverterTest {
+ @Test
+ void testUnknownKeepsDisplayIdentityAndUsesNullExecutionType() {
+ Type unknown =
ConnectorColumnConverter.convertType(ConnectorType.of("UNKNOWN"));
+ Assertions.assertInstanceOf(ConnectorUnknownType.class, unknown);
+ Assertions.assertTrue(unknown.isNull());
+ Assertions.assertTrue(unknown.isSupported());
+ Assertions.assertEquals("unknown", unknown.toSql());
+ Assertions.assertEquals("unknown", unknown.toString());
+
Assertions.assertEquals(org.apache.doris.nereids.types.NullType.INSTANCE,
+
org.apache.doris.nereids.types.DataType.fromCatalogType(unknown));
+
Assertions.assertEquals(org.apache.doris.thrift.TPrimitiveType.NULL_TYPE,
+
unknown.toThrift().getTypes().get(0).getScalarType().getType());
+ Assertions.assertEquals("UNKNOWN",
ConnectorColumnConverter.toConnectorType(unknown).getTypeName());
+ String json =
org.apache.doris.persist.gson.GsonUtils.GSON.toJson(unknown, Type.class);
+ Type restored =
org.apache.doris.persist.gson.GsonUtils.GSON.fromJson(json, Type.class);
+ Assertions.assertInstanceOf(ConnectorUnknownType.class, restored);
+ Assertions.assertEquals("unknown", restored.toSql());
+ // Unknown result cells use the same client metadata contract as an
ordinary SQL NULL.
+ Assertions.assertEquals(Type.NULL.getPrimitiveType().toMysqlType(),
+ unknown.getPrimitiveType().toMysqlType());
+ }
+
+ @Test
+ void testNestedUnknownRoundtripAndConcreteRebindDoNotMutateOldColumns() {
+ ConnectorType unknown = ConnectorType.of("UNKNOWN");
+ List<ConnectorType> types = Arrays.asList(unknown,
ConnectorType.arrayOf(unknown, true),
+ ConnectorType.mapOf(ConnectorType.of("STRING"), unknown, true),
+ ConnectorType.structOf(Collections.singletonList("u"),
Collections.singletonList(unknown),
+ Collections.singletonList(true),
Collections.singletonList("null only")));
+ for (ConnectorType type : types) {
+ Type converted = ConnectorColumnConverter.convertType(type);
+ Assertions.assertEquals(type,
ConnectorColumnConverter.toConnectorType(converted));
+ Assertions.assertTrue(converted.toSql().contains("unknown"));
+ }
+ Column oldColumn = ConnectorColumnConverter.convertColumn(
+ new ConnectorColumn("u", unknown, "", true,
null).withUniqueId(7));
+ Column newColumn = ConnectorColumnConverter.convertColumn(
+ new ConnectorColumn("u", ConnectorType.of("BIGINT"), "", true,
null).withUniqueId(7));
+ Assertions.assertEquals(oldColumn.getUniqueId(),
newColumn.getUniqueId());
+ Assertions.assertEquals(Type.BIGINT, newColumn.getType());
+ Assertions.assertEquals("unknown", oldColumn.getType().toSql());
+ }
+
@Test
void connectorWriteDefaultSqlSurvivesColumnRoundtrip() {
ConnectorColumn connectorColumn = new ConnectorColumn(
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_unknown_read.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_unknown_read.out
new file mode 100644
index 00000000000..963d467459b
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_unknown_read.out
@@ -0,0 +1,219 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !parquet_read_unknown_rows --
+1 \N \N \N \N \N \N
+2 \N {"known":20, "u":null} [{"known":20, "u":null}, null]
{"key":{"known":20, "u":null}} \N \N
+3 \N \N [] {} \N \N
+
+-- !parquet_read_unknown_only --
+\N
+\N
+\N
+
+-- !parquet_read_unknown_count --
+3 0
+
+-- !parquet_read_unknown_is_null --
+1
+2
+3
+
+-- !parquet_read_unknown_null_safe --
+1
+2
+3
+
+-- !parquet_read_unknown_is_not_null --
+0
+
+-- !parquet_read_unknown_nested --
+1 \N \N \N true true true \N \N \N
\N \N \N
+2 \N \N \N false false false 2 1 \N
\N \N \N
+3 \N \N \N true false false 0 0 \N
\N \N \N
+
+-- !parquet_read_unknown_desc --
+a array<struct<known:int,u:unknown>> Yes true \N
+id int No true \N
+m map<text,struct<known:int,u:unknown>> Yes true \N
+null_array array<unknown> Yes true \N
+null_map map<text,unknown> Yes true \N
+s struct<known:int,u:unknown> Yes true \N
+u unknown Yes true \N
+
+-- !parquet_read_unknown_show_create --
+ `u` unknown NULL, `s` struct<known:int,u:unknown> NULL, `a`
array<struct<known:int,u:unknown>> NULL, `m`
map<text,struct<known:int,u:unknown>> NULL, `null_array` array<unknown> NULL,
`null_map` map<text,unknown> NULL
+
+-- !parquet_physical_unknown_rows --
+1 \N \N \N \N \N \N
+2 \N {"known":20, "u":null} [{"known":20, "u":null}, null]
{"key":{"known":20, "u":null}} [null, null] {"key":null}
+3 \N \N [] {} [] {}
+
+-- !parquet_physical_unknown_only --
+\N
+\N
+\N
+
+-- !parquet_physical_unknown_count --
+3 0
+
+-- !parquet_physical_unknown_is_null --
+1
+2
+3
+
+-- !parquet_physical_unknown_null_safe --
+1
+2
+3
+
+-- !parquet_physical_unknown_is_not_null --
+0
+
+-- !parquet_physical_unknown_nested --
+1 \N \N \N true true true \N \N \N
\N \N \N
+2 \N \N \N false false false 2 1 2
1 \N \N
+3 \N \N \N true false false 0 0 0
0 \N \N
+
+-- !parquet_physical_unknown_desc --
+a array<struct<known:int,u:unknown>> Yes true \N
+id int No true \N
+m map<text,struct<known:int,u:unknown>> Yes true \N
+null_array array<unknown> Yes true \N
+null_map map<text,unknown> Yes true \N
+s struct<known:int,u:unknown> Yes true \N
+u unknown Yes true \N
+
+-- !parquet_physical_unknown_show_create --
+ `u` unknown NULL, `s` struct<known:int,u:unknown> NULL, `a`
array<struct<known:int,u:unknown>> NULL, `m`
map<text,struct<known:int,u:unknown>> NULL, `null_array` array<unknown> NULL,
`null_map` map<text,unknown> NULL
+
+-- !parquet_unknown_empty --
+0 0
+
+-- !orc_read_unknown_rows --
+1 \N \N \N \N \N \N
+2 \N {"known":20, "u":null} [{"known":20, "u":null}, null]
{"key":{"known":20, "u":null}} \N \N
+3 \N \N [] {} \N \N
+
+-- !orc_read_unknown_only --
+\N
+\N
+\N
+
+-- !orc_read_unknown_count --
+3 0
+
+-- !orc_read_unknown_is_null --
+1
+2
+3
+
+-- !orc_read_unknown_null_safe --
+1
+2
+3
+
+-- !orc_read_unknown_is_not_null --
+0
+
+-- !orc_read_unknown_nested --
+1 \N \N \N true true true \N \N \N
\N \N \N
+2 \N \N \N false false false 2 1 \N
\N \N \N
+3 \N \N \N true false false 0 0 \N
\N \N \N
+
+-- !orc_read_unknown_desc --
+a array<struct<known:int,u:unknown>> Yes true \N
+id int No true \N
+m map<text,struct<known:int,u:unknown>> Yes true \N
+null_array array<unknown> Yes true \N
+null_map map<text,unknown> Yes true \N
+s struct<known:int,u:unknown> Yes true \N
+u unknown Yes true \N
+
+-- !orc_read_unknown_show_create --
+ `u` unknown NULL, `s` struct<known:int,u:unknown> NULL, `a`
array<struct<known:int,u:unknown>> NULL, `m`
map<text,struct<known:int,u:unknown>> NULL, `null_array` array<unknown> NULL,
`null_map` map<text,unknown> NULL
+
+-- !orc_physical_unknown_rows --
+1 \N \N \N \N \N \N
+2 \N {"known":20, "u":null} [{"known":20, "u":null}, null]
{"key":{"known":20, "u":null}} [null, null] {"key":null}
+3 \N \N [] {} [] {}
+
+-- !orc_physical_unknown_only --
+\N
+\N
+\N
+
+-- !orc_physical_unknown_count --
+3 0
+
+-- !orc_physical_unknown_is_null --
+1
+2
+3
+
+-- !orc_physical_unknown_null_safe --
+1
+2
+3
+
+-- !orc_physical_unknown_is_not_null --
+0
+
+-- !orc_physical_unknown_nested --
+1 \N \N \N true true true \N \N \N
\N \N \N
+2 \N \N \N false false false 2 1 2
1 \N \N
+3 \N \N \N true false false 0 0 0
0 \N \N
+
+-- !orc_physical_unknown_desc --
+a array<struct<known:int,u:unknown>> Yes true \N
+id int No true \N
+m map<text,struct<known:int,u:unknown>> Yes true \N
+null_array array<unknown> Yes true \N
+null_map map<text,unknown> Yes true \N
+s struct<known:int,u:unknown> Yes true \N
+u unknown Yes true \N
+
+-- !orc_physical_unknown_show_create --
+ `u` unknown NULL, `s` struct<known:int,u:unknown> NULL, `a`
array<struct<known:int,u:unknown>> NULL, `m`
map<text,struct<known:int,u:unknown>> NULL, `null_array` array<unknown> NULL,
`null_map` map<text,unknown> NULL
+
+-- !orc_unknown_empty --
+0 0
+
+-- !parquet_unknown_promoted --
+1 \N
+2 \N
+3 \N
+4 now known
+
+-- !parquet_unknown_promoted_desc --
+a array<struct<known:int,u:text>> Yes true \N
+id int No true \N
+m map<text,struct<known:int,u:text>> Yes true \N
+null_array array<text> Yes true \N
+null_map map<text,text> Yes true \N
+s struct<known:int,u:text> Yes true \N
+u text Yes true \N
+
+-- !parquet_unknown_historical --
+1 \N
+2 \N
+3 \N
+
+-- !orc_unknown_promoted --
+1 \N
+2 \N
+3 \N
+4 now known
+
+-- !orc_unknown_promoted_desc --
+a array<struct<known:int,u:text>> Yes true \N
+id int No true \N
+m map<text,struct<known:int,u:text>> Yes true \N
+null_array array<text> Yes true \N
+null_map map<text,text> Yes true \N
+s struct<known:int,u:text> Yes true \N
+u text Yes true \N
+
+-- !orc_unknown_historical --
+1 \N
+2 \N
+3 \N
+
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_unknown_read.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_unknown_read.groovy
new file mode 100644
index 00000000000..578fb80bf4d
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_unknown_read.groovy
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_unknown_read", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+ String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ sql "drop catalog if exists test_iceberg_unknown_read"
+ sql """
+ CREATE CATALOG test_iceberg_unknown_read PROPERTIES (
+ 'type' = 'iceberg',
+ 'iceberg.catalog.type' = 'rest',
+ 'uri' = 'http://${externalEnvIp}:${restPort}',
+ 's3.access_key' = 'admin',
+ 's3.secret_key' = 'password',
+ 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}',
+ 's3.region' = 'us-east-1',
+ 's3.path.style.access' = 'true',
+ 's3.connection.ssl.enabled' = 'false'
+ )
+ """
+ def executeCommandWithStatus = { String cmd, int timeoutSeconds = 300,
+ Boolean logFailure = true, Boolean logCommand = true ->
+ StringBuilder stdout = new StringBuilder()
+ StringBuilder stderr = new StringBuilder()
+ try {
+ if (logCommand) {
+ logger.info("execute ${cmd}")
+ }
+ def proc = new ProcessBuilder("/bin/bash", "-c", cmd).start()
+ proc.consumeProcessOutput(stdout, stderr)
+ proc.waitForOrKill(timeoutSeconds * 1000)
+ int exitCode = proc.exitValue()
+ String output = stdout.toString()
+ String error = stderr.toString()
+ if (exitCode != 0 && logFailure) {
+ logger.info("exit code: ${exitCode}, stdout\n:
${output}\nstderr\n: ${error}")
+ }
+ return [exitCode: exitCode, stdout: output, stderr: error]
+ } catch (IOException e) {
+ assertTrue(false, "Execute failed: ${cmd}, err: ${e.message}")
+ }
+ }
+
+ def executeCommand = { String cmd, Boolean mustSucceed, int timeoutSeconds
= 300 ->
+ def result = executeCommandWithStatus(cmd, timeoutSeconds)
+ if (mustSucceed && result.exitCode != 0) {
+ assertTrue(false,
+ "Execute failed:
${cmd}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}")
+ }
+ return result.stdout
+ }
+
+ String dockerCommand =
context.config.otherConfigs.get("externalDockerCommand") ?: "docker"
+ def listDockerContainers = {
+ String containers = executeCommand(
+ "${dockerCommand} ps --format
'{{.ID}}\t{{.Names}}\t{{.Image}}'", true, 30) ?: ""
+ return containers.readLines().collect { it.trim() }.findAll {
!it.isEmpty() }
+ }
+
+ def findSparkContainer = {
+ String configuredContainer =
context.config.otherConfigs.get("icebergSparkContainer")
+ String probeCommand = "command -v spark-sql >/dev/null && test -f
/mnt/SUCCESS && " +
+ "test -f /mnt/scripts/java/CreateIcebergUnknownFixtures.java"
+ if (configuredContainer != null && !configuredContainer.isEmpty()) {
+ def probe = executeCommandWithStatus(
+ "${dockerCommand} exec ${configuredContainer} bash -lc
'${probeCommand}'", 30)
+ assertEquals(0, probe.exitCode,
+ "Configured Spark Iceberg container ${configuredContainer}
is not usable")
+ return configuredContainer
+ }
+
+ def matchedContainers = []
+ listDockerContainers().each { String containerLine ->
+ def fields = containerLine.split(/\t/, 3)
+ assertTrue(fields.length >= 2, "Unexpected docker ps output:
${containerLine}")
+ String containerId = fields[0].trim()
+ String containerName = fields[1].trim()
+ String containerImage = fields.length >= 3 ? fields[2].trim() : ""
+ def probe = executeCommandWithStatus(
+ "${dockerCommand} exec ${containerId} bash -lc
'${probeCommand}'",
+ 30,
+ false,
+ false)
+ if (probe.exitCode == 0) {
+ matchedContainers.add(
+ [id: containerId, name: containerName, image:
containerImage])
+ }
+ }
+
+ assertFalse(matchedContainers.isEmpty(),
+ "No usable Spark Iceberg container found. Set
icebergSparkContainer or start it.")
+ assertEquals(1, matchedContainers.size(),
+ "Multiple usable Spark Iceberg containers found:
${matchedContainers}. " +
+ "Set icebergSparkContainer to the exact container
name.")
+ logger.info("use Spark Iceberg container ${matchedContainers[0].name}
" +
+ "(${matchedContainers[0].image})")
+ return matchedContainers[0].id
+ }
+
+ String sparkContainer = findSparkContainer()
+ def runInSparkContainer = { String command, int timeoutSeconds = 300 ->
+ executeCommand(
+ "${dockerCommand} exec ${sparkContainer} bash -lc
'${command}'",
+ true,
+ timeoutSeconds)
+ }
+
+ // Generate fixtures at test runtime. The source is mounted by the
existing Iceberg Docker setup.
+ runInSparkContainer(
+ "javac -cp \"/opt/spark/jars/*\"
/mnt/scripts/java/CreateIcebergUnknownFixtures.java && " +
+ "java -cp \"/mnt/scripts/java:/opt/spark/jars/*\"
CreateIcebergUnknownFixtures create format_v3")
+ sql "switch test_iceberg_unknown_read"
+ sql "use format_v3"
+ // Exercise more than one scan batch, including a projection with no
physical output column.
+ sql "set batch_size = 2"
+
+ ["parquet", "orc"].each { String format ->
+ ["read", "physical"].each { String kind ->
+ String table = "unknown_${kind}_${format}"
+ "order_qt_${format}_${kind}_unknown_rows" "select * from ${table}
order by id"
+ "order_qt_${format}_${kind}_unknown_only" "select u from ${table}
order by id"
+ "qt_${format}_${kind}_unknown_count" "select count(*), count(u)
from ${table}"
+ "order_qt_${format}_${kind}_unknown_is_null" "select id from
${table} where u is null order by id"
+ "order_qt_${format}_${kind}_unknown_null_safe" "select id from
${table} where u <=> null order by id"
+ "qt_${format}_${kind}_unknown_is_not_null" "select count(*) from
${table} where u is not null"
+ "order_qt_${format}_${kind}_unknown_nested" """
+ select id, struct_element(s, 'u'),
+ struct_element(element_at(a, 1), 'u'),
+ struct_element(element_at(m, 'key'), 'u'),
+ s is null, a is null, m is null, array_size(a),
map_size(m),
+ array_size(null_array), map_size(null_map),
element_at(null_array, 1),
+ element_at(null_map, 'key')
+ from ${table} order by id
+ """
+ "order_qt_${format}_${kind}_unknown_desc" "desc ${table}"
+ // SHOW CREATE contains a remote fixture location; record only the
relevant type clauses.
+ def create = sql "show create table ${table}"
+ def typeLines = create[0][1].readLines().findAll {
it.trim().startsWith('`') && it.contains('unknown') }
+ "qt_${format}_${kind}_unknown_show_create" "select
'${typeLines.join(' ').replace("'", "''")}'"
+ [
+ "insert into ${table}(id) values (9)",
+ "insert overwrite table ${table} select * from ${table}",
+ "update ${table} set id = 9 where id = 1",
+ "delete from ${table} where id = 1",
+ "delete from ${table}",
+ "merge into ${table} t using (select 1 id) s on t.id = s.id
when matched then delete",
+ "alter table ${table} execute rewrite_data_files()"
+ ].each { String statement ->
+ test {
+ sql statement
+ exception "unknown type columns"
+ }
+ }
+ }
+ "qt_${format}_unknown_empty" "select count(*), count(u) from
unknown_empty_${format}"
+ }
+ sql "drop table if exists test_iceberg_unknown_create"
+ test {
+ sql "create table test_iceberg_unknown_create (u unknown)"
+ exception "unknown"
+ }
+
+ Map<String, Long> oldSnapshots = [:]
+ ["parquet", "orc"].each { String format ->
+ oldSnapshots[format] = (sql "select snapshot_id from
unknown_read_${format}\$snapshots")[0][0] as Long
+ }
+ runInSparkContainer(
+ "java -cp \"/mnt/scripts/java:/opt/spark/jars/*\"
CreateIcebergUnknownFixtures evolve format_v3")
+ ["parquet", "orc"].each { String format ->
+ sql "refresh table unknown_read_${format}"
+ "order_qt_${format}_unknown_promoted" "select id, u from
unknown_read_${format} order by id"
+ "order_qt_${format}_unknown_promoted_desc" "desc
unknown_read_${format}"
+ "order_qt_${format}_unknown_historical" """
+ select id, u from unknown_read_${format}
+ for version as of ${oldSnapshots[format]} order by id
+ """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]