This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 88549d0033627aeb4ac25f2dfe61b28e6ab1c569 Author: Gabriel <[email protected]> AuthorDate: Thu Oct 8 17:25:24 2026 +0800 [fix](catalog) Validate external table partition models on branch-4.1 (#68769) ### What problem does this PR solve? On branch-4.1, external CREATE TABLE can silently discard unsupported partition expressions or explicit partition definitions. For example, a Paimon table declared with `date_trunc(ts, 'day')` may be created without partition keys. Validate partition models through a shared external-table entry point before descriptor conversion and remote creation. Hive, Paimon, and MaxCompute accept column partitions; Iceberg retains its supported transforms and schema-specific validation. Reject explicit Doris partition definitions for these formats, preserve Elasticsearch's existing partition checks, and reject partition clauses for engines that do not support them. Also remove identical duplicate Nimbus and Hadoop COS dependency declarations inherited from the base branch. Maven 3.10 rejects these declarations before Checkstyle can run; dependency versions and exclusions are preserved. ### Release note External CREATE TABLE reports errors for unsupported partition models instead of silently ignoring them. ### Check List (For Author) - Test - [x] Regression test: a standalone P0 suite covers invalid models for all external engines without external services. Catalog suites cover Paimon, Hive, Iceberg, and MaxCompute, including inferred engines and supported partition behavior. - [x] Unit Test: shared analysis checks, Paimon rejection before remote creation, and supported partition syntax. - Behavior changed: - [x] Yes. Unsupported external partition models fail before table creation. - Does this need documentation? - [x] No. ### Validation - New rejection tests failed before the corresponding validation changes. - All 38 selected FE tests pass: 24 Paimon metadata tests, nine Iceberg table-creation tests, and five external-table partition analysis tests. - The full `mvn clean checkstyle:check` passes with Maven 3.10.0 (the CI version), with zero violations across all 28 modules. - All five new or updated Groovy suites compile with Groovy 4.0.19. SQL regression suites were not executed locally; catalog integration tests require their configured external services. --- fe/be-java-extensions/preload-extensions/pom.xml | 13 +-- .../trees/plans/commands/info/CreateTableInfo.java | 9 +- .../plans/commands/info/PartitionTableInfo.java | 49 ++++++++ .../datasource/paimon/PaimonMetadataOpsTest.java | 59 ++++++++++ .../trees/plans/CreateTableCommandTest.java | 48 ++++++++ fe/pom.xml | 6 +- .../test_external_partition_validation.groovy | 58 ++++++++++ .../hive/ddl/test_hive_ddl.groovy | 20 ++++ .../test_iceberg_write_transform_partitions.groovy | 31 +++++ .../test_paimon_create_partition_validation.groovy | 128 +++++++++++++++++++++ .../write/test_mc_write_partitions.groovy | 16 +++ 11 files changed, 414 insertions(+), 23 deletions(-) diff --git a/fe/be-java-extensions/preload-extensions/pom.xml b/fe/be-java-extensions/preload-extensions/pom.xml index 8113589e99c..772a98dd653 100644 --- a/fe/be-java-extensions/preload-extensions/pom.xml +++ b/fe/be-java-extensions/preload-extensions/pom.xml @@ -124,6 +124,7 @@ under the License. <artifactId>HikariCP</artifactId> </dependency> <!-- For BE CosN Access --> + <!-- Keep a single declaration: Maven 3.10+ rejects duplicate dependency coordinates. --> <dependency> <groupId>com.qcloud.cos</groupId> <artifactId>hadoop-cos</artifactId> @@ -139,18 +140,6 @@ under the License. <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> - <!-- For BE CosN Access --> - <dependency> - <groupId>com.qcloud.cos</groupId> - <artifactId>hadoop-cos</artifactId> - <version>${tencentcos.version}</version> - <exclusions> - <exclusion> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - </exclusion> - </exclusions> - </dependency> <!-- these dependencies are for iceberg-aws --> <dependency> diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index c34f2c6d2d3..f702023d153 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -1131,6 +1131,9 @@ public class CreateTableInfo { * analyzeEngine */ public void analyzeEngine() { + if (isExternal) { + partitionTableInfo.validateExternalPartition(engineName); + } this.partitionDesc = partitionTableInfo.convertToPartitionDesc(isExternal); this.distributionDesc = distribution != null ? distribution.translateToCatalogStyle() : null; @@ -1147,12 +1150,6 @@ public class CreateTableInfo { throw new AnalysisException("Create " + engineName + " table should not contain distribution desc"); } - if (!engineName.equals(ENGINE_HIVE) && !engineName.equals(ENGINE_ICEBERG) - && !engineName.equals(ENGINE_PAIMON) && !engineName.equals(ENGINE_MAXCOMPUTE) - && partitionDesc != null) { - throw new AnalysisException("Create " + engineName - + " table should not contain partition desc"); - } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionTableInfo.java index 8c54b56018e..ec829dcebf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionTableInfo.java @@ -103,6 +103,55 @@ public class PartitionTableInfo { return partitionType; } + /** + * Validate external partition models before conversion can discard unsupported information. + * Schema-specific checks, including Iceberg transform binding, remain with the metadata builders. + */ + public void validateExternalPartition(String engineName) { + if (partitionType.equalsIgnoreCase(PartitionType.UNPARTITIONED.name())) { + return; + } + + String formatName; + boolean supportsTransforms = false; + switch (engineName) { + case CreateTableInfo.ENGINE_HIVE: + if (!partitionType.equalsIgnoreCase(PartitionType.LIST.name())) { + throw new AnalysisException("Only support 'LIST' partition type in hive catalog."); + } + formatName = "Hive"; + break; + case CreateTableInfo.ENGINE_PAIMON: + formatName = "Paimon"; + break; + case CreateTableInfo.ENGINE_MAXCOMPUTE: + formatName = "MaxCompute"; + break; + case CreateTableInfo.ENGINE_ICEBERG: + formatName = "Iceberg"; + supportsTransforms = true; + break; + case CreateTableInfo.ENGINE_ELASTICSEARCH: + // Elasticsearch mapping partitions are validated by EsUtil after descriptor conversion. + return; + default: + throw new AnalysisException("Create " + engineName + " table should not contain partition desc"); + } + + // Column-only formats must not silently lose function expressions from their partition keys. + if (!supportsTransforms && partitionList != null + && partitionList.stream().anyMatch(expr -> !(expr instanceof UnboundSlot))) { + throw new AnalysisException(formatName + " only supports partitioning by columns; " + + "partition expressions are not supported"); + } + if (partitionDefs != null && !partitionDefs.isEmpty()) { + if (engineName.equals(CreateTableInfo.ENGINE_HIVE)) { + throw new AnalysisException("Partition values expressions is not supported in hive catalog."); + } + throw new AnalysisException(formatName + " does not support explicit partition definitions"); + } + } + /** * check partitions types. */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java index ac8b33281c7..5ce12fe698e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java @@ -26,6 +26,7 @@ import org.apache.doris.datasource.CatalogFactory; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; @@ -338,6 +339,64 @@ public class PaimonMetadataOpsTest { Assert.assertEquals(1, table.primaryKeys().size()); } + @Test + public void testRejectPartitionExpressionsBeforeCreatingTable() throws Exception { + for (String partitionClause : Arrays.asList( + "auto partition by range(date_trunc(ts, 'day')) ()", + "auto partition by list(date_trunc(ts, 'day')) ()", + "partition by (date_trunc(ts, 'day')) ()", + "partition by (dt, date_trunc(ts, 'day')) ()")) { + assertPartitionRejected(partitionClause, "Paimon only supports partitioning by columns"); + } + } + + @Test + public void testRejectPartitionDefinitionsBeforeCreatingTable() throws Exception { + for (String partitionClause : Arrays.asList( + "partition by range(dt) (partition p1 values less than ('2026-01-02'))", + "partition by range(dt) (partition p1 values [('2026-01-01'), ('2026-01-02')))", + "partition by list(dt) (partition p1 values in ('2026-01-01'))", + "partition by range(dt) (from ('2026-01-01') to ('2026-01-03') interval 1 day)", + "auto partition by range(dt) (partition p1 values less than ('2026-01-02'))")) { + assertPartitionRejected(partitionClause, "Paimon does not support explicit partition definitions"); + } + } + + private void assertPartitionRejected(String partitionClause, String message) throws Exception { + String tableName = getTableName(); + String sql = "create table " + dbName + "." + tableName + + " (id int, ts datetime not null, dt date not null) engine = paimon " + partitionClause; + AnalysisException exception = Assert.assertThrows(AnalysisException.class, () -> createTable(sql)); + Assert.assertTrue(exception.getMessage(), exception.getMessage().contains(message)); + Assert.assertThrows(Catalog.TableNotExistException.class, + () -> ops.getCatalog().getTable(new Identifier(dbName, tableName))); + } + + @Test + public void testIdentityPartitionSyntaxes() throws Exception { + for (String partitionClause : Arrays.asList( + "partition by (dt) ()", + "partition by range(dt) ()", + "partition by list(dt) ()", + "auto partition by range(dt) ()", + "auto partition by list(dt) ()")) { + String tableName = getTableName(); + createTable("create table " + dbName + "." + tableName + + " (id int, dt date not null) engine = paimon " + partitionClause); + Assert.assertEquals(Arrays.asList("dt"), + ops.getCatalog().getTable(new Identifier(dbName, tableName)).partitionKeys()); + } + } + + @Test + public void testMultipleIdentityPartitionColumns() throws Exception { + String tableName = getTableName(); + createTable("create table " + dbName + "." + tableName + + " (id int, region string, dt date not null) engine = paimon partition by (region, dt) ()"); + Assert.assertEquals(Arrays.asList("region", "dt"), + ops.getCatalog().getTable(new Identifier(dbName, tableName)).partitionKeys()); + } + @Test public void testPartitionPreservesNonLowercaseColumnNames() throws Exception { String tableName = getTableName(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java index e423c30b507..b630f15f1b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java @@ -838,6 +838,54 @@ public class CreateTableCommandTest extends TestWithFeService { return createTableInfo.getPartitionDesc(); } + @Test + public void testExternalIdentityPartitionValidation() { + for (String engine : new String[] {"hive", "paimon", "maxcompute"}) { + String prefix = "create table partition_validation (id int, ts datetime, dt date) engine=" + engine; + PartitionDesc partition = getCreateTableStmt(prefix + " partition by (dt) ()"); + Assertions.assertEquals(java.util.Collections.singletonList("dt"), partition.getPartitionColNames()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> getCreateTableStmt(prefix + " partition by (date_trunc(ts, 'day')) ()")); + Assertions.assertTrue(exception.getMessage().contains("only supports partitioning by columns")); + } + } + + @Test + public void testExternalPartitionDefinitionsRejected() { + for (String engine : new String[] {"hive", "paimon", "maxcompute", "iceberg"}) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> getCreateTableStmt("create table partition_validation (id int, dt date) engine=" + engine + + " partition by list(dt) (partition p1 values in ('2026-01-01'))")); + String expected = engine.equals("hive") ? "Partition values expressions is not supported in hive catalog" + : "does not support explicit partition definitions"; + Assertions.assertTrue(exception.getMessage().contains(expected), exception.getMessage()); + } + } + + @Test + public void testExternalPartitionModelRestrictions() { + AnalysisException hiveException = Assertions.assertThrows(AnalysisException.class, + () -> getCreateTableStmt("create table partition_validation (id int, dt date) engine=hive" + + " partition by range(dt) ()")); + Assertions.assertTrue(hiveException.getMessage().contains("Only support 'LIST' partition type in hive catalog")); + + PartitionDesc iceberg = getCreateTableStmt("create table partition_validation (id int, ts datetime)" + + " engine=iceberg partition by (bucket(4, id), day(ts)) ()"); + Assertions.assertEquals(2, iceberg.getPartitionExprs().size()); + + PartitionDesc elasticsearch = getCreateTableStmt("create table partition_validation (id int, dt date)" + + " engine=elasticsearch partition by range(dt) ()"); + Assertions.assertEquals(java.util.Collections.singletonList("dt"), elasticsearch.getPartitionColNames()); + Assertions.assertThrows(AnalysisException.class, + () -> getCreateTableStmt("create table partition_validation (id int, dt date) engine=elasticsearch" + + " partition by list(dt) ()")); + for (String engine : new String[] {"jdbc", "odbc", "mysql", "broker"}) { + Assertions.assertThrows(AnalysisException.class, + () -> getCreateTableStmt("create table partition_validation (id int, dt date) engine=" + engine + + " partition by list(dt) ()")); + } + } + @Test public void testPartitionCheckForHive() { try { diff --git a/fe/pom.xml b/fe/pom.xml index 002e3e60552..5f1223aa793 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -627,6 +627,7 @@ under the License. <artifactId>quartz</artifactId> <version>${quartz.version}</version> </dependency> + <!-- Maven 3.10+ rejects duplicate coordinates in dependencyManagement. --> <dependency> <groupId>com.nimbusds</groupId> <artifactId>nimbus-jose-jwt</artifactId> @@ -1928,11 +1929,6 @@ under the License. </exclusion> </exclusions> </dependency> - <dependency> - <groupId>com.nimbusds</groupId> - <artifactId>nimbus-jose-jwt</artifactId> - <version>${nimbusds.version}</version> - </dependency> <dependency> <groupId>jakarta.annotation</groupId> <artifactId>jakarta.annotation-api</artifactId> diff --git a/regression-test/suites/ddl_p0/test_external_partition_validation.groovy b/regression-test/suites/ddl_p0/test_external_partition_validation.groovy new file mode 100644 index 00000000000..d89e2248498 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_external_partition_validation.groovy @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_external_partition_validation", "p0") { + // Invalid external partition models must fail in analysis without contacting a remote catalog. + def rejectPartition = { String engine, String partition, String message -> + String tableName = "test_external_partition_validation_${engine}" + sql """DROP TABLE IF EXISTS `${tableName}`""" + try { + test { + sql """ + CREATE TABLE `${tableName}` (id INT, ts DATETIME, dt DATE) + ENGINE=${engine} ${partition} + """ + exception message + } + assertEquals([], sql("""SHOW TABLES LIKE '${tableName}'""")) + } finally { + sql """DROP TABLE IF EXISTS `${tableName}`""" + } + } + + ["hive", "paimon", "maxcompute"].each { engine -> + rejectPartition(engine, "PARTITION BY (date_trunc(ts, 'day')) ()", + "only supports partitioning by columns") + } + ["hive", "paimon", "maxcompute", "iceberg"].each { engine -> + String message = engine == "hive" ? "Partition values expressions is not supported in hive catalog" + : "does not support explicit partition definitions" + rejectPartition(engine, "PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('2026-01-01'))", message) + } + ["paimon", "maxcompute", "iceberg"].each { engine -> + rejectPartition(engine, "PARTITION BY RANGE(dt) (PARTITION p1 VALUES LESS THAN ('2026-01-02'))", + "does not support explicit partition definitions") + } + rejectPartition("hive", "PARTITION BY RANGE(dt) ()", "Only support 'LIST' partition type in hive catalog") + rejectPartition("elasticsearch", "PARTITION BY LIST(dt) ()", "Elasticsearch table only permit range partition") + rejectPartition("elasticsearch", "PARTITION BY RANGE(id, dt) ()", + "Elasticsearch table's partition column could only be a single column") + rejectPartition("jdbc", "PARTITION BY LIST(dt) ()", "Create jdbc table should not contain partition desc") + ["mysql", "odbc", "broker"].each { engine -> + rejectPartition(engine, "PARTITION BY LIST(dt) ()", "odbc, mysql and broker table is no longer supported") + } +} diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy index a3bf1410fbf..34efc5a86fe 100644 --- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy +++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy @@ -488,6 +488,26 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive") sql """ create database if not exists `test_hive_db_tbl` """; sql """use `${catalog_name}`.`test_hive_db_tbl`""" + // Inferred Hive engines must reject unsupported models before creating HMS metadata. + String rejectedTable = "partition_model_rejected_${file_format}" + sql """DROP TABLE IF EXISTS `${rejectedTable}`""" + try { + [ + ["PARTITION BY (date_trunc(ts, 'day')) ()", "Hive only supports partitioning by columns"], + ["PARTITION BY RANGE(dt) ()", "Only support 'LIST' partition type in hive catalog"], + ["PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('2026-01-01'))", + "Partition values expressions is not supported in hive catalog"] + ].each { entry -> + test { + sql """CREATE TABLE `${rejectedTable}` (id INT, ts DATETIME, dt DATE) ${entry[0]}""" + exception entry[1] + } + assertEquals([], sql("""SHOW TABLES LIKE '${rejectedTable}'""")) + } + } finally { + sql """DROP TABLE IF EXISTS `${rejectedTable}`""" + } + sql """ drop table if exists unpart_tbl_${file_format}""" sql """ CREATE TABLE unpart_tbl_${file_format}( diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy index 2d8e265222e..dc8eb69a81d 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy @@ -50,6 +50,37 @@ suite("test_iceberg_write_transform_partitions", "p0,external,iceberg,external_d } try { + String validationTable = "create_partition_validation" + sql """DROP TABLE IF EXISTS `${validationTable}`""" + try { + // Iceberg supports transforms, but explicit Doris partition bounds have no Iceberg equivalent. + [ + ["PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('2026-01-01'))", + "Iceberg does not support explicit partition definitions"], + ["PARTITION BY RANGE(dt) (PARTITION p1 VALUES LESS THAN ('2026-01-02'))", + "Iceberg does not support explicit partition definitions"], + ["PARTITION BY (date_trunc(ts, 'day')) ()", "unsupported partition for date_trunc"] + ].each { entry -> + test { + sql """CREATE TABLE `${validationTable}` (id INT, ts DATETIME, dt DATE) ${entry[0]}""" + exception entry[1] + } + assertEquals([], sql("""SHOW TABLES LIKE '${validationTable}'""")) + } + + sql """ + CREATE TABLE `${validationTable}` (id INT, ts DATETIME) + PARTITION BY (bucket(4, id), day(ts)) () + """ + String ddl = sql("""SHOW CREATE TABLE `${validationTable}`""")[0][1].toString().toLowerCase() + assertTrue(ddl.contains("bucket")) + assertTrue(ddl.contains("day")) + sql """INSERT INTO `${validationTable}` VALUES (1, '2026-01-01 12:00:00')""" + assertEquals(1, sql("""SELECT COUNT(*) FROM `${validationTable}`""")[0][0].intValue()) + } finally { + sql """DROP TABLE IF EXISTS `${validationTable}`""" + } + sql """ set time_zone = 'Asia/Shanghai'; """ test_write_transform_partitions("bucket_int_4"); test_write_transform_partitions("bucket_bigint_4"); diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_create_partition_validation.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_create_partition_validation.groovy new file mode 100644 index 00000000000..15682a782db --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_create_partition_validation.groovy @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_create_partition_validation", + "p0,external,paimon,external_docker,external_docker_paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_paimon_create_partition_validation" + String dbName = "paimon_create_partition_validation_db" + String expressionError = "Paimon only supports partitioning by columns" + String definitionError = "Paimon does not support explicit partition definitions" + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + + try { + sql """SWITCH `${catalogName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """CREATE DATABASE `${dbName}`""" + sql """USE `${dbName}`""" + + // Both expressions and explicit bounds used to disappear from the remote schema. + def unsupportedPartitions = [ + ["expr_range", "AUTO PARTITION BY RANGE(date_trunc(ts, 'day')) ()", expressionError], + ["expr_list", "AUTO PARTITION BY LIST(date_trunc(ts, 'day')) ()", expressionError], + ["expr_plain", "PARTITION BY (date_trunc(ts, 'day')) ()", expressionError], + ["expr_mixed", "PARTITION BY (dt, date_trunc(ts, 'day')) ()", expressionError], + ["expr_bucket", "PARTITION BY (bucket(4, id)) ()", expressionError], + ["range_less", "PARTITION BY RANGE(dt) (PARTITION p1 VALUES LESS THAN ('2026-01-02'))", + definitionError], + ["range_fixed", "PARTITION BY RANGE(dt) (PARTITION p1 VALUES [('2026-01-01'), ('2026-01-02')))", + definitionError], + ["list_values", "PARTITION BY LIST(dt) (PARTITION p1 VALUES IN ('2026-01-01'))", definitionError], + ["range_step", "PARTITION BY RANGE(dt) (FROM ('2026-01-01') TO ('2026-01-03') INTERVAL 1 DAY)", + definitionError], + ["auto_bounds", "AUTO PARTITION BY RANGE(dt) (PARTITION p1 VALUES LESS THAN ('2026-01-02'))", + definitionError] + ] + unsupportedPartitions.each { entry -> + test { + sql """ + CREATE TABLE `${entry[0]}` (id INT, ts DATETIME NOT NULL, dt DATE NOT NULL) + ENGINE=paimon ${entry[1]} + """ + exception entry[2] + } + assertEquals([], sql("""SHOW TABLES LIKE '${entry[0]}'""")) + } + + // Catalog-inferred engines and CTAS must use the same validation before publishing metadata. + test { + sql """ + CREATE TABLE inferred_engine (id INT, ts DATETIME NOT NULL) + AUTO PARTITION BY RANGE(date_trunc(ts, 'day')) () + """ + exception expressionError + } + test { + sql """ + CREATE TABLE ctas_expression ENGINE=paimon + AUTO PARTITION BY RANGE(date_trunc(ts, 'day')) () + AS SELECT 1 AS id, CAST('2026-01-01 12:00:00' AS DATETIME) AS ts + """ + exception expressionError + } + assertEquals([], sql("SHOW TABLES")) + assertEquals([], spark_paimon("""SHOW TABLES IN paimon.${dbName}""")) + + def supportedPartitions = [ + ["unpartitioned", "", []], + ["identity_plain", "PARTITION BY (dt) ()", ["dt"]], + ["identity_range", "PARTITION BY RANGE(dt) ()", ["dt"]], + ["identity_list", "PARTITION BY LIST(dt) ()", ["dt"]], + ["identity_auto_range", "AUTO PARTITION BY RANGE(dt) ()", ["dt"]], + ["identity_auto_list", "AUTO PARTITION BY LIST(dt) ()", ["dt"]], + ["identity_multiple", "PARTITION BY (region, dt) ()", ["region", "dt"]] + ] + supportedPartitions.each { entry -> + sql """ + CREATE TABLE `${entry[0]}` (id INT, region STRING, dt DATE NOT NULL) + ENGINE=paimon ${entry[1]} + """ + def schemas = sql """ + SELECT partition_keys FROM `${entry[0]}\$schemas` + ORDER BY schema_id DESC LIMIT 1 + """ + assertEquals(1, schemas.size()) + assertEquals(entry[2], parseJson(schemas[0][0].toString())) + } + } finally { + try { + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + } finally { + sql "SWITCH internal" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } + } +} diff --git a/regression-test/suites/external_table_p2/maxcompute/write/test_mc_write_partitions.groovy b/regression-test/suites/external_table_p2/maxcompute/write/test_mc_write_partitions.groovy index 9bcc48ba199..9613d13e162 100644 --- a/regression-test/suites/external_table_p2/maxcompute/write/test_mc_write_partitions.groovy +++ b/regression-test/suites/external_table_p2/maxcompute/write/test_mc_write_partitions.groovy @@ -50,6 +50,22 @@ suite("test_mc_write_partitions", "p2,external,maxcompute,external_remote,extern sql """use ${db}""" try { + // Inferred MaxCompute engines must not silently ignore transforms or partition bounds. + String rejectedTable = "partition_model_rejected_${uuid}" + [ + ["PARTITION BY (date_trunc(ts, 'day')) ()", "MaxCompute only supports partitioning by columns"], + ["PARTITION BY LIST(ds) (PARTITION p1 VALUES IN ('20260101'))", + "MaxCompute does not support explicit partition definitions"], + ["PARTITION BY RANGE(ds) (PARTITION p1 VALUES LESS THAN ('20260102'))", + "MaxCompute does not support explicit partition definitions"] + ].each { entry -> + test { + sql """CREATE TABLE `${rejectedTable}` (id INT, ts DATETIME, ds STRING) ${entry[0]}""" + exception entry[1] + } + assertEquals([], sql("""SHOW TABLES LIKE '${rejectedTable}'""")) + } + // Test 1: Single partition column INSERT String tb1 = "single_part_${uuid}" sql """DROP TABLE IF EXISTS ${tb1}""" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
