This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new f14b453f675 branch-3.1: [fix][paimon] Fix Paimon date partition
conversion failure #54043 (#54250)
f14b453f675 is described below
commit f14b453f675ba72e78e97417589e5c15fb7530f7
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Aug 4 14:13:42 2025 +0800
branch-3.1: [fix][paimon] Fix Paimon date partition conversion failure
#54043 (#54250)
Cherry-picked from #54043
Co-authored-by: Petrichor <[email protected]>
---
.../apache/doris/datasource/paimon/PaimonUtil.java | 21 +++++++++++++++------
regression-test/data/mtmv_p0/test_paimon_mtmv.out | Bin 1353 -> 1394 bytes
.../suites/mtmv_p0/test_paimon_mtmv.groovy | 14 ++++++++++----
3 files changed, 25 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index 800c0d9413e..610cde051d8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -59,6 +59,7 @@ import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VarCharType;
+import org.apache.paimon.utils.DateTimeUtils;
import org.apache.paimon.utils.InstantiationUtil;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.Projection;
@@ -116,12 +117,23 @@ public class PaimonUtil {
Map<String, PartitionItem> nameToPartitionItem = Maps.newHashMap();
Map<String, Partition> nameToPartition = Maps.newHashMap();
PaimonPartitionInfo partitionInfo = new
PaimonPartitionInfo(nameToPartitionItem, nameToPartition);
+ List<Type> types = partitionColumns.stream()
+ .map(Column::getType)
+ .collect(Collectors.toList());
+ Map<String, Type> columnNameToType = partitionColumns.stream()
+ .collect(Collectors.toMap(Column::getName, Column::getType));
for (Partition partition : paimonPartitions) {
Map<String, String> spec = partition.spec();
StringBuilder sb = new StringBuilder();
for (Map.Entry<String, String> entry : spec.entrySet()) {
-
sb.append(entry.getKey()).append("=").append(entry.getValue()).append("/");
+ sb.append(entry.getKey()).append("=");
+ // Paimon stores DATE type as days since 1970-01-01 (epoch),
so we convert the integer to a date string.
+ if (columnNameToType.getOrDefault(entry.getKey(),
Type.NULL).isDateV2()) {
+
sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/");
+ } else {
+ sb.append(entry.getValue()).append("/");
+ }
}
if (sb.length() > 0) {
sb.deleteCharAt(sb.length() - 1);
@@ -131,7 +143,7 @@ public class PaimonUtil {
try {
// partition values return by paimon api, may have problem,
// to avoid affecting the query, we catch exceptions here
- nameToPartitionItem.put(partitionName,
toListPartitionItem(partitionName, partitionColumns));
+ nameToPartitionItem.put(partitionName,
toListPartitionItem(partitionName, types));
} catch (Exception e) {
LOG.warn("toListPartitionItem failed, partitionColumns: {},
partitionValues: {}",
partitionColumns, partition.spec(), e);
@@ -140,11 +152,8 @@ public class PaimonUtil {
return partitionInfo;
}
- public static ListPartitionItem toListPartitionItem(String partitionName,
List<Column> partitionColumns)
+ public static ListPartitionItem toListPartitionItem(String partitionName,
List<Type> types)
throws AnalysisException {
- List<Type> types = partitionColumns.stream()
- .map(Column::getType)
- .collect(Collectors.toList());
// Partition name will be in format: nation=cn/city=beijing
// parse it to get values "cn" and "beijing"
List<String> partitionValues =
HiveUtil.toPartitionValues(partitionName);
diff --git a/regression-test/data/mtmv_p0/test_paimon_mtmv.out
b/regression-test/data/mtmv_p0/test_paimon_mtmv.out
index 0cfe4bd293c..5c7547c0687 100644
Binary files a/regression-test/data/mtmv_p0/test_paimon_mtmv.out and
b/regression-test/data/mtmv_p0/test_paimon_mtmv.out differ
diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
index abd0f64450e..c4ea889f923 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
@@ -268,8 +268,7 @@ suite("test_paimon_mtmv",
"p0,external,mtmv,external_docker,external_docker_dori
// date type will has problem
order_qt_date_partition_base_table "SELECT * FROM
${catalogName}.`test_paimon_spark`.date_partition"
- test {
- sql """
+ sql """
CREATE MATERIALIZED VIEW ${mvName}
BUILD DEFERRED REFRESH AUTO ON MANUAL
partition by (`create_date`)
@@ -278,8 +277,15 @@ suite("test_paimon_mtmv",
"p0,external,mtmv,external_docker,external_docker_dori
AS
SELECT * FROM
${catalogName}.`test_paimon_spark`.date_partition;
"""
- exception "Unable to find a suitable base table"
- }
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} auto;
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ def showPaimonDateTypePartitionsResult = sql """show partitions from
${mvName}"""
+ logger.info("showPaimonDateTypePartitionsResult: " +
showPaimonDateTypePartitionsResult.toString())
+
assertTrue(showPaimonDateTypePartitionsResult.toString().contains("p_20200101"))
+ order_qt_date_type_partition "select * FROM ${mvName}"
+ sql """drop materialized view if exists ${mvName};"""
sql """
CREATE MATERIALIZED VIEW ${mvName}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
