This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new c016eb49c56 [enhance](mtmv)When obtaining the partition list fails,
treat the pai… (#46708)
c016eb49c56 is described below
commit c016eb49c56acb22498e890dc4a3af0abba7ee89
Author: zhangdong <[email protected]>
AuthorDate: Fri Jan 10 10:46:09 2025 +0800
[enhance](mtmv)When obtaining the partition list fails, treat the pai…
(#46708)
…mon table as an unpartitioned table (#46641)
pick: https://github.com/apache/doris/pull/46641
---
.../create_preinstalled_scripts/paimon/run01.sql | 47 +++++++++++++++++++++-
.../datasource/paimon/PaimonExternalTable.java | 8 ++++
.../datasource/paimon/PaimonPartitionInfo.java | 5 +++
.../apache/doris/datasource/paimon/PaimonUtil.java | 11 ++++-
4 files changed, 68 insertions(+), 3 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql
index 7aa4170eab0..7722d094636 100644
---
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql
@@ -22,4 +22,49 @@ insert into test_tb_mix_format values
(1,1,'b'),(2,1,'b'),(3,1,'b'),(4,1,'b'),(5
-- update some data, these splits will be readed by jni
insert into test_tb_mix_format values
(1,2,'b'),(2,2,'b'),(3,2,'b'),(4,2,'b'),(5,2,'b');
-- delete foramt in table properties, doris should get format by file name
-alter table test_tb_mix_format unset TBLPROPERTIES ('file.format');
\ No newline at end of file
+alter table test_tb_mix_format unset TBLPROPERTIES ('file.format');
+
+drop table if exists two_partition;
+CREATE TABLE two_partition (
+ id BIGINT,
+ create_date STRING,
+ region STRING
+) PARTITIONED BY (create_date,region) TBLPROPERTIES (
+ 'primary-key' = 'create_date,region,id',
+ 'bucket'=10,
+ 'file.format'='orc'
+);
+
+insert into two_partition values(1,'2020-01-01','bj');
+insert into two_partition values(2,'2020-01-01','sh');
+insert into two_partition values(3,'2038-01-01','bj');
+insert into two_partition values(4,'2038-01-01','sh');
+insert into two_partition values(5,'2038-01-02','bj');
+
+drop table if exists null_partition;
+CREATE TABLE null_partition (
+ id BIGINT,
+ region STRING
+) PARTITIONED BY (region) TBLPROPERTIES (
+ 'primary-key' = 'region,id',
+ 'bucket'=10,
+ 'file.format'='orc'
+);
+-- null NULL "null" all will be in partition [null]
+insert into null_partition values(1,'bj');
+insert into null_partition values(2,null);
+insert into null_partition values(3,NULL);
+insert into null_partition values(4,'null');
+insert into null_partition values(5,'NULL');
+
+drop table if exists date_partition;
+CREATE TABLE date_partition (
+ id BIGINT,
+ create_date DATE
+) PARTITIONED BY (create_date) TBLPROPERTIES (
+ 'primary-key' = 'create_date,id',
+ 'bucket'=10,
+ 'file.format'='orc'
+);
+
+insert into date_partition values(1,date '2020-01-01');
\ No newline at end of file
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index 3f22ce4c46b..efd8e6bb7a7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -144,9 +144,17 @@ public class PaimonExternalTable extends ExternalTable
implements MvccTable {
@Override
public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
+ if (isPartitionInvalid(snapshot)) {
+ return Collections.emptyList();
+ }
return getPaimonSchemaCacheValue(snapshot).getPartitionColumns();
}
+ private boolean isPartitionInvalid(Optional<MvccSnapshot> snapshot) {
+ PaimonSnapshotCacheValue paimonSnapshotCacheValue =
getOrFetchSnapshotCacheValue(snapshot);
+ return
paimonSnapshotCacheValue.getPartitionInfo().isPartitionInvalid();
+ }
+
@Override
public MvccSnapshot loadSnapshot() {
return new PaimonMvccSnapshot(getPaimonSnapshotCacheValue());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
index 4d3326f8e48..88515a2510d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
@@ -45,4 +45,9 @@ public class PaimonPartitionInfo {
public Map<String, PaimonPartition> getNameToPartition() {
return nameToPartition;
}
+
+ public boolean isPartitionInvalid() {
+ // when transfer to partitionItem failed, will not equal
+ return nameToPartitionItem.size() != nameToPartition.size();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index 1f7576dca51..b3df41bc5ce 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -117,7 +117,7 @@ public class PaimonUtil {
}
public static PaimonPartitionInfo generatePartitionInfo(List<Column>
partitionColumns,
- List<PaimonPartition> paimonPartitions) throws AnalysisException {
+ List<PaimonPartition> paimonPartitions) {
Map<String, PartitionItem> nameToPartitionItem = Maps.newHashMap();
Map<String, PaimonPartition> nameToPartition = Maps.newHashMap();
PaimonPartitionInfo partitionInfo = new
PaimonPartitionInfo(nameToPartitionItem, nameToPartition);
@@ -127,7 +127,14 @@ public class PaimonUtil {
for (PaimonPartition paimonPartition : paimonPartitions) {
String partitionName = getPartitionName(partitionColumns,
paimonPartition.getPartitionValues());
nameToPartition.put(partitionName, paimonPartition);
- nameToPartitionItem.put(partitionName,
toListPartitionItem(partitionName, partitionColumns));
+ try {
+ // partition values return by paimon api, may have problem,
+ // to avoid affecting the query, we catch exceptions here
+ nameToPartitionItem.put(partitionName,
toListPartitionItem(partitionName, partitionColumns));
+ } catch (Exception e) {
+ LOG.warn("toListPartitionItem failed, partitionColumns: {},
partitionValues: {}", partitionColumns,
+ paimonPartition.getPartitionValues(), e);
+ }
}
return partitionInfo;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
