This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 743ff3e9fc2 [fix](truncate table) truncate skip empty partition #28229
(#28295)
743ff3e9fc2 is described below
commit 743ff3e9fc21080c45b1e73693c952d049c20cc7
Author: yujun <yu.jun.re...@gmail.com>
AuthorDate: Sat Dec 16 18:37:23 2023 +0800
[fix](truncate table) truncate skip empty partition #28229 (#28295)
---
.../apache/doris/datasource/InternalCatalog.java | 23 ++++++++---
.../apache/doris/catalog/TruncateTableTest.java | 14 ++++---
.../data/ddl_p0/test_truncate_table.out | 12 ++++++
.../suites/ddl_p0/test_truncate_table.groovy | 46 ++++++++++++++--------
4 files changed, 69 insertions(+), 26 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index b9e952a6207..9fb391b0e80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2879,19 +2879,31 @@ public class InternalCatalog implements
CatalogIf<Database> {
if (partition == null) {
throw new DdlException("Partition " + partName + "
does not exist");
}
+ // If need absolutely correct, should check running txn
here.
+ // But if the txn is in prepare state, cann't known which
partitions had load data.
+ if (!partition.hasData()) {
+ continue;
+ }
origPartitions.put(partName, partition.getId());
partitionsDistributionInfo.put(partition.getId(),
partition.getDistributionInfo());
rowsToTruncate += partition.getBaseIndex().getRowCount();
}
} else {
for (Partition partition : olapTable.getPartitions()) {
+ // If need absolutely correct, should check running txn
here.
+ // But if the txn is in prepare state, cann't known which
partitions had load data.
+ if (!partition.hasData()) {
+ continue;
+ }
origPartitions.put(partition.getName(), partition.getId());
partitionsDistributionInfo.put(partition.getId(),
partition.getDistributionInfo());
}
}
- // if table currently has no partitions, this sql like empty
command and do nothing, should return directly
- // at the same time, it will avoid throwing IllegalStateException
when `bufferSize` equals zero
- if (origPartitions.isEmpty()) {
+ // if table currently has no partitions, this sql like empty
command and do nothing, should return directly.
+ // but if truncate whole table, the temporary partitions also need
drop
+ if (origPartitions.isEmpty() && (!truncateEntireTable ||
olapTable.getTempPartitions().isEmpty())) {
+ LOG.info("finished to truncate table {}, no partition contains
data, do nothing",
+ tblRef.getName().toSql());
return;
}
copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(),
IndexExtState.VISIBLE, false);
@@ -2905,9 +2917,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
List<Partition> newPartitions = Lists.newArrayList();
// tabletIdSet to save all newly created tablet ids.
Set<Long> tabletIdSet = Sets.newHashSet();
- long bufferSize =
IdGeneratorUtil.getBufferSizeForTruncateTable(copiedTbl,
origPartitions.values());
- IdGeneratorBuffer idGeneratorBuffer =
Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize);
try {
+ long bufferSize =
IdGeneratorUtil.getBufferSizeForTruncateTable(copiedTbl,
origPartitions.values());
+ IdGeneratorBuffer idGeneratorBuffer =
+ origPartitions.isEmpty() ? null :
Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize);
for (Map.Entry<String, Long> entry : origPartitions.entrySet()) {
// the new partition must use new id
// If we still use the old partition id, the behavior of
current load jobs on this partition
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
index 0a05d7c618a..e5708eded29 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TruncateTableTest.java
@@ -88,9 +88,12 @@ public class TruncateTableTest {
OlapTable tbl = db.getOlapTableOrDdlException("case_sensitive_table");
long p20211006Id = tbl.getPartition("P20211006").getId();
long p20211007Id = tbl.getPartition("P20211007").getId();
- long p20211008Id = tbl.getPartition("p20211008").getId();
- // truncate p20211008(real name is P20211008)
- String truncateStr = "TRUNCATE TABLE test.case_sensitive_table
PARTITION p20211008; \n";
+ long p20211008Id = tbl.getPartition("P20211008").getId();
+ // truncate P20211008(real name is P20211008)
+ Partition p20211008 = tbl.getPartition("P20211008");
+ p20211008.updateVisibleVersion(2L);
+ p20211008.setNextVersion(p20211008.getVisibleVersion() + 1);
+ String truncateStr = "TRUNCATE TABLE test.case_sensitive_table
PARTITION P20211008; \n";
TruncateTableStmt truncateTableStmt
= (TruncateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
Env.getCurrentEnv().truncateTable(truncateTableStmt);
@@ -100,10 +103,11 @@ public class TruncateTableTest {
truncateTableStmt = (TruncateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(truncateStr, connectContext);
Env.getCurrentEnv().truncateTable(truncateTableStmt);
Assert.assertEquals(3, tbl.getPartitionInfo().idToDataProperty.size());
- Assert.assertNotEquals(p20211007Id,
tbl.getPartition("p20211007").getId());
Assert.assertEquals(p20211006Id,
tbl.getPartition("p20211006").getId());
+ Assert.assertEquals(p20211007Id,
tbl.getPartition("P20211007").getId());
Assert.assertNotNull(tbl.getPartition("p20211006"));
- Assert.assertNotNull(tbl.getPartition("P20211006"));
+ Assert.assertNotNull(tbl.getPartition("P20211007"));
+ Assert.assertNotNull(tbl.getPartition("P20211008"));
}
@Test
diff --git a/regression-test/data/ddl_p0/test_truncate_table.out
b/regression-test/data/ddl_p0/test_truncate_table.out
new file mode 100644
index 00000000000..95ed72f34af
--- /dev/null
+++ b/regression-test/data/ddl_p0/test_truncate_table.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_1 --
+2020-01-01 1.00 a 1
+2020-03-10 1.00 a 1
+
+-- !select_2 --
+
+-- !select_3 --
+2020-02-10 1.00 a 1
+
+-- !select_4 --
+
diff --git a/regression-test/suites/ddl_p0/test_truncate_table.groovy
b/regression-test/suites/ddl_p0/test_truncate_table.groovy
index e983915f224..7a79d04c70e 100644
--- a/regression-test/suites/ddl_p0/test_truncate_table.groovy
+++ b/regression-test/suites/ddl_p0/test_truncate_table.groovy
@@ -40,24 +40,38 @@ suite("test_truncate_table") {
"replication_num" = "1"
);
"""
- List<List<Object>> result = sql "show partitions from ${testTable}"
- logger.info("${result}")
- assertEquals(result.size(), 3)
- assertEquals(result.get(0).get(1), "p1")
+
+ def getPartitionIds = { ->
+ def result = sql_return_maparray("show partitions from ${testTable}")
+ return result.collectEntries { [it.PartitionName, it.PartitionId as
long] }
+ }
+
+ def partitionIds1 = getPartitionIds()
+ assertEquals(["p1", "p2", "p3"].toSet(), partitionIds1.keySet())
+
+ sql "insert into ${testTable} values ('2020-01-01', 1.0, 'a', 1)"
+ sql "insert into ${testTable} values ('2020-03-10', 1.0, 'a', 1)"
+ order_qt_select_1 "SELECT * FROM ${testTable}"
sql """truncate table ${testTable};"""
- result = sql "show partitions from ${testTable}"
- logger.info("${result}")
- assertEquals(result.size(), 3)
- assertEquals(result.get(0).get(1), "p1")
-
- sql """truncate table ${testTable} partitions (p1, P1);"""
-
- result = sql "show partitions from ${testTable}"
- logger.info("${result}")
- assertEquals(result.size(), 3)
- assertEquals(result.get(0).get(1), "p1")
-
+ def partitionIds2 = getPartitionIds()
+ assertEquals(["p1", "p2", "p3"].toSet(), partitionIds2.keySet())
+ assertNotEquals(partitionIds1.get("p1"), partitionIds2.get("p1"))
+ assertEquals(partitionIds1.get("p2"), partitionIds2.get("p2"))
+ assertNotEquals(partitionIds1.get("p3"), partitionIds2.get("p3"))
+ order_qt_select_2 "SELECT * FROM ${testTable}"
+
+ sql "insert into ${testTable} values ('2020-02-10', 1.0, 'a', 1)"
+ order_qt_select_3 "SELECT * FROM ${testTable}"
+ sql """truncate table ${testTable} partitions (p1, p2);"""
+ order_qt_select_4 "SELECT * FROM ${testTable}"
+
+ def partitionIds3 = getPartitionIds()
+ assertEquals(["p1", "p2", "p3"].toSet(), partitionIds3.keySet())
+ assertEquals(partitionIds2.get("p1"), partitionIds3.get("p1"))
+ assertNotEquals(partitionIds2.get("p2"), partitionIds3.get("p2"))
+ assertEquals(partitionIds2.get("p3"), partitionIds3.get("p3"))
+
sql "DROP TABLE IF EXISTS ${testTable}"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
