This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3af5ef661fd [improve](backup) Add config ignore_backup_tmp_partitions
(#45240)
3af5ef661fd is described below
commit 3af5ef661fdd946772493bfe007e86e35153d822
Author: walter <maoch...@selectdb.com>
AuthorDate: Thu Dec 12 10:36:00 2024 +0800
[improve](backup) Add config ignore_backup_tmp_partitions (#45240)
To filter tmp partitions, instead of report exception.
---
.../main/java/org/apache/doris/common/Config.java | 9 ++++++
.../org/apache/doris/backup/BackupHandler.java | 4 +--
.../java/org/apache/doris/backup/BackupJob.java | 15 ++++++----
.../java/org/apache/doris/catalog/OlapTable.java | 8 +++++
...est_backup_restore_backup_temp_partition.groovy | 34 ++++++++++++++++++++++
5 files changed, 63 insertions(+), 7 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index b02305bdbfc..42a64a2770f 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1638,6 +1638,15 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static boolean ignore_backup_not_support_table_type = false;
+ /**
+ * whether to ignore temp partitions when backup, and not report exception.
+ */
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "是否忽略备份临时分区，不报异常",
+ "Whether to ignore temp partitions when backup, and not report
exception."
+ })
+ public static boolean ignore_backup_tmp_partitions = false;
+
/**
* A internal config, to control the update interval of backup handler.
Only used to speed up tests.
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 6a12eee3a78..6f88881e3cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -446,14 +446,14 @@ public class BackupHandler extends MasterDaemon
implements Writable {
OlapTable olapTbl = (OlapTable) tbl;
tbl.readLock();
try {
- if (olapTbl.existTempPartitions()) {
+ if (!Config.ignore_backup_tmp_partitions &&
olapTbl.existTempPartitions()) {
ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
"Do not support backup table " + olapTbl.getName()
+ " with temp partitions");
}
PartitionNames partitionNames = tblRef.getPartitionNames();
if (partitionNames != null) {
- if (partitionNames.isTemp()) {
+ if (!Config.ignore_backup_tmp_partitions &&
partitionNames.isTemp()) {
ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
"Do not support backup temp partitions in
table " + tblRef.getName());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index d0a957dcd9a..1c897368624 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -597,10 +597,15 @@ public class BackupJob extends AbstractJob implements
GsonPostProcessable {
// check backup table again
if (backupTableRef.getPartitionNames() != null) {
for (String partName :
backupTableRef.getPartitionNames().getPartitionNames()) {
- Partition partition = olapTable.getPartition(partName);
+ Partition partition = olapTable.getPartition(partName, false);
// exclude tmp partitions
if (partition == null) {
- status = new Status(ErrCode.NOT_FOUND, "partition " +
partName
- + " does not exist in table" +
backupTableRef.getName().getTbl());
+ if (olapTable.getPartition(partName, true) != null) {
+ status = new Status(ErrCode.NOT_FOUND, "backup tmp
partition " + partName
+ + " in table " +
backupTableRef.getName().getTbl() + " is not supported");
+ } else {
+ status = new Status(ErrCode.NOT_FOUND, "partition " +
partName
+ + " does not exist in table " +
backupTableRef.getName().getTbl());
+ }
return;
}
}
@@ -609,10 +614,10 @@ public class BackupJob extends AbstractJob implements
GsonPostProcessable {
// create snapshot tasks
List<Partition> partitions = Lists.newArrayList();
if (backupTableRef.getPartitionNames() == null) {
- partitions.addAll(olapTable.getPartitions());
+ partitions.addAll(olapTable.getPartitions()); // no temp
partitions in OlapTable.getPartitions()
} else {
for (String partName :
backupTableRef.getPartitionNames().getPartitionNames()) {
- Partition partition = olapTable.getPartition(partName);
+ Partition partition = olapTable.getPartition(partName, false);
// exclude tmp partitions
partitions.add(partition);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index f3d27908bdd..92d9aa4e9c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2023,6 +2023,14 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
}
}
+ if (isForBackup) {
+ // drop all tmp partitions in copied table
+ for (Partition partition :
copied.tempPartitions.getAllPartitions()) {
+ copied.partitionInfo.dropPartition(partition.getId());
+ }
+ copied.tempPartitions = new TempPartitions();
+ }
+
if (reservedPartitions == null || reservedPartitions.isEmpty()) {
// reserve all
return copied;
diff --git
a/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
b/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
index 61514bedd88..8f06ff4eb8b 100644
---
a/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
+++
b/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
@@ -62,6 +62,7 @@ suite("test_backup_restore_backup_temp_partition",
"backup_restore") {
ALTER TABLE ${dbName}.${tableName} ADD TEMPORARY PARTITION tp1 VALUES
LESS THAN ("70")
"""
+ sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'false')"
test {
sql """
BACKUP SNAPSHOT ${dbName}.${snapshotName}
@@ -71,6 +72,39 @@ suite("test_backup_restore_backup_temp_partition",
"backup_restore") {
exception "Do not support backup table ${tableName} with temp
partitions"
}
+ // ignore the tmp partitions
+ sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'true')"
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tableName})
+ """
+
+ syncer.waitSnapshotFinish(dbName)
+ def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ // The restored table has no tmp partitions
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON (
+ `${tableName}` PARTITION (p1, p2, p3)
+ )
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "reserve_replica" = "true"
+ )
+ """
+
+ syncer.waitAllRestoreFinish(dbName)
+ def res = sql "SHOW TEMPORARY PARTITIONS FROM ${dbName}.${tableName}"
+ assertTrue(res.size() == 0);
+
+ sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'false')"
sql "DROP TABLE ${dbName}.${tableName} FORCE"
sql "DROP DATABASE ${dbName} FORCE"
sql "DROP REPOSITORY `${repoName}`"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
