This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 900850fb1604af164cf6864f254312eeb973b47a Author: walter <[email protected]> AuthorDate: Thu May 30 09:46:05 2024 +0800 [fix](backup) save finished state for local repo backup (#35491) The backup finished state in the local repo is not logged. So if there exist multiple local repo backups, and FE was restarted, then these backup records will be reset to the UPLOAD_INFO state. --- .../org/apache/doris/backup/BackupHandler.java | 1 + .../java/org/apache/doris/backup/BackupJob.java | 35 ++++++++++++---------- .../java/org/apache/doris/persist/EditLog.java | 8 ++--- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 0273c3c6a5a..1c6cb2d80c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -764,6 +764,7 @@ public class BackupHandler extends MasterDaemon implements Writable { // We use replayed job, not the existing job, to do the replayRun(). // Because if we use the existing job to run again, // for example: In restore job, PENDING will transfer to SNAPSHOTING, not DOWNLOAD. + job.setEnv(env); job.replayRun(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 5404fcca064..fe13e02d9c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -258,8 +258,11 @@ public class BackupJob extends AbstractJob { @Override public synchronized void replayRun() { - // Backup process does not change any current catalog state, - // So nothing need to be done when replaying log + LOG.info("replay run backup job: {}", this); + if (state == BackupJobState.FINISHED && repoId == Repository.KEEP_ON_LOCAL_REPO_ID) { + Snapshot snapshot = new Snapshot(label, metaInfoBytes, jobInfoBytes); + env.getBackupHandler().addSnapshot(label, snapshot); + } } @Override @@ -812,22 +815,18 @@ public class BackupJob extends AbstractJob { } private void uploadMetaAndJobInfoFile() { - if (repoId == Repository.KEEP_ON_LOCAL_REPO_ID) { - state = BackupJobState.FINISHED; - Snapshot snapshot = new Snapshot(label, metaInfoBytes, jobInfoBytes); - env.getBackupHandler().addSnapshot(label, snapshot); - return; - } + if (repoId != Repository.KEEP_ON_LOCAL_REPO_ID) { + String remoteMetaInfoFile = repo.assembleMetaInfoFilePath(label); + if (!uploadFile(localMetaInfoFilePath, remoteMetaInfoFile)) { + return; + } - String remoteMetaInfoFile = repo.assembleMetaInfoFilePath(label); - if (!uploadFile(localMetaInfoFilePath, remoteMetaInfoFile)) { - return; + String remoteJobInfoFile = repo.assembleJobInfoFilePath(label, createTime); + if (!uploadFile(localJobInfoFilePath, remoteJobInfoFile)) { + return; + } } - String remoteJobInfoFile = repo.assembleJobInfoFilePath(label, createTime); - if (!uploadFile(localJobInfoFilePath, remoteJobInfoFile)) { - return; - } finishedTime = System.currentTimeMillis(); state = BackupJobState.FINISHED; @@ -835,6 +834,12 @@ public class BackupJob extends AbstractJob { // log env.getEditLog().logBackupJob(this); LOG.info("job is finished. {}", this); + + if (repoId == Repository.KEEP_ON_LOCAL_REPO_ID) { + Snapshot snapshot = new Snapshot(label, metaInfoBytes, jobInfoBytes); + env.getBackupHandler().addSnapshot(label, snapshot); + return; + } } private boolean uploadFile(String localFilePath, String remoteFilePath) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index e7491e9d478..5aa10946dba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -167,7 +167,7 @@ public class EditLog { short opCode = journal.getOpCode(); if (opCode != OperationType.OP_SAVE_NEXTID && opCode != OperationType.OP_TIMESTAMP) { if (LOG.isDebugEnabled()) { - LOG.debug("replay journal op code: {}", opCode); + LOG.debug("replay journal op code: {}, log id: {}", opCode, logId); } } try { @@ -1202,7 +1202,7 @@ public class EditLog { } default: { IOException e = new IOException(); - LOG.error("UNKNOWN Operation Type {}", opCode, e); + LOG.error("UNKNOWN Operation Type {}, log id: {}", opCode, logId, e); throw e; } } @@ -1226,9 +1226,9 @@ public class EditLog { * log a warning here to debug when happens. This could happen to other meta * like DB. */ - LOG.warn("[INCONSISTENT META] replay failed {}: {}", journal, e.getMessage(), e); + LOG.warn("[INCONSISTENT META] replay log {} failed, journal {}: {}", logId, journal, e.getMessage(), e); } catch (Exception e) { - LOG.error("Operation Type {}", opCode, e); + LOG.error("replay Operation Type {}, log id: {}", opCode, logId, e); System.exit(-1); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
