[doris] branch master updated: [fix](cooldown) Fix potential data loss when clone task's dst tablet is cooldown replica (#17644)

[dataroaring]

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 9d2fc78bd5 [fix](cooldown) Fix potential data loss when clone task's 
dst tablet is cooldown replica (#17644)
9d2fc78bd5 is described below

commit 9d2fc78bd5d268a37a5cc61472bdb2c6cc559c27
Author: plat1ko <platonekos...@gmail.com>
AuthorDate: Fri Sep 1 15:27:52 2023 +0800

    [fix](cooldown) Fix potential data loss when clone task's dst tablet is 
cooldown replica (#17644)
    
    
    
    Co-authored-by: Yongqiang YANG 
<98214048+dataroar...@users.noreply.github.com>
    Co-authored-by: Kang <kxiao.ti...@gmail.com>
---
 be/src/olap/tablet.cpp                                  |  3 ++-
 be/src/olap/task/engine_clone_task.cpp                  | 17 ++++++++++++++++-
 .../java/org/apache/doris/clone/TabletSchedCtx.java     |  3 +++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 45d94f857f..afae3fa7e2 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2144,7 +2144,8 @@ Status Tablet::_read_cooldown_meta(const 
std::shared_ptr<io::RemoteFileSystem>&
     RETURN_IF_ERROR(tablet_meta_reader->read_at(0, {buf.get(), file_size}, 
&bytes_read));
     tablet_meta_reader->close();
     if (!tablet_meta_pb->ParseFromArray(buf.get(), file_size)) {
-        return Status::InternalError("malformed tablet meta");
+        return Status::InternalError("malformed tablet meta, path={}/{}", 
fs->root_path().native(),
+                                     remote_meta_path);
     }
     return Status::OK();
 }
diff --git a/be/src/olap/task/engine_clone_task.cpp 
b/be/src/olap/task/engine_clone_task.cpp
index 9b03375756..75e17e433e 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -735,7 +735,22 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet,
         RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs));
         to_add.push_back(std::move(rs));
     }
-    
tablet->tablet_meta()->set_cooldown_meta_id(cloned_tablet_meta->cooldown_meta_id());
+    {
+        std::shared_lock cooldown_conf_rlock(tablet->get_cooldown_conf_lock());
+        if (tablet->cooldown_conf_unlocked().first == tablet->replica_id()) {
+            // If this replica is cooldown replica, MUST generate a new 
`cooldown_meta_id` to avoid use `cooldown_meta_id`
+            // generated in old cooldown term which may lead to such situation:
+            // Replica A is cooldown replica, cooldown_meta_id=2,
+            // Replica B: cooldown_replica=A, cooldown_meta_id=1
+            // Replica A: full clone Replica A, cooldown_meta_id=1, but remote 
cooldown_meta is still with cooldown_meta_id=2
+            // After tablet report. FE finds all replicas' cooldowned data is 
consistent
+            // Replica A: confirm_unused_remote_files, delete some cooldowned 
data of cooldown_meta_id=2
+            // Replica B: follow_cooldown_data, cooldown_meta_id=2, data lost
+            tablet->tablet_meta()->set_cooldown_meta_id(UniqueId::gen_uid());
+        } else {
+            
tablet->tablet_meta()->set_cooldown_meta_id(cloned_tablet_meta->cooldown_meta_id());
+        }
+    }
     if (tablet->enable_unique_key_merge_on_write()) {
         tablet->tablet_meta()->delete_bitmap() = 
cloned_tablet_meta->delete_bitmap();
     }
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java 
b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
index 5b1ad25e68..f3c03b3e1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
@@ -925,6 +925,8 @@ public class TabletSchedCtx implements 
Comparable<TabletSchedCtx> {
                     committedVersion, /* use committed version as last failed 
version */
                     -1 /* last success version */);
 
+            LOG.info("create clone task to make new replica, tabletId={}, 
replicaId={}", tabletId,
+                    cloneReplica.getId());
             // addReplica() method will add this replica to tablet inverted 
index too.
             tablet.addReplica(cloneReplica);
             replicaId = cloneReplica.getId();
@@ -950,6 +952,7 @@ public class TabletSchedCtx implements 
Comparable<TabletSchedCtx> {
                 replicaId, schemaHash, Lists.newArrayList(tSrcBe), 
storageMedium,
                 visibleVersion, (int) (taskTimeoutMs / 1000));
         cloneTask.setPathHash(srcPathHash, destPathHash);
+        LOG.info("create clone task to repair replica, tabletId={}, 
replicaId={}", tabletId, replicaId);
 
         this.state = State.RUNNING;
         return cloneTask;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org