This is an automated email from the ASF dual-hosted git repository.

deardeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new e87bc301743 [fix](cloud) Update node decommission status atomically 
(#70224)
e87bc301743 is described below

commit e87bc301743969fdc123b6117c8077771ffc84f7
Author: deardeng <[email protected]>
AuthorDate: Tue Nov 17 15:35:39 2026 +0800

    [fix](cloud) Update node decommission status atomically (#70224)
    
    Problem Summary: Both start-decommission and completion updated node
    status by deleting the node and re-adding it in separate transactions.
    Each transaction independently committed and refreshed the MS cache,
    exposing a state with the target node missing. A concurrent rowset
    commit could then fail with "failed to commit rowset: empty
    instance_id". Update the existing node's status in one transaction and
    add deterministic regression coverage for both transitions.
---
 cloud/src/meta-service/injection_point_http.cpp    |  17 +-
 cloud/src/meta-service/meta_service.cpp            |   1 +
 cloud/src/meta-service/meta_service_resource.cpp   | 188 ++++++++-----------
 cloud/src/resource-manager/resource_manager.cpp    |   7 +-
 cloud/src/resource-manager/resource_manager.h      |   4 +-
 cloud/test/meta_service_http_test.cpp              |  97 ++++++++--
 cloud/test/mock_resource_manager.h                 |   3 +-
 cloud/test/resource_test.cpp                       | 201 +++++++++++++++++++++
 .../cloud_p0/node_mgr/test_cloud_decommission.out  |   7 +
 .../node_mgr/test_cloud_decommission.groovy        | 115 +++++++++++-
 10 files changed, 501 insertions(+), 139 deletions(-)

diff --git a/cloud/src/meta-service/injection_point_http.cpp 
b/cloud/src/meta-service/injection_point_http.cpp
index 3c2ae4fee33..fdca4f83bc7 100644
--- a/cloud/src/meta-service/injection_point_http.cpp
+++ b/cloud/src/meta-service/injection_point_http.cpp
@@ -268,6 +268,17 @@ HttpResponse set_sleep(const std::string& point, const 
brpc::URI& uri) {
     return http_json_reply(MetaServiceCode::OK, "OK");
 }
 
+HttpResponse set_wait(const std::string& point) {
+    auto* sp = SyncPoint::get_instance();
+    sp->set_call_back(point, [point, sp](auto&&) {
+        LOG(INFO) << "injection point waiting, point=" << point;
+        while (sp->get_enable() && sp->has_point(point)) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        }
+    });
+    return http_json_reply(MetaServiceCode::OK, "OK");
+}
+
 HttpResponse set_return(const std::string& point, const brpc::URI& uri) {
     auto sp = SyncPoint::get_instance();
     sp->set_call_back(point, [point](auto&& args) {
@@ -293,7 +304,9 @@ HttpResponse handle_set(const brpc::URI& uri) {
     if (behavior.empty()) {
         return http_json_reply(MetaServiceCode::INVALID_ARGUMENT, "empty 
behavior");
     }
-    if (behavior == "sleep") {
+    if (behavior == "wait") {
+        return set_wait(point);
+    } else if (behavior == "sleep") {
         return set_sleep(point, uri);
     } else if (behavior == "return") {
         return set_return(point, uri);
@@ -349,6 +362,8 @@ HttpResponse handle_disable(const brpc::URI& uri) {
 // curl 
"ms_ip:port/MetaService/http/v1/injection_point?token=greedisgood9999&op=disable"
 // ```
 
+// behavior=wait blocks at a point until it is cleared or injection is 
disabled.
+//
 // clear all injection points
 // ```
 // curl 
"ms_ip:port/MetaService/http/v1/injection_point?token=greedisgood9999&op=clear"
diff --git a/cloud/src/meta-service/meta_service.cpp 
b/cloud/src/meta-service/meta_service.cpp
index dac3e2e6e2a..38e759e5374 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -2728,6 +2728,7 @@ void 
MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
         msg = "no rowset meta";
         return;
     }
+    TEST_INJECTION_POINT("commit_rowset::before_get_instance_id:" + 
request->cloud_unique_id());
     instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id());
     if (instance_id.empty()) {
         code = MetaServiceCode::INVALID_ARGUMENT;
diff --git a/cloud/src/meta-service/meta_service_resource.cpp 
b/cloud/src/meta-service/meta_service_resource.cpp
index b0b5b1db74d..df7adab3090 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -3314,133 +3314,95 @@ void handle_drop_node(const std::string& instance_id, 
const AlterClusterRequest*
     msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
 }
 
-void handle_decommission_node(const std::string& instance_id, const 
AlterClusterRequest* request,
-                              std::shared_ptr<ResourceManager> resource_mgr, 
std::string& msg,
-                              MetaServiceCode& code) {
-    resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false, 
false);
-    if (msg != "") {
-        LOG(WARNING) << msg;
-        return;
+static bool matches_node(const NodeInfoPB& node, const NodeInfoPB& requested) {
+    if (node.cloud_unique_id() != requested.cloud_unique_id() ||
+        node.heartbeat_port() != requested.heartbeat_port()) {
+        return false;
+    }
+    if (node.has_ip() && requested.has_ip()) {
+        return node.ip() == requested.ip();
     }
+    return node.has_host() && requested.has_host() && node.host() == 
requested.host();
+}
 
-    std::string be_unique_id = 
(request->cluster().nodes())[0].cloud_unique_id();
+static std::string resolve_cluster_from_node(ResourceManager& resource_mgr,
+                                             const NodeInfoPB& requested, 
ClusterPB& cluster) {
     std::vector<NodeInfo> nodes;
-    std::string err = resource_mgr->get_node(be_unique_id, &nodes);
-    if (!err.empty()) {
-        LOG(INFO) << "failed to check instance info, err=" << err;
-        msg = err;
-        return;
-    }
-
-    std::vector<NodeInfo> decomission_nodes;
-    for (auto& node : nodes) {
-        for (auto req_node : request->cluster().nodes()) {
-            bool ip_processed = false;
-            if (node.node_info.has_ip() && req_node.has_ip()) {
-                std::string endpoint =
-                        node.node_info.ip() + ":" + 
std::to_string(node.node_info.heartbeat_port());
-                std::string req_endpoint =
-                        req_node.ip() + ":" + 
std::to_string(req_node.heartbeat_port());
-                if (endpoint == req_endpoint) {
-                    decomission_nodes.push_back(node);
-                    
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
-                }
-                ip_processed = true;
-            }
-
-            if (!ip_processed && node.node_info.has_host() && 
req_node.has_host()) {
-                std::string endpoint = node.node_info.host() + ":" +
-                                       
std::to_string(node.node_info.heartbeat_port());
-                std::string req_endpoint =
-                        req_node.host() + ":" + 
std::to_string(req_node.heartbeat_port());
-                if (endpoint == req_endpoint) {
-                    decomission_nodes.push_back(node);
-                    
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
-                }
-            }
+    auto msg = resource_mgr.get_node(requested.cloud_unique_id(), &nodes);
+    if (!msg.empty()) {
+        return msg;
+    }
+    for (const auto& node : nodes) {
+        if (matches_node(node.node_info, requested)) {
+            cluster.set_cluster_id(node.cluster_id);
+            cluster.set_cluster_name(node.cluster_name);
+            return "";
         }
     }
+    return "failed to find node to update: " + requested.ShortDebugString();
+}
 
-    {
-        std::vector<NodeInfo> to_add;
-        std::vector<NodeInfo>& to_del = decomission_nodes;
-        msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
-    }
-    {
-        std::vector<NodeInfo>& to_add = decomission_nodes;
-        std::vector<NodeInfo> to_del;
-        for (auto& node : to_add) {
-            
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
-            LOG(INFO) << "decomission node, "
-                      << "size: " << to_add.size() << " " << 
node.node_info.DebugString() << " "
-                      << node.cluster_id << " " << node.cluster_name;
+static std::string update_cluster_node_statuses(ClusterPB& cluster,
+                                                const ClusterPB& 
requested_cluster,
+                                                NodeStatusPB status) {
+    const auto mtime = 
duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
+    auto* nodes = cluster.mutable_nodes();
+    for (const auto& requested : requested_cluster.nodes()) {
+        NodeInfoPB* matched = nullptr;
+        for (auto& node : *nodes) {
+            if (matches_node(node, requested)) {
+                matched = &node;
+                break;
+            }
+        }
+        if (matched == nullptr) {
+            return "failed to find node to update: " + 
requested.ShortDebugString();
         }
-        msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
+        matched->set_status(status);
+        matched->set_mtime(mtime);
     }
+    return "";
 }
 
-void handle_notify_decommissioned(const std::string& instance_id,
-                                  const AlterClusterRequest* request,
-                                  std::shared_ptr<ResourceManager> 
resource_mgr, std::string& msg,
-                                  MetaServiceCode& code) {
-    resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false, 
false);
-    if (msg != "") {
+void handle_update_node_status(const std::string& instance_id, const 
AlterClusterRequest* request,
+                               std::shared_ptr<ResourceManager> resource_mgr, 
NodeStatusPB status,
+                               std::string& msg) {
+    const auto& requested_cluster = request->cluster();
+    resource_mgr->check_cluster_params_valid(requested_cluster, &msg, false, 
false);
+    if (!msg.empty()) {
         LOG(WARNING) << msg;
         return;
     }
-
-    std::string be_unique_id = 
(request->cluster().nodes())[0].cloud_unique_id();
-    std::vector<NodeInfo> nodes;
-    std::string err = resource_mgr->get_node(be_unique_id, &nodes);
-    if (!err.empty()) {
-        LOG(INFO) << "failed to check instance info, err=" << err;
-        msg = err;
-        return;
-    }
-
-    std::vector<NodeInfo> decomission_nodes;
-    for (auto& node : nodes) {
-        for (auto req_node : request->cluster().nodes()) {
-            bool ip_processed = false;
-            if (node.node_info.has_ip() && req_node.has_ip()) {
-                std::string endpoint =
-                        node.node_info.ip() + ":" + 
std::to_string(node.node_info.heartbeat_port());
-                std::string req_endpoint =
-                        req_node.ip() + ":" + 
std::to_string(req_node.heartbeat_port());
-                if (endpoint == req_endpoint) {
-                    decomission_nodes.push_back(node);
-                }
-                ip_processed = true;
-            }
-
-            if (!ip_processed && node.node_info.has_host() && 
req_node.has_host()) {
-                std::string endpoint = node.node_info.host() + ":" +
-                                       
std::to_string(node.node_info.heartbeat_port());
-                std::string req_endpoint =
-                        req_node.host() + ":" + 
std::to_string(req_node.heartbeat_port());
-                if (endpoint == req_endpoint) {
-                    decomission_nodes.push_back(node);
-                }
-            }
-        }
+    if (requested_cluster.nodes().empty()) {
+        msg = "no nodes to update";
+        return;
     }
 
-    {
-        std::vector<NodeInfo> to_add;
-        std::vector<NodeInfo>& to_del = decomission_nodes;
-        msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
-    }
-    {
-        std::vector<NodeInfo>& to_add = decomission_nodes;
-        std::vector<NodeInfo> to_del;
-        for (auto& node : to_add) {
-            
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
-            LOG(INFO) << "notify node decomissioned, "
-                      << " size: " << to_add.size() << " " << 
node.node_info.DebugString() << " "
-                      << node.cluster_id << " " << node.cluster_name;
+    ClusterInfo cluster;
+    cluster.cluster.CopyFrom(requested_cluster);
+    if (cluster.cluster.cluster_id().empty()) {
+        // Older HTTP callers can identify the cluster through the node alone.
+        msg = resolve_cluster_from_node(*resource_mgr, 
requested_cluster.nodes(0), cluster.cluster);
+        if (!msg.empty()) {
+            return;
         }
-        msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
     }
+
+    const auto& cluster_id = cluster.cluster.cluster_id();
+    const auto& cluster_name = cluster.cluster.cluster_name();
+    // HTTP-created clusters without a stored ID are distinguished by name.
+    auto match_cluster = [&cluster_id, &cluster_name](const ClusterPB& 
candidate) {
+        return candidate.cluster_id() == cluster_id &&
+               (!cluster_id.empty() || candidate.cluster_name() == 
cluster_name);
+    };
+    auto update_nodes = [&requested_cluster, status](ClusterPB& current, 
std::vector<ClusterPB>&) {
+        return update_cluster_node_statuses(current, requested_cluster, 
status);
+    };
+
+    // Publish only the final state: removing and re-adding nodes breaks 
instance lookup.
+    msg = resource_mgr->update_cluster(instance_id, cluster, match_cluster, 
update_nodes,
+                                       
/*replace_if_existing_empty_target_cluster=*/false,
+                                       /*allow_cluster_name_fallback=*/true);
 }
 
 void handle_rename_cluster(const std::string& instance_id, const ClusterInfo& 
cluster,
@@ -3738,10 +3700,12 @@ void 
MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller,
         handle_drop_node(instance_id, request, resource_mgr(), msg, code);
         break;
     case AlterClusterRequest::DECOMMISSION_NODE:
-        handle_decommission_node(instance_id, request, resource_mgr(), msg, 
code);
+        handle_update_node_status(instance_id, request, resource_mgr(),
+                                  NodeStatusPB::NODE_STATUS_DECOMMISSIONING, 
msg);
         break;
     case AlterClusterRequest::NOTIFY_DECOMMISSIONED:
-        handle_notify_decommissioned(instance_id, request, resource_mgr(), 
msg, code);
+        handle_update_node_status(instance_id, request, resource_mgr(),
+                                  NodeStatusPB::NODE_STATUS_DECOMMISSIONED, 
msg);
         break;
     case AlterClusterRequest::RENAME_CLUSTER: {
         // SQL mode, cluster cluster name eq empty cluster name, need drop 
empty cluster first.
diff --git a/cloud/src/resource-manager/resource_manager.cpp 
b/cloud/src/resource-manager/resource_manager.cpp
index fafd4ae09e8..3c6487e4fff 100644
--- a/cloud/src/resource-manager/resource_manager.cpp
+++ b/cloud/src/resource-manager/resource_manager.cpp
@@ -781,14 +781,15 @@ std::string ResourceManager::update_cluster(
         const std::string& instance_id, const ClusterInfo& cluster,
         std::function<bool(const ClusterPB&)> filter,
         std::function<std::string(ClusterPB&, std::vector<ClusterPB>& 
clusters_in_instance)> action,
-        bool replace_if_existing_empty_target_cluster) {
+        bool replace_if_existing_empty_target_cluster, bool 
allow_cluster_name_fallback) {
     std::stringstream ss;
     std::string msg;
 
     std::string cluster_id = cluster.cluster.has_cluster_id() ? 
cluster.cluster.cluster_id() : "";
     std::string cluster_name =
             cluster.cluster.has_cluster_name() ? 
cluster.cluster.cluster_name() : "";
-    if (cluster_id.empty()) {
+    const bool can_fallback_to_name = allow_cluster_name_fallback && 
!cluster_name.empty();
+    if (cluster_id.empty() && !can_fallback_to_name) {
         ss << "missing cluster_id=" << cluster_id;
         msg = ss.str();
         LOG(INFO) << msg;
@@ -916,6 +917,7 @@ std::string ResourceManager::update_cluster(
               << " instance json=" << proto_to_json(instance);
 
     refresh_instance(instance_id, instance);
+    TEST_INJECTION_POINT("ResourceManager::node_update:after_refresh");
 
     return msg;
 }
@@ -1383,6 +1385,7 @@ std::string ResourceManager::modify_nodes(const 
std::string& instance_id,
     }
 
     refresh_instance(instance_id, instance);
+    TEST_INJECTION_POINT("ResourceManager::node_update:after_refresh");
 
     return "";
 }
diff --git a/cloud/src/resource-manager/resource_manager.h 
b/cloud/src/resource-manager/resource_manager.h
index ca33a3f0983..f8a3a75da80 100644
--- a/cloud/src/resource-manager/resource_manager.h
+++ b/cloud/src/resource-manager/resource_manager.h
@@ -89,6 +89,7 @@ public:
      * @param cluster cluster to update, only cluster name and cluster id are 
concered
      * @param action update operation code snippet
      * @param replace_if_existing_empty_target_cluster, find 
cluster.cluster_name is a empty cluster(no node), drop it
+     * @param allow_cluster_name_fallback allow an empty ID when the cluster 
name is set
      * @filter filter condition
      * @return empty string for success, otherwise failure reason returned
      */
@@ -97,7 +98,8 @@ public:
             std::function<bool(const ClusterPB&)> filter,
             std::function<std::string(ClusterPB&, std::vector<ClusterPB>& 
clusters_in_instance)>
                     action,
-            bool replace_if_existing_empty_target_cluster = false);
+            bool replace_if_existing_empty_target_cluster = false,
+            bool allow_cluster_name_fallback = false);
 
     /**
      * Get instance from underlying storage with given transaction.
diff --git a/cloud/test/meta_service_http_test.cpp 
b/cloud/test/meta_service_http_test.cpp
index 9b7c05c125c..3d0199b1e68 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -954,49 +954,45 @@ TEST(MetaServiceHttpTest, AlterClusterTest) {
     {
         AlterClusterRequest req;
         req.set_instance_id(mock_instance);
-        req.mutable_cluster()->set_cluster_name(mock_cluster_name);
+        req.mutable_cluster()->set_cluster_name("rename_cluster_name");
         req.mutable_cluster()->set_cluster_id(mock_cluster_id);
         req.mutable_cluster()->set_type(ClusterPB::COMPUTE);
         auto node = req.mutable_cluster()->add_nodes();
         node->set_ip("127.0.0.1");
         node->set_heartbeat_port(9999);
         node->set_cloud_unique_id("cloud_unique_id");
-        auto& meta_service = ctx.meta_service_;
-        NodeInfoPB npb;
-        npb.set_heartbeat_port(9999);
-        npb.set_ip("127.0.0.1");
-        npb.set_cloud_unique_id("cloud_unique_id");
-        meta_service->resource_mgr()->node_info_.insert(
-                {"cloud_unique_id", NodeInfo {Role::COMPUTE_NODE, 
mock_instance,
-                                              "rename_cluster_name", 
mock_cluster_id, npb}});
+        auto [add_status_code, add_resp] = 
ctx.forward<MetaServiceResponseStatus>("add_node", req);
+        ASSERT_EQ(add_status_code, 200);
+        ASSERT_EQ(add_resp.code(), MetaServiceCode::OK);
         auto [status_code, resp] = 
ctx.forward<MetaServiceResponseStatus>("decommission_node", req);
         ASSERT_EQ(status_code, 200);
         ASSERT_EQ(resp.code(), MetaServiceCode::OK);
+
+        auto instance = ctx.get_instance_info(mock_instance);
+        ASSERT_EQ(instance.clusters(0).nodes_size(), 1);
+        EXPECT_EQ(instance.clusters(0).nodes(0).status(),
+                  NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
     }
 
     // notify_decommissioned
     {
         AlterClusterRequest req;
         req.set_instance_id(mock_instance);
-        req.mutable_cluster()->set_cluster_name(mock_cluster_name);
+        req.mutable_cluster()->set_cluster_name("rename_cluster_name");
         req.mutable_cluster()->set_cluster_id(mock_cluster_id);
         req.mutable_cluster()->set_type(ClusterPB::COMPUTE);
         auto node = req.mutable_cluster()->add_nodes();
         node->set_ip("127.0.0.1");
-        node->set_heartbeat_port(9996);
+        node->set_heartbeat_port(9999);
         node->set_cloud_unique_id("cloud_unique_id");
-        auto& meta_service = ctx.meta_service_;
-        NodeInfoPB npb;
-        npb.set_heartbeat_port(9996);
-        npb.set_ip("127.0.0.1");
-        npb.set_cloud_unique_id("cloud_unique_id");
-        meta_service->resource_mgr()->node_info_.insert(
-                {"cloud_unique_id", NodeInfo {Role::COMPUTE_NODE, 
mock_instance,
-                                              "rename_cluster_name", 
mock_cluster_id, npb}});
         auto [status_code, resp] =
                 
ctx.forward<MetaServiceResponseStatus>("notify_decommissioned", req);
         ASSERT_EQ(status_code, 200);
         ASSERT_EQ(resp.code(), MetaServiceCode::OK);
+
+        auto instance = ctx.get_instance_info(mock_instance);
+        ASSERT_EQ(instance.clusters(0).nodes_size(), 1);
+        EXPECT_EQ(instance.clusters(0).nodes(0).status(), 
NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
     }
 
     // update_cluster_endpoint
@@ -1014,6 +1010,69 @@ TEST(MetaServiceHttpTest, AlterClusterTest) {
     }
 }
 
+class NodeStatusWithoutClusterIdHttpTest : public 
testing::TestWithParam<std::tuple<bool, bool>> {};
+
+TEST_P(NodeStatusWithoutClusterIdHttpTest, 
PreservesNamedClusterWithoutStoredId) {
+    const auto [explicit_empty_id, notify_decommissioned] = GetParam();
+    HttpContext ctx;
+    CreateInstanceRequest create_req;
+    create_req.set_instance_id("legacy_node_status_instance");
+    create_req.set_user_id("test_user");
+    create_req.set_name("test_name");
+    auto [create_http_status, create_resp] =
+            ctx.forward<MetaServiceResponseStatus>("create_instance", 
create_req);
+    ASSERT_EQ(create_http_status, 200);
+    ASSERT_EQ(create_resp.code(), MetaServiceCode::OK) << create_resp.msg();
+
+    AlterClusterRequest req;
+    req.set_instance_id(create_req.instance_id());
+    auto* cluster = req.mutable_cluster();
+    cluster->set_type(ClusterPB::COMPUTE);
+    cluster->set_cluster_name("legacy_named_cluster");
+    if (explicit_empty_id) {
+        cluster->set_cluster_id("");
+    }
+    auto* node = cluster->add_nodes();
+    node->set_cloud_unique_id("legacy_node_uid");
+    node->set_ip("127.0.0.1");
+    node->set_heartbeat_port(9050);
+    node->set_status(NodeStatusPB::NODE_STATUS_RUNNING);
+    auto [add_http_status, add_resp] = 
ctx.forward<MetaServiceResponseStatus>("add_cluster", req);
+    ASSERT_EQ(add_http_status, 200);
+    ASSERT_EQ(add_resp.code(), MetaServiceCode::OK) << add_resp.msg();
+    auto persisted = ctx.get_instance_info(create_req.instance_id());
+    ASSERT_EQ(persisted.clusters_size(), 1);
+    ASSERT_TRUE(persisted.clusters(0).cluster_id().empty());
+    ASSERT_EQ(persisted.clusters(0).has_cluster_id(), explicit_empty_id);
+
+    // Seed an old timestamp so the refresh is observable even within the same 
second.
+    constexpr int64_t old_mtime = 200;
+    persisted.mutable_clusters(0)->mutable_nodes(0)->set_mtime(old_mtime);
+    std::unique_ptr<Transaction> txn;
+    ASSERT_EQ(ctx.meta_service_->txn_kv()->create_txn(&txn), 
TxnErrorCode::TXN_OK);
+    txn->put(instance_key({create_req.instance_id()}), 
persisted.SerializeAsString());
+    ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+    auto expected = persisted.clusters(0);
+
+    const auto* operation = notify_decommissioned ? "notify_decommissioned" : 
"decommission_node";
+    auto [http_status, resp] = 
ctx.forward<MetaServiceResponseStatus>(operation, req);
+    ASSERT_EQ(http_status, 200) << resp.ShortDebugString();
+    ASSERT_EQ(resp.code(), MetaServiceCode::OK) << resp.msg();
+    persisted = ctx.get_instance_info(create_req.instance_id());
+    ASSERT_EQ(persisted.clusters_size(), 1);
+    ASSERT_EQ(persisted.clusters(0).nodes_size(), 1);
+    const auto mtime = persisted.clusters(0).nodes(0).mtime();
+    EXPECT_GT(mtime, old_mtime);
+    auto* expected_node = expected.mutable_nodes(0);
+    expected_node->set_mtime(mtime);
+    expected_node->set_status(notify_decommissioned ? 
NodeStatusPB::NODE_STATUS_DECOMMISSIONED
+                                                    : 
NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
+    EXPECT_EQ(persisted.clusters(0).SerializeAsString(), 
expected.SerializeAsString());
+}
+
+INSTANTIATE_TEST_SUITE_P(StoredClusterId, NodeStatusWithoutClusterIdHttpTest,
+                         testing::Combine(testing::Bool(), testing::Bool()));
+
 TEST(MetaServiceHttpTest, GetClusterTest) {
     HttpContext ctx(true);
 
diff --git a/cloud/test/mock_resource_manager.h 
b/cloud/test/mock_resource_manager.h
index 59d0365c8e6..a072b0578b5 100644
--- a/cloud/test/mock_resource_manager.h
+++ b/cloud/test/mock_resource_manager.h
@@ -61,7 +61,8 @@ public:
             std::function<bool(const ClusterPB&)> filter,
             std::function<std::string(ClusterPB&, std::vector<ClusterPB>& 
clusters_in_instance)>
                     action,
-            bool replace_if_existing_empty_target_cluster) override {
+            bool replace_if_existing_empty_target_cluster,
+            bool allow_cluster_name_fallback) override {
         return "";
     }
 
diff --git a/cloud/test/resource_test.cpp b/cloud/test/resource_test.cpp
index 206f563a20a..75dbfd3288b 100644
--- a/cloud/test/resource_test.cpp
+++ b/cloud/test/resource_test.cpp
@@ -28,6 +28,7 @@
 #include <atomic>
 #include <condition_variable>
 #include <cstdint>
+#include <functional>
 #include <memory>
 #include <optional>
 #include <random>
@@ -217,6 +218,206 @@ static void drop_cluster(MetaServiceProxy* ms, const 
std::string& instance_id,
     ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
 }
 
+class NodeStatusResourceManager : public ResourceManager {
+public:
+    using ResourceManager::ResourceManager;
+
+    void refresh_instance(const std::string& instance_id, const 
InstanceInfoPB& instance) override {
+        ResourceManager::refresh_instance(instance_id, instance);
+        after_refresh();
+    }
+
+    std::function<void()> after_refresh = [] {};
+};
+
+class NodeStatusTest : public testing::TestWithParam<bool> {
+protected:
+    static constexpr int64_t initial_node_mtime = 200;
+
+    void SetUp() override {
+        txn_kv = create_txn_kv();
+        resource_mgr = std::make_shared<NodeStatusResourceManager>(txn_kv);
+        ASSERT_EQ(resource_mgr->init(), 0);
+        auto rate_limiter = std::make_shared<RateLimiter>();
+        auto snapshot = std::make_shared<SnapshotManager>(txn_kv);
+        meta_service = std::make_unique<MetaServiceProxy>(
+                std::make_unique<MetaServiceImpl>(txn_kv, resource_mgr, 
rate_limiter, snapshot));
+
+        instance.set_instance_id("node_status_instance");
+        for (int cluster_idx = 0; cluster_idx < 2; ++cluster_idx) {
+            auto* cluster = instance.add_clusters();
+            cluster->set_cluster_id("cluster_" + std::to_string(cluster_idx));
+            cluster->set_cluster_name("cluster_" + 
std::to_string(cluster_idx));
+            cluster->set_type(ClusterPB::COMPUTE);
+            for (int node_idx = 0; node_idx < 3; ++node_idx) {
+                auto* node = cluster->add_nodes();
+                node->set_cloud_unique_id(node_idx == 0
+                                                  ? "unique_node_" + 
std::to_string(cluster_idx)
+                                                  : "shared_node");
+                GetParam() ? node->set_host("host_" + 
std::to_string(cluster_idx))
+                           : node->set_ip("127.0.0." + 
std::to_string(cluster_idx + 1));
+                node->set_heartbeat_port(10000 + node_idx);
+                node->set_status(NodeStatusPB::NODE_STATUS_RUNNING);
+                node->set_ctime(100);
+                node->set_mtime(initial_node_mtime);
+            }
+        }
+        std::unique_ptr<Transaction> txn;
+        ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+        txn->put(instance_key({instance.instance_id()}), 
instance.SerializeAsString());
+        ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+        resource_mgr->refresh_instance(instance.instance_id(), instance);
+    }
+
+    AlterClusterRequest request_for_node(int node_idx) {
+        AlterClusterRequest req;
+        req.set_instance_id(instance.instance_id());
+        req.mutable_cluster()->CopyFrom(instance.clusters(0));
+        req.mutable_cluster()->clear_nodes();
+        
req.mutable_cluster()->add_nodes()->CopyFrom(instance.clusters(0).nodes(node_idx));
+        return req;
+    }
+
+    void alter_node(const AlterClusterRequest& req, MetaServiceCode 
expected_code) {
+        brpc::Controller cntl;
+        AlterClusterResponse resp;
+        meta_service->alter_cluster(&cntl, &req, &resp, brpc::DoNothing());
+        ASSERT_FALSE(cntl.Failed());
+        ASSERT_EQ(resp.status().code(), expected_code) << resp.status().msg();
+    }
+
+    void check_nodes(NodeInfoPB* updated_node = nullptr) {
+        InstanceInfoPB persisted;
+        get_instance_info(meta_service.get(), &persisted, 
instance.instance_id());
+        ASSERT_EQ(persisted.clusters_size(), instance.clusters_size());
+        for (int i = 0; i < instance.clusters_size(); ++i) {
+            ASSERT_EQ(persisted.clusters(i).nodes_size(), 
instance.clusters(i).nodes_size());
+            for (int j = 0; j < instance.clusters(i).nodes_size(); ++j) {
+                const auto& expected_node = instance.clusters(i).nodes(j);
+                const auto& persisted_node = persisted.clusters(i).nodes(j);
+                if (&expected_node == updated_node) {
+                    EXPECT_GT(persisted_node.mtime(), initial_node_mtime);
+                    updated_node->set_mtime(persisted_node.mtime());
+                }
+                EXPECT_EQ(persisted_node.SerializeAsString(), 
expected_node.SerializeAsString());
+                EXPECT_EQ(get_instance_id(resource_mgr, 
expected_node.cloud_unique_id()),
+                          instance.instance_id());
+            }
+        }
+    }
+
+    std::shared_ptr<TxnKv> txn_kv;
+    InstanceInfoPB instance;
+    std::shared_ptr<NodeStatusResourceManager> resource_mgr;
+    std::unique_ptr<MetaServiceProxy> meta_service;
+};
+
+TEST_P(NodeStatusTest, KeepsNodesResolvableAtEveryPublishedUpdate) {
+    int refreshes = 0;
+    NodeInfoPB* updated_node = nullptr;
+    // Observe a reader immediately after each committed metadata/cache update.
+    resource_mgr->after_refresh = [&] {
+        ++refreshes;
+        EXPECT_EQ(get_instance_id(resource_mgr, "unique_node_0"), 
instance.instance_id());
+        check_nodes(updated_node);
+    };
+    for (int node_idx : {0, 1}) {
+        for (auto op :
+             {AlterClusterRequest::DECOMMISSION_NODE, 
AlterClusterRequest::NOTIFY_DECOMMISSIONED}) {
+            auto req = request_for_node(node_idx);
+            req.set_op(op);
+            updated_node = 
instance.mutable_clusters(0)->mutable_nodes(node_idx);
+            updated_node->set_status(op == 
AlterClusterRequest::DECOMMISSION_NODE
+                                             ? 
NodeStatusPB::NODE_STATUS_DECOMMISSIONING
+                                             : 
NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
+            int before_refreshes = refreshes;
+            alter_node(req, MetaServiceCode::OK);
+            EXPECT_EQ(refreshes, before_refreshes + 1);
+            check_nodes();
+            // Preserve HTTP requests that identify a cluster by the node 
rather than its ID.
+            req.mutable_cluster()->clear_cluster_id();
+            alter_node(req, MetaServiceCode::OK);
+            check_nodes();
+        }
+    }
+    resource_mgr->after_refresh = [] {};
+}
+
+TEST_P(NodeStatusTest, UpdatesNodesWithoutStoredClusterIdAtomically) {
+    for (bool explicit_empty_id : {false, true}) {
+        SCOPED_TRACE(explicit_empty_id);
+        auto* cluster = instance.mutable_clusters(0);
+        cluster->clear_cluster_id();
+        if (explicit_empty_id) {
+            cluster->set_cluster_id("");
+        }
+        std::unique_ptr<Transaction> txn;
+        ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+        txn->put(instance_key({instance.instance_id()}), 
instance.SerializeAsString());
+        ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+        resource_mgr->refresh_instance(instance.instance_id(), instance);
+
+        int refreshes = 0;
+        auto* updated_node = cluster->mutable_nodes(1);
+        resource_mgr->after_refresh = [&] {
+            ++refreshes;
+            check_nodes(updated_node);
+        };
+        for (auto op :
+             {AlterClusterRequest::DECOMMISSION_NODE, 
AlterClusterRequest::NOTIFY_DECOMMISSIONED}) {
+            SCOPED_TRACE(op);
+            // A shared UID must resolve this named cluster without changing 
the other cluster.
+            auto req = request_for_node(1);
+            req.mutable_cluster()->clear_cluster_name();
+            req.set_op(op);
+            updated_node->set_status(op == 
AlterClusterRequest::DECOMMISSION_NODE
+                                             ? 
NodeStatusPB::NODE_STATUS_DECOMMISSIONING
+                                             : 
NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
+            int before_refreshes = refreshes;
+            alter_node(req, MetaServiceCode::OK);
+            EXPECT_EQ(refreshes, before_refreshes + 1);
+            check_nodes();
+            InstanceInfoPB persisted;
+            get_instance_info(meta_service.get(), &persisted, 
instance.instance_id());
+            EXPECT_EQ(persisted.clusters(0).SerializeAsString(),
+                      instance.clusters(0).SerializeAsString());
+        }
+        resource_mgr->after_refresh = [] {};
+    }
+}
+
+TEST_P(NodeStatusTest, RejectsMissingNodesWithoutPublishingPartialUpdates) {
+    int refreshes = 0;
+    resource_mgr->after_refresh = [&] { ++refreshes; };
+    for (auto op :
+         {AlterClusterRequest::DECOMMISSION_NODE, 
AlterClusterRequest::NOTIFY_DECOMMISSIONED}) {
+        auto req = request_for_node(0);
+        req.set_op(op);
+        
req.mutable_cluster()->add_nodes()->CopyFrom(instance.clusters(0).nodes(1));
+        req.mutable_cluster()->mutable_nodes(1)->set_heartbeat_port(12345);
+        alter_node(req, MetaServiceCode::INVALID_ARGUMENT);
+        EXPECT_EQ(refreshes, 0);
+        check_nodes();
+
+        req = request_for_node(0);
+        req.set_op(op);
+        
req.mutable_cluster()->mutable_nodes(0)->set_cloud_unique_id("missing_uid");
+        alter_node(req, MetaServiceCode::INVALID_ARGUMENT);
+        EXPECT_EQ(refreshes, 0);
+        check_nodes();
+
+        req = request_for_node(0);
+        req.set_op(op);
+        req.mutable_cluster()->set_cluster_id("missing_cluster");
+        alter_node(req, MetaServiceCode::INVALID_ARGUMENT);
+        EXPECT_EQ(refreshes, 0);
+        check_nodes();
+    }
+    resource_mgr->after_refresh = [] {};
+}
+
+INSTANTIATE_TEST_SUITE_P(NodeAddress, NodeStatusTest, testing::Bool());
+
 // test cluster's node addr use ip
 TEST(ResourceTest, ModifyNodesIpTest) {
     auto meta_service = get_meta_service();
diff --git a/regression-test/data/cloud_p0/node_mgr/test_cloud_decommission.out 
b/regression-test/data/cloud_p0/node_mgr/test_cloud_decommission.out
new file mode 100644
index 00000000000..84e20211dba
--- /dev/null
+++ b/regression-test/data/cloud_p0/node_mgr/test_cloud_decommission.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you 
want to edit this
+-- !decommission_atomic --
+1      1
+
+-- !decommissioned_atomic --
+1      1
+
diff --git 
a/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy 
b/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy
index d2d4947f859..0033d06d11c 100644
--- a/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy
+++ b/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy
@@ -17,6 +17,7 @@
 import groovy.json.JsonSlurper
 import groovy.json.JsonOutput
 import org.apache.doris.regression.suite.ClusterOptions
+import java.util.concurrent.TimeUnit
 
 suite("cloud_decommission", 'p0, docker') {
     if (!isCloudMode()) {
@@ -51,7 +52,7 @@ suite("cloud_decommission", 'p0, docker') {
         }
     }
 
-    def check = { Closure beforeDecommissionActionSupplier, Closure 
afterDecommissionActionSupplier, int beNum ->
+    def checkDecommission = { Closure beforeDecommissionActionSupplier, 
Closure afterDecommissionActionSupplier, int beNum ->
         def begin = System.currentTimeMillis()
         setFeConfig("cloud_balance_tablet_percent_per_run", 0.5)
 
@@ -160,7 +161,7 @@ suite("cloud_decommission", 'p0, docker') {
 
     def checkDifferentAction = { Closure beforeDecommissionActionSupplier, 
Closure afterDecommissionActionSupplier, int atLeastCost, int waitTime, int 
beNum ->
         def begin = System.currentTimeMillis()
-        def cost = check.call(beforeDecommissionActionSupplier, 
afterDecommissionActionSupplier, beNum)
+        def cost = checkDecommission.call(beforeDecommissionActionSupplier, 
afterDecommissionActionSupplier, beNum)
         log.info("in check, inner cost {}", cost)
         cost = System.currentTimeMillis() - begin
         log.info("in check, outter cost {}", cost)
@@ -251,6 +252,114 @@ suite("cloud_decommission", 'p0, docker') {
         assertEquals(2, ret.size())
     }
 
+    // Keep the target RPC in flight while a node-state update is published.
+    // Both builds use the same gates: the old one publishes a missing node,
+    // while the fixed one publishes the same node with its new status.
+    for (String operation : ["decommission_node", "notify_decommissioned"]) {
+        def atomicOptions = new ClusterOptions()
+        atomicOptions.setFeNum(1)
+        atomicOptions.setBeNum(1)
+        atomicOptions.cloudMode = true
+        atomicOptions.feConfigs += 
['cloud_tablet_rebalancer_interval_second=3600']
+        docker(atomicOptions) {
+            sql "USE @compute_cluster"
+            sql "DROP TABLE IF EXISTS decommission_atomic_table"
+            sql """CREATE TABLE decommission_atomic_table (id INT)
+                   DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES 
("replication_num"="1")"""
+            // Wait for the automatic decommission checker to finish its first 
round.
+            // Its next round is an hour away, so only this test changes node 
status.
+            def feLog = new File(cluster.getFeByIndex(1).getLogFilePath())
+            awaitUntil(60) { feLog.text.contains("rebalancer sche interval 
3600 s") }
+
+            def be = (sql_return_maparray("SHOW BACKENDS"))[0]
+            def tag = new JsonSlurper().parseText(be.Tag as String)
+            def ms = cluster.getAllMetaservices().get(0)
+            def msLog = new File(ms.getLogFilePath())
+            def nodePoint = "ResourceManager::node_update:after_refresh"
+            def commitPoint = 
"commit_rowset::before_get_instance_id:${tag.cloud_unique_id}"
+            def inject = { String action, String point = "" ->
+                httpTest {
+                    op "get"
+                    endpoint "${ms.host}:${ms.httpPort}"
+                    uri 
"/MetaService/http/v1/injection_point?token=${token}&op=${action}&name=${point}&behavior=wait"
+                    check { code, response ->
+                        assertEquals(200, code)
+                        assertEquals("OK", parseJson(response).code)
+                    }
+                }
+            }
+            def waitForPoint = { String point, long offset ->
+                awaitUntil(60) {
+                    def log = new RandomAccessFile(msLog, "r")
+                    try {
+                        log.seek(offset)
+                        byte[] bytes = new byte[(int) (log.length() - offset)]
+                        log.readFully(bytes)
+                        new String(bytes, "UTF-8").contains("injection point 
waiting, point=${point}")
+                    } finally {
+                        log.close()
+                    }
+                }
+                logger.info("Reached MS injection point {}", point)
+            }
+            def alterStatus = { String action ->
+                httpTest {
+                    endpoint "${ms.host}:${ms.httpPort}"
+                    uri "/MetaService/http/${action}?token=${token}"
+                    body JsonOutput.toJson([
+                        instance_id: context.config.instanceId,
+                        cluster: [
+                            type: "COMPUTE",
+                            cluster_id: tag.compute_group_id,
+                            cluster_name: tag.compute_group_name,
+                            nodes: [[cloud_unique_id: tag.cloud_unique_id,
+                                     ip: be.Host, heartbeat_port: 
be.HeartbeatPort as int]]
+                        ]
+                    ])
+                    check { code, response ->
+                        assertEquals(200, code)
+                        assertEquals("OK", parseJson(response).code)
+                    }
+                }
+            }
+            def load = null
+            def alter = null
+            inject("enable")
+            try {
+                long loadOffset = msLog.length()
+                inject("set", commitPoint)
+                load = thread { sql "INSERT INTO decommission_atomic_table 
VALUES (1)" }
+                waitForPoint(commitPoint, loadOffset)
+                if (operation == "notify_decommissioned") {
+                    alterStatus("decommission_node")
+                }
+
+                long updateOffset = msLog.length()
+                inject("set", nodePoint)
+                alter = thread { alterStatus(operation) }
+                waitForPoint(nodePoint, updateOffset)
+                // The node-state request stays blocked until the load 
finishes.
+                inject("clear", commitPoint)
+                load.get(60, TimeUnit.SECONDS)
+                if (operation == "decommission_node") {
+                    qt_decommission_atomic "SELECT count(*), sum(id) FROM 
decommission_atomic_table"
+                } else {
+                    qt_decommissioned_atomic "SELECT count(*), sum(id) FROM 
decommission_atomic_table"
+                }
+            } finally {
+                inject("clear", commitPoint)
+                inject("clear", nodePoint)
+                inject("disable")
+                if (load != null && !load.isDone()) {
+                    load.get(60, TimeUnit.SECONDS)
+                }
+                if (alter != null) {
+                    alter.get(60, TimeUnit.SECONDS)
+                }
+            }
+        }
+    }
+
     def clusterOptions = [
         new ClusterOptions(),
         new ClusterOptions(),
@@ -279,7 +388,7 @@ suite("cloud_decommission", 'p0, docker') {
         def noWalCheckCost = 0
         docker(clusterOptions[i]) {
             createTestTable.call()
-            noWalCheckCost = check(null, null, beNum)
+            noWalCheckCost = checkDecommission(null, null, beNum)
             checkFENormalAfterRestart.call()
         }
         log.info("no wal check cost {}", noWalCheckCost)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to