Copilot commented on code in PR #3750:
URL: https://github.com/apache/brpc/pull/3750#discussion_r4640668948


##########
src/brpc/rdma/rdma_endpoint.cpp:
##########
@@ -2012,34 +2063,85 @@ int RdmaEndpoint::PollingModeInitialize(bthread_tag_t 
tag,
 
         return nullptr;
     };
-    for (int i = 0; i < FLAGS_rdma_poller_num; ++i) {
-        auto args = new FnArgs{&pollers[i], &running};
+    int creation_error = 0;
+    for (size_t i = 0; i < pollers.size(); ++i) {
+        std::unique_ptr<FnArgs> args(
+            new (std::nothrow) FnArgs{&pollers[i], &group});
+        if (!args) {
+            creation_error = ENOMEM;
+            break;
+        }
         auto attr = FLAGS_rdma_disable_bthread ? BTHREAD_ATTR_PTHREAD
                                                : BTHREAD_ATTR_NORMAL;
         attr.tag = tag;
         bthread_attr_set_name(&attr, "RdmaPolling");
         pollers[i].callback = callback;
         pollers[i].init_fn = init_fn;
         pollers[i].release_fn = release_fn;
-        auto rc = bthread_start_background(&pollers[i].tid, &attr, fn, args);
+#if defined(UNIT_TEST)
+        if (g_before_rdma_poller_start_for_test) {
+            g_before_rdma_poller_start_for_test();
+        }
+        const int rc = g_start_rdma_poller_for_test(
+#else
+        const int rc = bthread_start_background(
+#endif
+            &pollers[i].tid, &attr, fn, args.get());
         if (rc != 0) {
-            LOG(ERROR) << "Fail to start rdma polling bthread";
-            return -1;
+            pollers[i].tid = INVALID_BTHREAD;
+            creation_error = rc;
+            break;
+        }
+        args.release();
+    }
+    if (creation_error != 0) {
+        group.state = PollerGroup::FAILED;
+        group.error = creation_error;
+        group.startup_result->store(PollerGroup::ABORTED,
+                                    butil::memory_order_release);
+        bthread::butex_wake_all(group.startup_result);
+        for (auto& poller : pollers) {
+            if (poller.tid != INVALID_BTHREAD) {
+                bthread_join(poller.tid, nullptr);
+                poller.tid = INVALID_BTHREAD;
+            }
         }
+        LOG(ERROR) << "Fail to start rdma polling bthread: "
+                   << berror(creation_error);
+        errno = creation_error;
+        return -1;
     }
+    running.store(true, std::memory_order_relaxed);
+    group.state = PollerGroup::STARTED;
+    group.startup_result->store(PollerGroup::SUCCEEDED,
+                                butil::memory_order_release);
+    bthread::butex_wake_all(group.startup_result);
     return 0;
 }
 
 void RdmaEndpoint::PollingModeRelease(bthread_tag_t tag) {
     if (!FLAGS_rdma_use_polling) {
         return;
     }
+    if (tag < 0 || static_cast<size_t>(tag) >= _poller_groups.size()) {
+        return;
+    }
     auto& group = _poller_groups[tag];
+    BAIDU_SCOPED_LOCK(group.lifecycle_mutex);
+    // Preserve the original creation error, including after final shutdown.
+    if (group.state == PollerGroup::FAILED ||
+        group.state == PollerGroup::RELEASED) {
+        return;
+    }
+    group.state = PollerGroup::RELEASED;
+    group.error = ESHUTDOWN;
     auto& pollers = group.pollers;
-    auto& running = group.running;
-    running.store(false, std::memory_order_relaxed);
-    for (int i = 0; i < FLAGS_rdma_poller_num; ++i) {
-        bthread_join(pollers[i].tid, nullptr);
+    group.running.store(false, std::memory_order_relaxed);
+    for (auto& poller : pollers) {
+        if (poller.tid != INVALID_BTHREAD) {
+            bthread_join(poller.tid, nullptr);

Review Comment:
   Holding `lifecycle_mutex` across this join can deadlock shutdown if a poller 
callback calls `PollingModeInitialize` for the same tag, including indirectly 
through RDMA channel initialization. The callback waits for the mutex while 
shutdown waits for that poller to exit. Publish shutdown under the lock, then 
join without holding it; coordinate shutdown completion separately so 
concurrent release calls still wait until cleanup finishes.



##########
test/brpc_rdma_unittest.cpp:
##########
@@ -3569,6 +3577,277 @@ TEST(RdmaInlineDataTest, should_post_inline) {
     EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0));
 }
 
+// Poller lifecycle tests do not need any RDMA queues or hardware.
+static std::atomic<int> g_poller_start_count{0};
+static int g_poller_fail_at = -1;
+
+static int StartRdmaPollerForTest(bthread_t* tid, const bthread_attr_t* attr,
+                                 void* (*fn)(void*), void* arg) {
+    if (g_poller_start_count.fetch_add(1) == g_poller_fail_at) {
+        return EAGAIN;
+    }
+    return bthread_start_background(tid, attr, fn, arg);
+}
+
+class RdmaPollingTest : public ::testing::TestWithParam<int> {
+protected:
+    void SetUp() override {
+        _saved_use_polling = rdma::FLAGS_rdma_use_polling;
+        _saved_poller_num = rdma::FLAGS_rdma_poller_num;
+        _saved_poller_yield = rdma::FLAGS_rdma_poller_yield;
+        _saved_disable_bthread = rdma::FLAGS_rdma_disable_bthread;
+        _saved_start_poller = rdma::g_start_rdma_poller_for_test;
+        rdma::FLAGS_rdma_use_polling = true;
+        rdma::FLAGS_rdma_poller_num = 3;
+        rdma::FLAGS_rdma_poller_yield = true;
+        _saved_groups.swap(rdma::RdmaEndpoint::_poller_groups);
+        ResetGroups();
+    }
+
+    void TearDown() override {
+        rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT);
+        rdma::g_start_rdma_poller_for_test = _saved_start_poller;
+        rdma::g_before_rdma_poller_start_for_test = nullptr;
+        _saved_groups.swap(rdma::RdmaEndpoint::_poller_groups);
+        rdma::FLAGS_rdma_use_polling = _saved_use_polling;
+        rdma::FLAGS_rdma_poller_num = _saved_poller_num;
+        rdma::FLAGS_rdma_poller_yield = _saved_poller_yield;
+        rdma::FLAGS_rdma_disable_bthread = _saved_disable_bthread;
+    }
+
+    void ResetGroups() {
+        std::vector<rdma::RdmaEndpoint::PollerGroup> fresh(1);
+        rdma::RdmaEndpoint::_poller_groups.swap(fresh);
+        g_poller_start_count = 0;
+        g_poller_fail_at = -1;
+        _initialized = 0;
+        _released = 0;
+        _callbacks = 0;
+        rdma::g_start_rdma_poller_for_test = StartRdmaPollerForTest;
+        rdma::g_before_rdma_poller_start_for_test = nullptr;
+    }
+
+    int Initialize(const std::function<void()>& init_fn = nullptr) {
+        auto callback = [this] { _callbacks.fetch_add(1); };
+        auto release_fn = [this] { _released.fetch_add(1); };
+        return rdma::RdmaEndpoint::PollingModeInitialize(
+            BTHREAD_TAG_DEFAULT, callback, init_fn, release_fn);
+    }
+
+    void ExpectStopped() {
+        const auto& group =
+            rdma::RdmaEndpoint::_poller_groups[BTHREAD_TAG_DEFAULT];
+        EXPECT_FALSE(group.running.load());
+        for (const auto& poller : group.pollers) {
+            EXPECT_EQ(INVALID_BTHREAD, poller.tid);
+        }
+    }
+
+    std::atomic<int> _initialized{0};
+    std::atomic<int> _released{0};
+    std::atomic<int> _callbacks{0};
+
+private:
+    bool _saved_use_polling;
+    int _saved_poller_num;
+    bool _saved_poller_yield;
+    bool _saved_disable_bthread;
+    decltype(&bthread_start_background) _saved_start_poller;
+    std::vector<rdma::RdmaEndpoint::PollerGroup> _saved_groups;
+};
+
+TEST_P(RdmaPollingTest, failed_creation_is_permanent) {
+    for (bool disable_bthread : {false, true}) {
+        SCOPED_TRACE(disable_bthread);
+        ResetGroups();
+        rdma::FLAGS_rdma_disable_bthread = disable_bthread;
+        g_poller_fail_at = GetParam();
+        auto init_fn = [this] { _initialized.fetch_add(1); };
+        EXPECT_EQ(-1, Initialize(init_fn));
+        EXPECT_EQ(EAGAIN, errno);
+        EXPECT_EQ(GetParam() + 1, g_poller_start_count.load());
+        ExpectStopped();
+        EXPECT_EQ(0, _initialized.load());
+        EXPECT_EQ(0, _released.load());
+        EXPECT_EQ(0, _callbacks.load());
+        EXPECT_EQ(rdma::RdmaEndpoint::PollerGroup::ABORTED,
+                  
rdma::RdmaEndpoint::_poller_groups[0].startup_result->load());
+
+        g_poller_fail_at = -1;
+        EXPECT_EQ(-1, Initialize(init_fn));
+        EXPECT_EQ(EAGAIN, errno);
+        rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT);
+        EXPECT_EQ(-1, Initialize(init_fn));
+        EXPECT_EQ(EAGAIN, errno);
+        EXPECT_EQ(GetParam() + 1, g_poller_start_count.load());
+    }
+}
+
+class RdmaPollingConcurrencyTest : public RdmaPollingTest {};
+
+TEST_F(RdmaPollingConcurrencyTest, initialize_waits_for_rollback_result) {
+    for (bool disable_bthread : {false, true}) {
+        SCOPED_TRACE(disable_bthread);
+        ResetGroups();
+        rdma::FLAGS_rdma_disable_bthread = disable_bthread;
+        auto init_fn = [this] { _initialized.fetch_add(1); };
+        g_poller_fail_at = 1;
+        bthread::CountdownEvent creation_paused;
+        bthread::CountdownEvent resume_creation;
+        std::atomic<int> before_start_calls{0};
+        rdma::g_before_rdma_poller_start_for_test = [&] {
+            if (before_start_calls.fetch_add(1) == 0) {
+                creation_paused.signal();
+                resume_creation.wait();
+            }
+        };
+        int rc = 0;
+        int competing_rc = 0;
+        int competing_errno = 0;
+        std::thread initializer([&] { rc = Initialize(init_fn); });
+        EXPECT_EQ(0, creation_paused.timed_wait(butil::seconds_from_now(5)));
+        auto& group = rdma::RdmaEndpoint::_poller_groups[0];
+        std::thread competing_initializer([&] {
+            competing_rc = Initialize(init_fn);
+            competing_errno = errno;
+        });
+        // The competing call must wait for rollback instead of returning
+        // success while its poller is about to be stopped.
+        ASSERT_FALSE(group.lifecycle_mutex.try_lock());

Review Comment:
   If this assertion fails, it returns with both `std::thread`s still joinable, 
causing `std::terminate` and aborting the test binary before creation is 
resumed. Unlock if `try_lock` succeeds and use a nonfatal expectation so the 
resume and join operations still run. Apply the same change at 
`test/brpc_rdma_unittest.cpp:3754`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to