Copilot commented on code in PR #3761:
URL: https://github.com/apache/brpc/pull/3761#discussion_r4625558238
##########
test/brpc_rdma_unittest.cpp:
##########
@@ -3995,6 +4004,193 @@ TEST(RdmaInlineDataTest, should_post_inline) {
EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0));
}
+
+// Regression tests for the endpoint lifetime: the endpoint must outlive the
+// CQ Socket, and PollCq() must not dereference a recycled main Socket.
+namespace {
+
+static int FakeGetCqEvent(ibv_comp_channel*, ibv_cq**, void**) {
+ errno = EAGAIN;
+ return -1;
+}
+static void FakeAckCqEvents(ibv_cq*, unsigned int) {}
+static int FakeDestroyCq(ibv_cq*) { return 0; }
+static int FakeDestroyQp(ibv_qp*) { return 0; }
+static int FakeDestroyCompChannel(ibv_comp_channel*) { return 0; }
+static int FakeModifyQp(ibv_qp*, ibv_qp_attr*, ibv_qp_attr_mask) { return 0; }
+
+
+struct IbvMockGuard {
+ decltype(rdma::IbvGetCqEvent) get_cq_event;
+ decltype(rdma::IbvAckCqEvents) ack_cq_events;
+ decltype(rdma::IbvDestroyCq) destroy_cq;
+ decltype(rdma::IbvDestroyQp) destroy_qp;
+ decltype(rdma::IbvDestroyCompChannel) destroy_comp_channel;
+ decltype(rdma::IbvModifyQp) modify_qp;
+ int32_t prepared_qp_cnt;
+ bool use_polling;
+
+ IbvMockGuard()
+ : get_cq_event(rdma::IbvGetCqEvent)
+ , ack_cq_events(rdma::IbvAckCqEvents)
+ , destroy_cq(rdma::IbvDestroyCq)
+ , destroy_qp(rdma::IbvDestroyQp)
+ , destroy_comp_channel(rdma::IbvDestroyCompChannel)
+ , modify_qp(rdma::IbvModifyQp)
+ , prepared_qp_cnt(rdma::FLAGS_rdma_prepared_qp_cnt)
+ , use_polling(rdma::FLAGS_rdma_use_polling) {
+ rdma::IbvGetCqEvent = FakeGetCqEvent;
+ rdma::IbvAckCqEvents = FakeAckCqEvents;
+ rdma::IbvDestroyCq = FakeDestroyCq;
+ rdma::IbvDestroyQp = FakeDestroyQp;
+ rdma::IbvDestroyCompChannel = FakeDestroyCompChannel;
+ rdma::IbvModifyQp = FakeModifyQp;
+ rdma::FLAGS_rdma_prepared_qp_cnt = 0;
+ rdma::FLAGS_rdma_use_polling = false;
+ }
+
+ ~IbvMockGuard() {
+ rdma::IbvGetCqEvent = get_cq_event;
+ rdma::IbvAckCqEvents = ack_cq_events;
+ rdma::IbvDestroyCq = destroy_cq;
+ rdma::IbvDestroyQp = destroy_qp;
+ rdma::IbvDestroyCompChannel = destroy_comp_channel;
+ rdma::IbvModifyQp = modify_qp;
+ rdma::FLAGS_rdma_prepared_qp_cnt = prepared_qp_cnt;
+ rdma::FLAGS_rdma_use_polling = use_polling;
+ }
+};
+
+// A main RDMA Socket with a mocked resource, plus its CQ Socket.
+struct FakeRdmaConnection {
+ IbvMockGuard mock_guard;
+ ibv_context fake_ctx;
+ ibv_cq fake_recv_cq;
+ ibv_cq fake_send_cq;
+ ibv_comp_channel fake_comp_channel;
+ int comp_fd;
+ SocketId main_sid;
+ SocketUniquePtr main_socket;
+ RdmaTransport* transport;
+ rdma::RdmaEndpoint* ep;
+
+ FakeRdmaConnection()
+ : fake_ctx()
+ , fake_recv_cq()
+ , fake_send_cq()
+ , fake_comp_channel()
+ , comp_fd(eventfd(0, 0))
+ , transport(nullptr)
+ , ep(nullptr) {
+ fake_recv_cq.context = &fake_ctx;
+ fake_send_cq.context = &fake_ctx;
+ fake_comp_channel.fd = comp_fd;
+ }
+ ~FakeRdmaConnection() {
+ if (comp_fd >= 0) {
+ close(comp_fd);
+ }
+ }
+
+ int Setup() {
+ SocketOptions main_options;
+ main_options.socket_mode = SOCKET_MODE_RDMA;
+ if (Socket::Create(main_options, &main_sid) != 0) {
+ return -1;
+ }
+ if (Socket::Address(main_sid, &main_socket) != 0) {
+ return -1;
+ }
+ transport = static_cast<RdmaTransport*>(main_socket->_transport.get());
+ ep = transport->_rdma_ep;
+ if (ep == nullptr) {
+ return -1;
+ }
+ ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED,
+ butil::memory_order_release);
+ rdma::RdmaResource* res = new rdma::RdmaResource();
+ res->recv_cq = &fake_recv_cq;
+ res->send_cq = &fake_send_cq;
+ res->comp_channel = &fake_comp_channel;
+ ep->_resource = res;
+ return ep->StartCqEvents();
+ }
+};
+
+} // namespace
+
+TEST_F(RdmaTest, endpoint_outlives_cq_socket_and_reset) {
+ const int live_before = rdma::g_live_endpoint_count.load();
+
+ // Case 1: the main Socket is recycled while a PollCq callback still holds
+ // a CQ Socket reference. The endpoint must outlive it, and a late callback
+ // must return safely.
+ {
+ FakeRdmaConnection conn;
+ ASSERT_GE(conn.comp_fd, 0);
+ ASSERT_EQ(0, conn.Setup());
+ rdma::RdmaEndpoint* ep = conn.ep;
+ ASSERT_NE(INVALID_SOCKET_ID, ep->_cq_sid);
+
+ // Hold a CQ Socket ref, as an in-flight PollCq would.
+ SocketUniquePtr cq_socket;
+ ASSERT_EQ(0, Socket::Address(ep->_cq_sid, &cq_socket));
+
+ // Recycle the main Socket as at exit: the drain must wait for that
ref.
+ butil::atomic<bool> main_recycled(false);
+ std::thread recycler([&] {
+ conn.main_socket->ReleaseAdditionalReference();
+ conn.main_socket.reset();
+ main_recycled.store(true);
+ });
+ ASSERT_TRUE(WaitUntil([&] { return cq_socket->Failed(); }));
+
+ rdma::RdmaEndpoint::PollCq(cq_socket.get());
Review Comment:
This call reaches `ibv_poll_cq` directly; unlike the mocked
`IbvGetCqEvent`/destroy hooks, it is not redirected through a test seam. The
test supplies zero-initialized stack `ibv_context`/`ibv_cq` objects, so
libibverbs can dereference an uninitialized provider callback here and crash
before the lifetime assertions run. Add an injectable poll wrapper and mock it,
or drive this test with a real CQ.
##########
src/brpc/rdma/rdma_endpoint.cpp:
##########
@@ -1475,6 +1482,9 @@ int RdmaEndpoint::StartCqEvents() {
return -1;
}
+ // Before Create(): OnCreated() may call BeforeRecycle() before returning.
+ _nref.fetch_add(1, butil::memory_order_relaxed);
Review Comment:
The extra endpoint reference is acquired before `Socket::Create`, but this
failure path does not balance it. `Socket::OnCreated()` can fail before it
assigns `options.user` (for example, `_io_event.Init()` fails), so
`BeforeRecycle()` is never called to execute `DropRef()`. `StartCqEvents()`
then returns `-1` with `_nref` still one higher; later transport release drops
only the transport reference and leaves the endpoint leaked (and the new
live-endpoint counter never returns to baseline). Track whether the CQ socket
took ownership of this reference and release it on create failure when it did
not, while avoiding a second drop when `BeforeRecycle()` already ran.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]