randomkang commented on code in PR #3144:
URL: https://github.com/apache/brpc/pull/3144#discussion_r4525855808
##########
example/rdma_performance/server.cpp:
##########
@@ -56,7 +68,35 @@ class PerfTestServiceImpl : public PerfTestService {
if (request->echo_attachment()) {
brpc::Controller* cntl =
static_cast<brpc::Controller*>(cntl_base);
- cntl->response_attachment().append(cntl->request_attachment());
+#ifdef BRPC_WITH_GDR
+ if (FLAGS_use_gdr && FLAGS_consume_on_gpu) {
+ // Copy the received attachment to GPU memory owned by the
+ // application, so the app can consume it with CUDA kernels.
+ // The data is consumed locally and never sent back, so no
+ // RDMA memory region registration is needed.
+ // Works for both GPU (device-to-device) and host
+ // (host-to-device) sources.
+ const size_t size = cntl->request_attachment().size();
+ if (size > 0) {
+ void* gpu_addr = nullptr;
+ if (cudaMalloc(&gpu_addr, size) != cudaSuccess) {
+ LOG(ERROR) << "Fail to alloc gpu memory for
attachment";
+ } else if (cntl->request_attachment().copy_from_gpu(
+ gpu_addr, size, 0, /*to_gpu=*/true) != size) {
Review Comment:
Right, when gdr connection fallback to tcp, the attachment will be on host
memory. We should consider this situation.Code has been updated.
##########
CMakeLists.txt:
##########
@@ -86,6 +87,41 @@ if(WITH_DEBUG_SYMBOLS)
list(APPEND BRPC_COMMON_COMPILE_OPTIONS -g)
endif()
+set(WITH_GDR_VAL "0")
+if(WITH_GDR)
+ # GDR is a special mode of RDMA; it implies RDMA.
+ if(NOT WITH_RDMA)
+ message(STATUS "WITH_GDR=ON implies WITH_RDMA=ON")
+ set(WITH_RDMA ON)
+ endif()
+ set(WITH_GDR_VAL "1")
+ if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ message(FATAL_ERROR "GDR (GPU Direct RDMA) is only supported on Linux")
+ endif()
+ message(STATUS "brpc compile with gdr")
+ # Honor CUDA_HOME if set by the user, otherwise fall back to common
defaults.
+ if(DEFINED ENV{CUDA_HOME} AND NOT CUDA_HOME)
+ set(CUDA_HOME $ENV{CUDA_HOME})
+ endif()
+ if(CUDA_HOME)
+ set(CUDA_INCLUDE_PATH ${CUDA_HOME}/include)
+ find_library(CUDA_LIB NAMES cuda PATHS ${CUDA_HOME}/lib64
${CUDA_HOME}/lib NO_DEFAULT_PATH)
+ find_library(CUDART_LIB NAMES cudart PATHS ${CUDA_HOME}/lib64
${CUDA_HOME}/lib NO_DEFAULT_PATH)
Review Comment:
ok
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]