chenBright commented on code in PR #3144: URL: https://github.com/apache/brpc/pull/3144#discussion_r4522096724
########## src/brpc/policy/baidu_rpc_protocol_gpu.cpp: ########## @@ -0,0 +1,312 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#if BRPC_WITH_GDR + +#include <google/protobuf/descriptor.h> // MethodDescriptor +#include <google/protobuf/message.h> // Message +#include <google/protobuf/io/zero_copy_stream_impl_lite.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/text_format.h> + +#include "butil/gpu/gpu_block_pool.h" +#include "butil/iobuf.h" // butil::IOBuf +#include "butil/logging.h" // LOG() +#include "butil/memory/scope_guard.h" +#include "butil/raw_pack.h" // RawPacker RawUnpacker +#include "butil/strings/string_util.h" + +#include "json2pb/json_to_pb.h" +#include "json2pb/pb_to_json.h" +#include "brpc/controller.h" // Controller +#include "brpc/socket.h" // Socket +#include "brpc/server.h" // Server +#include "brpc/span.h" +#include "brpc/compress.h" // ParseFromCompressedData +#include "brpc/checksum.h" +#include "brpc/stream_impl.h" +#include "brpc/rpc_dump.h" // SampledRequest +#include "brpc/rpc_pb_message_factory.h" +#include "brpc/policy/baidu_rpc_meta.pb.h" // RpcRequestMeta +#include "brpc/policy/baidu_rpc_protocol.h" +#include "brpc/policy/most_common_message.h" +#include "brpc/policy/streaming_rpc_protocol.h" +#include "brpc/details/usercode_backup_pool.h" +#include "brpc/details/controller_private_accessor.h" +#include "brpc/details/server_private_accessor.h" + +namespace brpc { +namespace policy { + +// Notes: +// 1. 12-byte header [PRPC][body_size][meta_size] +// 2. body_size and meta_size are in network byte order +// 3. Use service->full_name() + method_name to specify the method to call +// 4. `attachment_size' is set iff request/response has attachment +// 5. Not supported: chunk_info + +// Pack header into `buf' + +const int header_size = 12; +// if we recv data into gpu, the header/meta/body will be copied to cpu and processed. +// in to to limit the count of d2h, we will prefetch some bytes from gpu to cpu. +// if header_size + meta_size + body_size(without attachment) is less than the +// prefetch size, then one d2h is enough for one rpc. +// The prefetch size is derived from the CPU block pool's block size so that the +// allocation never exceeds the allocator's single-block limit and stays effective +// if the block size changes. + +static const int kDefaultPrefetchD2HSize = 512; + +static int GetPrefetchD2HSize(butil::gdr::BlockPoolAllocator* host_allocator) { + const int64_t block_size = host_allocator->get_block_size(); + return block_size > 0 ? static_cast<int>(block_size) + : kDefaultPrefetchD2HSize; +} + +ParseResult ParseRpcMessageGpu(butil::IOBuf* source, Socket* socket, + bool /*read_eof*/, const void*) { + + char header_buf[12]; + size_t n = 0; + uint32_t body_size; + uint32_t meta_size; + ParseError pe = PARSE_OK; + + void* prefetch_d2h_data = NULL; + bool is_gpu_memory = source->is_gpu_memory(); + if (!is_gpu_memory) { + // A non-GPU buffer here means the message arrived through a non-GDR + // path; reject it instead of aborting the whole process. + LOG(ERROR) << "RpcMessage is not in gpu!!!"; + return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG); + } + butil::gdr::BlockPoolAllocator* host_allocator = butil::gdr::BlockPoolAllocators::GetInstance()->get_cpu_allocator(); + const int prefetch_d2h_size = GetPrefetchD2HSize(host_allocator); + prefetch_d2h_data = host_allocator->AllocateRaw(prefetch_d2h_size); + if (prefetch_d2h_data == NULL) { + // Pool exhausted or single-block limit hit: ask the caller to retry + // instead of crashing the whole server. + LOG(ERROR) << "alloc host data for prefetch d2h failed, size=" + << prefetch_d2h_size; + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + + // n is the bytes we real frefetch, n maybe less than prefetch_d2h_size; + n = source->copy_from_gpu(prefetch_d2h_data, prefetch_d2h_size); + size_t copy_size = n > 12 ? 12 : n; + memcpy(header_buf, prefetch_d2h_data, copy_size); + + do { + // NOTE: unlike the CPU ParseRpcMessage, malformed GPU frames must NOT + // return PARSE_ERROR_TRY_OTHERS: InputMessengerProcessor would then + // dispatch the same GPU-resident IOBuf to non-GDR parsers whose + // copy_to cannot dereference device memory and would crash. A GDR + // connection only speaks baidu_std, so treat this as terminal. + if (n >= 4) { + void* dummy = header_buf; + if (*(const uint32_t*)dummy != *(const uint32_t*)"PRPC") { + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } else { + if (memcmp(header_buf, "PRPC", n) != 0) { + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } + if (n < sizeof(header_buf)) { + pe = PARSE_ERROR_NOT_ENOUGH_DATA; + break; + } + butil::RawUnpacker(header_buf + 4).unpack32(body_size).unpack32(meta_size); + if (body_size > FLAGS_max_body_size) { + // We need this log to report the body_size to give users some clues + // which is not printed in InputMessenger. + LOG(ERROR) << "body_size=" << body_size << " from " + << socket->remote_side() << " is too large"; + pe = PARSE_ERROR_TOO_BIG_DATA; + break; + } else if (source->length() < sizeof(header_buf) + body_size) { + pe = PARSE_ERROR_NOT_ENOUGH_DATA; + break; + } + if (meta_size > body_size) { + LOG(ERROR) << "meta_size=" << meta_size << " is bigger than body_size=" + << body_size; + // Pop the message + source->pop_front(sizeof(header_buf) + body_size); + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } while (0); + + if (pe != PARSE_OK) { + host_allocator->DeallocateRaw(prefetch_d2h_data); + return MakeParseError(pe); + } + + source->pop_front(sizeof(header_buf)); + MostCommonMessage* msg = MostCommonMessage::Get(); + + if (header_size + meta_size <= n) { + auto deleter = [host_allocator, prefetch_d2h_data](void* data) { host_allocator->DeallocateRaw(prefetch_d2h_data); }; + // `n` is the number of bytes actually prefetched from GPU. It is stored + // in the meta's `data_meta' so that FillReqBufGpu/FillResBufGpu can + // decide whether the prefetched host buffer also covers the body (and + // thus avoid a second D2H copy). This block is host memory (prefetched), + // NOT GPU memory, so we deliberately use append_user_data_with_meta + // (not append_user_gpu_data_with_meta) to keep is_gpu_memory() returning false. + if (msg->meta.append_user_data_with_meta( + (char*)prefetch_d2h_data + header_size, meta_size, deleter, n) != 0) { + // append failed (internal malloc): the prefetch buffer is NOT + // owned by anyone; release it back to the pool instead of leaking. + // The 12-byte header has already been popped from `source', so + // the stream can no longer be re-parsed from the frame header. + // Returning NOT_ENOUGH_DATA would make the caller wait for more + // input and re-parse starting at the meta bytes, permanently + // desynchronizing the stream; fail absolutely instead so the + // connection is closed. Return `msg' to its pool as well. + LOG(ERROR) << "Fail to append prefetched meta to IOBuf"; + host_allocator->DeallocateRaw(prefetch_d2h_data); + msg->Destroy(); + return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG); + } + source->pop_front(meta_size); + } else { + host_allocator->DeallocateRaw(prefetch_d2h_data); + if (source->cutn_from_gpu(&msg->meta, meta_size) != meta_size) { + // Host pool exhausted or the GPU copy failed: meta is NOT cut out + // of `source'. Continuing would make the following cutn treat the + // meta bytes as payload and silently corrupt the message. + LOG(ERROR) << "Fail to cut meta_size=" << meta_size << " from gpu"; + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); Review Comment: When this returns `PARSE_ERROR_NOT_ENOUGH_DATA`, the complete frame has already arrived, and line 163 has already removed the frame header. After the copy fails, parsing again starts at the metadata and cannot recover the original frame boundary; `msg` is also not freed. Please destroy the message and return a terminal parse error as in the small-metadata failure branch above, or ensure that the input is not consumed at all on failure. ########## src/brpc/policy/baidu_rpc_protocol_gpu.cpp: ########## @@ -0,0 +1,312 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#if BRPC_WITH_GDR + +#include <google/protobuf/descriptor.h> // MethodDescriptor +#include <google/protobuf/message.h> // Message +#include <google/protobuf/io/zero_copy_stream_impl_lite.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/text_format.h> + +#include "butil/gpu/gpu_block_pool.h" +#include "butil/iobuf.h" // butil::IOBuf +#include "butil/logging.h" // LOG() +#include "butil/memory/scope_guard.h" +#include "butil/raw_pack.h" // RawPacker RawUnpacker +#include "butil/strings/string_util.h" + +#include "json2pb/json_to_pb.h" +#include "json2pb/pb_to_json.h" +#include "brpc/controller.h" // Controller +#include "brpc/socket.h" // Socket +#include "brpc/server.h" // Server +#include "brpc/span.h" +#include "brpc/compress.h" // ParseFromCompressedData +#include "brpc/checksum.h" +#include "brpc/stream_impl.h" +#include "brpc/rpc_dump.h" // SampledRequest +#include "brpc/rpc_pb_message_factory.h" +#include "brpc/policy/baidu_rpc_meta.pb.h" // RpcRequestMeta +#include "brpc/policy/baidu_rpc_protocol.h" +#include "brpc/policy/most_common_message.h" +#include "brpc/policy/streaming_rpc_protocol.h" +#include "brpc/details/usercode_backup_pool.h" +#include "brpc/details/controller_private_accessor.h" +#include "brpc/details/server_private_accessor.h" + +namespace brpc { +namespace policy { + +// Notes: +// 1. 12-byte header [PRPC][body_size][meta_size] +// 2. body_size and meta_size are in network byte order +// 3. Use service->full_name() + method_name to specify the method to call +// 4. `attachment_size' is set iff request/response has attachment +// 5. Not supported: chunk_info + +// Pack header into `buf' + +const int header_size = 12; +// if we recv data into gpu, the header/meta/body will be copied to cpu and processed. +// in to to limit the count of d2h, we will prefetch some bytes from gpu to cpu. +// if header_size + meta_size + body_size(without attachment) is less than the +// prefetch size, then one d2h is enough for one rpc. +// The prefetch size is derived from the CPU block pool's block size so that the +// allocation never exceeds the allocator's single-block limit and stays effective +// if the block size changes. + +static const int kDefaultPrefetchD2HSize = 512; + +static int GetPrefetchD2HSize(butil::gdr::BlockPoolAllocator* host_allocator) { + const int64_t block_size = host_allocator->get_block_size(); + return block_size > 0 ? static_cast<int>(block_size) + : kDefaultPrefetchD2HSize; +} + +ParseResult ParseRpcMessageGpu(butil::IOBuf* source, Socket* socket, + bool /*read_eof*/, const void*) { + + char header_buf[12]; + size_t n = 0; + uint32_t body_size; + uint32_t meta_size; + ParseError pe = PARSE_OK; + + void* prefetch_d2h_data = NULL; + bool is_gpu_memory = source->is_gpu_memory(); + if (!is_gpu_memory) { + // A non-GPU buffer here means the message arrived through a non-GDR + // path; reject it instead of aborting the whole process. + LOG(ERROR) << "RpcMessage is not in gpu!!!"; + return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG); + } + butil::gdr::BlockPoolAllocator* host_allocator = butil::gdr::BlockPoolAllocators::GetInstance()->get_cpu_allocator(); + const int prefetch_d2h_size = GetPrefetchD2HSize(host_allocator); + prefetch_d2h_data = host_allocator->AllocateRaw(prefetch_d2h_size); + if (prefetch_d2h_data == NULL) { + // Pool exhausted or single-block limit hit: ask the caller to retry + // instead of crashing the whole server. + LOG(ERROR) << "alloc host data for prefetch d2h failed, size=" + << prefetch_d2h_size; + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + + // n is the bytes we real frefetch, n maybe less than prefetch_d2h_size; + n = source->copy_from_gpu(prefetch_d2h_data, prefetch_d2h_size); + size_t copy_size = n > 12 ? 12 : n; + memcpy(header_buf, prefetch_d2h_data, copy_size); + + do { + // NOTE: unlike the CPU ParseRpcMessage, malformed GPU frames must NOT + // return PARSE_ERROR_TRY_OTHERS: InputMessengerProcessor would then + // dispatch the same GPU-resident IOBuf to non-GDR parsers whose + // copy_to cannot dereference device memory and would crash. A GDR + // connection only speaks baidu_std, so treat this as terminal. + if (n >= 4) { + void* dummy = header_buf; + if (*(const uint32_t*)dummy != *(const uint32_t*)"PRPC") { + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } else { + if (memcmp(header_buf, "PRPC", n) != 0) { + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } + if (n < sizeof(header_buf)) { + pe = PARSE_ERROR_NOT_ENOUGH_DATA; + break; + } + butil::RawUnpacker(header_buf + 4).unpack32(body_size).unpack32(meta_size); + if (body_size > FLAGS_max_body_size) { + // We need this log to report the body_size to give users some clues + // which is not printed in InputMessenger. + LOG(ERROR) << "body_size=" << body_size << " from " + << socket->remote_side() << " is too large"; + pe = PARSE_ERROR_TOO_BIG_DATA; + break; + } else if (source->length() < sizeof(header_buf) + body_size) { + pe = PARSE_ERROR_NOT_ENOUGH_DATA; + break; + } + if (meta_size > body_size) { + LOG(ERROR) << "meta_size=" << meta_size << " is bigger than body_size=" + << body_size; + // Pop the message + source->pop_front(sizeof(header_buf) + body_size); + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } while (0); + + if (pe != PARSE_OK) { + host_allocator->DeallocateRaw(prefetch_d2h_data); + return MakeParseError(pe); + } + + source->pop_front(sizeof(header_buf)); + MostCommonMessage* msg = MostCommonMessage::Get(); + + if (header_size + meta_size <= n) { + auto deleter = [host_allocator, prefetch_d2h_data](void* data) { host_allocator->DeallocateRaw(prefetch_d2h_data); }; + // `n` is the number of bytes actually prefetched from GPU. It is stored + // in the meta's `data_meta' so that FillReqBufGpu/FillResBufGpu can + // decide whether the prefetched host buffer also covers the body (and + // thus avoid a second D2H copy). This block is host memory (prefetched), + // NOT GPU memory, so we deliberately use append_user_data_with_meta + // (not append_user_gpu_data_with_meta) to keep is_gpu_memory() returning false. + if (msg->meta.append_user_data_with_meta( + (char*)prefetch_d2h_data + header_size, meta_size, deleter, n) != 0) { + // append failed (internal malloc): the prefetch buffer is NOT + // owned by anyone; release it back to the pool instead of leaking. + // The 12-byte header has already been popped from `source', so + // the stream can no longer be re-parsed from the frame header. + // Returning NOT_ENOUGH_DATA would make the caller wait for more + // input and re-parse starting at the meta bytes, permanently + // desynchronizing the stream; fail absolutely instead so the + // connection is closed. Return `msg' to its pool as well. + LOG(ERROR) << "Fail to append prefetched meta to IOBuf"; + host_allocator->DeallocateRaw(prefetch_d2h_data); + msg->Destroy(); + return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG); + } + source->pop_front(meta_size); + } else { + host_allocator->DeallocateRaw(prefetch_d2h_data); + if (source->cutn_from_gpu(&msg->meta, meta_size) != meta_size) { + // Host pool exhausted or the GPU copy failed: meta is NOT cut out + // of `source'. Continuing would make the following cutn treat the + // meta bytes as payload and silently corrupt the message. + LOG(ERROR) << "Fail to cut meta_size=" << meta_size << " from gpu"; + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + } + source->cutn(&msg->payload, body_size - meta_size); + return MakeMessage(msg); +} + + +// Returns true on success. Returns false (nothing cut out of msg->payload) +// when the GPU -> host copy fails; the caller should fail the request. +bool FillReqBufGpu(butil::IOBuf* req_buf, MostCommonMessage* msg, int body_without_attachment_size) { + int meta_size = msg->meta.size(); + if (!msg->payload.is_gpu_memory()) { + // Should not happen on a GDR connection; do not crash the server. + LOG(ERROR) << "message is not on gpu!!!"; + return false; + } + int64_t real_prefetch_d2h_size = msg->meta.get_first_data_meta(); + if (header_size + meta_size + body_without_attachment_size <= real_prefetch_d2h_size) { + void* data = msg->meta.get_first_data_ptr(); + if (data == nullptr) { + LOG(ERROR) << "illegal prefetched data"; + return false; + } + req_buf->append((char*)data + meta_size, body_without_attachment_size); + msg->payload.pop_front(body_without_attachment_size); + } else { + if (msg->payload.cutn_from_gpu(req_buf, body_without_attachment_size) + != (size_t)body_without_attachment_size) { + LOG(ERROR) << "Fail to cut " << body_without_attachment_size + << " bytes from gpu"; + return false; + } + } + return true; +} + +// Returns true on success; on failure cntl is SetFailed and the caller must +// stop processing this response (res_buf_ptr is left untouched). +bool FillResBufGpu(butil::IOBuf* res_buf, MostCommonMessage* msg, const RpcMeta& meta, + butil::IOBuf** res_buf_ptr, Controller* cntl) { + const int res_size = msg->payload.length(); + int meta_size = msg->meta.size(); + if (!msg->payload.is_gpu_memory()) { + // Should not happen on a GDR connection; fail this response instead + // of crashing the whole server. + cntl->SetFailed(ERESPONSE, "response payload is not on gpu memory"); + return false; + } + if (meta.has_attachment_size()) { + // Mirror the CPU response path: attachment_size() < 0 must be + // rejected. Without this check a negative attachment_size makes + // body_without_attachment_size = res_size + 1, and the fast path + // would copy one byte beyond this frame (e.g. the 'P' of the next + // frame's PRPC header when the prefetch covers it) into the body + // and hand the corrupted response to the caller as a success. + if (meta.attachment_size() < 0 || + meta.attachment_size() > res_size) { + cntl->SetFailed( + ERESPONSE, "Invalid attachment_size=%d (response_size=%d)", + meta.attachment_size(), res_size); + return false; + } + int body_without_attachment_size = res_size - meta.attachment_size(); + + int64_t real_prefetch_d2h_size = msg->meta.get_first_data_meta(); + if (header_size + meta_size + body_without_attachment_size <= real_prefetch_d2h_size) { + void* data = msg->meta.get_first_data_ptr(); + if (data == nullptr) { + cntl->SetFailed(ERESPONSE, "illegal prefetched data in response"); + return false; + } + res_buf->append((char*)data + meta_size, body_without_attachment_size); + msg->payload.pop_front(body_without_attachment_size); + } else { + if (msg->payload.cutn_from_gpu(res_buf, body_without_attachment_size) + != (size_t)body_without_attachment_size) { + cntl->SetFailed(ERESPONSE, + "Fail to cut %d bytes from gpu", + body_without_attachment_size); + return false; + } + } + *res_buf_ptr = res_buf; + // NOTE: even when checksum_with_attachment is on, the attachment is + // kept in GPU memory; the CRC is computed over a temporary host copy + // inside ExtendCrc32c (see crc32c_checksum.cpp), so the GPU -> GPU + // data path is preserved for GPU-aware callers. + cntl->response_attachment().swap(msg->payload); + } else { + int64_t real_prefetch_d2h_size = msg->meta.get_first_data_meta(); + if (header_size + meta_size + res_size <= real_prefetch_d2h_size) { + void* data = msg->meta.get_first_data_ptr(); + if (data == nullptr) { + cntl->SetFailed(ERESPONSE, "illegal prefetched data in response"); + return false; + } + res_buf->append((char*)data + meta_size, res_size); + msg->payload.pop_front(res_size); Review Comment: `append()` returns `-1` when host block allocation fails, but this code still calls `pop_front()` and returns success, discarding the only copy of the original body. The `SerializedResponse` path can even pass an empty body to the caller as a successful response. Please consume the payload only after the append succeeds, and fix the corresponding operations in the request fast path and responses with attachments as well. ########## src/brpc/policy/baidu_rpc_protocol_gpu.cpp: ########## @@ -0,0 +1,312 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#if BRPC_WITH_GDR + +#include <google/protobuf/descriptor.h> // MethodDescriptor +#include <google/protobuf/message.h> // Message +#include <google/protobuf/io/zero_copy_stream_impl_lite.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/text_format.h> + +#include "butil/gpu/gpu_block_pool.h" +#include "butil/iobuf.h" // butil::IOBuf +#include "butil/logging.h" // LOG() +#include "butil/memory/scope_guard.h" +#include "butil/raw_pack.h" // RawPacker RawUnpacker +#include "butil/strings/string_util.h" + +#include "json2pb/json_to_pb.h" +#include "json2pb/pb_to_json.h" +#include "brpc/controller.h" // Controller +#include "brpc/socket.h" // Socket +#include "brpc/server.h" // Server +#include "brpc/span.h" +#include "brpc/compress.h" // ParseFromCompressedData +#include "brpc/checksum.h" +#include "brpc/stream_impl.h" +#include "brpc/rpc_dump.h" // SampledRequest +#include "brpc/rpc_pb_message_factory.h" +#include "brpc/policy/baidu_rpc_meta.pb.h" // RpcRequestMeta +#include "brpc/policy/baidu_rpc_protocol.h" +#include "brpc/policy/most_common_message.h" +#include "brpc/policy/streaming_rpc_protocol.h" +#include "brpc/details/usercode_backup_pool.h" +#include "brpc/details/controller_private_accessor.h" +#include "brpc/details/server_private_accessor.h" + +namespace brpc { +namespace policy { + +// Notes: +// 1. 12-byte header [PRPC][body_size][meta_size] +// 2. body_size and meta_size are in network byte order +// 3. Use service->full_name() + method_name to specify the method to call +// 4. `attachment_size' is set iff request/response has attachment +// 5. Not supported: chunk_info + +// Pack header into `buf' + +const int header_size = 12; +// if we recv data into gpu, the header/meta/body will be copied to cpu and processed. +// in to to limit the count of d2h, we will prefetch some bytes from gpu to cpu. +// if header_size + meta_size + body_size(without attachment) is less than the +// prefetch size, then one d2h is enough for one rpc. +// The prefetch size is derived from the CPU block pool's block size so that the +// allocation never exceeds the allocator's single-block limit and stays effective +// if the block size changes. + +static const int kDefaultPrefetchD2HSize = 512; + +static int GetPrefetchD2HSize(butil::gdr::BlockPoolAllocator* host_allocator) { + const int64_t block_size = host_allocator->get_block_size(); + return block_size > 0 ? static_cast<int>(block_size) + : kDefaultPrefetchD2HSize; +} + +ParseResult ParseRpcMessageGpu(butil::IOBuf* source, Socket* socket, + bool /*read_eof*/, const void*) { + + char header_buf[12]; + size_t n = 0; + uint32_t body_size; + uint32_t meta_size; + ParseError pe = PARSE_OK; + + void* prefetch_d2h_data = NULL; + bool is_gpu_memory = source->is_gpu_memory(); + if (!is_gpu_memory) { + // A non-GPU buffer here means the message arrived through a non-GDR + // path; reject it instead of aborting the whole process. + LOG(ERROR) << "RpcMessage is not in gpu!!!"; + return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG); + } + butil::gdr::BlockPoolAllocator* host_allocator = butil::gdr::BlockPoolAllocators::GetInstance()->get_cpu_allocator(); + const int prefetch_d2h_size = GetPrefetchD2HSize(host_allocator); + prefetch_d2h_data = host_allocator->AllocateRaw(prefetch_d2h_size); + if (prefetch_d2h_data == NULL) { + // Pool exhausted or single-block limit hit: ask the caller to retry + // instead of crashing the whole server. + LOG(ERROR) << "alloc host data for prefetch d2h failed, size=" + << prefetch_d2h_size; + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + + // n is the bytes we real frefetch, n maybe less than prefetch_d2h_size; + n = source->copy_from_gpu(prefetch_d2h_data, prefetch_d2h_size); + size_t copy_size = n > 12 ? 12 : n; + memcpy(header_buf, prefetch_d2h_data, copy_size); + + do { + // NOTE: unlike the CPU ParseRpcMessage, malformed GPU frames must NOT + // return PARSE_ERROR_TRY_OTHERS: InputMessengerProcessor would then + // dispatch the same GPU-resident IOBuf to non-GDR parsers whose + // copy_to cannot dereference device memory and would crash. A GDR + // connection only speaks baidu_std, so treat this as terminal. + if (n >= 4) { + void* dummy = header_buf; + if (*(const uint32_t*)dummy != *(const uint32_t*)"PRPC") { + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } else { + if (memcmp(header_buf, "PRPC", n) != 0) { + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } + if (n < sizeof(header_buf)) { + pe = PARSE_ERROR_NOT_ENOUGH_DATA; + break; + } + butil::RawUnpacker(header_buf + 4).unpack32(body_size).unpack32(meta_size); + if (body_size > FLAGS_max_body_size) { + // We need this log to report the body_size to give users some clues + // which is not printed in InputMessenger. + LOG(ERROR) << "body_size=" << body_size << " from " + << socket->remote_side() << " is too large"; + pe = PARSE_ERROR_TOO_BIG_DATA; + break; + } else if (source->length() < sizeof(header_buf) + body_size) { + pe = PARSE_ERROR_NOT_ENOUGH_DATA; + break; + } + if (meta_size > body_size) { + LOG(ERROR) << "meta_size=" << meta_size << " is bigger than body_size=" + << body_size; + // Pop the message + source->pop_front(sizeof(header_buf) + body_size); + pe = PARSE_ERROR_ABSOLUTELY_WRONG; + break; + } + } while (0); + + if (pe != PARSE_OK) { + host_allocator->DeallocateRaw(prefetch_d2h_data); + return MakeParseError(pe); + } + + source->pop_front(sizeof(header_buf)); + MostCommonMessage* msg = MostCommonMessage::Get(); + + if (header_size + meta_size <= n) { + auto deleter = [host_allocator, prefetch_d2h_data](void* data) { host_allocator->DeallocateRaw(prefetch_d2h_data); }; + // `n` is the number of bytes actually prefetched from GPU. It is stored + // in the meta's `data_meta' so that FillReqBufGpu/FillResBufGpu can + // decide whether the prefetched host buffer also covers the body (and + // thus avoid a second D2H copy). This block is host memory (prefetched), + // NOT GPU memory, so we deliberately use append_user_data_with_meta + // (not append_user_gpu_data_with_meta) to keep is_gpu_memory() returning false. + if (msg->meta.append_user_data_with_meta( + (char*)prefetch_d2h_data + header_size, meta_size, deleter, n) != 0) { + // append failed (internal malloc): the prefetch buffer is NOT + // owned by anyone; release it back to the pool instead of leaking. + // The 12-byte header has already been popped from `source', so + // the stream can no longer be re-parsed from the frame header. + // Returning NOT_ENOUGH_DATA would make the caller wait for more + // input and re-parse starting at the meta bytes, permanently + // desynchronizing the stream; fail absolutely instead so the + // connection is closed. Return `msg' to its pool as well. + LOG(ERROR) << "Fail to append prefetched meta to IOBuf"; + host_allocator->DeallocateRaw(prefetch_d2h_data); + msg->Destroy(); + return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG); + } + source->pop_front(meta_size); + } else { + host_allocator->DeallocateRaw(prefetch_d2h_data); + if (source->cutn_from_gpu(&msg->meta, meta_size) != meta_size) { + // Host pool exhausted or the GPU copy failed: meta is NOT cut out + // of `source'. Continuing would make the following cutn treat the + // meta bytes as payload and silently corrupt the message. + LOG(ERROR) << "Fail to cut meta_size=" << meta_size << " from gpu"; + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + } + source->cutn(&msg->payload, body_size - meta_size); + return MakeMessage(msg); Review Comment: The payload returned here still holds GPU blocks, but `ProcessRpcRequest` passes it directly to `rpc_dump` before converting the body. `RpcDumpContext` then passes these addresses to a regular `writev`, so writing GPU requests fails and may even leave incomplete samples. Please convert sampled data to a host copy at the sampling boundary and handle copy failures. ########## example/rdma_performance/server.cpp: ########## @@ -56,7 +68,35 @@ class PerfTestServiceImpl : public PerfTestService { if (request->echo_attachment()) { brpc::Controller* cntl = static_cast<brpc::Controller*>(cntl_base); - cntl->response_attachment().append(cntl->request_attachment()); +#ifdef BRPC_WITH_GDR + if (FLAGS_use_gdr && FLAGS_consume_on_gpu) { + // Copy the received attachment to GPU memory owned by the + // application, so the app can consume it with CUDA kernels. + // The data is consumed locally and never sent back, so no + // RDMA memory region registration is needed. + // Works for both GPU (device-to-device) and host + // (host-to-device) sources. + const size_t size = cntl->request_attachment().size(); + if (size > 0) { + void* gpu_addr = nullptr; + if (cudaMalloc(&gpu_addr, size) != cudaSuccess) { + LOG(ERROR) << "Fail to alloc gpu memory for attachment"; + } else if (cntl->request_attachment().copy_from_gpu( + gpu_addr, size, 0, /*to_gpu=*/true) != size) { Review Comment: A GDR service can still handle ordinary TCP requests, in which case attachments are in host memory. However, `copy_from_gpu(..., true)` here ultimately always calls `cudaMemcpyDeviceToDevice`, so it cannot support H2D as the comment above says. Please select the copy direction based on the actual source memory type; the [CUDA documentation](https://docs.nvidia.com/cuda/archive/12.6.0/cuda-runtime-api/group__CUDART__MEMORY.html) defines a mismatch between the direction and the pointers as undefined behavior. ########## CMakeLists.txt: ########## @@ -86,6 +87,41 @@ if(WITH_DEBUG_SYMBOLS) list(APPEND BRPC_COMMON_COMPILE_OPTIONS -g) endif() +set(WITH_GDR_VAL "0") +if(WITH_GDR) + # GDR is a special mode of RDMA; it implies RDMA. + if(NOT WITH_RDMA) + message(STATUS "WITH_GDR=ON implies WITH_RDMA=ON") + set(WITH_RDMA ON) + endif() + set(WITH_GDR_VAL "1") + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "GDR (GPU Direct RDMA) is only supported on Linux") + endif() + message(STATUS "brpc compile with gdr") + # Honor CUDA_HOME if set by the user, otherwise fall back to common defaults. + if(DEFINED ENV{CUDA_HOME} AND NOT CUDA_HOME) + set(CUDA_HOME $ENV{CUDA_HOME}) + endif() + if(CUDA_HOME) + set(CUDA_INCLUDE_PATH ${CUDA_HOME}/include) + find_library(CUDA_LIB NAMES cuda PATHS ${CUDA_HOME}/lib64 ${CUDA_HOME}/lib NO_DEFAULT_PATH) + find_library(CUDART_LIB NAMES cudart PATHS ${CUDA_HOME}/lib64 ${CUDA_HOME}/lib NO_DEFAULT_PATH) Review Comment: `CUDA_HOME` points to the Toolkit, while `libcuda.so` may be installed by the driver in a system directory. `NO_DEFAULT_PATH` here excludes that directory, causing environments with the headers, Runtime, and driver all installed to still report that CUDA cannot be found. Please retain a fallback to system paths for the driver library and update the same logic in the example; see the [NVIDIA installation guide](https://docs.nvidia.com/cuda/archive/12.6.0/cuda-installation-guide-linux/index.html) for this layout. ########## src/butil/CMakeLists.txt: ########## @@ -289,6 +290,11 @@ if(WITH_GLOG) list(PREPEND butil_libraries ${GLOG_LIB}) set(butil_private_libs "-lglog ${butil_private_libs}") endif() +if(WITH_GDR) + # gpu_block_pool.cpp is a no-op translation unit without BRPC_WITH_GDR; + # only link CUDA libs when GDR is actually enabled. + list(APPEND butil_libraries ${CUDA_LIB} ${CUDART_LIB}) +endif() Review Comment: This only updates the internal CMake link dependencies and does not update `butil_private_libs`. As a result, the installed `brpc.pc` lacks the CUDA dependencies, and external programs linking the GDR version using only `pkg-config --static --libs brpc` will encounter unresolved CUDA symbols. Please also export the required libraries and search paths, and check a standalone consumer after installation. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
