This is an automated email from the ASF dual-hosted git repository.
deardeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6f38079e391 [fix](cloud) Hide token cloud_unique_id and ak/sk in meta
service logs (#70006)
6f38079e391 is described below
commit 6f38079e3914d55a57e2431b835573f6256a3e50
Author: deardeng <[email protected]>
AuthorDate: Thu Nov 19 20:38:21 2026 +0800
[fix](cloud) Hide token cloud_unique_id and ak/sk in meta service logs
(#70006)
Related PR: #69732
Problem Summary: Meta service and recycler write secrets to their logs,
and some into error messages, in plaintext:
- http admin token: `MetaServiceImpl::http` formats the whole http
request (every query string and header) and logs it at INFO for every
request, successful or not, so the `token` in the query string, which
can act on every instance, reaches the log. When the token check fails,
meta service and recycler also echo the supplied token in the 403 body
and the WARNING log.
- cloud_unique_id: since #69732 a cloud_unique_id
`${version}:${instance_id}:${node_secret}` authenticates a node, but
`begin_rpc` / `finish_rpc` log the request and response debug strings
(e.g. every `get_cluster` logs the cloud_unique_id of the caller and of
every node in the returned cluster), the http request log contains it,
about 60 log lines and error messages print it, and about 40 dump an
instance, cluster or node PB that contains it.
- ak/sk: when `alter_obj_store_info`, `alter_storage_vault`,
`alter_ram_user` or `update_ak_sk` rejects a request, the error message
embeds the request as JSON, ak/sk included; `finish_rpc` logs it and the
caller receives it. `begin_rpc` hides sk only for CreateInstance,
CreateStage and AlterObjStoreInfo, so `update_ak_sk` and
`alter_ram_user` log the sk.
Fix:
1. Mask the `token` query value as `***` in the logged http request, and
drop the echoed token from the 403 message (keep `token=(not given)`
when no token is passed).
2. Add `hide_cloud_unique_id()`, which keeps
`${version}:${instance_id}:` for debugging and replaces the node secret
with `***`, and `hide_cloud_unique_ids()`, which applies it to every
cloud_unique_id(s) value in a protobuf debug string, a JSON string or a
`cloud_unique_id=xxx` message. Apply the latter once to every debug
string in `begin_rpc` / `finish_rpc` and to the logged http request, and
apply them at the other log lines and error messages.
3. Add `request_to_json_for_log()`, which applies `encryt_sk()`,
`hide_ak()` and `hide_cloud_unique_ids()` to the request JSON, and use
it wherever meta_service_resource.cpp puts a request in an error message
or a log. Add UpdateAkSkRequest and AlterRamUserRequest to the requests
whose sk `begin_rpc` hides.
`hide_cloud_unique_ids()` only scans for "cloud_unique_id" and copies
the string once when there is a match. Compared with the
`ShortDebugString()` it wraps, on a Release build it adds 0.65us to a
91-byte request, 0.65ms (7%) to a 1.7MB request with one
cloud_unique_id, and 0.4ms (29%) to a `get_cluster` response with 2000
nodes.
Meta service logs and error messages show cloud_unique_id as
`1:<instance_id>:***`, the sk as an md5 and the ak partially hidden, and
no longer contain the http admin token. The 403 body of meta service and
recycler http APIs is `incorrect token` instead of `incorrect token,
token=xxx`.
---
cloud/src/common/http_helper.cpp | 27 +-
cloud/src/meta-service/meta_service.cpp | 64 ++--
cloud/src/meta-service/meta_service.h | 5 +-
cloud/src/meta-service/meta_service_helper.h | 163 ++++++++--
cloud/src/meta-service/meta_service_http.cpp | 50 ++-
cloud/src/meta-service/meta_service_job.cpp | 10 +-
cloud/src/meta-service/meta_service_resource.cpp | 314 ++++++++++---------
.../src/meta-service/meta_service_tablet_stats.cpp | 2 +-
cloud/src/meta-service/meta_service_txn.cpp | 71 ++---
cloud/src/recycler/recycler_service.cpp | 4 +-
cloud/src/resource-manager/resource_manager.cpp | 102 ++++---
cloud/test/meta_service_http_test.cpp | 336 ++++++++++++++++++++-
12 files changed, 797 insertions(+), 351 deletions(-)
diff --git a/cloud/src/common/http_helper.cpp b/cloud/src/common/http_helper.cpp
index 03f756f7e40..60c3ef7e6c9 100644
--- a/cloud/src/common/http_helper.cpp
+++ b/cloud/src/common/http_helper.cpp
@@ -54,9 +54,9 @@ static google::protobuf::util::Status
parse_json_message(const std::string& unre
static_assert(std::is_base_of_v<google::protobuf::Message, Message>);
auto st = google::protobuf::util::JsonStringToMessage(body, req);
if (!st.ok()) {
- std::string msg = "failed to strictly parse http request for '" +
unresolved_path +
- "' error: " + st.ToString();
- LOG_WARNING(msg).tag("body", encryt_sk(hide_access_key(body)));
+ LOG_WARNING("failed to strictly parse HTTP request")
+ .tag("path", unresolved_path)
+ .tag("parse_code", static_cast<int>(st.code()));
// ignore unknown fields
google::protobuf::util::JsonParseOptions json_parse_options;
@@ -66,16 +66,17 @@ static google::protobuf::util::Status
parse_json_message(const std::string& unre
return {};
}
-#define PARSE_MESSAGE_OR_RETURN(ctrl, req)
\
- do {
\
- std::string body = ctrl->request_attachment().to_string();
\
- auto& unresolved_path = ctrl->http_request().unresolved_path();
\
- auto st = parse_json_message(unresolved_path, body, &req);
\
- if (!st.ok()) {
\
- std::string msg = "parse http request '" + unresolved_path + "': "
+ st.ToString(); \
- LOG_WARNING(msg).tag("body", encryt_sk(hide_access_key(body)));
\
- return http_json_reply(MetaServiceCode::PROTOBUF_PARSE_ERR, msg);
\
- }
\
+#define PARSE_MESSAGE_OR_RETURN(ctrl, req)
\
+ do {
\
+ std::string body = ctrl->request_attachment().to_string();
\
+ auto& unresolved_path = ctrl->http_request().unresolved_path();
\
+ auto st = parse_json_message(unresolved_path, body, &req);
\
+ if (!st.ok()) {
\
+ std::string msg =
\
+ "parse http request '" + unresolved_path + "': invalid
request JSON"; \
+ LOG_WARNING(msg).tag("parse_code", static_cast<int>(st.code()));
\
+ return http_json_reply(MetaServiceCode::PROTOBUF_PARSE_ERR, msg);
\
+ }
\
} while (0)
const std::unordered_map<std::string_view, HttpHandlerInfo>&
get_http_handlers() {
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index 7ee10ce47bc..adfdbb45288 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -115,9 +115,7 @@ std::string get_instance_id(const
std::shared_ptr<ResourceManager>& rc_mgr,
// check it split by ':' c
auto [valid, id] =
ResourceManager::get_instance_id_by_cloud_unique_id(cloud_unique_id);
if (!valid) {
- LOG(WARNING) << "use degraded format cloud_unique_id, but
cloud_unique_id not degrade "
- "format, cloud_unique_id="
- << cloud_unique_id;
+ LOG(WARNING) << "cannot resolve instance_id: invalid degraded
cloud_unique_id format";
return "";
}
@@ -128,14 +126,14 @@ std::string get_instance_id(const
std::shared_ptr<ResourceManager>& rc_mgr,
if (code == TxnErrorCode::TXN_KEY_NOT_FOUND) {
LOG(WARNING)
<< "use degraded format cloud_unique_id, but instance
is not registered, "
- "cloud_unique_id="
- << cloud_unique_id;
+ "instance_id="
+ << id;
return "";
}
if (instance.has_status() && instance.status() ==
InstanceInfoPB::DELETED) {
msg = "instance status has been set delete, plz check it,
recycle_state=" +
InstanceRecycleState_Name(instance.recycle_state());
- LOG(WARNING) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(WARNING) << msg << ", instance_id=" << id;
return "";
}
}
@@ -144,8 +142,7 @@ std::string get_instance_id(const
std::shared_ptr<ResourceManager>& rc_mgr,
for (auto& node : nodes) {
if (!instance_id.empty() && instance_id != node.instance_id) {
- LOG(WARNING) << "cloud_unique_id is one-to-many instance_id, "
- << " cloud_unique_id=" << cloud_unique_id
+ LOG(WARNING) << "cloud_unique_id is one-to-many instance_id,"
<< " current_instance_id=" << instance_id
<< " later_instance_id=" << node.instance_id;
}
@@ -159,10 +156,12 @@ std::string get_instance_id(const
std::shared_ptr<ResourceManager>& rc_mgr,
}
if (id != node.instance_id || id != instance_id) {
- LOG(WARNING) << "in cache, node=" << node.node_info.DebugString()
- << ", cloud_unique_id=" << cloud_unique_id
- << " current_instance_id=" << instance_id
- << ", later_instance_id=" << node.instance_id;
+ LOG(WARNING)
+ << "cached node instance_id does not match the ID encoded
in cloud_unique_id"
+ << ", cached_instance_id=" << node.instance_id << ",
encoded_instance_id=" << id
+ << ", node_ip=" << node.node_info.ip()
+ << ", node_host=" << node.node_info.host()
+ << ", heartbeat_port=" << node.node_info.heartbeat_port();
continue;
}
}
@@ -313,7 +312,7 @@ void
MetaServiceImpl::get_version(::google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_version)
@@ -475,7 +474,7 @@ void
MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -1040,7 +1039,7 @@ void
MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -1224,7 +1223,7 @@ void
MetaServiceImpl::update_tablet(::google::protobuf::RpcController* controlle
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ LOG(WARNING) << msg;
return;
}
RPC_RATE_LIMIT(update_tablet)
@@ -1420,7 +1419,7 @@ void
MetaServiceImpl::get_tablet(::google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_tablet)
@@ -1615,7 +1614,7 @@ void
MetaServiceImpl::prepare_restore_job(::google::protobuf::RpcController* con
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -1791,7 +1790,7 @@ void
MetaServiceImpl::commit_restore_job(::google::protobuf::RpcController* cont
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -2305,7 +2304,7 @@ void
MetaServiceImpl::finish_restore_job(::google::protobuf::RpcController* cont
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -2587,7 +2586,7 @@ void
MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
doris::RowsetMetaCloudPB rowset_meta(request->rowset_meta());
@@ -2780,7 +2779,7 @@ void
MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
doris::RowsetMetaCloudPB rowset_meta(request->rowset_meta());
@@ -3019,7 +3018,7 @@ void
MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
doris::RowsetMetaCloudPB rowset_meta(request->rowset_meta());
@@ -3302,7 +3301,7 @@ void
MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_rowset)
@@ -3546,7 +3545,7 @@ void
MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_tablet_stats)
@@ -4483,7 +4482,7 @@ void
MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ LOG(WARNING) << msg;
return;
}
bool without_lock = request->has_without_lock() ? request->without_lock()
: false;
@@ -4752,7 +4751,7 @@ void
MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ LOG(WARNING) << msg;
return;
}
RPC_RATE_LIMIT(get_delete_bitmap)
@@ -6136,7 +6135,7 @@ void
MetaServiceImpl::remove_delete_bitmap(google::protobuf::RpcController* cont
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ LOG(WARNING) << msg;
return;
}
RPC_RATE_LIMIT(remove_delete_bitmap)
@@ -6187,9 +6186,8 @@ std::pair<MetaServiceCode, std::string>
MetaServiceImpl::get_instance_info(
// get instance_id by cloud_unique_id
cloned_instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (cloned_instance_id.empty()) {
- std::string msg =
- fmt::format("cannot find instance_id with
cloud_unique_id={}", cloud_unique_id);
- return {MetaServiceCode::INVALID_ARGUMENT, std::move(msg)};
+ return {MetaServiceCode::INVALID_ARGUMENT,
+ "cannot find instance_id from cloud_unique_id"};
}
}
@@ -6352,7 +6350,7 @@ void
MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ LOG(WARNING) << msg;
return;
}
@@ -6424,7 +6422,7 @@ void
MetaServiceImpl::update_packed_file_info(::google::protobuf::RpcController*
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
diff --git a/cloud/src/meta-service/meta_service.h
b/cloud/src/meta-service/meta_service.h
index ea07421e2ed..cff8e029e61 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -1201,13 +1201,11 @@ private:
auto dist =
std::uniform_int_distribution(-config::idempotent_request_replay_delay_range_ms,
config::idempotent_request_replay_delay_range_ms);
int64_t sleep_ms = config::idempotent_request_replay_delay_base_ms
+ dist(rng);
- std::string debug_string = req.ShortDebugString();
if constexpr (std::is_same_v<Request, GetTabletStatsRequest>) {
auto short_req = req;
if (short_req.tablet_idx_size() > 10) {
short_req.mutable_tablet_idx()->DeleteSubrange(10,
req.tablet_idx_size() - 10);
}
- debug_string = short_req.ShortDebugString();
TEST_SYNC_POINT_CALLBACK("idempotent_injection_short_debug_string_for_get_stats",
&short_req);
}
LOG(INFO) << " request_name=" << req.GetDescriptor()->name()
@@ -1216,8 +1214,7 @@ private:
<< " now_ts=" <<
duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count()
<< " idempotent_request_replay_delay_base_ms=" <<
config::idempotent_request_replay_delay_base_ms
<< " idempotent_request_replay_delay_range_ms=" <<
config::idempotent_request_replay_delay_range_ms
- << " idempotent_request_replay_delay_ms=" << sleep_ms
- << " request=" << debug_string;
+ << " idempotent_request_replay_delay_ms=" << sleep_ms;
if (sleep_ms < 0 || exclusion.count(req.GetDescriptor()->name()))
return;
brpc::Controller ctrl;
bthread_usleep(sleep_ms * 1000);
diff --git a/cloud/src/meta-service/meta_service_helper.h
b/cloud/src/meta-service/meta_service_helper.h
index 30c9cfce1fa..93cb7d1de16 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -19,11 +19,14 @@
#include <brpc/controller.h>
#include <fmt/core.h>
+#include <fmt/ranges.h>
#include <gen_cpp/cloud.pb.h>
#include <openssl/md5.h>
+#include <algorithm>
#include <iomanip>
#include <memory>
+#include <sstream>
#include <string>
#include <string_view>
#include <utility>
@@ -128,7 +131,7 @@ inline void process_ak_sk_pattern(std::string& str, const
std::string& pattern,
str.replace(value_start, value_end - value_start, key_value);
- pos = value_end + (key_value.length() - key_value.length());
+ pos = value_start + key_value.size() + 1;
}
};
@@ -162,6 +165,96 @@ inline std::string hide_ak(std::string debug_string) {
return debug_string;
}
+inline constexpr char kMaskedCloudUniqueId[] = "***MASKED***";
+
+inline void mask_cluster_cloud_unique_ids(ClusterPB& cluster) {
+ for (auto& node : *cluster.mutable_nodes()) {
+ if (node.has_cloud_unique_id()) {
+ node.set_cloud_unique_id(kMaskedCloudUniqueId);
+ }
+ }
+ auto& properties = *cluster.mutable_properties();
+ for (const char* key :
+ {"cloud_unique_id", "cloud_unique_ids", "cloudUniqueId",
"cloudUniqueIds"}) {
+ if (auto it = properties.find(key); it != properties.end()) {
+ it->second = kMaskedCloudUniqueId;
+ }
+ }
+}
+
+// Keep identifiers used to correlate RPC logs without dumping the message.
+template <class Message>
+std::string rpc_log_ids(const Message& message) {
+ if constexpr (std::is_same_v<Message, GetTabletRequest>) {
+ return fmt::format(" tablet_id={}", message.tablet_id());
+ } else if constexpr (std::is_same_v<Message, GetRowsetRequest>) {
+ return fmt::format(" tablet_id={} start_version={} end_version={}",
+ message.idx().tablet_id(), message.start_version(),
+ message.end_version());
+ } else if constexpr (std::is_same_v<Message, GetVersionRequest>) {
+ if (message.batch_mode()) {
+ return fmt::format(" db_ids=[{}] table_ids=[{}]
partition_ids=[{}]",
+ fmt::join(message.db_ids(), ","),
+ fmt::join(message.table_ids(), ","),
+ fmt::join(message.partition_ids(), ","));
+ }
+ return fmt::format(" db_id={} table_id={} partition_id={}",
message.db_id(),
+ message.table_id(), message.partition_id());
+ }
+
+ std::ostringstream log;
+ if constexpr (requires { message.has_instance_id(); }) {
+ if (message.has_instance_id()) {
+ log << " instance_id=" << message.instance_id();
+ }
+ }
+ if constexpr (requires { message.has_db_id(); }) {
+ if (message.has_db_id()) {
+ log << " db_id=" << message.db_id();
+ }
+ }
+ if constexpr (requires { message.has_table_id(); }) {
+ if (message.has_table_id()) {
+ log << " table_id=" << message.table_id();
+ }
+ }
+ if constexpr (requires { message.has_tablet_id(); }) {
+ if (message.has_tablet_id()) {
+ log << " tablet_id=" << message.tablet_id();
+ }
+ }
+ if constexpr (requires { message.has_txn_id(); }) {
+ if (message.has_txn_id()) {
+ log << " txn_id=" << message.txn_id();
+ }
+ }
+ if constexpr (requires { message.has_cluster_id(); }) {
+ if (message.has_cluster_id()) {
+ log << " cluster_id=" << message.cluster_id();
+ }
+ }
+ if constexpr (requires { message.has_job_id(); }) {
+ if (message.has_job_id()) {
+ log << " job_id=" << message.job_id();
+ }
+ }
+ if constexpr (requires { message.has_snapshot_id(); }) {
+ if (message.has_snapshot_id()) {
+ log << " snapshot_id=" << message.snapshot_id();
+ }
+ }
+ if constexpr (requires { message.has_stage_id(); }) {
+ if (message.has_stage_id()) {
+ log << " stage_id=" << message.stage_id();
+ }
+ }
+ return log.str();
+}
+
+template <class Request>
+inline constexpr bool is_credential_update_request =
+ std::is_same_v<Request, UpdateAkSkRequest> || std::is_same_v<Request,
AlterRamUserRequest>;
+
template <class Request, class Response>
void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const
Request* req,
Response* res) {
@@ -190,8 +283,7 @@ void begin_rpc(std::string_view func_name,
brpc::Controller* ctrl, const Request
std::is_same_v<Request, GetRowsetRequest> ||
std::is_same_v<Request, GetTabletRequest>) {
VLOG_DEBUG << "begin " << func_name << " remote_caller=" <<
ctrl->remote_side()
- << " original_client_ip=" << req->request_ip()
- << " request=" << req->ShortDebugString();
+ << " original_client_ip=" << req->request_ip() <<
rpc_log_ids(*req);
} else if constexpr (std::is_same_v<Request, RemoveDeleteBitmapRequest>) {
LOG(INFO) << "begin " << func_name << " remote_caller=" <<
ctrl->remote_side()
<< " original_client_ip=" << req->request_ip()
@@ -202,18 +294,37 @@ void begin_rpc(std::string_view func_name,
brpc::Controller* ctrl, const Request
<< " lock_id=" << req->lock_id() << " initiator=" <<
req->initiator()
<< " expiration=" << req->expiration()
<< " require_compaction_stats=" <<
req->require_compaction_stats();
+ } else if constexpr (std::is_same_v<Request, AlterClusterRequest>) {
+ LOG(INFO) << "begin " << func_name << " remote_caller=" <<
ctrl->remote_side()
+ << " original_client_ip=" << req->request_ip()
+ << " instance_id=" << req->instance_id() << " op=" <<
req->op()
+ << " cluster_id=" << req->cluster().cluster_id()
+ << " cluster_name=" << req->cluster().cluster_name()
+ << " node_count=" << req->cluster().nodes_size();
+ } else if constexpr (std::is_same_v<Request, CommitTxnRequest>) {
+ LOG(INFO) << "begin " << func_name << " remote_caller=" <<
ctrl->remote_side()
+ << " original_client_ip=" << req->request_ip() << " db_id="
<< req->db_id()
+ << " txn_id=" << req->txn_id() << " is_2pc=" << req->is_2pc()
+ << " sub_txn_count=" << req->sub_txn_infos_size();
+ } else if constexpr (is_credential_update_request<Request>) {
+ LOG(INFO) << "begin " << func_name << " instance_id=" <<
req->instance_id();
} else if constexpr (std::is_same_v<Request, CreateInstanceRequest> ||
std::is_same_v<Request, CreateStageRequest> ||
std::is_same_v<Request, AlterObjStoreInfoRequest>) {
- std::string debug_string = encryt_sk(req->ShortDebugString());
+ Request request_for_log = *req;
+ if constexpr (requires { request_for_log.has_cloud_unique_id(); }) {
+ if (request_for_log.has_cloud_unique_id()) {
+ request_for_log.set_cloud_unique_id(kMaskedCloudUniqueId);
+ }
+ }
+ std::string debug_string =
encryt_sk(request_for_log.ShortDebugString());
debug_string = hide_ak(debug_string);
TEST_SYNC_POINT_CALLBACK("sk_begin_rpc", &debug_string);
LOG(INFO) << "begin " << func_name << " remote_caller=" <<
ctrl->remote_side()
<< " original_client_ip=" << req->request_ip() << "
request=" << debug_string;
} else {
LOG(INFO) << "begin " << func_name << " remote_caller=" <<
ctrl->remote_side()
- << " original_client_ip=" << req->request_ip()
- << " request=" << req->ShortDebugString();
+ << " original_client_ip=" << req->request_ip() <<
rpc_log_ids(*req);
}
}
@@ -222,28 +333,29 @@ void finish_rpc(std::string_view func_name,
brpc::Controller* ctrl, const Reques
Response* res) {
if constexpr (std::is_same_v<Response, CommitTxnResponse>) {
LOG(INFO) << "finish " << func_name << " remote_caller=" <<
ctrl->remote_side()
- << " original_client_ip=" << req->request_ip()
- << " response=" << res->ShortDebugString();
- } else if constexpr (std::is_same_v<Response, GetRowsetResponse>) {
+ << " original_client_ip=" << req->request_ip() << " db_id="
<< req->db_id()
+ << " txn_id=" << req->txn_id() << " status=" <<
res->status().ShortDebugString()
+ << " txn_status=" << res->txn_info().status() << "
table_ids=["
+ << fmt::format("{}", fmt::join(res->table_ids(), ",")) << "]"
+ << " partition_ids=[" << fmt::format("{}",
fmt::join(res->partition_ids(), ","))
+ << "] versions=[" << fmt::format("{}",
fmt::join(res->versions(), ",")) << "]"
+ << " lazy_commit=" << res->is_lazy_commit()
+ << " lazy_commit_incomplete=" <<
res->is_lazy_commit_incomplete()
+ << " tso_fence=" << res->tso_fence();
+ } else if constexpr (std::is_same_v<Response, GetRowsetResponse> ||
+ std::is_same_v<Response, GetVersionResponse> ||
+ std::is_same_v<Response, GetTabletResponse>) {
LOG_IF(INFO, res->status().code() != MetaServiceCode::OK)
<< "finish " << func_name << " remote_caller=" <<
ctrl->remote_side()
- << " original_client_ip=" << req->request_ip()
- << " request=" << req->ShortDebugString()
+ << " original_client_ip=" << req->request_ip() <<
rpc_log_ids(*req)
<< " status=" << res->status().ShortDebugString();
} else if constexpr (std::is_same_v<Response, GetTabletStatsResponse>) {
LOG_IF(INFO, res->status().code() != MetaServiceCode::OK)
<< "finish " << func_name << " remote_caller=" <<
ctrl->remote_side()
<< " original_client_ip=" << req->request_ip()
- << " request=" << req->ShortDebugString()
+ << " requested_tablets=" << req->tablet_idx_size()
<< " status=" << res->status().ShortDebugString()
<< " num_tablets: " << res->tablet_stats().size();
- } else if constexpr (std::is_same_v<Response, GetVersionResponse> ||
- std::is_same_v<Response, GetTabletResponse>) {
- LOG_IF(INFO, res->status().code() != MetaServiceCode::OK)
- << "finish " << func_name << " remote_caller=" <<
ctrl->remote_side()
- << " request=" << req->ShortDebugString()
- << " original_client_ip=" << req->request_ip()
- << " response=" << res->ShortDebugString();
} else if constexpr (std::is_same_v<Response, GetDeleteBitmapResponse>) {
LOG(INFO) << "finish " << func_name << " remote_caller=" <<
ctrl->remote_side()
<< " original_client_ip=" << req->request_ip()
@@ -259,7 +371,15 @@ void finish_rpc(std::string_view func_name,
brpc::Controller* ctrl, const Reques
std::is_same_v<Response, GetInstanceResponse> ||
std::is_same_v<Response, BeginSnapshotResponse> ||
std::is_same_v<Response, CloneInstanceResponse>) {
- std::string debug_string = encryt_sk(res->DebugString());
+ Response response_for_log = *res;
+ if constexpr (std::is_same_v<Response, GetInstanceResponse>) {
+ if (response_for_log.has_instance()) {
+ for (auto& cluster :
*response_for_log.mutable_instance()->mutable_clusters()) {
+ mask_cluster_cloud_unique_ids(cluster);
+ }
+ }
+ }
+ std::string debug_string = encryt_sk(response_for_log.DebugString());
debug_string = hide_ak(debug_string);
TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string);
LOG(INFO) << "finish " << func_name << " remote_caller=" <<
ctrl->remote_side()
@@ -267,7 +387,8 @@ void finish_rpc(std::string_view func_name,
brpc::Controller* ctrl, const Reques
} else {
LOG(INFO) << "finish " << func_name << " remote_caller=" <<
ctrl->remote_side()
<< " original_client_ip=" << req->request_ip()
- << " response=" << res->ShortDebugString();
+ << " status=" << res->status().ShortDebugString() <<
rpc_log_ids(*req)
+ << rpc_log_ids(*res);
}
}
diff --git a/cloud/src/meta-service/meta_service_http.cpp
b/cloud/src/meta-service/meta_service_http.cpp
index 8db770c5b17..8191818f35e 100644
--- a/cloud/src/meta-service/meta_service_http.cpp
+++ b/cloud/src/meta-service/meta_service_http.cpp
@@ -26,16 +26,16 @@
#include <glog/logging.h>
#include <google/protobuf/message.h>
#include <google/protobuf/service.h>
-#include <google/protobuf/util/json_util.h>
#include <rapidjson/document.h>
-#include <rapidjson/error/en.h>
#include <rapidjson/prettywriter.h>
#include <rapidjson/stringbuffer.h>
+#include <algorithm>
#include <array>
#include <functional>
#include <memory>
#include <optional>
+#include <sstream>
#include <string>
#include <string_view>
#include <type_traits>
@@ -53,24 +53,6 @@ namespace doris::cloud {
extern void get_kv_range_boundaries_count(std::vector<std::string>&
partition_boundaries,
std::unordered_map<std::string,
size_t>& partition_count);
-template <typename Message>
-static google::protobuf::util::Status parse_json_message(const std::string&
unresolved_path,
- const std::string&
body, Message* req) {
- static_assert(std::is_base_of_v<google::protobuf::Message, Message>);
- auto st = google::protobuf::util::JsonStringToMessage(body, req);
- if (!st.ok()) {
- std::string msg = "failed to strictly parse http request for '" +
unresolved_path +
- "' error: " + st.ToString();
- LOG_WARNING(msg).tag("body", encryt_sk(hide_access_key(body)));
-
- // ignore unknown fields
- google::protobuf::util::JsonParseOptions json_parse_options;
- json_parse_options.ignore_unknown_fields = true;
- return google::protobuf::util::JsonStringToMessage(body, req,
json_parse_options);
- }
- return {};
-}
-
std::tuple<int, std::string_view> convert_ms_code_to_http_code(MetaServiceCode
ret) {
switch (ret) {
case OK:
@@ -151,7 +133,8 @@ static std::string format_http_request(brpc::Controller*
cntl) {
ss << "\nmethod=" << brpc::HttpMethod2Str(request.method());
ss << "\nquery strings:";
for (auto it = uri.QueryBegin(); it != uri.QueryEnd(); ++it) {
- ss << "\n" << it->first << "=" << it->second;
+ // The token is the admin credential of every instance, never log it
+ ss << "\n" << it->first << "=" << (it->first == "token" ? "***" :
it->second);
}
ss << "\nheaders:";
for (auto it = request.HeaderBegin(); it != request.HeaderEnd(); ++it) {
@@ -168,12 +151,11 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
auto* cntl = static_cast<brpc::Controller*>(controller);
brpc::ClosureGuard closure_guard(done);
- LOG(INFO) << "rpc from " << cntl->remote_side()
- << " request: " << cntl->http_request().uri().path();
- std::string http_request = format_http_request(cntl);
- std::string http_request_for_log = encryt_sk(http_request);
- http_request_for_log = hide_ak(http_request_for_log);
- const auto& unresolved_path = cntl->http_request().unresolved_path();
+ const auto& request = cntl->http_request();
+ const auto& path = request.uri().path();
+ LOG(INFO) << "rpc from " << cntl->remote_side() << " request: " << path;
+
+ const auto& unresolved_path = request.unresolved_path();
auto api_path = split_http_api_path(unresolved_path);
const auto& handlers = get_http_handlers();
auto it = handlers.find(api_path.route);
@@ -181,13 +163,14 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
// Auth
auto token = http_query(cntl->http_request().uri(), "token");
if (token != config::http_token) {
- std::string body = fmt::format("incorrect token, token={}",
- (token.empty() ? std::string_view("(not
given)") : token));
+ std::string body = token.empty() ? "incorrect token, token=(not
given)" : "incorrect token";
cntl->http_response().set_status_code(403);
cntl->response_attachment().append(body);
cntl->response_attachment().append("\n");
- LOG(WARNING) << "failed to handle http from " << cntl->remote_side()
- << " request: " << http_request_for_log << " msg: " <<
body;
+ LOG(WARNING) << "failed to handle http from " << cntl->remote_side()
<< " request: " << path
+ << " ret=403 msg: " << body;
+ VLOG_DEBUG << "http request details from " << cntl->remote_side()
+ << hide_ak(encryt_sk(format_http_request(cntl)));
return;
}
@@ -209,8 +192,9 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
int ret = cntl->http_response().status_code();
LOG(INFO) << (ret == 200 ? "succ to " : "failed to ") <<
__PRETTY_FUNCTION__ << " "
- << cntl->remote_side() << " request=\n"
- << http_request_for_log << "\n ret=" << ret << " msg=" << msg;
+ << cntl->remote_side() << " request: " << path << " ret=" << ret
<< " msg=" << msg;
+ VLOG_DEBUG << "http request details from " << cntl->remote_side()
+ << hide_ak(encryt_sk(format_http_request(cntl)));
}
} // namespace doris::cloud
diff --git a/cloud/src/meta-service/meta_service_job.cpp
b/cloud/src/meta-service/meta_service_job.cpp
index 34dda7ec920..a98cdcb90c5 100644
--- a/cloud/src/meta-service/meta_service_job.cpp
+++ b/cloud/src/meta-service/meta_service_job.cpp
@@ -561,9 +561,8 @@ void
MetaServiceImpl::start_tablet_job(::google::protobuf::RpcController* contro
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- SS << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = cloud_unique_id.empty() ? "cloud_unique_id is empty"
+ : "cannot find instance_id from
cloud_unique_id";
return;
}
RPC_RATE_LIMIT(start_tablet_job)
@@ -2274,9 +2273,8 @@ void
MetaServiceImpl::finish_tablet_job(::google::protobuf::RpcController* contr
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- SS << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = cloud_unique_id.empty() ? "cloud_unique_id is empty"
+ : "cannot find instance_id from
cloud_unique_id";
LOG(INFO) << msg;
return;
}
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index 6c5cd4c0fdb..6266434f839 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -25,10 +25,12 @@
#include <cctype>
#include <charconv>
#include <chrono>
+#include <initializer_list>
#include <numeric>
#include <queue>
#include <regex>
#include <string>
+#include <string_view>
#include <tuple>
#include <unordered_set>
@@ -88,7 +90,6 @@ static int encrypt_ak_sk_helper(const std::string plain_ak,
const std::string pl
MetaServiceCode& code, std::string& msg) {
std::string key;
int64_t key_id;
- LOG_INFO("enter encrypt_ak_sk_helper, plain_ak {}",
hide_access_key(plain_ak));
int ret = get_newest_encryption_key_for_ak_sk(&key_id, &key);
TEST_SYNC_POINT_CALLBACK("encrypt_ak_sk:get_encryption_key", &ret, &key,
&key_id);
if (ret != 0) {
@@ -425,17 +426,36 @@ static int
alter_instance_obj_store_info_by_id(InstanceInfoPB& instance,
return -1;
}
+static std::string invalid_field_names(
+ std::initializer_list<std::pair<bool, std::string_view>> fields) {
+ std::string names;
+ for (const auto& [invalid, name] : fields) {
+ if (invalid) {
+ if (!names.empty()) {
+ names += ", ";
+ }
+ names += name;
+ }
+ }
+ return names;
+}
+
+static std::string empty_ram_user_fields(const RamUserPB& ram_user) {
+ return invalid_field_names({{ram_user.user_id().empty(), "user_id"},
+ {ram_user.ak().empty(), "ak"},
+ {ram_user.sk().empty(), "sk"}});
+}
+
// Helper function to update AK/SK for a single instance
// Returns 0 on success, -1 on error
static int update_instance_ak_sk(InstanceInfoPB& instance, const
UpdateAkSkRequest* request,
- uint64_t time, MetaServiceCode& code,
std::string& msg,
- std::stringstream& update_record) {
+ uint64_t time, MetaServiceCode& code,
std::string& msg) {
// Update ram_user if has ram_user
if (request->has_ram_user()) {
- if (request->ram_user().user_id().empty() ||
request->ram_user().ak().empty() ||
- request->ram_user().sk().empty()) {
+ const auto& ram_user = request->ram_user();
+ if (auto fields = empty_ram_user_fields(ram_user); !fields.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "ram user info err " + proto_to_json(*request);
+ msg = "ram_user fields must not be empty: " + fields;
return -1;
}
if (!instance.has_ram_user()) {
@@ -443,7 +463,6 @@ static int update_instance_ak_sk(InstanceInfoPB& instance,
const UpdateAkSkReque
msg = "instance doesn't have ram user info";
return -1;
}
- auto& ram_user = request->ram_user();
EncryptionInfoPB encryption_info;
AkSkPair cipher_ak_sk_pair;
if (encrypt_ak_sk_helper(ram_user.ak(), ram_user.sk(),
&encryption_info, &cipher_ak_sk_pair,
@@ -457,9 +476,7 @@ static int update_instance_ak_sk(InstanceInfoPB& instance,
const UpdateAkSkReque
msg = "ram user_id err";
return -1;
}
- std::string old_ak = instance_ram_user.ak();
- std::string old_sk = instance_ram_user.sk();
- if (old_ak == ak && old_sk == sk) {
+ if (instance_ram_user.ak() == ak && instance_ram_user.sk() == sk) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "ak sk eq original, please check it";
return -1;
@@ -467,19 +484,18 @@ static int update_instance_ak_sk(InstanceInfoPB&
instance, const UpdateAkSkReque
instance_ram_user.set_ak(std::move(cipher_ak_sk_pair.first));
instance_ram_user.set_sk(std::move(cipher_ak_sk_pair.second));
instance_ram_user.mutable_encryption_info()->CopyFrom(encryption_info);
- update_record << "update ram_user's ak sk, instance_id: " <<
instance.instance_id()
- << " user_id: " << ram_user.user_id() << " old: cipher
ak: " << old_ak
- << " cipher sk: " << old_sk << " new: cipher ak: " << ak
- << " cipher sk: " << sk << "; ";
}
// Update internal_bucket_user
bool has_found_alter_obj_info = false;
- for (auto& alter_bucket_user : request->internal_bucket_user()) {
- if (!alter_bucket_user.has_ak() || !alter_bucket_user.has_sk() ||
- !alter_bucket_user.has_user_id()) {
+ for (int i = 0; i < request->internal_bucket_user_size(); ++i) {
+ const auto& alter_bucket_user = request->internal_bucket_user(i);
+ auto missing = invalid_field_names({{!alter_bucket_user.has_user_id(),
"user_id"},
+ {!alter_bucket_user.has_ak(),
"ak"},
+ {!alter_bucket_user.has_sk(),
"sk"}});
+ if (!missing.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 bucket info err " + proto_to_json(*request);
+ msg = fmt::format("internal_bucket_user[{}] is missing: {}", i,
missing);
return -1;
}
std::string user_id = alter_bucket_user.user_id();
@@ -493,16 +509,16 @@ static int update_instance_ak_sk(InstanceInfoPB&
instance, const UpdateAkSkReque
auto& obj_info =
const_cast<std::decay_t<decltype(instance.obj_info())>&>(instance.obj_info());
for (auto& it : obj_info) {
- std::string old_ak = it.ak();
- std::string old_sk = it.sk();
if (!it.has_user_id()) {
has_found_alter_obj_info = true;
// For compatibility, obj_info without a user_id only allow
// single internal_bucket_user to modify it.
if (request->internal_bucket_user_size() != 1) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "fail to update old instance's obj_info, s3 obj info
err " +
- proto_to_json(*request);
+ msg = fmt::format(
+ "updating obj_info without user_id requires
exactly one "
+ "internal_bucket_user, got {}",
+ request->internal_bucket_user_size());
return -1;
}
if (has_obj_credential(it) && (ak.empty() || sk.empty())) {
@@ -522,11 +538,6 @@ static int update_instance_ak_sk(InstanceInfoPB& instance,
const UpdateAkSkReque
it.set_sk(sk);
it.clear_credential();
it.mutable_encryption_info()->CopyFrom(encryption_info);
- update_record << "update obj_info's ak sk without user_id,
instance_id: "
- << instance.instance_id() << " obj_info_id: " <<
it.id()
- << " new user_id: " << user_id << " old: cipher
ak: " << old_ak
- << " cipher sk: " << old_sk << " new: cipher
ak: " << ak
- << " cipher sk: " << sk << "; ";
continue;
}
if (it.user_id() == user_id) {
@@ -547,10 +558,6 @@ static int update_instance_ak_sk(InstanceInfoPB& instance,
const UpdateAkSkReque
it.set_sk(sk);
it.clear_credential();
it.mutable_encryption_info()->CopyFrom(encryption_info);
- update_record << "update obj_info's ak sk, instance_id: " <<
instance.instance_id()
- << " obj_info_id: " << it.id() << " user_id: "
<< user_id
- << " old: cipher ak: " << old_ak << " cipher
sk: " << old_sk
- << " new: cipher ak: " << ak << " cipher sk: "
<< sk << "; ";
}
}
}
@@ -583,7 +590,7 @@ void
MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_obj_store_info)
@@ -812,11 +819,16 @@ static void create_object_info_with_encrypt(const
InstanceInfoPB& instance, Obje
return;
}
} else if (obj->has_role_arn()) {
- if (obj->role_arn().empty() || !obj->has_provider() ||
- obj->provider() != ObjectStoreInfoPB::S3 || bucket.empty() ||
endpoint.empty() ||
- region.empty()) {
+ auto invalid = invalid_field_names(
+ {{obj->role_arn().empty(), "role_arn"},
+ {!obj->has_provider() || obj->provider() !=
ObjectStoreInfoPB::S3,
+ "provider (must be S3)"},
+ {bucket.empty(), "bucket"},
+ {endpoint.empty(), "endpoint"},
+ {region.empty(), "region"}});
+ if (!invalid.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf info err with role_arn or cred provider, please
check it";
+ msg = "invalid object storage fields: " + invalid;
return;
}
if (!obj->has_cred_provider_type()) {
@@ -824,10 +836,16 @@ static void create_object_info_with_encrypt(const
InstanceInfoPB& instance, Obje
}
} else {
// ATTN: prefix may be empty
- if (plain_ak.empty() || plain_sk.empty() || bucket.empty() ||
endpoint.empty() ||
- region.empty() || !obj->has_provider() ||
external_endpoint.empty()) {
+ auto invalid = invalid_field_names({{plain_ak.empty(), "ak"},
+ {plain_sk.empty(), "sk"},
+ {bucket.empty(), "bucket"},
+ {endpoint.empty(), "endpoint"},
+ {region.empty(), "region"},
+ {!obj->has_provider(), "provider"},
+ {external_endpoint.empty(),
"external_endpoint"}});
+ if (!invalid.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf info err, please check it";
+ msg = "missing or empty object storage fields: " + invalid;
return;
}
@@ -1332,18 +1350,17 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
AkSkPair& cipher_ak_sk_pair) {
if (!request->has_obj() && (!request->has_vault() ||
!request->vault().has_obj_info())) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 obj info err " + proto_to_json(*request);
+ msg = "obj or vault.obj_info is required";
return -1;
}
const auto& obj = request->has_obj() ? request->obj() :
request->vault().obj_info();
- // obj size > 1k, refuse
- if (obj.ByteSizeLong() > 1024) {
+ if (const auto size = obj.ByteSizeLong(); size > 1024) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 obj info greater than 1k " + proto_to_json(*request);
+ msg = fmt::format("object storage configuration exceeds 1024 bytes,
got {}", size);
return -1;
- };
+ }
auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style, role_arn,
external_id] = obj_desc;
@@ -1362,9 +1379,11 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
use_credential_provider(obj)) ||
native_obj_add;
if (!obj.has_role_arn() && !use_credential_provider_for_add) {
- if (!obj.has_ak() || !obj.has_sk()) {
+ auto missing = invalid_field_names({{!obj.has_ak(), "ak"},
{!obj.has_sk(), "sk"}});
+ if (!missing.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 obj info err " + proto_to_json(*request);
+ msg = fmt::format("{} is missing: {}", request->has_obj() ? "obj"
: "vault.obj_info",
+ missing);
LOG(INFO) << msg;
return -1;
}
@@ -1471,23 +1490,22 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
// It should at least has one hdfs info or obj info inside storage
vault
if ((!request->has_vault())) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "storage vault is set " + proto_to_json(*request);
+ msg = "storage vault is required";
return;
}
break;
}
case AlterObjStoreInfoRequest::ADD_HDFS_INFO:
- case AlterObjStoreInfoRequest::DROP_HDFS_INFO: {
- if (!request->has_vault() || !request->vault().has_name()) {
+ case AlterObjStoreInfoRequest::DROP_HDFS_INFO:
+ case AlterObjStoreInfoRequest::SET_DEFAULT_VAULT: {
+ if (!request->has_vault()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "hdfs info is not found " + proto_to_json(*request);
+ msg = "vault is required";
return;
}
- } break;
- case AlterObjStoreInfoRequest::SET_DEFAULT_VAULT: {
- if (!request->has_vault() || !request->vault().has_name()) {
+ if (!request->vault().has_name()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "hdfs info is not found " + proto_to_json(*request);
+ msg = "vault.name is required";
return;
}
break;
@@ -1500,13 +1518,13 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
break;
case AlterObjStoreInfoRequest::UNKNOWN: {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "Unknown alter info " + proto_to_json(*request);
+ msg = "unknown alter operation";
return;
} break;
default:
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "Unknown alter obj store info, request info " +
proto_to_json(*request);
- LOG_WARNING("Unknown alter obj store info, request info {}",
request->DebugString());
+ msg = fmt::format("unknown alter operation, op={}",
static_cast<int>(request->op()));
+ LOG(WARNING) << msg;
return;
}
@@ -1523,7 +1541,7 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(alter_obj_store_info)
@@ -1583,7 +1601,7 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
auto& obj = request->has_obj() ? request->obj() :
request->vault().obj_info();
if (!obj.has_provider()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf lease provider info";
+ msg = "object storage provider is required";
return;
}
if (instance.obj_info().size() >= 10) {
@@ -1592,10 +1610,15 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
return;
}
// ATTN: prefix may be empty
- if (((ak.empty() || sk.empty()) && !use_credential_provider(obj)) ||
bucket.empty() ||
- endpoint.empty() || region.empty()) {
+ const bool requires_ak_sk = !use_credential_provider(obj);
+ auto invalid = invalid_field_names({{requires_ak_sk && ak.empty(),
"ak"},
+ {requires_ak_sk && sk.empty(),
"sk"},
+ {bucket.empty(), "bucket"},
+ {endpoint.empty(), "endpoint"},
+ {region.empty(), "region"}});
+ if (!invalid.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf info err, please check it";
+ msg = "missing or empty vault object storage fields: " + invalid;
return;
}
@@ -1752,7 +1775,7 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
}
LOG(INFO) << "instance " << instance_id << " has " <<
instance.obj_info().size()
- << " s3 history info, and instance = " <<
proto_to_json(instance);
+ << " s3 history entries";
val = instance.SerializeAsString();
if (val.empty()) {
@@ -1907,13 +1930,13 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
} break;
case AlterObjStoreInfoRequest::UNKNOWN: {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "Unknown alter info " + proto_to_json(*request);
+ msg = "unknown alter operation";
return;
} break;
default:
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "Unknown alter obj store info, request info " +
proto_to_json(*request);
- LOG_WARNING("Unknown alter obj store info, request info {}",
request->DebugString());
+ msg = fmt::format("unknown alter operation, op={}",
static_cast<int>(request->op()));
+ LOG(WARNING) << msg;
return;
}
@@ -1930,7 +1953,7 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(alter_obj_store_info)
@@ -2002,7 +2025,7 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
auto& obj = request->has_obj() ? request->obj() :
request->vault().obj_info();
if (!obj.has_provider()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf lease provider info";
+ msg = "object storage provider is required";
return;
}
if (instance.obj_info().size() >= 10) {
@@ -2010,11 +2033,16 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
msg = "this instance history has greater than 10 objs, please new
another instance";
return;
}
- // ATTN: prefix may be empty
- if (((ak.empty() || sk.empty()) && role_arn.empty() &&
!has_obj_credential(obj)) ||
- bucket.empty() || endpoint.empty() || region.empty() ||
prefix.empty()) {
+ const bool requires_ak_sk = role_arn.empty() &&
!has_obj_credential(obj);
+ auto invalid = invalid_field_names({{requires_ak_sk && ak.empty(),
"ak"},
+ {requires_ak_sk && sk.empty(),
"sk"},
+ {bucket.empty(), "bucket"},
+ {endpoint.empty(), "endpoint"},
+ {region.empty(), "region"},
+ {prefix.empty(), "prefix"}});
+ if (!invalid.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf info err, please check it";
+ msg = "missing or empty object storage fields: " + invalid;
return;
}
@@ -2052,7 +2080,7 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
}
LOG(INFO) << "instance " << instance_id << " has " <<
instance.obj_info().size()
- << " s3 history info, and instance = " <<
proto_to_json(instance);
+ << " s3 history entries";
val = instance.SerializeAsString();
if (val.empty()) {
@@ -2227,17 +2255,11 @@ void
MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller,
uint64_t time =
std::chrono::duration_cast<std::chrono::seconds>(now_time.time_since_epoch()).count();
- std::stringstream update_record;
-
// Update instance using helper function
- if (update_instance_ak_sk(instance, request, time, code, msg,
update_record) != 0) {
+ if (update_instance_ak_sk(instance, request, time, code, msg) != 0) {
return;
}
- LOG(INFO) << "instance " << instance_id << " has " <<
instance.obj_info().size()
- << " s3 history info, and " << instance.resource_ids_size() << "
vaults "
- << "instance = " << proto_to_json(instance);
-
val = instance.SerializeAsString();
if (val.empty()) {
msg = "failed to serialize";
@@ -2258,7 +2280,7 @@ void
MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller,
return;
}
- LOG(INFO) << update_record.str();
+ LOG(INFO) << "updated credentials for instance_id=" << instance_id;
async_notify_refresh_instance(txn_kv_, instance_id, true);
// Cascade update to derived instances using separate transactions
@@ -2321,11 +2343,10 @@ void
MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller,
}
// Update the cascade instance using helper function
- std::stringstream cascade_update_record;
MetaServiceCode cascade_code = MetaServiceCode::OK;
std::string cascade_msg;
- if (update_instance_ak_sk(cascade_instance, request, time,
cascade_code, cascade_msg,
- cascade_update_record) != 0) {
+ if (update_instance_ak_sk(cascade_instance, request, time,
cascade_code, cascade_msg) !=
+ 0) {
code = cascade_code;
msg = fmt::format("failed to update derived instance,
instance_id={}, msg={}",
cascade_id, cascade_msg);
@@ -2353,8 +2374,7 @@ void
MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller,
}
async_notify_refresh_instance(txn_kv_, cascade_id, true);
- LOG(INFO) << "cascade update for instance_id=" << cascade_id << " "
- << cascade_update_record.str();
+ LOG(INFO) << "updated credentials for derived instance_id=" <<
cascade_id;
}
}
@@ -2429,13 +2449,9 @@ void
MetaServiceImpl::create_instance_impl(google::protobuf::RpcController* cont
RPC_PREPROCESS(create_instance, get, put);
TEST_SYNC_POINT_RETURN_WITH_VOID("create_instance_sk_request_return");
if (request->has_ram_user()) {
- auto& ram_user = request->ram_user();
- std::string ram_user_id = ram_user.has_user_id() ? ram_user.user_id()
: "";
- std::string ram_user_ak = ram_user.has_ak() ? ram_user.ak() : "";
- std::string ram_user_sk = ram_user.has_sk() ? ram_user.sk() : "";
- if (ram_user_id.empty() || ram_user_ak.empty() || ram_user_sk.empty())
{
+ if (auto fields = empty_ram_user_fields(request->ram_user());
!fields.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "ram user info err, please check it";
+ msg = "ram_user fields must not be empty: " + fields;
return;
}
}
@@ -2552,7 +2568,7 @@ void
MetaServiceImpl::create_instance_impl(google::protobuf::RpcController* cont
obj_info.set_ak(hide_access_key(obj_info.ak()));
}
- LOG(INFO) << "xxx instance json=" << proto_to_json(instance);
+ LOG(INFO) << "create instance_id=" << instance_id;
txn->atomic_add(system_meta_service_instance_update_key(), 1);
txn->put(key, val);
@@ -2776,7 +2792,7 @@ static std::pair<MetaServiceCode, std::string>
drop_single_instance(const std::s
LOG(ERROR) << msg;
return {MetaServiceCode::PROTOBUF_SERIALIZE_ERR, std::move(msg)};
}
- LOG(INFO) << "drop instance_id=" << instance_id << " json=" <<
proto_to_json(*instance);
+ LOG(INFO) << "drop instance_id=" << instance_id;
if (instance->has_source_instance_id() &&
instance->has_source_snapshot_id() &&
!instance->source_instance_id().empty() &&
!instance->source_snapshot_id().empty()) {
@@ -2871,7 +2887,7 @@ static std::pair<MetaServiceCode, std::string>
drop_instance_chain(
return {MetaServiceCode::PROTOBUF_SERIALIZE_ERR, std::move(msg)};
}
LOG(INFO) << "drop instance_id=" << tail_instance_id << " and " <<
predecessors.size()
- << " predecessor instances, json=" <<
proto_to_json(*tail_instance);
+ << " predecessor instances";
return {MetaServiceCode::OK, std::move(serialized)};
}
@@ -2931,7 +2947,8 @@ void
MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller
StopWatch sw;
auto ctrl = static_cast<brpc::Controller*>(controller);
LOG(INFO) << __PRETTY_FUNCTION__ << " rpc from " << ctrl->remote_side()
- << " request=" << request->ShortDebugString();
+ << " instance_id=" << request->instance_id()
+ << " op=" << AlterInstanceRequest_Operation_Name(request->op());
brpc::ClosureGuard closure_guard(done);
MetaServiceCode code = MetaServiceCode::OK;
std::string msg = "OK";
@@ -2998,8 +3015,7 @@ void
MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller
LOG(ERROR) << msg;
return std::make_pair(MetaServiceCode::PROTOBUF_SERIALIZE_ERR,
msg);
}
- LOG(INFO) << "put instance_id=" << request->instance_id()
- << "rename instance json=" << proto_to_json(*instance);
+ LOG(INFO) << "rename instance_id=" << request->instance_id() << "
name=" << name;
return std::make_pair(MetaServiceCode::OK, ret);
});
} break;
@@ -3024,8 +3040,7 @@ void
MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller
LOG(ERROR) << msg;
return std::make_pair(MetaServiceCode::PROTOBUF_SERIALIZE_ERR,
msg);
}
- LOG(INFO) << "put instance_id=" << request->instance_id()
- << "instance enable sse json=" <<
proto_to_json(*instance);
+ LOG(INFO) << "enable sse for instance_id=" <<
request->instance_id();
return std::make_pair(MetaServiceCode::OK, ret);
});
} break;
@@ -3050,8 +3065,7 @@ void
MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller
LOG(ERROR) << msg;
return std::make_pair(MetaServiceCode::PROTOBUF_SERIALIZE_ERR,
msg);
}
- LOG(INFO) << "put instance_id=" << request->instance_id()
- << "instance disable sse json=" <<
proto_to_json(*instance);
+ LOG(INFO) << "disable sse for instance_id=" <<
request->instance_id();
return std::make_pair(MetaServiceCode::OK, ret);
});
} break;
@@ -3084,8 +3098,7 @@ void
MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller
LOG(WARNING) << msg;
return std::make_pair(MetaServiceCode::PROTOBUF_SERIALIZE_ERR,
msg);
}
- LOG(INFO) << "put instance_id=" << request->instance_id()
- << "set instance overdue json=" <<
proto_to_json(*instance);
+ LOG(INFO) << "set overdue for instance_id=" <<
request->instance_id();
return std::make_pair(MetaServiceCode::OK, ret);
});
} break;
@@ -3114,8 +3127,7 @@ void
MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller
LOG(WARNING) << msg;
return std::make_pair(MetaServiceCode::PROTOBUF_SERIALIZE_ERR,
msg);
}
- LOG(INFO) << "put instance_id=" << request->instance_id()
- << "set instance normal json=" <<
proto_to_json(*instance);
+ LOG(INFO) << "set normal for instance_id=" <<
request->instance_id();
return std::make_pair(MetaServiceCode::OK, ret);
});
} break;
@@ -3185,7 +3197,9 @@ void
MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller
return std::make_pair(MetaServiceCode::PROTOBUF_SERIALIZE_ERR,
msg);
}
LOG(INFO) << "put instance_id=" << request->instance_id()
- << "set instance snapshot property json=" <<
proto_to_json(*instance);
+ << " snapshot_switch_status=" <<
instance->snapshot_switch_status()
+ << " max_reserved_snapshot=" <<
instance->max_reserved_snapshot()
+ << " snapshot_interval_seconds=" <<
instance->snapshot_interval_seconds();
return std::make_pair(MetaServiceCode::OK, ret);
});
} break;
@@ -3218,7 +3232,7 @@ void
MetaServiceImpl::get_instance(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_instance);
@@ -3511,15 +3525,16 @@ void handle_rename_cluster(const std::string&
instance_id, const ClusterInfo& cl
if (it != cluster_names.end()) {
code = MetaServiceCode::INVALID_ARGUMENT;
ss << "failed to rename cluster, a cluster with the same
name already exists "
- "in this instance "
- << proto_to_json(c);
+ "in this instance, instance_id="
+ << instance_id << ", cluster_name=" <<
cluster.cluster.cluster_name();
msg = ss.str();
return msg;
}
if (c.cluster_name() == cluster.cluster.cluster_name()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "failed to rename cluster, name eq original name,
original cluster is "
- << proto_to_json(c);
+ ss << "failed to rename cluster, name equals original
name, instance_id="
+ << instance_id << ", cluster_id=" << c.cluster_id()
+ << ", cluster_name=" << c.cluster_name();
msg = ss.str();
return msg;
}
@@ -3723,9 +3738,9 @@ void
MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller,
auto [code1, get_msg] = resource_mgr_->get_instance(nullptr, id,
&instance);
{ TEST_SYNC_POINT_CALLBACK("is_instance_id_registered", &code1); }
if (code1 == TxnErrorCode::TXN_KEY_NOT_FOUND) {
- msg = "use degrade cloud_unique_id, but instance_id is not
registered, "
- "cloud_unique_id=" +
- cloud_unique_id;
+ msg = "use degraded format cloud_unique_id, but instance is
not registered, "
+ "instance_id=" +
+ id;
LOG(WARNING) << msg;
code = MetaServiceCode::INVALID_ARGUMENT;
return;
@@ -3734,8 +3749,8 @@ void
MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller,
msg = "instance status has been set delete, plz check it,
recycle_state=" +
InstanceRecycleState_Name(instance.recycle_state());
LOG(WARNING) << "use degraded format cloud_unique_id, but
check instance failed, "
- "cloud_unique_id="
- << cloud_unique_id << " code=" << code1 << "
info=" << get_msg
+ "instance_id="
+ << id << " code=" << code1 << " info=" << get_msg
<< " status=" << instance.status() << "
recycle_state="
<<
InstanceRecycleState_Name(instance.recycle_state());
LOG(WARNING) << msg;
@@ -3747,7 +3762,7 @@ void
MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
}
@@ -3867,12 +3882,13 @@ void
MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller,
instance_id = request->instance_id();
// FIXME(gavin): this mechanism benifits debugging and
// administration, is it dangerous?
- LOG(WARNING) << "failed to get instance_id with cloud_unique_id="
<< cloud_unique_id
- << " use the given instance_id=" << instance_id << "
instead";
+ LOG(WARNING)
+ << "failed to resolve instance from cloud_unique_id, using
given instance_id="
+ << instance_id;
} else {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
}
@@ -3894,7 +3910,7 @@ void
MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "failed to get instance_id with cloud_unique_id=" +
cloud_unique_id;
+ msg = "failed to get instance_id from cloud_unique_id";
return;
}
@@ -3971,8 +3987,7 @@ void
MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller,
}
txn->put(key, val);
- LOG(INFO) << "put instance_id=" << instance_id << " instance_key="
<< hex(key)
- << " json=" << proto_to_json(instance);
+ LOG(INFO) << "put instance_id=" << instance_id << " instance_key="
<< hex(key);
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::COMMIT>(err);
@@ -3983,9 +3998,7 @@ void
MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller,
}
if (response->cluster().empty()) {
- ss << "fail to get cluster with " << request->ShortDebugString();
- msg = ss.str();
- std::replace(msg.begin(), msg.end(), '\n', ' ');
+ msg = fmt::format("cluster not found in instance {}", instance_id);
code = MetaServiceCode::CLUSTER_NOT_FOUND;
}
} // get_cluster
@@ -4007,7 +4020,7 @@ void
MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(create_stage)
@@ -4091,8 +4104,8 @@ void
MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller
if (stage.type() == StagePB::INTERNAL) {
// check all internal stage format is right
if (s.type() == StagePB::INTERNAL && s.mysql_user_id_size() == 0) {
- LOG(WARNING) << "impossible, internal stage must have at least
one id instance="
- << proto_to_json(instance);
+ LOG(WARNING) << "internal stage is missing a MySQL user ID,
instance_id="
+ << instance_id << ", stage_id=" << s.stage_id();
}
if (s.type() == StagePB::INTERNAL &&
@@ -4170,7 +4183,7 @@ void
MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller
txn->atomic_add(system_meta_service_instance_update_key(), 1);
txn->put(key, val);
LOG(INFO) << "put instance_id=" << instance_id << " instance_key=" <<
hex(key)
- << " json=" << proto_to_json(instance);
+ << " stage_id=" << stage.stage_id();
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::COMMIT>(err);
@@ -4196,7 +4209,7 @@ void
MetaServiceImpl::get_stage(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_stage)
@@ -4414,8 +4427,7 @@ void
MetaServiceImpl::get_stage(google::protobuf::RpcController* controller,
}
}
- ss << "stage not found with " << proto_to_json(*request);
- msg = ss.str();
+ msg = "stage not found";
code = MetaServiceCode::STAGE_NOT_FOUND;
}
@@ -4424,7 +4436,8 @@ void
MetaServiceImpl::drop_stage(google::protobuf::RpcController* controller,
::google::protobuf::Closure* done) {
StopWatch sw;
auto ctrl = static_cast<brpc::Controller*>(controller);
- LOG(INFO) << "rpc from " << ctrl->remote_side() << " request=" <<
request->DebugString();
+ LOG(INFO) << "begin drop_stage remote_caller=" << ctrl->remote_side()
+ << " stage_name=" << request->stage_name();
brpc::ClosureGuard closure_guard(done);
int ret = 0;
MetaServiceCode code = MetaServiceCode::OK;
@@ -4453,7 +4466,7 @@ void
MetaServiceImpl::drop_stage(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(drop_stage)
@@ -4522,8 +4535,7 @@ void
MetaServiceImpl::drop_stage(google::protobuf::RpcController* controller,
}
}
if (idx == -1) {
- ss << "stage not found with " << proto_to_json(*request);
- msg = ss.str();
+ msg = "stage not found";
code = MetaServiceCode::STAGE_NOT_FOUND;
return;
}
@@ -4538,7 +4550,7 @@ void
MetaServiceImpl::drop_stage(google::protobuf::RpcController* controller,
}
txn->put(key, val);
LOG(INFO) << "put instance_id=" << instance_id << " instance_key=" <<
hex(key)
- << " json=" << proto_to_json(instance);
+ << " stage_id=" << stage.stage_id();
std::string key1;
std::string val1;
@@ -4582,7 +4594,7 @@ void
MetaServiceImpl::get_iam(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_iam)
@@ -4674,10 +4686,14 @@ void
MetaServiceImpl::alter_ram_user(google::protobuf::RpcController* controller
msg = "empty instance_id";
return;
}
- if (!request->has_ram_user() || request->ram_user().user_id().empty() ||
- request->ram_user().ak().empty() || request->ram_user().sk().empty()) {
+ if (!request->has_ram_user()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "ram_user is required";
+ return;
+ }
+ if (auto fields = empty_ram_user_fields(request->ram_user());
!fields.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "ram user info err " + proto_to_json(*request);
+ msg = "ram_user fields must not be empty: " + fields;
return;
}
auto& ram_user = request->ram_user();
@@ -4762,7 +4778,7 @@ void
MetaServiceImpl::begin_copy(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(begin_copy)
@@ -4878,7 +4894,7 @@ void
MetaServiceImpl::finish_copy(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(finish_copy)
@@ -4982,7 +4998,7 @@ void
MetaServiceImpl::get_copy_job(google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
@@ -5033,7 +5049,7 @@ void
MetaServiceImpl::get_copy_files(google::protobuf::RpcController* controller
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_copy_files)
@@ -5096,7 +5112,7 @@ void
MetaServiceImpl::filter_copy_files(google::protobuf::RpcController* control
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(filter_copy_files)
@@ -5157,7 +5173,7 @@ void
MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro
[&](const auto& it) {
std::string instance_id =
get_instance_id(resource_mgr_, it);
if (instance_id.empty()) {
- LOG(INFO) << "cant get instance_id from
cloud_unique_id : " << it;
+ LOG(INFO) << "failed to get instance_id from
cloud_unique_id";
return;
}
instance_ids.emplace_back(instance_id);
diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp
b/cloud/src/meta-service/meta_service_tablet_stats.cpp
index 282dabd7aac..1426c0e933d 100644
--- a/cloud/src/meta-service/meta_service_tablet_stats.cpp
+++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp
@@ -334,7 +334,7 @@ MetaServiceResponseStatus parse_fix_tablet_stats_param(
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id_str;
+ LOG(INFO) << msg;
st.set_code(code);
st.set_msg(msg);
return st;
diff --git a/cloud/src/meta-service/meta_service_txn.cpp
b/cloud/src/meta-service/meta_service_txn.cpp
index 2b71443e008..cf10c5eef98 100644
--- a/cloud/src/meta-service/meta_service_txn.cpp
+++ b/cloud/src/meta-service/meta_service_txn.cpp
@@ -450,8 +450,8 @@ void
MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller,
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id) << "
label=" << label;
+ ss << "cannot find instance_id from cloud_unique_id"
+ << " label=" << label;
msg = ss.str();
return;
}
@@ -725,8 +725,8 @@ void
MetaServiceImpl::precommit_txn(::google::protobuf::RpcController* controlle
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id) << "
txn_id=" << txn_id;
+ ss << "cannot find instance_id from cloud_unique_id"
+ << " txn_id=" << txn_id;
msg = ss.str();
return;
}
@@ -1026,7 +1026,7 @@ void
MetaServiceImpl::get_rl_task_commit_attach(::google::protobuf::RpcControlle
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_rl_task_commit_attach)
@@ -1042,7 +1042,7 @@ void
MetaServiceImpl::get_rl_task_commit_attach(::google::protobuf::RpcControlle
if (!request->has_db_id() || !request->has_job_id()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty db_id or job_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -1094,7 +1094,7 @@ void MetaServiceImpl::get_streaming_task_commit_attach(
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_streaming_task_commit_attach)
@@ -1110,7 +1110,7 @@ void MetaServiceImpl::get_streaming_task_commit_attach(
if (!request->has_db_id() || !request->has_job_id()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty db_id or job_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -1150,7 +1150,7 @@ void
MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(reset_rl_progress)
@@ -1166,7 +1166,7 @@ void
MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr
if (!request->has_db_id() || !request->has_job_id()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty db_id or job_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -1251,7 +1251,7 @@ void
MetaServiceImpl::reset_streaming_job_offset(::google::protobuf::RpcControll
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(reset_streaming_job_offset)
@@ -1267,7 +1267,7 @@ void
MetaServiceImpl::reset_streaming_job_offset(::google::protobuf::RpcControll
if (!request->has_db_id() || !request->has_job_id()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty db_id or job_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
@@ -1356,7 +1356,7 @@ void
MetaServiceImpl::delete_streaming_job(::google::protobuf::RpcController* co
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(delete_streaming_job)
@@ -1372,7 +1372,7 @@ void
MetaServiceImpl::delete_streaming_job(::google::protobuf::RpcController* co
if (!request->has_db_id() || !request->has_job_id()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "missing db_id or job_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
int64_t db_id = request->db_id();
@@ -3798,7 +3798,7 @@ void
MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller,
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id << "
txn_id=" << txn_id;
+ LOG(INFO) << msg << " txn_id=" << txn_id;
return;
}
RPC_RATE_LIMIT(commit_txn)
@@ -4114,9 +4114,8 @@ void
MetaServiceImpl::abort_txn(::google::protobuf::RpcController* controller,
instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id());
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id) << "
label=" << label
- << " txn_id=" << txn_id;
+ ss << "cannot find instance_id from cloud_unique_id"
+ << " label=" << label << " txn_id=" << txn_id;
msg = ss.str();
return;
}
@@ -4161,9 +4160,7 @@ void
MetaServiceImpl::get_txn(::google::protobuf::RpcController* controller,
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = "cannot find instance_id from cloud_unique_id";
return;
}
@@ -4280,7 +4277,7 @@ void
MetaServiceImpl::get_current_max_txn_id(::google::protobuf::RpcController*
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(get_current_max_txn_id)
@@ -4326,7 +4323,7 @@ void
MetaServiceImpl::create_meta_sync_point(::google::protobuf::RpcController*
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ LOG(INFO) << msg;
return;
}
RPC_RATE_LIMIT(create_meta_sync_point)
@@ -4407,9 +4404,7 @@ void
MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = "cannot find instance_id from cloud_unique_id";
return;
}
@@ -4608,9 +4603,7 @@ void
MetaServiceImpl::abort_sub_txn(::google::protobuf::RpcController* controlle
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = "cannot find instance_id from cloud_unique_id";
return;
}
@@ -4711,9 +4704,7 @@ void
MetaServiceImpl::abort_txn_with_coordinator(::google::protobuf::RpcControll
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = "cannot find instance_id from cloud_unique_id";
return;
}
RPC_RATE_LIMIT(abort_txn_with_coordinator);
@@ -4826,9 +4817,7 @@ void MetaServiceImpl::get_prepare_txn_by_coordinator(
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = "cannot find instance_id from cloud_unique_id";
return;
}
RPC_RATE_LIMIT(get_prepare_txn_by_coordinator);
@@ -5040,9 +5029,7 @@ void
MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = "cannot find instance_id from cloud_unique_id";
return;
}
RPC_RATE_LIMIT(check_txn_conflict)
@@ -5349,9 +5336,7 @@ void
MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = "cannot find instance_id from cloud_unique_id";
LOG(WARNING) << msg;
return;
}
@@ -5452,9 +5437,7 @@ void
MetaServiceImpl::get_txn_id(::google::protobuf::RpcController* controller,
instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "cannot find instance_id with cloud_unique_id="
- << (cloud_unique_id.empty() ? "(empty)" : cloud_unique_id);
- msg = ss.str();
+ msg = "cannot find instance_id from cloud_unique_id";
LOG(WARNING) << msg;
return;
}
diff --git a/cloud/src/recycler/recycler_service.cpp
b/cloud/src/recycler/recycler_service.cpp
index 8441418498d..0a3ab6a0135 100644
--- a/cloud/src/recycler/recycler_service.cpp
+++ b/cloud/src/recycler/recycler_service.cpp
@@ -611,8 +611,8 @@ void
RecyclerServiceImpl::http(::google::protobuf::RpcController* controller,
// Auth
const auto* token = cntl->http_request().uri().GetQuery("token");
if (token == nullptr || *token != config::http_token) {
- std::string msg = "incorrect token, token=" +
- (token == nullptr ? std::string("(not given)") :
*token);
+ std::string msg =
+ token == nullptr ? "incorrect token, token=(not given)" :
"incorrect token";
cntl->http_response().set_status_code(403);
cntl->response_attachment().append(msg);
cntl->response_attachment().append("\n");
diff --git a/cloud/src/resource-manager/resource_manager.cpp
b/cloud/src/resource-manager/resource_manager.cpp
index e1ed3730ff8..8125472db42 100644
--- a/cloud/src/resource-manager/resource_manager.cpp
+++ b/cloud/src/resource-manager/resource_manager.cpp
@@ -99,8 +99,7 @@ int ResourceManager::init() {
return -2;
}
- LOG(INFO) << "get an instance, instance_id=" << instance_id
- << " instance json=" << proto_to_json(inst);
+ LOG(INFO) << "get an instance, instance_id=" << instance_id;
++num_instances;
}
@@ -128,7 +127,7 @@ std::string ResourceManager::get_node(const std::string&
cloud_unique_id,
std::shared_lock l(mtx_);
auto [s, e] = node_info_.equal_range(cloud_unique_id);
if (s == node_info_.end() || s->first != cloud_unique_id) {
- LOG(INFO) << "cloud unique id not found cloud_unique_id=" <<
cloud_unique_id;
+ LOG(INFO) << "cloud_unique_id not found in node cache";
return "cloud_unique_id not found";
}
nodes->reserve(nodes->size() + node_info_.count(cloud_unique_id));
@@ -183,6 +182,11 @@ bool ResourceManager::validate_cluster_name(const
ClusterPB& cluster, std::strin
return true;
}
+static std::string node_log_endpoint(const NodeInfoPB& node) {
+ return fmt::format("node_host={} node_ip={} edit_log_port={}
heartbeat_port={}", node.host(),
+ node.ip(), node.edit_log_port(), node.heartbeat_port());
+}
+
bool ResourceManager::validate_nodes(const ClusterPB& cluster, std::string*
err,
bool check_master_num, std::string_view
creating_instance_id) {
std::stringstream ss;
@@ -204,15 +208,20 @@ bool ResourceManager::validate_nodes(const ClusterPB&
cluster, std::string* err,
auto [code, msg] = get_instance(nullptr, instance_id, &instance);
{ TEST_SYNC_POINT_CALLBACK("is_instance_id_registered", &code); }
if (code == TxnErrorCode::TXN_KEY_NOT_FOUND) {
- *err = "node=" + n.DebugString() +
- " cloud_unique_id use degrade format, but instance is
not registered";
- LOG(WARNING) << *err << ", cloud_unique_id=" <<
cloud_unique_id;
+ *err = fmt::format(
+ "cloud_unique_id uses degraded format but instance is
not "
+ "registered, instance_id={} {}",
+ instance_id, node_log_endpoint(n));
+ LOG(WARNING) << *err;
return false;
}
if (instance.has_status() && instance.status() ==
InstanceInfoPB::DELETED) {
- *err = "instance status has been set delete, plz check it,
recycle_state=" +
- InstanceRecycleState_Name(instance.recycle_state());
- LOG(WARNING) << *err << ", cloud_unique_id=" <<
cloud_unique_id;
+ *err = fmt::format(
+ "instance status has been set delete, plz check it, "
+ "instance_id={} recycle_state={} {}",
+ instance_id,
InstanceRecycleState_Name(instance.recycle_state()),
+ node_log_endpoint(n));
+ LOG(WARNING) << *err;
return false;
}
}
@@ -231,8 +240,8 @@ bool ResourceManager::validate_nodes(const ClusterPB&
cluster, std::string* err,
continue;
}
ss << "check cluster params failed, edit_log_port is required for
frontends while "
- "heatbeat_port is required for banckends, node : "
- << proto_to_json(n);
+ "heartbeat_port is required for backends, "
+ << node_log_endpoint(n) << " node_type=" << n.node_type();
*err = ss.str();
no_err = false;
break;
@@ -443,7 +452,8 @@ static std::pair<MetaServiceCode, std::string>
check_node_has_been_registered(
std::stringstream ss;
std::string msg;
if (addr == "" || port == -1) {
- ss << "add node input args node invalid, cluster=" <<
proto_to_json(node);
+ ss << "add node input args node invalid, cluster_type=" << type << " "
+ << node_log_endpoint(node);
LOG(WARNING) << ss.str();
msg = ss.str();
return std::make_pair(MetaServiceCode::INVALID_ARGUMENT, msg);
@@ -502,8 +512,7 @@ std::pair<MetaServiceCode, std::string>
ResourceManager::add_cluster(const std::
ss << "cloud_unique_id is already occupied by an instance,"
<< " instance_id=" << it->second.instance_id
<< " cluster_name=" << it->second.cluster_name
- << " cluster_id=" << it->second.cluster_id
- << " cloud_unique_id=" << it->first;
+ << " cluster_id=" << it->second.cluster_id << " " <<
node_log_endpoint(i);
msg = ss.str();
LOG(INFO) << msg;
return std::make_pair(MetaServiceCode::ALREADY_EXISTED,
msg);
@@ -535,12 +544,11 @@ std::pair<MetaServiceCode, std::string>
ResourceManager::add_cluster(const std::
}
auto& req_cluster = cluster.cluster;
- InstanceInfoPB instance_for_log {instance};
- for (auto& obj_info : *instance_for_log.mutable_obj_info()) {
- obj_info.set_ak(hide_access_key(obj_info.ak()));
- }
- LOG(INFO) << "cluster to add json=" << proto_to_json(req_cluster);
- LOG(INFO) << "json=" << proto_to_json(instance_for_log);
+ LOG(INFO) << "add cluster, instance_id=" << instance_id
+ << " cluster_id=" << req_cluster.cluster_id()
+ << " cluster_name=" << req_cluster.cluster_name()
+ << " cluster_type=" << req_cluster.type()
+ << " node_count=" << req_cluster.nodes_size();
// Check id and name, they need to be unique
// One cluster id per name, name is alias of cluster id
@@ -710,8 +718,9 @@ std::pair<MetaServiceCode, std::string>
ResourceManager::drop_cluster(
// check drop fe cluster
if (config::enable_check_fe_drop_in_safe_time &&
!is_sql_node_exceeded_safe_drop_time(fe_node)) {
- ss << "drop fe cluster not in safe time, try later,
cluster="
- << i.DebugString();
+ ss << "drop fe cluster not in safe time, try later,
instance_id="
+ << instance_id << " cluster_id=" << i.cluster_id()
<< " "
+ << node_log_endpoint(fe_node) << " node_ctime=" <<
fe_node.ctime();
msg = ss.str();
LOG(WARNING) << msg;
return
std::make_pair(MetaServiceCode::CLUSTER_NOT_FOUND, msg);
@@ -721,7 +730,7 @@ std::pair<MetaServiceCode, std::string>
ResourceManager::drop_cluster(
to_del.CopyFrom(i);
LOG(INFO) << "found a cluster to drop,"
<< " instance_id=" << instance_id << " cluster_id=" <<
i.cluster_id()
- << " cluster_name=" << i.cluster_name() << " cluster="
<< proto_to_json(i);
+ << " cluster_name=" << i.cluster_name();
found = true;
break;
}
@@ -838,7 +847,7 @@ std::string ResourceManager::update_cluster(
if (filter(i)) {
LOG(INFO) << "found a cluster to update,"
<< " instance_id=" << instance_id << " cluster_id=" <<
i.cluster_id()
- << " cluster_name=" << i.cluster_name() << " cluster="
<< proto_to_json(i);
+ << " cluster_name=" << i.cluster_name();
found = true;
break;
}
@@ -895,8 +904,12 @@ std::string ResourceManager::update_cluster(
uint64_t time =
std::chrono::duration_cast<std::chrono::seconds>(now_time.time_since_epoch()).count();
now.set_mtime(time);
- LOG(INFO) << "before update cluster original: " << proto_to_json(original)
- << " after update now: " << proto_to_json(now);
+ auto cluster_json_for_log = [](ClusterPB cluster) {
+ mask_cluster_cloud_unique_ids(cluster);
+ return proto_to_json(cluster);
+ };
+ LOG(INFO) << "before update cluster original: " <<
cluster_json_for_log(original)
+ << " after update now: " << cluster_json_for_log(now);
InstanceKeyInfo key_info {instance_id};
std::string key;
@@ -919,8 +932,7 @@ std::string ResourceManager::update_cluster(
return msg;
}
- LOG(INFO) << "update cluster instance_id=" << instance_id
- << " instance json=" << proto_to_json(instance);
+ LOG(INFO) << "update cluster instance_id=" << instance_id;
refresh_instance(instance_id, instance);
TEST_INJECTION_POINT("ResourceManager::node_update:after_refresh");
@@ -945,8 +957,10 @@ void ResourceManager::add_cluster_to_index_no_lock(const
std::string& instance_i
.cluster_id = c.cluster_id(),
.node_info = i};
LOG(WARNING) << (existed ? "duplicated cloud_unique_id " : "")
- << "instance_id=" << instance_id << " cloud_unique_id="
<< i.cloud_unique_id()
- << " node_info=" << proto_to_json(i);
+ << "instance_id=" << instance_id << " cluster_id=" <<
c.cluster_id()
+ << " node_host=" << i.host() << " node_ip=" << i.ip()
+ << " heartbeat_port=" << i.heartbeat_port()
+ << " edit_log_port=" << i.edit_log_port();
node_info_.insert({i.cloud_unique_id(), std::move(n)});
}
}
@@ -1062,7 +1076,8 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
return msg;
}
- LOG(INFO) << "instance json=" << proto_to_json(instance);
+ LOG(INFO) << "modify nodes, instance_id=" << instance_id << " add_count="
<< to_add.size()
+ << " drop_count=" << to_del.size();
if (!to_add.empty()) {
// add nodes
// Check duplicated nodes, one node cannot deploy on multiple clusters
@@ -1096,7 +1111,9 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
}
}
- LOG(INFO) << "node to modify json=" << proto_to_json(node.node_info);
+ LOG(INFO) << "modify node, instance_id=" << instance_id << "
cluster_id=" << cluster_id
+ << " cluster_name=" << cluster_name << " " <<
node_log_endpoint(node.node_info)
+ << " node_status=" << node.node_info.status();
for (auto& c : instance.clusters()) {
if ((c.has_cluster_name() && c.cluster_name() == cluster_name) ||
@@ -1130,8 +1147,8 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
s << "cloud_unique_id is already occupied by an instance,"
<< " instance_id=" << it->second.instance_id
<< " cluster_name=" << it->second.cluster_name
- << " cluster_id=" << it->second.cluster_id
- << " cloud_unique_id=" << n.node_info.cloud_unique_id();
+ << " cluster_id=" << it->second.cluster_id << " "
+ << node_log_endpoint(n.node_info);
err = s.str();
LOG(INFO) << err;
return err;
@@ -1215,9 +1232,9 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
std::stringstream s;
auto [start, end] =
node_info_.equal_range(n.node_info.cloud_unique_id());
if (start == node_info_.end() || start->first !=
n.node_info.cloud_unique_id()) {
- s << "can not find to drop nodes by cloud_unique_id=" <<
n.node_info.cloud_unique_id()
- << " instance_id=" << n.instance_id << " cluster_name=" <<
n.cluster_name
- << " cluster_id=" << n.cluster_id << " help Msg=" <<
cache_err_help_msg;
+ s << "node cloud_unique_id not found in cache, instance_id=" <<
n.instance_id
+ << " cluster_name=" << n.cluster_name << " cluster_id=" <<
n.cluster_id << " "
+ << node_log_endpoint(n.node_info) << " help Msg=" <<
cache_err_help_msg;
err = s.str();
LOG(WARNING) << err;
return std::string("not found ,") + err;
@@ -1264,7 +1281,7 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
if (!found) {
s << "cloud_unique_id can not find to drop node,"
<< " instance_id=" << n.instance_id << " cluster_name=" <<
n.cluster_name
- << " cluster_id=" << n.cluster_id << " node_info=" <<
n.node_info.DebugString()
+ << " cluster_id=" << n.cluster_id << " " <<
node_log_endpoint(n.node_info)
<< " help Msg=" << cache_err_help_msg;
err = s.str();
LOG(WARNING) << err;
@@ -1329,8 +1346,8 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
if (!found) {
s << "failed to find node to drop,"
<< " instance_id=" << instance.instance_id() << " cluster_id="
<< c.cluster_id()
- << " cluster_name=" << c.cluster_name() << " cluster=" <<
proto_to_json(c)
- << " help Msg =" << cache_err_help_msg;
+ << " cluster_name=" << c.cluster_name() << " " <<
node_log_endpoint(ni)
+ << " registered_nodes=" << c.nodes_size() << " help Msg =" <<
cache_err_help_msg;
err = s.str();
LOG(WARNING) << err;
return std::string("not found ,") + err;
@@ -1339,7 +1356,9 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
// check drop fe node
if (ClusterPB::SQL == c.type() &&
config::enable_check_fe_drop_in_safe_time &&
!is_sql_node_exceeded_safe_drop_time(copy_node)) {
- s << "drop fe node not in safe time, try later, node=" <<
copy_node.DebugString();
+ s << "drop fe node not in safe time, try later, instance_id=" <<
instance_id
+ << " cluster_id=" << c.cluster_id() << " " <<
node_log_endpoint(copy_node)
+ << " node_ctime=" << copy_node.ctime();
err = s.str();
LOG(WARNING) << err;
return err;
@@ -1361,7 +1380,6 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
}
}
- LOG(INFO) << "instance " << instance_id << " info: " <<
instance.DebugString();
// here, instance has been changed, not save in fdb
if ((!to_add.empty() || !to_del.empty()) && !is_instance_valid(instance)) {
msg = "instance invalid, cant modify, plz check";
diff --git a/cloud/test/meta_service_http_test.cpp
b/cloud/test/meta_service_http_test.cpp
index 333f2f6c975..8b20749fd0a 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -36,8 +36,11 @@
#include <cstddef>
#include <cstdint>
#include <filesystem>
+#include <mutex>
#include <optional>
#include <string>
+#include <string_view>
+#include <vector>
#include "common/config.h"
#include "common/configbase.h"
@@ -1590,8 +1593,6 @@ TEST(MetaServiceHttpTest, ToUnknownUrlTest) {
}
TEST(MetaServiceHttpTest, UnknownFields) {
- // LOG:
- // parse http request 'get_tablet_stats':
INVALID_ARGUMENT:an_unknown_field: Cannot find field. body="{"table_id": 1,
"an_unknown_field": "xxxx"}"
HttpContext ctx;
std::string body =
"{\"table_id\": 1, \"an_unknown_field\": \"xxxx\",
\"cloud_unique_id\": "
@@ -1675,10 +1676,339 @@ TEST(MetaServiceHttpTest, InvalidToken) {
HttpContext ctx(true);
auto [status_code, content] = ctx.query<std::string>("get_value",
"token=invalid_token", "");
ASSERT_EQ(status_code, 403);
- const char* invalid_token_output = "incorrect token,
token=invalid_token\n";
+ const char* invalid_token_output = "incorrect token\n";
ASSERT_EQ(content, invalid_token_output);
}
+// Captures what the service logs, skipping the logs this file writes about
the requests it sends.
+class ServerLogSink : public google::LogSink {
+public:
+ ServerLogSink() { google::AddLogSink(this); }
+ ~ServerLogSink() override { google::RemoveLogSink(this); }
+
+ void send(google::LogSeverity, const char*, const char* base_filename, int,
+ const google::LogMessageTime&, const char* message,
+ std::size_t message_len) override {
+ if (std::string_view(base_filename) == "meta_service_http_test.cpp") {
+ return;
+ }
+ std::lock_guard lock(mutex_);
+ messages_.emplace_back(message, message_len);
+ }
+
+ std::vector<std::string> messages() {
+ std::lock_guard lock(mutex_);
+ return messages_;
+ }
+
+private:
+ std::mutex mutex_;
+ std::vector<std::string> messages_;
+};
+
+TEST(MetaServiceHttpTest, HttpRequestLogsMaskCredentials) {
+ const int old_vlog_level = google::SetVLOGLevel("meta_service_http", -1);
+ DORIS_CLOUD_DEFER {
+ google::SetVLOGLevel("meta_service_http", old_vlog_level);
+ };
+ std::string origin_token = config::http_token;
+ config::http_token = "ms-http-token-secret";
+ DORIS_CLOUD_DEFER {
+ config::http_token = origin_token;
+ };
+ ServerLogSink sink;
+
+ HttpContext ctx(true);
+ const std::string_view body =
+
R"({"obj":{"ak":"ms-body-access-key","sk":"ms-body-secret","bucket":"review-bucket"}})";
+
+ auto [status_without_debug, ignored] =
+ ctx.query<std::string>("get_value", "token=ms-wrong-token-secret",
body);
+ ASSERT_EQ(status_without_debug, 403);
+ bool request_summary_logged = false;
+ for (const auto& message : sink.messages()) {
+ EXPECT_EQ(message.find("http request details from"),
std::string::npos) << message;
+ EXPECT_EQ(message.find("review-bucket"), std::string::npos) << message;
+ request_summary_logged |= message.find("/get_value ret=403") !=
std::string::npos;
+ }
+ EXPECT_TRUE(request_summary_logged);
+
+ google::SetVLOGLevel("meta_service_http", DEBUG);
+ ctx.query<std::string>("get_value",
"key_type=InstanceKey&instance_id=not_exist", body);
+ auto [status_code, content] =
+ ctx.query<std::string>("get_value", "token=ms-wrong-token-secret",
body);
+ ASSERT_EQ(status_code, 403);
+
+ int request_logs = 0;
+ for (const auto& message : sink.messages()) {
+ for (auto secret : {"ms-http-token-secret", "ms-wrong-token-secret",
"ms-body-access-key",
+ "ms-body-secret"}) {
+ EXPECT_EQ(message.find(secret), std::string::npos) << message;
+ }
+ if (message.find("\ntoken=***") != std::string::npos) {
+ ++request_logs;
+ EXPECT_NE(message.find("review-bucket"), std::string::npos) <<
message;
+ }
+ }
+ EXPECT_EQ(request_logs, 2);
+}
+
+TEST(MetaServiceHttpTest, MalformedRequestBodyNotLoggedWithoutDebug) {
+ const int old_vlog_level = google::SetVLOGLevel("meta_service_http", -1);
+ DORIS_CLOUD_DEFER {
+ google::SetVLOGLevel("meta_service_http", old_vlog_level);
+ };
+ HttpContext ctx(true);
+ ServerLogSink sink;
+ auto [status_code, body] = ctx.query<std::string>(
+ "get_cluster", "",
+
R"({"cloud_unique_id":"malformed-body-secret","sk":"another-secret")");
+ ASSERT_EQ(status_code, 400);
+ EXPECT_EQ(body.find("malformed-body-secret"), std::string::npos);
+ EXPECT_EQ(body.find("another-secret"), std::string::npos);
+ for (const auto& message : sink.messages()) {
+ EXPECT_EQ(message.find("malformed-body-secret"), std::string::npos) <<
message;
+ EXPECT_EQ(message.find("another-secret"), std::string::npos) <<
message;
+ EXPECT_EQ(message.find("http request details from"),
std::string::npos) << message;
+ }
+}
+
+TEST(MetaServiceHttpTest, CloudUniqueIdNotLoggedWithoutDebug) {
+ const int old_vlog_level = google::SetVLOGLevel("meta_service_http", -1);
+ DORIS_CLOUD_DEFER {
+ google::SetVLOGLevel("meta_service_http", old_vlog_level);
+ };
+ HttpContext ctx;
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(mock_instance);
+ ClusterPB* cluster = instance.add_clusters();
+ cluster->set_cluster_name(mock_cluster_name);
+ cluster->set_cluster_id(mock_cluster_id);
+ cluster->set_type(ClusterPB::COMPUTE);
+ cluster->set_cluster_status(ClusterStatus::NORMAL);
+ (*cluster->mutable_properties())["cloudUniqueId"] = "node-secret-property";
+ (*cluster->mutable_properties())["review_property"] = "retained-value";
+
cluster->add_nodes()->set_cloud_unique_id(fmt::format("1:{}:node-secret-b",
mock_instance));
+ std::string key;
+ instance_key({mock_instance}, &key);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(ctx.meta_service_->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ txn->put(key, instance.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ ServerLogSink sink;
+ GetClusterRequest req;
+ req.set_instance_id(mock_instance);
+ req.set_cloud_unique_id(fmt::format("1:{}:node-secret-a", mock_instance));
+ req.set_cluster_id(mock_cluster_id);
+ auto [status_code, resp] =
ctx.forward_with_result<ClusterPB>("get_cluster", req);
+ ASSERT_EQ(status_code, 200);
+ ASSERT_TRUE(resp.result.has_value());
+ ASSERT_EQ(resp.result->nodes(0).cloud_unique_id(),
+ fmt::format("1:{}:node-secret-b", mock_instance));
+
+ AlterClusterRequest alter_req;
+ alter_req.set_instance_id(mock_instance);
+ alter_req.set_op(AlterClusterRequest::SET_CLUSTER_STATUS);
+ alter_req.mutable_cluster()->set_cluster_id(mock_cluster_id);
+ alter_req.mutable_cluster()->set_cluster_status(ClusterStatus::SUSPENDED);
+ auto [alter_code, alter_status] =
+ ctx.forward<MetaServiceResponseStatus>("set_cluster_status",
alter_req);
+ ASSERT_EQ(alter_code, 200);
+ ASSERT_EQ(alter_status.code(), MetaServiceCode::OK);
+
+ auto [updated_code, updated] =
ctx.forward_with_result<ClusterPB>("get_cluster", req);
+ ASSERT_EQ(updated_code, 200);
+ ASSERT_TRUE(updated.result.has_value());
+ EXPECT_EQ(updated.result->cluster_status(), ClusterStatus::SUSPENDED);
+ EXPECT_EQ(updated.result->properties().at("cloudUniqueId"),
"node-secret-property");
+ ASSERT_EQ(updated.result->nodes_size(), 1);
+ EXPECT_EQ(updated.result->nodes(0).cloud_unique_id(),
+ fmt::format("1:{}:node-secret-b", mock_instance));
+
+ bool cluster_update_logged = false;
+ bool rpc_logs_cluster = false;
+ for (const auto& message : sink.messages()) {
+ EXPECT_EQ(message.find("node-secret"), std::string::npos) << message;
+ if (message.find("before update cluster original:") !=
std::string::npos) {
+ EXPECT_NE(message.find(" after update now: "), std::string::npos)
<< message;
+ EXPECT_NE(message.find("NORMAL"), std::string::npos) << message;
+ EXPECT_NE(message.find("SUSPENDED"), std::string::npos) << message;
+ EXPECT_NE(message.find("retained-value"), std::string::npos) <<
message;
+ EXPECT_NE(message.find(R"("cloud_unique_id":"***MASKED***")"),
std::string::npos)
+ << message;
+ EXPECT_NE(message.find(R"("cloudUniqueId":"***MASKED***")"),
std::string::npos)
+ << message;
+ cluster_update_logged = true;
+ }
+ rpc_logs_cluster |=
+ message.find("begin get_cluster ") != std::string::npos &&
+ message.find("cluster_id=" + std::string(mock_cluster_id)) !=
std::string::npos;
+ }
+ EXPECT_TRUE(rpc_logs_cluster);
+ EXPECT_TRUE(cluster_update_logged);
+}
+
+TEST(MetaServiceHttpTest, AkSkNotLogged) {
+ ScopedLegacyInstanceRpc legacy_instance_rpc;
+ HttpContext ctx(true);
+ auto* ms = ctx.meta_service_.get();
+ const std::string ak = "AK_ID_VALUE_1234567";
+ const std::string sk = "SK_SECRET_VALUE_42";
+ ServerLogSink sink;
+
+ // Rejected requests are logged, but their status messages only describe
the error.
+ AlterObjStoreInfoRequest alter_obj_req;
+ alter_obj_req.set_cloud_unique_id("test_cloud_unique_id");
+ alter_obj_req.set_op(AlterObjStoreInfoRequest::UNKNOWN);
+ alter_obj_req.mutable_obj()->set_ak(ak);
+ alter_obj_req.mutable_obj()->set_sk(sk);
+ {
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ ms->alter_obj_store_info(&cntl, &alter_obj_req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(res.status().msg(), "unknown alter operation");
+ }
+ {
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ ms->alter_storage_vault(&cntl, &alter_obj_req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(res.status().msg(), "unknown alter operation");
+ }
+ {
+ AlterRamUserRequest req;
+ req.set_instance_id(mock_instance);
+ req.mutable_ram_user()->set_ak(ak);
+ req.mutable_ram_user()->set_sk(sk);
+ brpc::Controller cntl;
+ AlterRamUserResponse res;
+ ms->alter_ram_user(&cntl, &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(res.status().msg(), "ram_user fields must not be empty:
user_id");
+ }
+ {
+ InstanceInfoPB instance;
+ instance.set_instance_id(mock_instance);
+ std::string key;
+ instance_key({mock_instance}, &key);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(ms->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->put(key, instance.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ UpdateAkSkRequest req;
+ req.set_instance_id(mock_instance);
+ req.mutable_ram_user()->set_ak(ak);
+ req.mutable_ram_user()->set_sk(sk);
+ brpc::Controller cntl;
+ UpdateAkSkResponse res;
+ ms->update_ak_sk(&cntl, &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(res.status().msg(), "ram_user fields must not be empty:
user_id");
+ }
+
+ {
+ AlterRamUserRequest req;
+ req.set_instance_id(mock_instance);
+ req.mutable_ram_user()->set_user_id("ram_user");
+ brpc::Controller cntl;
+ AlterRamUserResponse res;
+ ms->alter_ram_user(&cntl, &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(res.status().msg(), "ram_user fields must not be empty: ak,
sk");
+ }
+ {
+ UpdateAkSkRequest req;
+ req.set_instance_id(mock_instance);
+ auto* user = req.add_internal_bucket_user();
+ user->set_user_id("bucket_user");
+ user->set_ak(ak);
+ brpc::Controller cntl;
+ UpdateAkSkResponse res;
+ ms->update_ak_sk(&cntl, &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(res.status().msg(), "internal_bucket_user[0] is missing:
sk");
+ }
+
+ for (const auto& message : sink.messages()) {
+ EXPECT_EQ(message.find(sk), std::string::npos) << message;
+ EXPECT_EQ(message.find(ak), std::string::npos) << message;
+ }
+}
+
+TEST(MetaServiceHttpTest, CredentialUpdateLogs) {
+ HttpContext ctx(true);
+ const std::string old_ak = "old-access-key-marker";
+ const std::string old_sk = "old-stored-secret-marker";
+ const std::string update_ak = "update-access-key-marker";
+ const std::string update_sk = "update-secret-marker";
+ const std::string alter_ak = "alter-access-key-marker";
+ const std::string alter_sk = "alter-secret-marker";
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(mock_instance);
+ auto* ram_user = instance.mutable_ram_user();
+ ram_user->set_user_id("ram_user");
+ ram_user->set_ak(old_ak);
+ ram_user->set_sk(old_sk);
+ std::string key;
+ instance_key({mock_instance}, &key);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(ctx.meta_service_->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ txn->put(key, instance.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ ServerLogSink sink;
+ UpdateAkSkRequest update;
+ update.set_instance_id(mock_instance);
+ update.mutable_ram_user()->set_user_id("ram_user");
+ update.mutable_ram_user()->set_ak(update_ak);
+ update.mutable_ram_user()->set_sk(update_sk);
+ auto [update_http_code, update_status] =
+ ctx.forward<MetaServiceResponseStatus>("update_ak_sk", update);
+ ASSERT_EQ(update_http_code, 200);
+ ASSERT_EQ(update_status.code(), MetaServiceCode::OK);
+ const auto updated_instance = ctx.get_instance_info(mock_instance);
+ ASSERT_EQ(updated_instance.ram_user().ak(), update_ak);
+ ASSERT_FALSE(updated_instance.ram_user().sk().empty());
+
+ AlterRamUserRequest alter;
+ alter.set_instance_id(mock_instance);
+ alter.mutable_ram_user()->set_user_id("ram_user");
+ alter.mutable_ram_user()->set_ak(alter_ak);
+ alter.mutable_ram_user()->set_sk(alter_sk);
+ auto [alter_http_code, alter_status] =
+ ctx.forward<MetaServiceResponseStatus>("alter_ram_user", alter);
+ ASSERT_EQ(alter_http_code, 200);
+ ASSERT_EQ(alter_status.code(), MetaServiceCode::OK);
+ const auto altered_instance = ctx.get_instance_info(mock_instance);
+ ASSERT_EQ(altered_instance.ram_user().ak(), alter_ak);
+ ASSERT_FALSE(altered_instance.ram_user().sk().empty());
+
+ const auto messages = sink.messages();
+ for (auto rpc : {"update_ak_sk", "alter_ram_user"}) {
+ EXPECT_NE(std::ranges::find(messages,
+ fmt::format("begin {} instance_id={}",
rpc, mock_instance)),
+ messages.end());
+ }
+ const std::string_view credentials[] = {old_ak,
+ old_sk,
+ update_ak,
+ update_sk,
+ alter_ak,
+ alter_sk,
+ updated_instance.ram_user().sk(),
+ altered_instance.ram_user().sk()};
+ for (const auto& message : messages) {
+ for (std::string_view credential : credentials) {
+ EXPECT_EQ(message.find(credential), std::string::npos) << message;
+ }
+ }
+}
+
TEST(MetaServiceHttpTest, TxnLazyCommit) {
HttpContext ctx;
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]