This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fe4efdda463 [fix](cloud) Fix possible incorrect merged tablet stats
while iterating detached tablet stats (#40494)
fe4efdda463 is described below
commit fe4efdda46346869283e57f9eafbfcec18ab145c
Author: Gavin Chou <[email protected]>
AuthorDate: Wed Sep 11 12:32:08 2024 +0800
[fix](cloud) Fix possible incorrect merged tablet stats while iterating
detached tablet stats (#40494)
Previous impl. of `get_detached_tablet_stats()` may miss some detached
KVs or tablet stats due to KV iterating paging (`RangeGetIterator.more()
== true`), which leads to zero detached stats and produce buggy data
size report to FE.
---
cloud/src/meta-service/http_encode_key.cpp | 52 +-
cloud/src/meta-service/meta_service_job.cpp | 25 +-
.../src/meta-service/meta_service_tablet_stats.cpp | 75 +-
cloud/src/meta-service/meta_service_tablet_stats.h | 21 +-
cloud/test/http_encode_key_test.cpp | 844 +++++++++++----------
5 files changed, 565 insertions(+), 452 deletions(-)
diff --git a/cloud/src/meta-service/http_encode_key.cpp
b/cloud/src/meta-service/http_encode_key.cpp
index 165fff7ca44..4d05f0121b0 100644
--- a/cloud/src/meta-service/http_encode_key.cpp
+++ b/cloud/src/meta-service/http_encode_key.cpp
@@ -135,32 +135,50 @@ static std::string parse_tablet_schema(const ValueBuf&
buf) {
static std::string parse_tablet_stats(const ValueBuf& buf) {
if (buf.iters.empty()) {
- return "";
+ return "stats_kvs not found\n";
}
- TabletStatsPB stats;
- auto&& it = buf.iters[0];
- if (!it->has_next()) {
- return "";
+ std::vector<std::pair<std::string, std::string>> stats_kvs;
+ stats_kvs.reserve(5);
+ for (auto& i : buf.iters) {
+ while (i->has_next()) {
+ auto [k, v] = i->next();
+ stats_kvs.emplace_back(std::string {k.data(), k.size()},
+ std::string {v.data(), v.size()});
+ }
+ }
+
+ if (stats_kvs.empty()) {
+ return "stats_kvs not found\n";
}
- auto [k, v] = it->next();
+ TabletStatsPB stats;
+ auto [k, v] = stats_kvs[0];
stats.ParseFromArray(v.data(), v.size());
+ std::string json;
+ json += "aggregated_stats: " + proto_to_json(stats) + "\n";
+
// Parse split tablet stats
TabletStats detached_stats;
- int ret = get_detached_tablet_stats(*it, detached_stats);
+ int ret = get_detached_tablet_stats(stats_kvs, detached_stats);
if (ret != 0) {
- return "";
+ json += "failed to get detached_stats, ret=" + std::to_string(ret) +
"\n";
+ return json;
}
+ TabletStatsPB detached_stats_pb;
+ merge_tablet_stats(detached_stats_pb, detached_stats); // convert to pb
+ json += "detached_stats: " + proto_to_json(detached_stats_pb) + "\n";
merge_tablet_stats(stats, detached_stats);
- return proto_to_json(stats);
+ json += "merged_stats: " + proto_to_json(stats) + "\n";
+ return json;
}
// See keys.h to get all types of key, e.g: MetaRowsetKeyInfo
-// key_type -> {{param1, param2 ...}, encoding_func, value_parsing_func}
+// key_type -> {{param1, param2 ...}, key_encoding_func, value_parsing_func}
+// where params are the input for key_encoding_func
// clang-format off
static std::unordered_map<std::string_view,
std::tuple<std::vector<std::string_view>,
std::function<std::string(param_type&)>, std::function<std::string(const
ValueBuf&)>>> param_set {
@@ -250,11 +268,17 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const
brpc::URI& uri) {
ValueBuf value;
if (key_type == "StatsTabletKey") {
// FIXME(plat1ko): hard code
- std::string end_key {key};
- encode_bytes("\xff", &end_key);
+ std::string begin_key {key};
+ std::string end_key = key + "\xff";
std::unique_ptr<RangeGetIterator> it;
- err = txn->get(key, end_key, &it, true);
- value.iters.push_back(std::move(it));
+ bool more = false;
+ do {
+ err = txn->get(begin_key, end_key, &it, true);
+ if (err != TxnErrorCode::TXN_OK) break;
+ begin_key = it->next_begin_key();
+ more = it->more();
+ value.iters.push_back(std::move(it));
+ } while (more);
} else {
err = cloud::get(txn.get(), key, &value, true);
}
diff --git a/cloud/src/meta-service/meta_service_job.cpp
b/cloud/src/meta-service/meta_service_job.cpp
index 15b4e4c368f..d1c8df15870 100644
--- a/cloud/src/meta-service/meta_service_job.cpp
+++ b/cloud/src/meta-service/meta_service_job.cpp
@@ -733,16 +733,33 @@ void process_compaction_job(MetaServiceCode& code,
std::string& msg, std::string
}
auto stats_key = stats_tablet_key({instance_id, table_id, index_id,
partition_id, tablet_id});
auto stats_val = stats->SerializeAsString();
+
+ VLOG_DEBUG << "data size, tablet_id=" << tablet_id << " stats.num_rows="
<< stats->num_rows()
+ << " stats.data_size=" << stats->data_size()
+ << " stats.num_rowsets=" << stats->num_rowsets()
+ << " stats.num_segments=" << stats->num_segments()
+ << " detached_stats.num_rows=" << detached_stats.num_rows
+ << " detached_stats.data_size=" << detached_stats.data_size
+ << " detached_stats.num_rowset=" << detached_stats.num_rowsets
+ << " detached_stats.num_segments=" << detached_stats.num_segs
+ << " compaction.size_output_rowsets=" <<
compaction.size_output_rowsets()
+ << " compaction.size_input_rowsets=" <<
compaction.size_input_rowsets();
txn->put(stats_key, stats_val);
- merge_tablet_stats(*stats, detached_stats);
+ merge_tablet_stats(*stats, detached_stats); // this is to check
if (stats->data_size() < 0 || stats->num_rowsets() < 1) [[unlikely]] {
INSTANCE_LOG(ERROR) << "buggy data size, tablet_id=" << tablet_id
+ << " stats.num_rows=" << stats->num_rows()
<< " stats.data_size=" << stats->data_size()
+ << " stats.num_rowsets=" << stats->num_rowsets()
+ << " stats.num_segments=" << stats->num_segments()
+ << " detached_stats.num_rows=" <<
detached_stats.num_rows
+ << " detached_stats.data_size=" <<
detached_stats.data_size
+ << " detached_stats.num_rowset=" <<
detached_stats.num_rowsets
+ << " detached_stats.num_segments=" <<
detached_stats.num_segs
<< " compaction.size_output_rowsets="
<< compaction.size_output_rowsets()
- << " compaction.size_input_rowsets= "
- << compaction.size_input_rowsets();
- DCHECK(false) << "buggy data size";
+ << " compaction.size_input_rowsets=" <<
compaction.size_input_rowsets();
+ DCHECK(false) << "buggy data size, tablet_id=" << tablet_id;
}
VLOG_DEBUG << "update tablet stats tablet_id=" << tablet_id << " key=" <<
hex(stats_key)
diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp
b/cloud/src/meta-service/meta_service_tablet_stats.cpp
index 868f89e3559..572ee6ebb68 100644
--- a/cloud/src/meta-service/meta_service_tablet_stats.cpp
+++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp
@@ -30,49 +30,63 @@ namespace doris::cloud {
void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg,
Transaction* txn,
const std::string& instance_id, const
TabletIndexPB& idx,
TabletStatsPB& stats, TabletStats&
detached_stats, bool snapshot) {
- auto begin_key = stats_tablet_key(
- {instance_id, idx.table_id(), idx.index_id(), idx.partition_id(),
idx.tablet_id()});
- auto end_key = stats_tablet_key(
- {instance_id, idx.table_id(), idx.index_id(), idx.partition_id(),
idx.tablet_id() + 1});
+ // clang-format off
+ auto begin_key = stats_tablet_key({instance_id, idx.table_id(),
idx.index_id(), idx.partition_id(), idx.tablet_id()});
+ auto begin_key_check = begin_key;
+ auto end_key = stats_tablet_key({instance_id, idx.table_id(),
idx.index_id(), idx.partition_id(), idx.tablet_id() + 1});
+ // clang-format on
+ std::vector<std::pair<std::string, std::string>> stats_kvs;
+ stats_kvs.reserve(5); // aggregate + data_size + num_rows + num_rowsets +
num_segments
+
std::unique_ptr<RangeGetIterator> it;
- TxnErrorCode err = txn->get(begin_key, end_key, &it, snapshot);
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::READ>(err);
- msg = fmt::format("failed to get tablet stats, err={} tablet_id={}",
err, idx.tablet_id());
- return;
- }
- if (!it->has_next()) {
+ do {
+ TxnErrorCode err = txn->get(begin_key, end_key, &it, snapshot);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ msg = fmt::format("failed to get tablet stats, err={}
tablet_id={}", err,
+ idx.tablet_id());
+ return;
+ }
+ while (it->has_next()) {
+ auto [k, v] = it->next();
+ stats_kvs.emplace_back(std::string {k.data(), k.size()},
+ std::string {v.data(), v.size()});
+ }
+ begin_key = it->next_begin_key();
+ } while (it->more());
+
+ if (stats_kvs.empty()) {
code = MetaServiceCode::TABLET_NOT_FOUND;
msg = fmt::format("tablet stats not found, tablet_id={}",
idx.tablet_id());
return;
}
- auto [k, v] = it->next();
- // First key MUST be tablet stats key
- DCHECK(k == begin_key) << hex(k) << " vs " << hex(begin_key);
+
+ auto& [first_stats_key, v] = stats_kvs[0];
+ // First key MUST be tablet stats key, the original non-detached one
+ DCHECK(first_stats_key == begin_key_check)
+ << hex(first_stats_key) << " vs " << hex(begin_key_check);
if (!stats.ParseFromArray(v.data(), v.size())) {
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- msg = fmt::format("marformed tablet stats value, key={}", hex(k));
+ msg = fmt::format("marformed tablet stats value, key={}",
hex(first_stats_key));
return;
}
// Parse split tablet stats
- int ret = get_detached_tablet_stats(*it, detached_stats);
+ int ret = get_detached_tablet_stats(stats_kvs, detached_stats);
+
if (ret != 0) {
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- msg = fmt::format("marformed splitted tablet stats kv, key={}",
hex(k));
+ msg = fmt::format("marformed splitted tablet stats kv, key={}",
hex(first_stats_key));
return;
}
}
-int get_detached_tablet_stats(RangeGetIterator& iter, TabletStats&
detached_stats) {
- while (iter.has_next()) {
- auto [k, v] = iter.next();
- int64_t val;
- if (v.size() != sizeof(val)) [[unlikely]] {
- LOG(WARNING) << "malformed tablet stats value. key=" << hex(k);
- return -1;
- }
-
- // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "data_size"
+int get_detached_tablet_stats(const std::vector<std::pair<std::string,
std::string>>& stats_kvs,
+ TabletStats& detached_stats) {
+ if (stats_kvs.size() != 5 && stats_kvs.size() != 1) {
+ LOG(WARNING) << "incorrect tablet stats_kvs, it should be 1 or 5
size=" << stats_kvs.size();
+ }
+ for (size_t i = 1; i < stats_kvs.size(); ++i) {
+ std::string_view k(stats_kvs[i].first), v(stats_kvs[i].second);
k.remove_prefix(1);
constexpr size_t key_parts = 8;
std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>>
out;
@@ -84,9 +98,14 @@ int get_detached_tablet_stats(RangeGetIterator& iter,
TabletStats& detached_stat
auto* suffix = std::get_if<std::string>(&std::get<0>(out.back()));
if (!suffix) [[unlikely]] {
LOG(WARNING) << "malformed tablet stats key. key=" << hex(k);
- return -1;
+ return -2;
}
+ int64_t val = 0;
+ if (v.size() != sizeof(val)) [[unlikely]] {
+ LOG(WARNING) << "malformed tablet stats value v.size=" << v.size()
<< " key=" << hex(k);
+ return -3;
+ }
std::memcpy(&val, v.data(), sizeof(val));
if constexpr (std::endian::native == std::endian::big) {
val = bswap_64(val);
diff --git a/cloud/src/meta-service/meta_service_tablet_stats.h
b/cloud/src/meta-service/meta_service_tablet_stats.h
index 3e8b4f82a0b..5726cf50b76 100644
--- a/cloud/src/meta-service/meta_service_tablet_stats.h
+++ b/cloud/src/meta-service/meta_service_tablet_stats.h
@@ -47,8 +47,23 @@ void internal_get_tablet_stats(MetaServiceCode& code,
std::string& msg, Transact
const std::string& instance_id, const
TabletIndexPB& idx,
TabletStatsPB& stats, bool snapshot = false);
-// Get detached tablet stats via `iter`, `iter.next` SHOULD be the first
splitted tablet stats KV.
-// Return 0 if success, otherwise error.
-[[nodiscard]] int get_detached_tablet_stats(RangeGetIterator& iter,
TabletStats& detached_stats);
+// clang-format off
+/**
+ * Get detached tablet stats via with given stats_kvs
+ *
+ * stats_kvs stores the following KVs, see keys.h for more details
+ * 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} -> TabletStatsPB
+ * 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "data_size" -> int64
+ * 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "num_rows" -> int64
+ * 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "num_rowsets" -> int64
+ * 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "num_segments" -> int64
+ *
+ * @param stats_kvs the tablet stats kvs to process, it is in size of 5 or 1
+ * @param detached_stats output param for the detached stats
+ * @return 0 for success otherwise error
+ */
+[[nodiscard]] int get_detached_tablet_stats(const
std::vector<std::pair<std::string, std::string>>& stats_kvs,
+ TabletStats& detached_stats);
+// clang-format on
} // namespace doris::cloud
diff --git a/cloud/test/http_encode_key_test.cpp
b/cloud/test/http_encode_key_test.cpp
index fdb8d76ee1b..77f80084607 100644
--- a/cloud/test/http_encode_key_test.cpp
+++ b/cloud/test/http_encode_key_test.cpp
@@ -111,383 +111,445 @@ v v v v
v
struct Input {
std::string_view key_type;
std::string_view param;
- std::string_view key;
- std::function<std::string()> gen_value;
+ std::vector<std::string> key;
+ std::function<std::vector<std::string>()> gen_value;
std::string_view value;
};
+
// clang-format off
static auto test_inputs = std::array {
- Input {
- "InstanceKey",
- "instance_id=gavin-instance",
- "0110696e7374616e6365000110676176696e2d696e7374616e63650001",
- []() -> std::string {
- InstanceInfoPB pb;
- pb.set_instance_id("gavin-instance");
- return pb.SerializeAsString();
- },
- R"({"instance_id":"gavin-instance"})",
- },
- Input {
- "TxnLabelKey",
- "instance_id=gavin-instance&db_id=10086&label=test-label",
-
"011074786e000110676176696e2d696e7374616e636500011074786e5f6c6162656c000112000000000000276610746573742d6c6162656c0001",
- []() -> std::string {
- TxnLabelPB pb;
- pb.add_txn_ids(123456789);
- auto val = pb.SerializeAsString();
- MemTxnKv::gen_version_timestamp(123456790, 0, &val);
- return val;
- },
- R"({"txn_ids":["123456789"]}
+ Input {
+ "InstanceKey",
+ "instance_id=gavin-instance",
+ {"0110696e7374616e6365000110676176696e2d696e7374616e63650001"},
+ []() -> std::vector<std::string> {
+ InstanceInfoPB pb;
+ pb.set_instance_id("gavin-instance");
+ return {pb.SerializeAsString()};
+ },
+ R"({"instance_id":"gavin-instance"})",
+ },
+ Input {
+ "TxnLabelKey",
+ "instance_id=gavin-instance&db_id=10086&label=test-label",
+
{"011074786e000110676176696e2d696e7374616e636500011074786e5f6c6162656c000112000000000000276610746573742d6c6162656c0001"},
+ []() -> std::vector<std::string> {
+ TxnLabelPB pb;
+ pb.add_txn_ids(123456789);
+ auto val = pb.SerializeAsString();
+ MemTxnKv::gen_version_timestamp(123456790, 0, &val);
+ return {val};
+ },
+ R"({"txn_ids":["123456789"]}
txn_id=126419752960)",
+ },
+ Input {
+ "TxnInfoKey",
+ "instance_id=gavin-instance&db_id=10086&txn_id=10010",
+
{"011074786e000110676176696e2d696e7374616e636500011074786e5f696e666f000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ TxnInfoPB pb;
+ pb.set_db_id(10086);
+ pb.set_txn_id(10010);
+ return {pb.SerializeAsString()};
+ },
+ R"({"db_id":"10086","txn_id":"10010"})",
+ },
+ Input {
+ "TxnIndexKey",
+ "instance_id=gavin-instance&txn_id=10086",
+
{"011074786e000110676176696e2d696e7374616e636500011074786e5f696e6465780001120000000000002766"},
+ []() -> std::vector<std::string> {
+ TxnIndexPB pb;
+ pb.mutable_tablet_index()->set_db_id(10086);
+ return {pb.SerializeAsString()};
+ },
+ R"({"tablet_index":{"db_id":"10086"}})",
+ },
+ Input {
+ "TxnRunningKey",
+ "instance_id=gavin-instance&db_id=10086&txn_id=10010",
+
{"011074786e000110676176696e2d696e7374616e636500011074786e5f72756e6e696e67000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ TxnRunningPB pb;
+ pb.add_table_ids(10001);
+ return {pb.SerializeAsString()};
+ },
+ R"({"table_ids":["10001"]})",
+ },
+ Input {
+ "PartitionVersionKey",
+
"instance_id=gavin-instance&db_id=10086&tbl_id=10010&partition_id=10000",
+
{"011076657273696f6e000110676176696e2d696e7374616e6365000110706172746974696f6e000112000000000000276612000000000000271a120000000000002710"},
+ []() -> std::vector<std::string> {
+ VersionPB pb;
+ pb.set_version(10);
+ return {pb.SerializeAsString()};
+ },
+ R"({"version":"10"})",
+ },
+ Input {
+ "TableVersionKey",
+ "instance_id=gavin-instance&db_id=10086&tbl_id=10010",
+
{"011076657273696f6e000110676176696e2d696e7374616e63650001107461626c65000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ VersionPB pb;
+ pb.set_version(10);
+ return {pb.SerializeAsString()};
+ },
+ R"({"version":"10"})",
+ },
+ Input {
+ "MetaRowsetKey",
+ "instance_id=gavin-instance&tablet_id=10086&version=10010",
+
{"01106d657461000110676176696e2d696e7374616e6365000110726f77736574000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ doris::RowsetMetaCloudPB pb;
+ pb.set_rowset_id(0);
+ pb.set_rowset_id_v2("rowset_id_1");
+ pb.set_tablet_id(10086);
+ pb.set_start_version(10010);
+ pb.set_end_version(10010);
+ return {pb.SerializeAsString()};
+ },
+
R"({"rowset_id":"0","tablet_id":"10086","start_version":"10010","end_version":"10010","rowset_id_v2":"rowset_id_1"})",
+ },
+ Input {
+ "MetaRowsetTmpKey",
+ "instance_id=gavin-instance&txn_id=10086&tablet_id=10010",
+
{"01106d657461000110676176696e2d696e7374616e6365000110726f777365745f746d70000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ doris::RowsetMetaCloudPB pb;
+ pb.set_rowset_id(0);
+ pb.set_rowset_id_v2("rowset_id_1");
+ pb.set_tablet_id(10010);
+ pb.set_txn_id(10086);
+ pb.set_start_version(2);
+ pb.set_end_version(2);
+ return {pb.SerializeAsString()};
+ },
+
R"({"rowset_id":"0","tablet_id":"10010","txn_id":"10086","start_version":"2","end_version":"2","rowset_id_v2":"rowset_id_1"})",
+ },
+ Input {
+ "MetaTabletKey",
+
"instance_id=gavin-instance&table_id=10086&index_id=100010&part_id=10000&tablet_id=1008601",
+
{"01106d657461000110676176696e2d696e7374616e63650001107461626c657400011200000000000027661200000000000186aa1200000000000027101200000000000f63d9"},
+ []() -> std::vector<std::string> {
+ doris::TabletMetaCloudPB pb;
+ pb.set_table_id(10086);
+ pb.set_index_id(100010);
+ pb.set_partition_id(10000);
+ pb.set_tablet_id(1008601);
+ return {pb.SerializeAsString()};
+ },
+
R"({"table_id":"10086","partition_id":"10000","tablet_id":"1008601","index_id":"100010"})",
+ },
+ Input {
+ "MetaTabletIdxKey",
+ "instance_id=gavin-instance&tablet_id=10086",
+
{"01106d657461000110676176696e2d696e7374616e63650001107461626c65745f696e6465780001120000000000002766"},
+ []() -> std::vector<std::string> {
+ TabletIndexPB pb;
+ pb.set_table_id(10006);
+ pb.set_index_id(100010);
+ pb.set_partition_id(10000);
+ pb.set_tablet_id(10086);
+ return {pb.SerializeAsString()};
+ },
+
R"({"table_id":"10006","index_id":"100010","partition_id":"10000","tablet_id":"10086"})",
+ },
+ Input {
+ "RecycleIndexKey",
+ "instance_id=gavin-instance&index_id=10086",
+
{"011072656379636c65000110676176696e2d696e7374616e6365000110696e6465780001120000000000002766"},
+ []() -> std::vector<std::string> {
+ RecycleIndexPB pb;
+ pb.set_creation_time(12345);
+ pb.set_table_id(10000);
+ return {pb.SerializeAsString()};
+ },
+ R"({"table_id":"10000","creation_time":"12345"})",
+ },
+ Input {
+ "RecyclePartKey",
+ "instance_id=gavin-instance&part_id=10086",
+
{"011072656379636c65000110676176696e2d696e7374616e6365000110706172746974696f6e0001120000000000002766"},
+ []() -> std::vector<std::string> {
+ RecyclePartitionPB pb;
+ pb.set_creation_time(12345);
+ pb.set_table_id(10000);
+ pb.add_index_id(10001);
+ return {pb.SerializeAsString()};
+ },
+ R"({"table_id":"10000","index_id":["10001"],"creation_time":"12345"})",
+ },
+ Input {
+ "RecycleRowsetKey",
+ "instance_id=gavin-instance&tablet_id=10086&rowset_id=10010",
+
{"011072656379636c65000110676176696e2d696e7374616e6365000110726f7773657400011200000000000027661031303031300001"},
+ []() -> std::vector<std::string> {
+ RecycleRowsetPB pb;
+ pb.set_creation_time(12345);
+ auto rs = pb.mutable_rowset_meta();
+ rs->set_rowset_id(0);
+ rs->set_rowset_id_v2("10010");
+ rs->set_tablet_id(10086);
+ return {pb.SerializeAsString()};
+ },
+
R"({"creation_time":"12345","rowset_meta":{"rowset_id":"0","tablet_id":"10086","rowset_id_v2":"10010"}})",
+ },
+ Input {
+ "RecycleTxnKey",
+ "instance_id=gavin-instance&db_id=10086&txn_id=10010",
+
{"011072656379636c65000110676176696e2d696e7374616e636500011074786e000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ RecycleTxnPB pb;
+ pb.set_label("label_1");
+ pb.set_creation_time(12345);
+ return {pb.SerializeAsString()};
+ },
+ R"({"creation_time":"12345","label":"label_1"})",
+ },
+ Input { // aggregated_stats + full detached_stats, there are 5 KVs in total
+ "StatsTabletKey",
+
"instance_id=gavin-instance&table_id=10086&index_id=10010&part_id=10000&tablet_id=1008601",
+ {
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9",
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d910646174615f73697a650001",
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9106e756d5f726f77730001",
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9106e756d5f726f77736574730001",
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9106e756d5f736567730001",
+ },
+ []() -> std::vector<std::string> {
+ TabletStatsPB pb;
+ auto idx = pb.mutable_idx();
+ idx->set_table_id(10086);
+ idx->set_index_id(100010);
+ idx->set_partition_id(10000);
+ idx->set_tablet_id(1008601);
+ pb.set_data_size(1);
+ pb.set_num_rows(10);
+ pb.set_num_rowsets(11);
+ pb.set_num_segments(12);
+ return {pb.SerializeAsString(),
{"\x01\x00\x00\x00\x00\x00\x00\x00",8}, {"\x02\x00\x00\x00\x00\x00\x00\x00",8},
{"\x03\x00\x00\x00\x00\x00\x00\x00",8}, {"\x04\x00\x00\x00\x00\x00\x00\x00",8}};
+ },
+ R"(aggregated_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"1","num_rows":"10","num_rowsets":"11","num_segments":"12"}
+detached_stats:
{"data_size":"1","num_rows":"2","num_rowsets":"3","num_segments":"4"}
+merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"2","num_rows":"12","num_rowsets":"14","num_segments":"16"}
+)",
+ },
+ Input { // aggregated_stats + half detached_stats (num_segs == 0, there is
num_rowsets detached stats)
+ "StatsTabletKey",
+
"instance_id=gavin-instance&table_id=10086&index_id=10010&part_id=10000&tablet_id=1008602",
+ {
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63da",
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63da106e756d5f726f77736574730001",
+ },
+ []() -> std::vector<std::string> {
+ TabletStatsPB pb;
+ auto idx = pb.mutable_idx();
+ idx->set_table_id(10086);
+ idx->set_index_id(100010);
+ idx->set_partition_id(10000);
+ idx->set_tablet_id(1008602);
+ pb.set_data_size(1);
+ pb.set_num_rows(10);
+ pb.set_num_rowsets(11);
+ pb.set_num_segments(12);
+ return {pb.SerializeAsString(),
{"\x03\x00\x00\x00\x00\x00\x00\x00",8}};
+ },
+ R"(aggregated_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"1","num_rows":"10","num_rowsets":"11","num_segments":"12"}
+detached_stats:
{"data_size":"0","num_rows":"0","num_rowsets":"3","num_segments":"0"}
+merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"1","num_rows":"10","num_rowsets":"14","num_segments":"12"}
+)",
+ },
+ Input { // aggregated_stats only, the legacy
+ "StatsTabletKey",
+
"instance_id=gavin-instance&table_id=10086&index_id=10010&part_id=10000&tablet_id=1008603",
+ {
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63db",
+ },
+ []() -> std::vector<std::string> {
+ TabletStatsPB pb;
+ auto idx = pb.mutable_idx();
+ idx->set_table_id(10086);
+ idx->set_index_id(100010);
+ idx->set_partition_id(10000);
+ idx->set_tablet_id(1008602);
+ pb.set_data_size(1);
+ pb.set_num_rows(10);
+ pb.set_num_rowsets(11);
+ pb.set_num_segments(12);
+ return {pb.SerializeAsString()};
},
- Input {
- "TxnInfoKey",
- "instance_id=gavin-instance&db_id=10086&txn_id=10010",
-
"011074786e000110676176696e2d696e7374616e636500011074786e5f696e666f000112000000000000276612000000000000271a",
- []() -> std::string {
- TxnInfoPB pb;
- pb.set_db_id(10086);
- pb.set_txn_id(10010);
- return pb.SerializeAsString();
- },
- R"({"db_id":"10086","txn_id":"10010"})",
- },
- Input {
- "TxnIndexKey",
- "instance_id=gavin-instance&txn_id=10086",
-
"011074786e000110676176696e2d696e7374616e636500011074786e5f696e6465780001120000000000002766",
- []() -> std::string {
- TxnIndexPB pb;
- pb.mutable_tablet_index()->set_db_id(10086);
- return pb.SerializeAsString();
- },
- R"({"tablet_index":{"db_id":"10086"}})",
- },
- Input {
- "TxnRunningKey",
- "instance_id=gavin-instance&db_id=10086&txn_id=10010",
-
"011074786e000110676176696e2d696e7374616e636500011074786e5f72756e6e696e67000112000000000000276612000000000000271a",
- []() -> std::string {
- TxnRunningPB pb;
- pb.add_table_ids(10001);
- return pb.SerializeAsString();
- },
- R"({"table_ids":["10001"]})",
- },
- Input {
- "PartitionVersionKey",
-
"instance_id=gavin-instance&db_id=10086&tbl_id=10010&partition_id=10000",
-
"011076657273696f6e000110676176696e2d696e7374616e6365000110706172746974696f6e000112000000000000276612000000000000271a120000000000002710",
- []() -> std::string {
- VersionPB pb;
- pb.set_version(10);
- return pb.SerializeAsString();
- },
- R"({"version":"10"})",
- },
- Input {
- "TableVersionKey",
- "instance_id=gavin-instance&db_id=10086&tbl_id=10010",
-
"011076657273696f6e000110676176696e2d696e7374616e63650001107461626c65000112000000000000276612000000000000271a",
- []() -> std::string {
- VersionPB pb;
- pb.set_version(10);
- return pb.SerializeAsString();
- },
- R"({"version":"10"})",
- },
- Input {
- "MetaRowsetKey",
- "instance_id=gavin-instance&tablet_id=10086&version=10010",
-
"01106d657461000110676176696e2d696e7374616e6365000110726f77736574000112000000000000276612000000000000271a",
- []() -> std::string {
- doris::RowsetMetaCloudPB pb;
- pb.set_rowset_id(0);
- pb.set_rowset_id_v2("rowset_id_1");
- pb.set_tablet_id(10086);
- pb.set_start_version(10010);
- pb.set_end_version(10010);
- return pb.SerializeAsString();
- },
-
R"({"rowset_id":"0","tablet_id":"10086","start_version":"10010","end_version":"10010","rowset_id_v2":"rowset_id_1"})",
- },
- Input {
- "MetaRowsetTmpKey",
- "instance_id=gavin-instance&txn_id=10086&tablet_id=10010",
-
"01106d657461000110676176696e2d696e7374616e6365000110726f777365745f746d70000112000000000000276612000000000000271a",
- []() -> std::string {
- doris::RowsetMetaCloudPB pb;
- pb.set_rowset_id(0);
- pb.set_rowset_id_v2("rowset_id_1");
- pb.set_tablet_id(10010);
- pb.set_txn_id(10086);
- pb.set_start_version(2);
- pb.set_end_version(2);
- return pb.SerializeAsString();
- },
-
R"({"rowset_id":"0","tablet_id":"10010","txn_id":"10086","start_version":"2","end_version":"2","rowset_id_v2":"rowset_id_1"})",
- },
- Input {
- "MetaTabletKey",
-
"instance_id=gavin-instance&table_id=10086&index_id=100010&part_id=10000&tablet_id=1008601",
-
"01106d657461000110676176696e2d696e7374616e63650001107461626c657400011200000000000027661200000000000186aa1200000000000027101200000000000f63d9",
- []() -> std::string {
- doris::TabletMetaCloudPB pb;
- pb.set_table_id(10086);
- pb.set_index_id(100010);
- pb.set_partition_id(10000);
- pb.set_tablet_id(1008601);
- return pb.SerializeAsString();
- },
-
R"({"table_id":"10086","partition_id":"10000","tablet_id":"1008601","index_id":"100010"})",
- },
- Input {
- "MetaTabletIdxKey",
- "instance_id=gavin-instance&tablet_id=10086",
-
"01106d657461000110676176696e2d696e7374616e63650001107461626c65745f696e6465780001120000000000002766",
- []() -> std::string {
- TabletIndexPB pb;
- pb.set_table_id(10006);
- pb.set_index_id(100010);
- pb.set_partition_id(10000);
- pb.set_tablet_id(10086);
- return pb.SerializeAsString();
- },
-
R"({"table_id":"10006","index_id":"100010","partition_id":"10000","tablet_id":"10086"})",
- },
- Input {
- "RecycleIndexKey",
- "instance_id=gavin-instance&index_id=10086",
-
"011072656379636c65000110676176696e2d696e7374616e6365000110696e6465780001120000000000002766",
- []() -> std::string {
- RecycleIndexPB pb;
- pb.set_creation_time(12345);
- pb.set_table_id(10000);
- return pb.SerializeAsString();
- },
- R"({"table_id":"10000","creation_time":"12345"})",
- },
- Input {
- "RecyclePartKey",
- "instance_id=gavin-instance&part_id=10086",
-
"011072656379636c65000110676176696e2d696e7374616e6365000110706172746974696f6e0001120000000000002766",
- []() -> std::string {
- RecyclePartitionPB pb;
- pb.set_creation_time(12345);
- pb.set_table_id(10000);
- pb.add_index_id(10001);
- return pb.SerializeAsString();
- },
-
R"({"table_id":"10000","index_id":["10001"],"creation_time":"12345"})",
- },
- Input {
- "RecycleRowsetKey",
- "instance_id=gavin-instance&tablet_id=10086&rowset_id=10010",
-
"011072656379636c65000110676176696e2d696e7374616e6365000110726f7773657400011200000000000027661031303031300001",
- []() -> std::string {
- RecycleRowsetPB pb;
- pb.set_creation_time(12345);
- auto rs = pb.mutable_rowset_meta();
- rs->set_rowset_id(0);
- rs->set_rowset_id_v2("10010");
- rs->set_tablet_id(10086);
- return pb.SerializeAsString();
- },
-
R"({"creation_time":"12345","rowset_meta":{"rowset_id":"0","tablet_id":"10086","rowset_id_v2":"10010"}})",
- },
- Input {
- "RecycleTxnKey",
- "instance_id=gavin-instance&db_id=10086&txn_id=10010",
-
"011072656379636c65000110676176696e2d696e7374616e636500011074786e000112000000000000276612000000000000271a",
- []() -> std::string {
- RecycleTxnPB pb;
- pb.set_label("label_1");
- pb.set_creation_time(12345);
- return pb.SerializeAsString();
- },
- R"({"creation_time":"12345","label":"label_1"})",
- },
- Input {
- "StatsTabletKey",
-
"instance_id=gavin-instance&table_id=10086&index_id=10010&part_id=10000&tablet_id=1008601",
-
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9",
- []() -> std::string {
- TabletStatsPB pb;
- auto idx = pb.mutable_idx();
- idx->set_table_id(10086);
- idx->set_index_id(100010);
- idx->set_partition_id(10000);
- idx->set_tablet_id(1008601);
- pb.set_num_rowsets(10);
- return pb.SerializeAsString();
- },
-
R"({"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"0","num_rows":"0","num_rowsets":"10","num_segments":"0"})",
- },
- Input {
- "JobTabletKey",
-
"instance_id=gavin-instance&table_id=10086&index_id=10010&part_id=10000&tablet_id=1008601",
-
"01106a6f62000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9",
- []() -> std::string {
- TabletJobInfoPB pb;
- auto idx = pb.mutable_idx();
- idx->set_table_id(10086);
- idx->set_index_id(100010);
- idx->set_partition_id(10000);
- idx->set_tablet_id(1008601);
- auto c = pb.add_compaction();
- c->set_id("compaction_1");
- return pb.SerializeAsString();
- },
-
R"({"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"compaction":[{"id":"compaction_1"}]})",
- },
- Input {
- "CopyJobKey",
-
"instance_id=gavin-instance&stage_id=10086&table_id=10010©_id=10000&group_id=1008601",
-
"0110636f7079000110676176696e2d696e7374616e63650001106a6f620001103130303836000112000000000000271a10313030303000011200000000000f63d9",
- []() -> std::string {
- CopyJobPB pb;
- pb.set_stage_type(StagePB::EXTERNAL);
- pb.set_start_time_ms(12345);
- return pb.SerializeAsString();
- },
- R"({"stage_type":"EXTERNAL","start_time_ms":"12345"})",
- },
- Input {
- "CopyFileKey",
-
"instance_id=gavin-instance&stage_id=10086&table_id=10010&obj_key=10000&obj_etag=1008601",
-
"0110636f7079000110676176696e2d696e7374616e63650001106c6f6164696e675f66696c650001103130303836000112000000000000271a103130303030000110313030383630310001",
- []() -> std::string {
- CopyFilePB pb;
- pb.set_copy_id("copy_id_1");
- pb.set_group_id(10000);
- return pb.SerializeAsString();
- },
- R"({"copy_id":"copy_id_1","group_id":10000})",
- },
- Input {
- "RecycleStageKey",
- "instance_id=gavin-instance&stage_id=10086",
-
"011072656379636c65000110676176696e2d696e7374616e6365000110737461676500011031303038360001",
- []() -> std::string {
- RecycleStagePB pb;
- pb.set_instance_id("gavin-instance");
- pb.set_reason("reason");
- return pb.SerializeAsString();
- },
- R"({"instance_id":"gavin-instance","reason":"reason"})",
- },
- Input {
- "JobRecycleKey",
- "instance_id=gavin-instance",
-
"01106a6f62000110676176696e2d696e7374616e6365000110636865636b0001",
- []() -> std::string {
- JobRecyclePB pb;
- pb.set_instance_id("gavin-instance");
- pb.set_ip_port("host_1");
- return pb.SerializeAsString();
- },
- R"({"instance_id":"gavin-instance","ip_port":"host_1"})",
- },
- Input {
- "MetaSchemaKey",
-
"instance_id=gavin-instance&index_id=10086&schema_version=10010",
-
"01106d657461000110676176696e2d696e7374616e6365000110736368656d61000112000000000000276612000000000000271a",
- []() -> std::string {
- doris::TabletSchemaCloudPB pb;
- pb.set_schema_version(10010);
- auto col = pb.add_column();
- col->set_unique_id(6789);
- col->set_name("col_1");
- col->set_type("INT");
- return pb.SerializeAsString();
- },
-
R"({"column":[{"unique_id":6789,"name":"col_1","type":"INT"}],"schema_version":10010})",
- },
- Input {
- "MetaDeleteBitmap",
-
"instance_id=gavin-instance&tablet_id=10086&rowest_id=10010&version=10000&seg_id=1008601",
-
"01106d657461000110676176696e2d696e7374616e636500011064656c6574655f6269746d6170000112000000000000276610313030313000011200000000000027101200000000000f63d9",
- []() -> std::string {
- return "abcdefg";
- },
- "61626364656667",
- },
- Input {
- "MetaDeleteBitmapUpdateLock",
- "instance_id=gavin-instance&table_id=10086&partition_id=10010",
-
"01106d657461000110676176696e2d696e7374616e636500011064656c6574655f6269746d61705f6c6f636b000112000000000000276612000000000000271a",
- []() -> std::string {
- DeleteBitmapUpdateLockPB pb;
- pb.set_lock_id(12345);
- pb.add_initiators(114115);
- return pb.SerializeAsString();
- },
- R"({"lock_id":"12345","initiators":["114115"]})",
- },
- Input {
- "MetaPendingDeleteBitmap",
- "instance_id=gavin-instance&tablet_id=10086",
-
"01106d657461000110676176696e2d696e7374616e636500011064656c6574655f6269746d61705f70656e64696e670001120000000000002766",
- []() -> std::string {
- PendingDeleteBitmapPB pb;
- pb.add_delete_bitmap_keys("key_1");
- return pb.SerializeAsString();
- },
- R"({"delete_bitmap_keys":["a2V5XzE="]})",
- },
- Input {
- "RLJobProgressKey",
- "instance_id=gavin-instance&db_id=10086&job_id=10010",
-
"01106a6f62000110676176696e2d696e7374616e6365000110726f7574696e655f6c6f61645f70726f6772657373000112000000000000276612000000000000271a",
- []() -> std::string {
- RoutineLoadProgressPB pb;
- auto map = pb.mutable_partition_to_offset();
- map->insert({1000, 1234});
- return pb.SerializeAsString();
- },
- R"({"partition_to_offset":{"1000":"1234"}})",
- },
- Input {
- "MetaServiceRegistryKey",
- "",
-
"021073797374656d0001106d6574612d7365727669636500011072656769737472790001",
- []() -> std::string {
- ServiceRegistryPB pb;
- auto i = pb.add_items();
- i->set_host("host_1");
- i->set_ctime_ms(123456);
- return pb.SerializeAsString();
- },
- R"({"items":[{"ctime_ms":"123456","host":"host_1"}]})",
- },
- Input {
- "MetaServiceArnInfoKey",
- "",
-
"021073797374656d0001106d6574612d7365727669636500011061726e5f696e666f0001",
- []() -> std::string {
- RamUserPB pb;
- pb.set_user_id("user_1");
- pb.set_ak("ak");
- pb.set_sk("sk");
- return pb.SerializeAsString();
- },
- R"({"user_id":"user_1","ak":"ak","sk":"sk"})",
- },
- Input {
- "MetaServiceEncryptionKey",
- "",
-
"021073797374656d0001106d6574612d73657276696365000110656e6372797074696f6e5f6b65795f696e666f0001",
- []() -> std::string {
- EncryptionKeyInfoPB pb;
- auto i = pb.add_items();
- i->set_key_id(23456);
- i->set_key("key_1");
- return pb.SerializeAsString();
- },
- R"({"items":[{"key_id":"23456","key":"key_1"}]})",
+ R"(aggregated_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"1","num_rows":"10","num_rowsets":"11","num_segments":"12"}
+detached_stats:
{"data_size":"0","num_rows":"0","num_rowsets":"0","num_segments":"0"}
+merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"1","num_rows":"10","num_rowsets":"11","num_segments":"12"}
+)",
+ },
+ Input {
+ "JobTabletKey",
+
"instance_id=gavin-instance&table_id=10086&index_id=10010&part_id=10000&tablet_id=1008601",
+
{"01106a6f62000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9"},
+ []() -> std::vector<std::string> {
+ TabletJobInfoPB pb;
+ auto idx = pb.mutable_idx();
+ idx->set_table_id(10086);
+ idx->set_index_id(100010);
+ idx->set_partition_id(10000);
+ idx->set_tablet_id(1008601);
+ auto c = pb.add_compaction();
+ c->set_id("compaction_1");
+ return {pb.SerializeAsString()};
},
+
R"({"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"compaction":[{"id":"compaction_1"}]})",
+ },
+ Input {
+ "CopyJobKey",
+
"instance_id=gavin-instance&stage_id=10086&table_id=10010©_id=10000&group_id=1008601",
+
{"0110636f7079000110676176696e2d696e7374616e63650001106a6f620001103130303836000112000000000000271a10313030303000011200000000000f63d9"},
+ []() -> std::vector<std::string> {
+ CopyJobPB pb;
+ pb.set_stage_type(StagePB::EXTERNAL);
+ pb.set_start_time_ms(12345);
+ return {pb.SerializeAsString()};
+ },
+ R"({"stage_type":"EXTERNAL","start_time_ms":"12345"})",
+ },
+ Input {
+ "CopyFileKey",
+
"instance_id=gavin-instance&stage_id=10086&table_id=10010&obj_key=10000&obj_etag=1008601",
+
{"0110636f7079000110676176696e2d696e7374616e63650001106c6f6164696e675f66696c650001103130303836000112000000000000271a103130303030000110313030383630310001"},
+ []() -> std::vector<std::string> {
+ CopyFilePB pb;
+ pb.set_copy_id("copy_id_1");
+ pb.set_group_id(10000);
+ return {pb.SerializeAsString()};
+ },
+ R"({"copy_id":"copy_id_1","group_id":10000})",
+ },
+ Input {
+ "RecycleStageKey",
+ "instance_id=gavin-instance&stage_id=10086",
+
{"011072656379636c65000110676176696e2d696e7374616e6365000110737461676500011031303038360001"},
+ []() -> std::vector<std::string> {
+ RecycleStagePB pb;
+ pb.set_instance_id("gavin-instance");
+ pb.set_reason("reason");
+ return {pb.SerializeAsString()};
+ },
+ R"({"instance_id":"gavin-instance","reason":"reason"})",
+ },
+ Input {
+ "JobRecycleKey",
+ "instance_id=gavin-instance",
+ {"01106a6f62000110676176696e2d696e7374616e6365000110636865636b0001"},
+ []() -> std::vector<std::string> {
+ JobRecyclePB pb;
+ pb.set_instance_id("gavin-instance");
+ pb.set_ip_port("host_1");
+ return {pb.SerializeAsString()};
+ },
+ R"({"instance_id":"gavin-instance","ip_port":"host_1"})",
+ },
+ Input {
+ "MetaSchemaKey",
+ "instance_id=gavin-instance&index_id=10086&schema_version=10010",
+
{"01106d657461000110676176696e2d696e7374616e6365000110736368656d61000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ doris::TabletSchemaCloudPB pb;
+ pb.set_schema_version(10010);
+ auto col = pb.add_column();
+ col->set_unique_id(6789);
+ col->set_name("col_1");
+ col->set_type("INT");
+ return {pb.SerializeAsString()};
+ },
+
R"({"column":[{"unique_id":6789,"name":"col_1","type":"INT"}],"schema_version":10010})",
+ },
+ Input {
+ "MetaDeleteBitmap",
+
"instance_id=gavin-instance&tablet_id=10086&rowest_id=10010&version=10000&seg_id=1008601",
+
{"01106d657461000110676176696e2d696e7374616e636500011064656c6574655f6269746d6170000112000000000000276610313030313000011200000000000027101200000000000f63d9"},
+ []() -> std::vector<std::string> {
+ return {"abcdefg"};
+ },
+ "61626364656667",
+ },
+ Input {
+ "MetaDeleteBitmapUpdateLock",
+ "instance_id=gavin-instance&table_id=10086&partition_id=10010",
+
{"01106d657461000110676176696e2d696e7374616e636500011064656c6574655f6269746d61705f6c6f636b000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ DeleteBitmapUpdateLockPB pb;
+ pb.set_lock_id(12345);
+ pb.add_initiators(114115);
+ return {pb.SerializeAsString()};
+ },
+ R"({"lock_id":"12345","initiators":["114115"]})",
+ },
+ Input {
+ "MetaPendingDeleteBitmap",
+ "instance_id=gavin-instance&tablet_id=10086",
+
{"01106d657461000110676176696e2d696e7374616e636500011064656c6574655f6269746d61705f70656e64696e670001120000000000002766"},
+ []() -> std::vector<std::string> {
+ PendingDeleteBitmapPB pb;
+ pb.add_delete_bitmap_keys("key_1");
+ return {pb.SerializeAsString()};
+ },
+ R"({"delete_bitmap_keys":["a2V5XzE="]})",
+ },
+ Input {
+ "RLJobProgressKey",
+ "instance_id=gavin-instance&db_id=10086&job_id=10010",
+
{"01106a6f62000110676176696e2d696e7374616e6365000110726f7574696e655f6c6f61645f70726f6772657373000112000000000000276612000000000000271a"},
+ []() -> std::vector<std::string> {
+ RoutineLoadProgressPB pb;
+ auto map = pb.mutable_partition_to_offset();
+ map->insert({1000, 1234});
+ return {pb.SerializeAsString()};
+ },
+ R"({"partition_to_offset":{"1000":"1234"}})",
+ },
+ Input {
+ "MetaServiceRegistryKey",
+ "",
+
{"021073797374656d0001106d6574612d7365727669636500011072656769737472790001"},
+ []() -> std::vector<std::string> {
+ ServiceRegistryPB pb;
+ auto i = pb.add_items();
+ i->set_host("host_1");
+ i->set_ctime_ms(123456);
+ return {pb.SerializeAsString()};
+ },
+ R"({"items":[{"ctime_ms":"123456","host":"host_1"}]})",
+ },
+ Input {
+ "MetaServiceArnInfoKey",
+ "",
+
{"021073797374656d0001106d6574612d7365727669636500011061726e5f696e666f0001"},
+ []() -> std::vector<std::string> {
+ RamUserPB pb;
+ pb.set_user_id("user_1");
+ pb.set_ak("ak");
+ pb.set_sk("sk");
+ return {pb.SerializeAsString()};
+ },
+ R"({"user_id":"user_1","ak":"ak","sk":"sk"})",
+ },
+ Input {
+ "MetaServiceEncryptionKey",
+ "",
+
{"021073797374656d0001106d6574612d73657276696365000110656e6372797074696f6e5f6b65795f696e666f0001"},
+ []() -> std::vector<std::string> {
+ EncryptionKeyInfoPB pb;
+ auto i = pb.add_items();
+ i->set_key_id(23456);
+ i->set_key("key_1");
+ return {pb.SerializeAsString()};
+ },
+ R"({"items":[{"key_id":"23456","key":"key_1"}]})",
+ },
};
// clang-format on
@@ -504,8 +566,8 @@ TEST(HttpEncodeKeyTest,
process_http_encode_key_test_cover_all_template) {
EXPECT_EQ(uri.SetHttpURL(url.str()), 0); // clear and set query string
auto http_res = process_http_encode_key(uri);
EXPECT_EQ(http_res.status_code, 200);
- EXPECT_NE(http_res.body.find(input.key), std::string::npos)
- << "real full text: " << http_res.body << "\nexpect contains:
" << input.key;
+ EXPECT_NE(http_res.body.find(input.key[0]), std::string::npos)
+ << "real full text: " << http_res.body << "\nexpect contains:
" << input.key[0];
}
}
@@ -516,9 +578,12 @@ TEST(HttpGetValueTest,
process_http_get_value_test_cover_all_template) {
std::unique_ptr<Transaction> txn;
ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
for (auto&& input : test_inputs) {
- auto key = unhex(input.key);
- auto val = input.gen_value();
- txn->put(key, val);
+ auto keys = input.key;
+ auto vals = input.gen_value();
+ EXPECT_EQ(keys.size(), vals.size()) << input.key_type;
+ for (int i = 0; i < keys.size(); ++i) {
+ txn->put(unhex(keys[i]), vals[i]);
+ }
}
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
@@ -530,7 +595,7 @@ TEST(HttpGetValueTest,
process_http_get_value_test_cover_all_template) {
url << "localhost:5000/MetaService/http?key_type=" << input.key_type;
// Key mode
if (!use_param) {
- url << "&key=" << input.key;
+ url << "&key=" << input.key[0];
} else if (!input.param.empty()) {
url << "&" << input.param;
}
@@ -554,31 +619,4 @@ TEST(HttpGetValueTest,
process_http_get_value_test_cover_all_template) {
// std::cout << http_res.body << std::endl;
EXPECT_EQ(http_res.body, input.value);
}
-
- // Test splitted tablet stats KV
- ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
-
- TabletStatsPB stats;
- stats.set_cumulative_compaction_cnt(10);
- stats.set_num_rowsets(100);
- auto key = stats_tablet_key({"gavin-instance", 10001, 10002, 10003,
10004});
- txn->put(key, stats.SerializeAsString());
- std::string num_rowsets_key;
- stats_tablet_num_rowsets_key({"gavin-instance", 10001, 10002, 10003,
10004}, &num_rowsets_key);
- txn->atomic_add(num_rowsets_key, 300);
- ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
-
- Input input {
- .key_type = "StatsTabletKey",
- .param =
"instance_id=gavin-instance&table_id=10001&index_id=10002&part_id=10003&"
- "tablet_id="
- "10004",
- .value =
-
R"({"data_size":"0","num_rows":"0","num_rowsets":"400","num_segments":"0","cumulative_compaction_cnt":"10"})",
- };
- auto url = gen_url(input, true);
- ASSERT_EQ(uri.SetHttpURL(url), 0); // clear and set query string
- auto http_res = process_http_get_value(txn_kv.get(), uri);
- EXPECT_EQ(http_res.status_code, 200);
- EXPECT_EQ(http_res.body, input.value);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
