This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 293a8866cd6 [improve](restore) Link existing rowset files
bidirectionally (#49233)
293a8866cd6 is described below
commit 293a8866cd6e9f83b07a1c25ed5f41b0918023f7
Author: Uniqueyou <[email protected]>
AuthorDate: Thu Apr 3 13:36:42 2025 +0800
[improve](restore) Link existing rowset files bidirectionally (#49233)
### What problem does this PR solve?
Issue Number: close #xxx
```
I20250324 17:49:24.592511 1013706 snapshot_loader.cpp:534] remote rowset
020000000000001f34486e234cc2d59301468c0cef156c81 was derived from local tablet
1742809606877 rowset 020000000000000234486e234cc2d59301468c0cef156c81, skip
downloading these files
I20250324 17:49:24.592561 1013706 snapshot_loader.cpp:534] remote rowset
020000000000002034486e234cc2d59301468c0cef156c81 was derived from local tablet
1742809606877 rowset 020000000000001934486e234cc2d59301468c0cef156c81, skip
downloading these files
I20250324 17:49:24.592617 1013706 snapshot_loader.cpp:554] link file from
/mnt/disk2/wangyixuan/doris_sim/doris_master/be/0/storage/snapshot/20250324174918.6.86400/1742809606971/978928441/020000000000001934486e234cc2d59301468c0cef156c81_0.dat
to
/mnt/disk2/wangyixuan/doris_sim/doris_master/be/0/storage/snapshot/20250324174918.6.86400/1742809606971/978928441/020000000000002034486e234cc2d59301468c0cef156c81_0.dat
I20250324 17:49:24.592677 1013706 snapshot_loader.cpp:579] skip download
remote file 020000000000002034486e234cc2d59301468c0cef156c81_0.dat as its local
version 020000000000001934486e234cc2d59301468c0cef156c81_0.dat already exists
I20250324 17:49:24.592702 1013706 snapshot_loader.cpp:534] remote rowset
020000000000002134486e234cc2d59301468c0cef156c81 was derived from local tablet
1742809606877 rowset 020000000000001a34486e234cc2d59301468c0cef156c81, skip
downloading these files
I20250324 17:49:24.592752 1013706 snapshot_loader.cpp:554] link file from
/mnt/disk2/wangyixuan/doris_sim/doris_master/be/0/storage/snapshot/20250324174918.6.86400/1742809606971/978928441/020000000000001a34486e234cc2d59301468c0cef156c81_0.dat
to
/mnt/disk2/wangyixuan/doris_sim/doris_master/be/0/storage/snapshot/20250324174918.6.86400/1742809606971/978928441/020000000000002134486e234cc2d59301468c0cef156c81_0.dat
I20250324 17:49:24.592796 1013706 snapshot_loader.cpp:579] skip download
remote file 020000000000002134486e234cc2d59301468c0cef156c81_0.dat as its local
version 020000000000001a34486e234cc2d59301468c0cef156c81_0.dat already exists
I20250324 17:49:24.592815 1013706 snapshot_loader.cpp:534] remote rowset
020000000000002234486e234cc2d59301468c0cef156c81 was derived from local tablet
1742809606877 rowset 020000000000001b34486e234cc2d59301468c0cef156c81, skip
downloading these files
I20250324 17:49:24.592852 1013706 snapshot_loader.cpp:554] link file from
/mnt/disk2/wangyixuan/doris_sim/doris_master/be/0/storage/snapshot/20250324174918.6.86400/1742809606971/978928441/020000000000001b34486e234cc2d59301468c0cef156c81_0.dat
to
/mnt/disk2/wangyixuan/doris_sim/doris_master/be/0/storage/snapshot/20250324174918.6.86400/1742809606971/978928441/020000000000002234486e234cc2d59301468c0cef156c81_0.dat
I20250324 17:49:24.592887 1013706 snapshot_loader.cpp:579] skip download
remote file 020000000000002234486e234cc2d59301468c0cef156c81_0.dat as its local
version 020000000000001b34486e234cc2d59301468c0cef156c81_0.dat already exists
I20250324 17:49:24.592907 1013706 snapshot_loader.cpp:534] remote rowset
020000000000002334486e234cc2d59301468c0cef156c81 was derived from local tablet
1742809606877 rowset 020000000000001c34486e234cc2d59301468c0cef156c81, skip
downloading these files
I20250324 17:49:24.592942 1013706 snapshot_loader.cpp:554] link file from
/mnt/disk2/wangyixuan/doris_sim/doris_master/be/0/storage/snapshot/20250324174918.6.86400/1742809606971/978928441/020000000000001c34486e234cc2d59301468c0cef156c81_0.dat
to
/mnt/disk2/wangyixuan/doris_sim/doris_master/be/0/storage/snapshot/20250324174918.6.86400/1742809606971/978928441/020000000000002334486e234cc2d59301468c0cef156c81_0.dat
I20250324 17:49:24.592978 1013706 snapshot_loader.cpp:579] skip download
remote file 020000000000002334486e234cc2d59301468c0cef156c81_0.dat as its local
version 020000000000001c34486e234cc2d59301468c0cef156c81_0.dat already exists
```
```
I20250324 17:49:30.599040 1013717 rowset.cpp:76] rowset is loaded.
020000000000002234486e234cc2d59301468c0cef156c81, rowset version:[4-4], state
from ROWSET_UNLOADED to ROWSET_LOADED. tabletid:1742809606952
I20250324 17:49:30.599387 1013717 rowset.cpp:76] rowset is loaded.
020000000000002b34486e234cc2d59301468c0cef156c81, rowset version:[4-4], state
from ROWSET_UNLOADED to ROWSET_LOADED. tabletid:1742809606952
I20250324 17:49:30.599442 1013717 beta_rowset.cpp:221] begin to remove
files in rowset 020000000000002234486e234cc2d59301468c0cef156c81, version:4-4,
tabletid:1742809606952
```
ut_test
```
I20250328 15:22:21.599613 2738237 snapshot_loader.cpp:1347] finished to
list files in local path:
/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/snapshot/20250328152221.18446744073709551615.172800/222/333,
file num: 2
I20250328 15:22:21.599749 2738237 snapshot_loader.cpp:342] list remote
files:
http://127.0.0.1:1234/api/_tablet/_download?token=fake_token&channel=ingest_binlog&file=/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/remote_snapshot/222/333,
job: 3, task id: 222, remote be: TNetworkAddress(hostname=127.0.0.1, port=1234)
I20250328 15:22:21.602023 2738237 snapshot_loader.cpp:247] clone begin to
download file from:
http://127.0.0.1:1234/api/_tablet/_download?token=fake_token&channel=ingest_binlog&file=/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/remote_snapshot/222/333/333.hdr
to:
/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/snapshot/20250328152221.18446744073709551615.172800/222/333/333.hdr.tmp.
size(B): 1001, timeout(s): 300
I20250328 15:22:21.603295 2738237 snapshot_loader.cpp:418] link rowset
files by compare 222.hdr and 333.hdr
I20250328 15:22:21.603370 2738237 snapshot_loader.cpp:510] remote rowset
0200000000000003d94d5d45759f5448b85a280cc659c6b6 was derived from local tablet
222 rowset 0200000000000001d94d5d45759f5448b85a280cc659c6b6, skip downloading
these files
I20250328 15:22:21.603389 2738237 snapshot_loader.cpp:510] remote rowset
0200000000000004d94d5d45759f5448b85a280cc659c6b6 was derived from local tablet
222 rowset 0200000000000002d94d5d45759f5448b85a280cc659c6b6, skip downloading
these files
I20250328 15:22:21.603447 2738237 snapshot_loader.cpp:530] link file from
/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/snapshot/20250328152221.18446744073709551615.172800/222/333/0200000000000002d94d5d45759f5448b85a280cc659c6b6_0.dat
to
/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/snapshot/20250328152221.18446744073709551615.172800/222/333/0200000000000004d94d5d45759f5448b85a280cc659c6b6_0.dat
I20250328 15:22:21.604249 2738237 snapshot_loader.cpp:579] remote file:
0200000000000004d94d5d45759f5448b85a280cc659c6b6_0.dat, size: 232, md5:
3b61ab8303632eccddb9758e460e00a6
I20250328 15:22:21.604286 2738237 snapshot_loader.cpp:597] file
0200000000000004d94d5d45759f5448b85a280cc659c6b6_0.dat already exists, skip
download url
http://127.0.0.1:1234/api/_tablet/_download?token=fake_token&channel=ingest_binlog&file=/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/remote_snapshot/222/333/0200000000000004d94d5d45759f5448b85a280cc659c6b6_0.dat
I20250328 15:22:21.604517 2738237 snapshot_loader.cpp:686] begin to delete
local snapshot file:
/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/snapshot/20250328152221.18446744073709551615.172800/222/333/0200000000000002d94d5d45759f5448b85a280cc659c6b6_0.dat,
it does not exist in remote
I20250328 15:22:21.604579 2738237 snapshot_loader.cpp:1099] finished to
download snapshots. job: 3, task id: 222
```
```
slot_desc: col1
total column 1
before publish, tablet row nums:0
start to add rowset version:3-3
start to publish txn
start to add inc rowset:0200000000000002d4402edb1b9ebb582b6731034765a49e,
num rows:1, version:3-3
Original version: 3, 3
snapshot_path:
/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/snapshot/20250328175354.18446744073709551615.172800
Found linked file:
/mnt/disk2/wangyixuan/doris_/doris/snapshot_data_test/snapshot/20250328175354.18446744073709551615.172800/222/333/0200000000000004d4402edb1b9ebb582b6731034765a49e_0.dat,
size: 232 bytes
```
when switching between upstream and downstream, we hope to use the
existing rowset to avoid downloading
---
be/src/runtime/snapshot_loader.cpp | 94 ++++++++++++++++++++++-
be/src/runtime/snapshot_loader.h | 7 ++
be/test/runtime/snapshot_loader_test.cpp | 125 ++++++++++++++++++++++++++++++-
3 files changed, 220 insertions(+), 6 deletions(-)
diff --git a/be/src/runtime/snapshot_loader.cpp
b/be/src/runtime/snapshot_loader.cpp
index 8bb43f3214e..14824ecace8 100644
--- a/be/src/runtime/snapshot_loader.cpp
+++ b/be/src/runtime/snapshot_loader.cpp
@@ -29,6 +29,7 @@
#include <gen_cpp/Types_types.h>
#include <algorithm>
+#include <cstddef>
#include <cstring>
#include <filesystem>
#include <istream>
@@ -97,6 +98,8 @@ public:
_report_progress_callback = std::move(report_progress);
}
+ size_t get_download_file_num() { return _need_download_files.size(); }
+
Status download();
private:
@@ -426,6 +429,14 @@ Status SnapshotHttpDownloader::_link_same_rowset_files() {
remote_rowset_metas.insert({rowset_meta.rowset_id_v2(), rowset_meta});
}
+ std::unordered_map<std::string, const RowsetMetaPB&> local_rowset_metas;
+ for (const auto& rowset_meta : local_tablet_meta.rs_metas()) {
+ if (rowset_meta.has_resource_id()) {
+ continue;
+ }
+ local_rowset_metas.insert({rowset_meta.rowset_id_v2(), rowset_meta});
+ }
+
for (const auto& local_rowset_meta : local_tablet_meta.rs_metas()) {
if (local_rowset_meta.has_resource_id() ||
!local_rowset_meta.has_source_rowset_id()) {
continue;
@@ -479,6 +490,66 @@ Status SnapshotHttpDownloader::_link_same_rowset_files() {
}
}
+ for (const auto& remote_rowset_meta : remote_tablet_meta.rs_metas()) {
+ if (remote_rowset_meta.has_resource_id() ||
!remote_rowset_meta.has_source_rowset_id()) {
+ continue;
+ }
+
+ auto local_rowset_meta =
local_rowset_metas.find(remote_rowset_meta.source_rowset_id());
+ if (local_rowset_meta == local_rowset_metas.end()) {
+ continue;
+ }
+
+ const auto& local_rowset_id = local_rowset_meta->first;
+ const auto& local_rowset_meta_pb = local_rowset_meta->second;
+ const auto& remote_rowset_id = remote_rowset_meta.rowset_id_v2();
+ auto local_tablet_id = local_rowset_meta_pb.tablet_id();
+
+ if (remote_rowset_meta.start_version() !=
local_rowset_meta_pb.start_version() ||
+ remote_rowset_meta.end_version() !=
local_rowset_meta_pb.end_version()) {
+ continue;
+ }
+
+ LOG(INFO) << "remote rowset " << remote_rowset_id << " was derived
from local tablet "
+ << local_tablet_id << " rowset " << local_rowset_id
+ << ", skip downloading these files";
+
+ for (const auto& remote_file : _remote_file_list) {
+ if (!remote_file.starts_with(remote_rowset_id)) {
+ continue;
+ }
+
+ std::string local_file = remote_file;
+ local_file.replace(0, remote_rowset_id.size(), local_rowset_id);
+ std::string local_file_path = _local_path + "/" + local_file;
+ std::string remote_file_path = _local_path + "/" + remote_file;
+
+ bool exist = false;
+
RETURN_IF_ERROR(io::global_local_filesystem()->exists(remote_file_path,
&exist));
+ if (exist) {
+ continue;
+ }
+
+ LOG(INFO) << "link file from " << local_file_path << " to " <<
remote_file_path;
+ if (!io::global_local_filesystem()->link_file(local_file_path,
remote_file_path)) {
+ std::string msg = fmt::format("link file failed from {} to {},
err: {}",
+ local_file_path,
remote_file_path, strerror(errno));
+ LOG(WARNING) << msg;
+ return Status::InternalError(std::move(msg));
+ } else {
+ auto it = _local_files.find(local_file);
+ if (it != _local_files.end()) {
+ _local_files[remote_file] = it->second;
+ } else {
+ std::string msg =
+ fmt::format("local file {} don't exist in
_local_files, err: {}",
+ local_file, strerror(errno));
+ LOG(WARNING) << msg;
+ return Status::InternalError(std::move(msg));
+ }
+ }
+ }
+ }
return Status::OK();
}
@@ -831,7 +902,9 @@ Status SnapshotLoader::download(const std::map<std::string,
std::string>& src_to
int32_t schema_hash = 0;
RETURN_IF_ERROR(_get_tablet_id_and_schema_hash_from_file_path(local_path,
&local_tablet_id,
&schema_hash));
- downloaded_tablet_ids->push_back(local_tablet_id);
+ if (downloaded_tablet_ids != nullptr) {
+ downloaded_tablet_ids->push_back(local_tablet_id);
+ }
int64_t remote_tablet_id;
RETURN_IF_ERROR(_get_tablet_id_from_remote_path(remote_path,
&remote_tablet_id));
@@ -983,13 +1056,13 @@ Status SnapshotLoader::remote_http_download(
const std::vector<TRemoteTabletSnapshot>& remote_tablet_snapshots,
std::vector<int64_t>* downloaded_tablet_ids) {
// check if job has already been cancelled
+
+#ifndef BE_TEST
int tmp_counter = 1;
RETURN_IF_ERROR(_report_every(0, &tmp_counter, 0, 0,
TTaskType::type::DOWNLOAD));
+#endif
Status status = Status::OK();
- int report_counter = 0;
- int finished_num = 0;
- int total_num = remote_tablet_snapshots.size();
for (const auto& remote_tablet_snapshot : remote_tablet_snapshots) {
auto local_tablet_id = remote_tablet_snapshot.local_tablet_id;
const auto& local_path = remote_tablet_snapshot.local_snapshot_path;
@@ -1006,15 +1079,28 @@ Status SnapshotLoader::remote_http_download(
return Status::RuntimeError(std::move(msg));
}
+ if (downloaded_tablet_ids != nullptr) {
+ downloaded_tablet_ids->push_back(local_tablet_id);
+ }
+
SnapshotHttpDownloader downloader(remote_tablet_snapshot,
std::move(tablet), *this);
+#ifndef BE_TEST
+ int report_counter = 0;
+ int finished_num = 0;
+ int total_num = remote_tablet_snapshots.size();
downloader.set_report_progress_callback(
[this, &report_counter, &finished_num, &total_num]() {
return _report_every(10, &report_counter, finished_num,
total_num,
TTaskType::type::DOWNLOAD);
});
+#endif
+
RETURN_IF_ERROR(downloader.download());
+ _set_http_download_files_num(downloader.get_download_file_num());
+#ifndef BE_TEST
++finished_num;
+#endif
}
LOG(INFO) << "finished to download snapshots. job: " << _job_id << ", task
id: " << _task_id;
diff --git a/be/src/runtime/snapshot_loader.h b/be/src/runtime/snapshot_loader.h
index 4913af6dd86..ca150cf82c4 100644
--- a/be/src/runtime/snapshot_loader.h
+++ b/be/src/runtime/snapshot_loader.h
@@ -19,6 +19,7 @@
#include <gen_cpp/Types_types.h>
+#include <cstdint>
#include <map>
#include <memory>
#include <string>
@@ -86,6 +87,8 @@ public:
Status move(const std::string& snapshot_path, TabletSharedPtr tablet, bool
overwrite);
+ int64_t get_http_download_files_num() const { return
_http_download_files_num; }
+
private:
Status _get_tablet_id_and_schema_hash_from_file_path(const std::string&
src_path,
int64_t* tablet_id,
int32_t* schema_hash);
@@ -106,6 +109,8 @@ private:
Status _list_with_checksum(const std::string& dir, std::map<std::string,
FileStat>* md5_files);
+ void _set_http_download_files_num(int64_t num) { _http_download_files_num
= num; }
+
private:
StorageEngine& _engine;
ExecEnv* _env = nullptr;
@@ -114,6 +119,8 @@ private:
const TNetworkAddress _broker_addr;
const std::map<std::string, std::string> _prop;
std::shared_ptr<io::RemoteFileSystem> _remote_fs;
+ // for test remote_http_download
+ size_t _http_download_files_num;
};
} // end namespace doris
diff --git a/be/test/runtime/snapshot_loader_test.cpp
b/be/test/runtime/snapshot_loader_test.cpp
index cb75760bb6d..36af0f5d70d 100644
--- a/be/test/runtime/snapshot_loader_test.cpp
+++ b/be/test/runtime/snapshot_loader_test.cpp
@@ -17,22 +17,34 @@
#include "runtime/snapshot_loader.h"
+#include <fmt/core.h>
#include <gen_cpp/AgentService_types.h>
#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/Types_types.h>
#include <gen_cpp/internal_service.pb.h>
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
#include <gtest/gtest_pred_impl.h>
+#include <boost/algorithm/string/replace.hpp>
+#include <cstddef>
#include <cstdint>
#include <filesystem>
#include <iostream>
#include <string>
+#include <system_error>
+#include <utility>
#include "common/config.h"
#include "common/object_pool.h"
#include "exec/tablet_info.h"
+#include "http/action/download_action.h"
+#include "http/ev_http_server.h"
+#include "http/http_channel.h"
+#include "http/http_handler.h"
+#include "http/http_request.h"
+#include "io/fs/file_reader.h"
#include "io/fs/local_file_system.h"
#include "olap/data_dir.h"
#include "olap/delta_writer.h"
@@ -49,6 +61,7 @@
#include "olap/tablet_manager.h"
#include "olap/task/engine_publish_version_task.h"
#include "olap/txn_manager.h"
+#include "runtime/cluster_info.h"
#include "runtime/define_primitive_type.h"
#include "runtime/descriptor_helper.h"
#include "runtime/descriptors.h"
@@ -56,13 +69,28 @@
#include "vec/columns/column.h"
#include "vec/core/block.h"
#include "vec/core/column_with_type_and_name.h"
-#include "vec/runtime/vdatetime_value.h"
namespace doris {
+static std::string storage_root_path;
+
+class MockDownloadHandler : public HttpHandler {
+public:
+ MockDownloadHandler() = default;
+ void handle(HttpRequest* req) override {
+ std::vector<std::string> allow_path {fmt::format("{}",
storage_root_path)};
+ DownloadAction action(ExecEnv::GetInstance(), nullptr, allow_path);
+ action.handle(req);
+ }
+};
+
static const uint32_t MAX_PATH_LEN = 1024;
static StorageEngine* engine_ref = nullptr;
-static std::string storage_root_path;
+static EvHttpServer* s_server = nullptr;
+static MockDownloadHandler mock_download_handler;
+static std::string hostname;
+static std::string address;
+static ClusterInfo cluster_info;
static void set_up() {
char buffer[MAX_PATH_LEN];
@@ -85,8 +113,18 @@ static void set_up() {
ASSERT_TRUE(s.ok()) << s;
ExecEnv* exec_env = doris::ExecEnv::GetInstance();
+ cluster_info.token = "fake_token";
+ exec_env->set_cluster_info(&cluster_info);
exec_env->set_memtable_memory_limiter(new MemTableMemoryLimiter());
exec_env->set_storage_engine(std::move(engine));
+ s_server = new EvHttpServer(1234, 3);
+ s_server->register_handler(GET, "/api/_tablet/_download",
&mock_download_handler);
+ s_server->register_handler(POST, "/api/_tablet/_download",
&mock_download_handler);
+ s_server->register_handler(HEAD, "/api/_tablet/_download",
&mock_download_handler);
+
+ static_cast<void>(s_server->start());
+ address = "127.0.0.1:" + std::to_string(1234);
+ hostname = "http://"; + address;
}
static void tear_down() {
@@ -99,6 +137,9 @@ static void tear_down() {
return;
}
+ Status st =
io::global_local_filesystem()->delete_directory(storage_root_path);
+ ASSERT_TRUE(st.ok()) << st;
+ delete s_server;
// Status s =
io::global_local_filesystem()->delete_directory(storage_root_path);
// EXPECT_TRUE(s.ok()) << "delete directory " << s;
}
@@ -339,5 +380,85 @@ TEST_F(SnapshotLoaderTest, DirMoveTaskIsIdempotent) {
ASSERT_EQ(version.first, last_version.first);
ASSERT_EQ(version.second, last_version.second);
}
+TEST_F(SnapshotLoaderTest, TestLinkSameRowsetFiles) {
+ // 1. Create a tablet
+ int64_t tablet_id = 222;
+ int32_t schema_hash = 333;
+ int64_t partition_id = 444;
+ TCreateTabletReq req = create_tablet(partition_id, tablet_id, schema_hash);
+ RuntimeProfile profile("CreateTablet");
+ Status status = engine_ref->create_tablet(req, &profile);
+ EXPECT_TRUE(status.ok());
+ TabletSharedPtr tablet =
engine_ref->tablet_manager()->get_tablet(tablet_id);
+ EXPECT_TRUE(tablet != nullptr);
+
+ // 2. Add a rowset to the tablet
+ add_rowset(tablet_id, schema_hash, partition_id, 100, 100);
+ auto version = tablet->max_version();
+ std::cout << "Original version: " << version.first << ", " <<
version.second << std::endl;
+
+ // 3. Make a snapshot of the tablet
+ string snapshot_path;
+ bool allow_incremental_clone = false;
+ TSnapshotRequest snapshot_request;
+ snapshot_request.tablet_id = tablet_id;
+ snapshot_request.schema_hash = schema_hash;
+ snapshot_request.version = version.second;
+ status = engine_ref->snapshot_mgr()->make_snapshot(snapshot_request,
&snapshot_path,
+
&allow_incremental_clone);
+ ASSERT_TRUE(status.ok());
+ std::cout << "snapshot_path: " << snapshot_path << std::endl;
+ snapshot_path = fmt::format("{}/{}/{}", snapshot_path, tablet_id,
schema_hash);
+
+ // 4. Create a destination path for "remote" snapshot
+ std::string remote_snapshot_dir = storage_root_path + "/remote_snapshot";
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(remote_snapshot_dir).ok());
+ std::string remote_tablet_path =
+ fmt::format("{}/{}/{}", remote_snapshot_dir, tablet_id,
schema_hash);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(remote_tablet_path).ok());
+
+ // 5. Copy snapshot files to remote path and calls convert_rowset_ids
+ std::vector<io::FileInfo> snapshot_files;
+ bool is_exists = false;
+ ASSERT_TRUE(io::global_local_filesystem()
+ ->list(snapshot_path, true, &snapshot_files,
&is_exists)
+ .ok());
+ for (const auto& file : snapshot_files) {
+ std::string src_file = snapshot_path + "/" + file.file_name;
+ std::string dst_file = remote_tablet_path + "/" + file.file_name;
+ ASSERT_TRUE(io::global_local_filesystem()->copy_path(src_file,
dst_file).ok());
+ }
+
+ int64_t dest_tablet_id = 333;
+ int32_t dest_schema_hash = 444;
+ std::string dest_path = fmt::format("{}/dest_snapshot/{}/{}",
storage_root_path, dest_tablet_id,
+ dest_schema_hash);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(dest_path).ok());
+
+ std::string src_hdr = remote_tablet_path + "/" + std::to_string(tablet_id)
+ ".hdr";
+ std::string dst_hdr = remote_tablet_path + "/" +
std::to_string(dest_tablet_id) + ".hdr";
+ ASSERT_TRUE(io::global_local_filesystem()->rename(src_hdr, dst_hdr).ok());
+ auto guards = engine_ref->snapshot_mgr()->convert_rowset_ids(
+ remote_tablet_path, dest_tablet_id, 0, 0, partition_id,
dest_schema_hash);
+
+ // 7. Setup a remote tablet snapshot for download
+ TRemoteTabletSnapshot remote_snapshot;
+ remote_snapshot.remote_tablet_id = dest_tablet_id;
+ remote_snapshot.local_tablet_id = tablet_id;
+ remote_snapshot.local_snapshot_path = snapshot_path;
+ remote_snapshot.remote_snapshot_path = remote_tablet_path;
+ remote_snapshot.remote_be_addr.hostname = "127.0.0.1";
+ remote_snapshot.remote_be_addr.port = 1234;
+ remote_snapshot.remote_token = "fake_token";
+
+ // 8. Download the snapshot
+ std::vector<TRemoteTabletSnapshot> remote_snapshots = {remote_snapshot};
+ std::vector<int64_t> downloaded_tablet_ids;
+ SnapshotLoader loader(*engine_ref, ExecEnv::GetInstance(), 3L, tablet_id);
+ status = loader.remote_http_download(remote_snapshots,
&downloaded_tablet_ids);
+ ASSERT_TRUE(status.ok());
+ // 9. Verify skip download files
+ ASSERT_EQ(loader.get_http_download_files_num(), 0);
+}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
