This is an automated email from the ASF dual-hosted git repository.
lingmiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a1ae399 [Refactor] Refactor storage medium migration task process
(#4475)
a1ae399 is described below
commit a1ae399737f16d6cb9fdb2ca9dd616e1e2c485b0
Author: Mingyu Chen <morningman....@gmail.com>
AuthorDate: Thu Nov 12 10:00:43 2020 +0800
[Refactor] Refactor storage medium migration task process (#4475)
This CL refactor the storage medium migration task process in BE.
I did not modify the execution logic. Just extract part of the logic
in the migration task and put it in task_work_pool.
In this way, the migration task is only used to process the migration
from the specified tablet to the specified data dir.
Later, we can use this task to migrate of tablets between different disks.
#4476
---
be/src/agent/task_worker_pool.cpp | 88 +++++++++--
be/src/agent/task_worker_pool.h | 5 +
be/src/olap/snapshot_manager.cpp | 16 +-
be/src/olap/tablet_meta.cpp | 10 +-
be/src/olap/tablet_meta_manager.cpp | 2 +-
be/src/olap/task/engine_storage_migration_task.cpp | 171 ++++++++-------------
be/src/olap/task/engine_storage_migration_task.h | 22 +--
gensrc/thrift/AgentService.thrift | 3 +
8 files changed, 180 insertions(+), 137 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 0b04b39..575be14 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -916,19 +916,26 @@ void
TaskWorkerPool::_storage_medium_migrate_worker_thread_callback() {
}
TStatusCode::type status_code = TStatusCode::OK;
- vector<string> error_msgs;
- TStatus task_status;
- EngineStorageMigrationTask engine_task(storage_medium_migrate_req);
- OLAPStatus res = _env->storage_engine()->execute_task(&engine_task);
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "storage media migrate failed. status: " << res
- << ", signature: " << agent_task_req.signature;
- status_code = TStatusCode::RUNTIME_ERROR;
+ // check request and get info
+ TabletSharedPtr tablet;
+ DataDir* dest_store;
+ if (_check_migrate_requset(storage_medium_migrate_req, tablet,
&dest_store) != OLAP_SUCCESS) {
+ status_code = TStatusCode::RUNTIME_ERROR;
} else {
- LOG(INFO) << "storage media migrate success. status:" << res << ","
- << ", signature:" << agent_task_req.signature;
+ EngineStorageMigrationTask engine_task(tablet, dest_store);
+ OLAPStatus res =
_env->storage_engine()->execute_task(&engine_task);
+ if (res != OLAP_SUCCESS) {
+ LOG(WARNING) << "storage media migrate failed. status: " << res
+ << ", signature: " << agent_task_req.signature;
+ status_code = TStatusCode::RUNTIME_ERROR;
+ } else {
+ LOG(INFO) << "storage media migrate success. status:" << res
<< ","
+ << ", signature:" << agent_task_req.signature;
+ }
}
+ TStatus task_status;
+ vector<string> error_msgs;
task_status.__set_status_code(status_code);
task_status.__set_error_msgs(error_msgs);
@@ -943,6 +950,67 @@ void
TaskWorkerPool::_storage_medium_migrate_worker_thread_callback() {
}
}
+OLAPStatus TaskWorkerPool::_check_migrate_requset(
+ const TStorageMediumMigrateReq& req,
+ TabletSharedPtr& tablet,
+ DataDir** dest_store) {
+
+ int64_t tablet_id = req.tablet_id;
+ int32_t schema_hash = req.schema_hash;
+ tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id, schema_hash);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "can't find tablet. tablet_id= " << tablet_id
+ << " schema_hash=" << schema_hash;
+ return OLAP_ERR_TABLE_NOT_FOUND;
+ }
+
+ if (req.__isset.data_dir) {
+ // request specify the data dir
+ *dest_store = StorageEngine::instance()->get_store(req.data_dir);
+ if (*dest_store == nullptr) {
+ LOG(WARNING) << "data dir not found: " << req.data_dir;
+ return OLAP_ERR_DIR_NOT_EXIST;
+ }
+ } else {
+ // this is a storage medium
+ // get data dir by storage medium
+
+ // judge case when no need to migrate
+ uint32_t count =
StorageEngine::instance()->available_storage_medium_type_count();
+ if (count <= 1) {
+ LOG(INFO) << "available storage medium type count is less than 1, "
+ << "no need to migrate. count=" << count;
+ return OLAP_REQUEST_FAILED;
+ }
+ // check current tablet storage medium
+ TStorageMedium::type storage_medium = req.storage_medium;
+ TStorageMedium::type src_storage_medium =
tablet->data_dir()->storage_medium();
+ if (src_storage_medium == storage_medium) {
+ LOG(INFO) << "tablet is already on specified storage medium. "
+ << "storage_medium=" << storage_medium;
+ return OLAP_REQUEST_FAILED;
+ }
+ // get a random store of specified storage medium
+ auto stores =
StorageEngine::instance()->get_stores_for_create_tablet(storage_medium);
+ if (stores.empty()) {
+ LOG(WARNING) << "fail to get root path for create tablet.";
+ return OLAP_ERR_INVALID_ROOT_PATH;
+ }
+
+ *dest_store = stores[0];
+ }
+
+ // check disk capacity
+ int64_t tablet_size = tablet->tablet_footprint();
+ if ((*dest_store)->reach_capacity_limit(tablet_size)) {
+ LOG(WARNING) << "reach the capacity limit of path: " <<
(*dest_store)->path()
+ << ", tablet size: " << tablet_size;
+ return OLAP_ERR_DISK_REACH_CAPACITY_LIMIT;
+ }
+
+ return OLAP_SUCCESS;
+}
+
void TaskWorkerPool::_check_consistency_worker_thread_callback() {
while (_is_work) {
TAgentTaskRequest agent_task_req;
diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h
index 7cb1d0a..c40e6e1 100644
--- a/be/src/agent/task_worker_pool.h
+++ b/be/src/agent/task_worker_pool.h
@@ -168,6 +168,11 @@ private:
bool overwrite,
std::vector<std::string>* error_msgs);
+ OLAPStatus _check_migrate_requset(
+ const TStorageMediumMigrateReq& req,
+ TabletSharedPtr& tablet,
+ DataDir** dest_store);
+
private:
std::string _name;
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index f77528b..10173a2 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -126,8 +126,13 @@ OLAPStatus SnapshotManager::release_snapshot(const string&
snapshot_path) {
}
// TODO support beta rowset
-OLAPStatus SnapshotManager::convert_rowset_ids(const string& clone_dir,
int64_t tablet_id,
- const int32_t& schema_hash) {
+// For now, alpha and beta rowset meta have same fields, so we can just use
+// AlphaRowsetMeta here.
+OLAPStatus SnapshotManager::convert_rowset_ids(
+ const string& clone_dir,
+ int64_t tablet_id,
+ const int32_t& schema_hash) {
+
OLAPStatus res = OLAP_SUCCESS;
// check clone dir existed
if (!FileUtils::check_exist(clone_dir)) {
@@ -198,6 +203,11 @@ OLAPStatus SnapshotManager::_rename_rowset_id(const
RowsetMetaPB& rs_meta_pb, co
TabletSchema& tablet_schema, const RowsetId& rowset_id, RowsetMetaPB*
new_rs_meta_pb) {
OLAPStatus res = OLAP_SUCCESS;
// TODO use factory to obtain RowsetMeta when
SnapshotManager::convert_rowset_ids supports beta rowset
+ // TODO(cmy): now we only has AlphaRowsetMeta, and no BetaRowsetMeta.
+ // AlphaRowsetMeta only add some functions about segment group,
and no addition fields.
+ // So we can use AlphaRowsetMeta here even if this is a beta
rowset.
+ // And the `rowset_type` field indicates the real type of
rowset, so that the correct rowset
+ // can be created.
RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta());
alpha_rowset_meta->init_from_pb(rs_meta_pb);
RowsetSharedPtr org_rowset;
@@ -271,6 +281,8 @@ OLAPStatus SnapshotManager::_calc_snapshot_id_path(
return res;
}
+// location: /path/to/data/DATA_PREFIX/shard_id
+// return: /path/to/data/DATA_PREFIX/shard_id/tablet_id/schema_hash
string SnapshotManager::get_schema_hash_full_path(
const TabletSharedPtr& ref_tablet,
const string& location) const {
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index ded6521..5169cfb 100755
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -216,21 +216,21 @@ OLAPStatus TabletMeta::create_from_file(const string&
file_path) {
return OLAP_SUCCESS;
}
-OLAPStatus TabletMeta::reset_tablet_uid(const string& file_path) {
+OLAPStatus TabletMeta::reset_tablet_uid(const string& header_file) {
OLAPStatus res = OLAP_SUCCESS;
TabletMeta tmp_tablet_meta;
- if ((res = tmp_tablet_meta.create_from_file(file_path)) != OLAP_SUCCESS) {
+ if ((res = tmp_tablet_meta.create_from_file(header_file)) != OLAP_SUCCESS)
{
LOG(WARNING) << "fail to load tablet meta from file"
- << ", meta_file=" << file_path;
+ << ", meta_file=" << header_file;
return res;
}
TabletMetaPB tmp_tablet_meta_pb;
tmp_tablet_meta.to_meta_pb(&tmp_tablet_meta_pb);
*(tmp_tablet_meta_pb.mutable_tablet_uid()) =
TabletUid::gen_uid().to_proto();
- res = save(file_path, tmp_tablet_meta_pb);
+ res = save(header_file, tmp_tablet_meta_pb);
if (res != OLAP_SUCCESS) {
LOG(FATAL) << "fail to save tablet meta pb to "
- << " meta_file=" << file_path;
+ << " meta_file=" << header_file;
return res;
}
return res;
diff --git a/be/src/olap/tablet_meta_manager.cpp
b/be/src/olap/tablet_meta_manager.cpp
index b6a1b20..2476063 100755
--- a/be/src/olap/tablet_meta_manager.cpp
+++ b/be/src/olap/tablet_meta_manager.cpp
@@ -44,7 +44,7 @@ using rocksdb::kDefaultColumnFamilyName;
namespace doris {
-// should use tablet's generate tablet meta copy method to get a copy of
current tablet meta
+// should use tablet->generate_tablet_meta_copy() method to get a copy of
current tablet meta
// there are some rowset meta in local meta store and in in-memory tablet meta
// but not in tablet meta in local meta store
OLAPStatus TabletMetaManager::get_meta(
diff --git a/be/src/olap/task/engine_storage_migration_task.cpp
b/be/src/olap/task/engine_storage_migration_task.cpp
index 7e35a13..7a47324 100644
--- a/be/src/olap/task/engine_storage_migration_task.cpp
+++ b/be/src/olap/task/engine_storage_migration_task.cpp
@@ -24,179 +24,137 @@ namespace doris {
using std::stringstream;
-EngineStorageMigrationTask::EngineStorageMigrationTask(TStorageMediumMigrateReq&
storage_medium_migrate_req) :
- _storage_medium_migrate_req(storage_medium_migrate_req) {
+EngineStorageMigrationTask::EngineStorageMigrationTask(
+ const TabletSharedPtr& tablet, DataDir* dest_store):
+ _tablet(tablet),
+ _dest_store(dest_store) {
}
OLAPStatus EngineStorageMigrationTask::execute() {
- return _storage_medium_migrate(
- _storage_medium_migrate_req.tablet_id,
- _storage_medium_migrate_req.schema_hash,
- _storage_medium_migrate_req.storage_medium);
+ return _migrate();
}
-OLAPStatus EngineStorageMigrationTask::_storage_medium_migrate(
- TTabletId tablet_id, TSchemaHash schema_hash,
- TStorageMedium::type storage_medium) {
- LOG(INFO) << "begin to process storage media migrate. "
- << "tablet_id=" << tablet_id << ", schema_hash=" << schema_hash
- << ", dest_storage_medium=" << storage_medium;
+OLAPStatus EngineStorageMigrationTask::_migrate() {
+ int64_t tablet_id = _tablet->tablet_id();
+ int32_t schema_hash = _tablet->schema_hash();
+ LOG(INFO) << "begin to process tablet migrate. "
+ << "tablet_id=" << tablet_id
+ << ", dest_store=" << _dest_store->path();
+
DorisMetrics::instance()->storage_migrate_requests_total->increment(1);
+ // try hold migration lock first
OLAPStatus res = OLAP_SUCCESS;
- TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id, schema_hash);
- if (tablet == nullptr) {
- LOG(WARNING) << "can't find tablet. tablet_id= " << tablet_id
- << " schema_hash=" << schema_hash;
- return OLAP_ERR_TABLE_NOT_FOUND;
- }
-
- // judge case when no need to migrate
- uint32_t count =
StorageEngine::instance()->available_storage_medium_type_count();
- if (count <= 1) {
- LOG(INFO) << "available storage medium type count is less than 1, "
- << "no need to migrate. count=" << count;
- return OLAP_SUCCESS;
- }
-
- TStorageMedium::type src_storage_medium =
tablet->data_dir()->storage_medium();
- if (src_storage_medium == storage_medium) {
- LOG(INFO) << "tablet is already on specified storage medium. "
- << "storage_medium=" << storage_medium;
- return OLAP_SUCCESS;
- }
-
- WriteLock migration_wlock(tablet->get_migration_lock_ptr(), TRY_LOCK);
+ WriteLock migration_wlock(_tablet->get_migration_lock_ptr(), TRY_LOCK);
if (!migration_wlock.own_lock()) {
return OLAP_ERR_RWLOCK_ERROR;
}
+ // check if this tablet has related running txns. if yes, can not do
migration.
int64_t partition_id;
std::set<int64_t> transaction_ids;
-
StorageEngine::instance()->txn_manager()->get_tablet_related_txns(tablet->tablet_id(),
- tablet->schema_hash(), tablet->tablet_uid(), &partition_id,
&transaction_ids);
+
StorageEngine::instance()->txn_manager()->get_tablet_related_txns(tablet_id,
+ schema_hash, _tablet->tablet_uid(), &partition_id, &transaction_ids);
if (transaction_ids.size() > 0) {
LOG(WARNING) << "could not migration because has unfinished txns, "
- << " tablet=" << tablet->full_name();
+ << " tablet=" << _tablet->full_name();
return OLAP_ERR_HEADER_HAS_PENDING_DATA;
}
- tablet->obtain_push_lock();
+ _tablet->obtain_push_lock();
// TODO(ygl): the tablet should not under schema change or rollup or load
do {
// get all versions to be migrate
- tablet->obtain_header_rdlock();
- const RowsetSharedPtr last_version = tablet->rowset_with_max_version();
+ _tablet->obtain_header_rdlock();
+ const RowsetSharedPtr last_version =
_tablet->rowset_with_max_version();
if (last_version == nullptr) {
- tablet->release_header_lock();
+ _tablet->release_header_lock();
res = OLAP_ERR_VERSION_NOT_EXIST;
- LOG(WARNING) << "tablet has not any version.";
+ LOG(WARNING) << "failed to get rowset with max version, tablet="
<< _tablet->full_name();
break;
}
int32_t end_version = last_version->end_version();
vector<RowsetSharedPtr> consistent_rowsets;
- res = tablet->capture_consistent_rowsets(Version(0, end_version),
&consistent_rowsets);
+ res = _tablet->capture_consistent_rowsets(Version(0, end_version),
&consistent_rowsets);
if (consistent_rowsets.empty()) {
- tablet->release_header_lock();
+ _tablet->release_header_lock();
res = OLAP_ERR_VERSION_NOT_EXIST;
- LOG(WARNING) << "fail to capture consistent rowsets. tablet=" <<
tablet->full_name()
+ LOG(WARNING) << "fail to capture consistent rowsets. tablet=" <<
_tablet->full_name()
<< ", version=" << end_version;
break;
}
- tablet->release_header_lock();
-
- // get a random store of specified storage medium
- auto stores =
StorageEngine::instance()->get_stores_for_create_tablet(storage_medium);
- if (stores.empty()) {
- res = OLAP_ERR_INVALID_ROOT_PATH;
- LOG(WARNING) << "fail to get root path for create tablet.";
- break;
- }
-
- // check disk capacity
- int64_t tablet_size = tablet->tablet_footprint();
- if (stores[0]->reach_capacity_limit(tablet_size)) {
- res = OLAP_ERR_DISK_REACH_CAPACITY_LIMIT;
- break;
- }
+ _tablet->release_header_lock();
- // get shard
uint64_t shard = 0;
- res = stores[0]->get_shard(&shard);
+ res = _dest_store->get_shard(&shard);
if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to get root path shard. res=" << res;
+ LOG(WARNING) << "fail to get shard from store: " <<
_dest_store->path();
break;
}
-
stringstream root_path_stream;
- root_path_stream << stores[0]->path() << DATA_PREFIX << "/" << shard;
- string schema_hash_path =
SnapshotManager::instance()->get_schema_hash_full_path(tablet,
root_path_stream.str());
- // if dir already exist then return err, it should not happen
- // should not remove the dir directly
- if (FileUtils::check_exist(schema_hash_path)) {
+ root_path_stream << _dest_store->path() << DATA_PREFIX << "/" << shard;
+ string full_path =
SnapshotManager::instance()->get_schema_hash_full_path(_tablet,
root_path_stream.str());
+ // if dir already exist then return err, it should not happen.
+ // should not remove the dir directly, for safety reason.
+ if (FileUtils::check_exist(full_path)) {
LOG(INFO) << "schema hash path already exist, skip this path. "
- << "schema_hash_path=" << schema_hash_path;
+ << "full_path=" << full_path;
res = OLAP_ERR_FILE_ALREADY_EXIST;
break;
}
-
- TabletMetaSharedPtr new_tablet_meta(new(std::nothrow) TabletMeta());
- res = TabletMetaManager::get_meta(stores[0], tablet->tablet_id(),
tablet->schema_hash(), new_tablet_meta);
- if (res != OLAP_ERR_META_KEY_NOT_FOUND) {
- LOG(WARNING) << "tablet_meta already exists. "
- << "data_dir:" << stores[0]->path()
- << "tablet:" << tablet->full_name();
- res = OLAP_ERR_META_ALREADY_EXIST;
- break;
- }
- Status st = FileUtils::create_dir(schema_hash_path);
+ Status st = FileUtils::create_dir(full_path);
if (!st.ok()) {
res = OLAP_ERR_CANNOT_CREATE_DIR;
- LOG(WARNING) << "fail to create path. path=" << schema_hash_path
<< ", error:" << st.to_string();
+ LOG(WARNING) << "fail to create path. path=" << full_path << ",
error:" << st.to_string();
break;
}
// migrate all index and data files but header file
- res = _copy_index_and_data_files(schema_hash_path, tablet,
consistent_rowsets);
+ res = _copy_index_and_data_files(full_path, consistent_rowsets);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "fail to copy index and data files when migrate.
res=" << res;
break;
}
- tablet->obtain_header_rdlock();
- _generate_new_header(stores[0], shard, tablet, consistent_rowsets,
new_tablet_meta);
- tablet->release_header_lock();
- std::string new_meta_file = schema_hash_path + "/" +
std::to_string(tablet_id) + ".hdr";
+
+ // generate new tablet meta and write to hdr file
+ TabletMetaSharedPtr new_tablet_meta(new(std::nothrow) TabletMeta());
+ {
+ _tablet->obtain_header_rdlock();
+ _generate_new_header(shard, consistent_rowsets, new_tablet_meta);
+ _tablet->release_header_lock();
+ }
+ std::string new_meta_file = full_path + "/" +
std::to_string(tablet_id) + ".hdr";
res = new_tablet_meta->save(new_meta_file);
if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "failed to save meta to path" << new_meta_file;
+ LOG(WARNING) << "failed to save meta to path: " << new_meta_file;
break;
}
+ // reset tablet id and rowset id
res = TabletMeta::reset_tablet_uid(new_meta_file);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "errors while set tablet uid: '" << new_meta_file;
break;
}
-
// it will change rowset id and its create time
// rowset create time is useful when load tablet from meta to check
which tablet is the tablet to load
- res =
SnapshotManager::instance()->convert_rowset_ids(schema_hash_path, tablet_id,
schema_hash);
+ res = SnapshotManager::instance()->convert_rowset_ids(full_path,
tablet_id, schema_hash);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "failed to convert rowset id when do storage
migration"
- << " path = " << schema_hash_path;
+ << " path = " << full_path;
break;
}
- res =
StorageEngine::instance()->tablet_manager()->load_tablet_from_dir(stores[0],
- tablet_id, schema_hash, schema_hash_path, false);
+ res =
StorageEngine::instance()->tablet_manager()->load_tablet_from_dir(_dest_store,
+ tablet_id, schema_hash, full_path, false);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "failed to load tablet from new path. tablet_id="
<< tablet_id
<< " schema_hash=" << schema_hash
- << " path = " << schema_hash_path;
+ << " path = " << full_path;
break;
}
@@ -204,12 +162,12 @@ OLAPStatus
EngineStorageMigrationTask::_storage_medium_migrate(
// else the schema change status of the new tablet is FAILED
TabletSharedPtr new_tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id, schema_hash);
if (new_tablet == nullptr) {
- LOG(WARNING) << "get null tablet. tablet_id=" << tablet_id
+ LOG(WARNING) << "tablet not found. tablet_id=" << tablet_id
<< " schema_hash=" << schema_hash;
res = OLAP_ERR_TABLE_NOT_FOUND;
break;
}
- AlterTabletTaskSharedPtr alter_task = tablet->alter_task();
+ AlterTabletTaskSharedPtr alter_task = _tablet->alter_task();
if (alter_task != nullptr) {
if (alter_task->alter_state() == ALTER_FINISHED) {
new_tablet->set_alter_state(ALTER_FINISHED);
@@ -219,19 +177,17 @@ OLAPStatus
EngineStorageMigrationTask::_storage_medium_migrate(
}
} while (0);
- tablet->release_push_lock();
+ _tablet->release_push_lock();
return res;
}
// TODO(ygl): lost some information here, such as cumulative layer point
void EngineStorageMigrationTask::_generate_new_header(
- DataDir* store, const uint64_t new_shard,
- const TabletSharedPtr& tablet,
+ uint64_t new_shard,
const std::vector<RowsetSharedPtr>& consistent_rowsets,
TabletMetaSharedPtr new_tablet_meta) {
- DCHECK(store != nullptr);
- tablet->generate_tablet_meta_copy_unlocked(new_tablet_meta);
+ _tablet->generate_tablet_meta_copy_unlocked(new_tablet_meta);
vector<RowsetMetaSharedPtr> rs_metas;
for (auto& rs : consistent_rowsets) {
@@ -245,17 +201,16 @@ void EngineStorageMigrationTask::_generate_new_header(
}
OLAPStatus EngineStorageMigrationTask::_copy_index_and_data_files(
- const string& schema_hash_path,
- const TabletSharedPtr& ref_tablet,
+ const string& full_path,
const std::vector<RowsetSharedPtr>& consistent_rowsets) const {
OLAPStatus status = OLAP_SUCCESS;
for (const auto& rs : consistent_rowsets) {
- status = rs->copy_files_to(schema_hash_path);
+ status = rs->copy_files_to(full_path);
if (status != OLAP_SUCCESS) {
- Status ret = FileUtils::remove_all(schema_hash_path);
+ Status ret = FileUtils::remove_all(full_path);
if (!ret.ok()) {
LOG(FATAL) << "remove storage migration path failed. "
- << "schema_hash_path:" << schema_hash_path
+ << "full_path:" << full_path
<< " error: " << ret.to_string();
}
break;
diff --git a/be/src/olap/task/engine_storage_migration_task.h
b/be/src/olap/task/engine_storage_migration_task.h
index 7a449f2..7535b53 100644
--- a/be/src/olap/task/engine_storage_migration_task.h
+++ b/be/src/olap/task/engine_storage_migration_task.h
@@ -24,36 +24,36 @@
namespace doris {
-// base class for storage engine
-// add "Engine" as task prefix to prevent duplicate name with agent task
+/// This task is used to migrate the specified tablet to the specified data
directory.
+// Usually used for storage medium migration, or migration of tablets between
disks.
class EngineStorageMigrationTask : public EngineTask {
public:
virtual OLAPStatus execute();
public:
- EngineStorageMigrationTask(TStorageMediumMigrateReq&
storage_medium_migrate_req);
+ EngineStorageMigrationTask(const TabletSharedPtr& tablet, DataDir*
dest_store);
~EngineStorageMigrationTask() {}
private:
- OLAPStatus _storage_medium_migrate(
- TTabletId tablet_id, TSchemaHash schema_hash,
- TStorageMedium::type storage_medium);
+ OLAPStatus _migrate();
- void _generate_new_header(DataDir* store, const uint64_t new_shard,
- const TabletSharedPtr& tablet,
+ void _generate_new_header(uint64_t new_shard,
const std::vector<RowsetSharedPtr>&
consistent_rowsets,
TabletMetaSharedPtr new_tablet_meta);
// TODO: hkp
// rewrite this function
OLAPStatus _copy_index_and_data_files(
- const std::string& header_path,
- const TabletSharedPtr& ref_tablet,
+ const std::string& full_path,
const std::vector<RowsetSharedPtr>& consistent_rowsets) const;
private:
- const TStorageMediumMigrateReq& _storage_medium_migrate_req;
+ // tablet to do migrated
+ TabletSharedPtr _tablet;
+ // destination data dir
+ DataDir* _dest_store;
+
}; // EngineTask
} // doris
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index bcbc576..5eef2d1 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -161,6 +161,9 @@ struct TStorageMediumMigrateReq {
1: required Types.TTabletId tablet_id
2: required Types.TSchemaHash schema_hash
3: required Types.TStorageMedium storage_medium
+ // if data dir is specified, the storage_medium is meaning less,
+ // Doris will try to migrate the tablet to the specified data dir.
+ 4: optional string data_dir
}
struct TCancelDeleteDataReq {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
