kangpinghuang commented on a change in pull request #567: Add publish version
task
URL: https://github.com/apache/incubator-doris/pull/567#discussion_r249740794
##########
File path: be/src/olap/storage_engine.cpp
##########
@@ -115,96 +115,140 @@ StorageEngine::~StorageEngine() {
clear();
}
+// TODO(ygl): deal with rowsets and tablets when load failed
OLAPStatus StorageEngine::_load_data_dir(DataDir* data_dir) {
std::string data_dir_path = data_dir->path();
LOG(INFO) <<"start to load tablets from data_dir_path:" << data_dir_path;
-
bool is_header_converted = false;
- OLAPStatus res = TabletMetaManager::get_header_converted(data_dir,
is_header_converted);
+ OLAPStatus res = TabletMetaManager::get_header_converted(data_dir,
&is_header_converted);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "get convert flag from meta failed";
return res;
}
+ if (!is_header_converted) {
+ // ygl: could not be compatable with old doris data. User has to use
previous version to parse
+ // header file into meta env first.
+ LOG(WARNING) << "header is not converted to tablet meta yet, could not
use this Doris version. "
+ << "[dir path =" << data_dir_path << "]";
+ return OLAP_ERR_INIT_FAILED;
+ }
- // load rowset meta from metaenv and create rowset
+ // load rowset meta from meta env and create rowset
// COMMITTED: add to txn manager
// VISIBLE: add to tablet
// if one rowset load failed, then the total data dir will not be loaded
- std::vector<std::shared_ptr<RowsetMeta>> dir_rowset_metas;
- if (is_header_converted) {
- LOG(INFO) << "begin loading rowset from meta";
- bool has_error = false;
- auto load_rowset_func = [this, data_dir, &has_error,
&dir_rowset_metas](RowsetId rowset_id, const std::string& meta_str) -> bool {
- std::shared_ptr<RowsetMeta> rowset_meta(new RowsetMeta());
- bool parsed = rowset_meta->init(meta_str);
- if (!parsed) {
- LOG(WARNING) << "parse rowset meta string failed for
rowset_id:" << rowset_id;
- has_error = true;
- // return false will break meta iterator
- return false;
- }
- dir_rowset_metas.push_back(rowset_meta);
+ std::vector<RowsetMetaSharedPtr> dir_rowset_metas;
+ LOG(INFO) << "begin loading rowset from meta";
+ auto load_rowset_func = [this, data_dir, &dir_rowset_metas](RowsetId
rowset_id, const std::string& meta_str) -> bool {
+ RowsetMetaSharedPtr rowset_meta(new RowsetMeta());
+ bool parsed = rowset_meta->init(meta_str);
+ if (!parsed) {
+ LOG(WARNING) << "parse rowset meta string failed for rowset_id:"
<< rowset_id;
+ // return false will break meta iterator
return true;
- };
- OLAPStatus s =
RowsetMetaManager::traverse_rowset_metas(data_dir->get_meta(),
load_rowset_func);
- if (has_error) {
- LOG(WARNING) << "errors when load rowset meta from meta env, skip
this data dir:" << data_dir_path;
- return OLAP_ERR_META_ITERATOR;
- }
- LOG(INFO) << "load header from meta finished";
- if (s != OLAP_SUCCESS) {
- LOG(WARNING) << "errors when load rowset meta from meta env, skip
this data dir:" << data_dir_path;
- return s;
- } else {
- return OLAP_SUCCESS;
}
+ dir_rowset_metas.push_back(rowset_meta);
+ return true;
+ };
+ OLAPStatus load_rowset_status =
RowsetMetaManager::traverse_rowset_metas(data_dir->get_meta(),
load_rowset_func);
+
+ if (load_rowset_status != OLAP_SUCCESS) {
+ LOG(WARNING) << "errors when load rowset meta from meta env, skip this
data dir:" << data_dir_path;
+ } else {
+ LOG(INFO) << "load rowset from meta finished, data dir: " <<
data_dir_path;
}
+
// load tablet
// create tablet from tablet meta and add it to tablet mgr
- if (is_header_converted) {
- LOG(INFO) << "load header from meta";
- auto load_tablet_func = [this, data_dir](long tablet_id,
- long schema_hash, const std::string& value) -> bool {
-
- OLAPStatus status =
TabletManager::instance()->load_tablet_from_header(data_dir, tablet_id,
schema_hash, value);
- if (status != OLAP_SUCCESS) {
- LOG(WARNING) << "load tablet from header failed. status:" <<
status
- << "tablet=" << tablet_id << "." << schema_hash;
- };
- return true;
+ LOG(INFO) << "begin loading tablet from meta";
+ auto load_tablet_func = [this, data_dir](long tablet_id,
+ long schema_hash, const std::string& value) -> bool {
+ OLAPStatus status =
TabletManager::instance()->load_tablet_from_meta(data_dir, tablet_id,
schema_hash, value);
+ if (status != OLAP_SUCCESS) {
+ LOG(WARNING) << "load tablet from header failed. status:" << status
+ << "tablet=" << tablet_id << "." << schema_hash;
};
- OLAPStatus s =
TabletMetaManager::traverse_headers(data_dir->get_meta(), load_tablet_func);
- LOG(INFO) << "load header from meta finished";
- if (s != OLAP_SUCCESS) {
- LOG(WARNING) << "there is failure when loading tablet headers,
path:" << data_dir_path;
- return s;
- } else {
- return OLAP_SUCCESS;
- }
+ return true;
+ };
+ OLAPStatus load_tablet_status =
TabletMetaManager::traverse_headers(data_dir->get_meta(), load_tablet_func);
+ if (load_tablet_status != OLAP_SUCCESS) {
+ LOG(WARNING) << "there is failure when loading tablet headers, path:"
<< data_dir_path;
} else {
- // ygl: could not be compatable with old doris data. User has to use
previous version to parse
- // header file into meta env first.
- LOG(WARNING) << "header is not converted to tablet meta yet, could not
use this Doris version. "
- << "[dir path =" << data_dir_path << "]";
- return OLAP_ERR_INIT_FAILED;
+ LOG(INFO) << "load rowset from meta finished, data dir: " <<
data_dir_path;
}
+ // tranverse rowset
+ // 1. add committed rowset to txn map
+ // 2. add visible rowset to tablet
+ // ignore any errors when load tablet or rowset, because fe will repair
them after report
for (auto rowset_meta : dir_rowset_metas) {
- // TODO(ygl)
- // 1. build rowset from meta
- // 2. add committed rowset to txn
- // 3. add visible rowset to tablet
+ TabletSharedPtr tablet =
TabletManager::instance()->get_tablet(rowset_meta->tablet_id(),
rowset_meta->tablet_schema_hash());
+ // tablet maybe dropped, but not drop related rowset meta
+ if (tablet.get() == NULL) {
+ LOG(WARNING) << "could not find tablet id: " <<
rowset_meta->tablet_id()
+ << " schema hash: " <<
rowset_meta->tablet_schema_hash()
+ << " for rowset: " << rowset_meta->rowset_id()
+ << " skip this rowset";
+ continue;
+ }
+ // TODO(ygl): build rowset from rowset meta using tablet info
+ RowsetSharedPtr rowset;
+ if (rowset_meta->rowset_state() == RowsetStatePB::COMMITTED) {
+ // build rowset from meta and create
+ PUniqueId load_id;
+ load_id.set_hi(0);
+ load_id.set_lo(0);
+ // TODO(ygl): create rowset from rowset meta
+ OLAPStatus add_txn_status = TxnManager::instance()->add_txn(
+ rowset_meta->partition_id(), rowset_meta->txn_id(),
+ rowset_meta->tablet_id(), rowset_meta->tablet_schema_hash(),
+ load_id, NULL);
+ if (add_txn_status != OLAP_SUCCESS && add_txn_status !=
OLAP_ERR_PUSH_TRANSACTION_ALREADY_EXIST) {
+ LOG(WARNING) << "failed to add committed rowset: " <<
rowset_meta->rowset_id()
+ << " to tablet: " << rowset_meta->tablet_id()
+ << " for txn: " << rowset_meta->txn_id();
+ } else {
+ LOG(INFO) << "successfully to add committed rowset: " <<
rowset_meta->rowset_id()
+ << " to tablet: " << rowset_meta->tablet_id()
+ << " schema hash: " <<
rowset_meta->tablet_schema_hash()
+ << " for txn: " << rowset_meta->txn_id();
+ }
+ } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE) {
+ // add visible rowset to tablet, it maybe use in the future
+ // there should be only preparing rowset in meta env because
visible
+ // rowset is persist with tablet meta currently
+ OLAPStatus publish_status = tablet->add_inc_rowset(*rowset);
+ if (publish_status != OLAP_SUCCESS) {
+ LOG(WARNING) << "add visilbe rowset to tablet failed
rowset_id:" << rowset->rowset_id()
+ << " tablet id: " << rowset_meta->tablet_id()
+ << " txn id:" << rowset_meta->txn_id()
+ << " start_version: " <<
rowset_meta->version().first
+ << " end_version: " <<
rowset_meta->version().second;
+ } else {
+ LOG(INFO) << "successfully to add visible rowset: " <<
rowset_meta->rowset_id()
+ << " to tablet: " << rowset_meta->tablet_id()
+ << " txn id:" << rowset_meta->txn_id()
+ << " start_version: " << rowset_meta->version().first
+ << " end_version: " << rowset_meta->version().second;
+ }
+ } else {
+ LOG(WARNING) << "find invalid rowset: " << rowset_meta->rowset_id()
+ << " with tablet id: " << rowset_meta->tablet_id()
+ << " schema hash: " <<
rowset_meta->tablet_schema_hash()
+ << " txn: " << rowset_meta->txn_id();
Review comment:
I think this log should print the invalid rowset state.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@doris.apache.org
For additional commands, e-mail: dev-h...@doris.apache.org
