This is an automated email from the ASF dual-hosted git repository.
lichaoyong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0462607 StorageEngine: unused_rowsets use unordered_multimap (#3207)
0462607 is described below
commit 0462607d8d537afea73b0f31e31d598cfe1c04c1
Author: HuangWei <[email protected]>
AuthorDate: Fri Mar 27 14:30:31 2020 +0800
StorageEngine: unused_rowsets use unordered_multimap (#3207)
---
be/src/olap/rowset/rowset.h | 2 ++
be/src/olap/storage_engine.cpp | 39 +++++++++++++++++++--------------------
be/src/olap/storage_engine.h | 1 +
3 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 2c509eb..30cf314 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -204,6 +204,8 @@ public:
// return whether `path` is one of the files in this rowset
virtual bool check_path(const std::string& path) = 0;
+ std::string rowset_path() const { return _rowset_path; }
+
// return an unique identifier string for this rowset
std::string unique_id() const {
return _rowset_path + "/" + rowset_id().to_string();
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 9290807..d8be355 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -713,36 +713,41 @@ void StorageEngine::_parse_default_rowset_type() {
}
void StorageEngine::start_delete_unused_rowset() {
- _gc_mutex.lock();
+ MutexLock lock(&_gc_mutex);
for (auto it = _unused_rowsets.begin(); it != _unused_rowsets.end();) {
if (it->second.use_count() != 1) {
++it;
} else if (it->second->need_delete_file()) {
VLOG(3) << "start to remove rowset:" << it->second->rowset_id()
- << ", version:" << it->second->version().first << "-" <<
it->second->version().second;
+ << ", version:" << it->second->version().first << "-"
+ << it->second->version().second;
OLAPStatus status = it->second->remove();
- VLOG(3) << "remove rowset:" << it->second->rowset_id() << "
finished. status:" << status;
+ VLOG(3) << "remove rowset:" << it->second->rowset_id()
+ << " finished. status:" << status;
it = _unused_rowsets.erase(it);
}
}
- _gc_mutex.unlock();
}
void StorageEngine::add_unused_rowset(RowsetSharedPtr rowset) {
- if (rowset == nullptr) { return; }
- _gc_mutex.lock();
+ if (rowset == nullptr) {
+ return;
+ }
+
VLOG(3) << "add unused rowset, rowset id:" << rowset->rowset_id()
- << ", version:" << rowset->version().first
- << "-" << rowset->version().second
+ << ", version:" << rowset->version().first << "-" <<
rowset->version().second
<< ", unique id:" << rowset->unique_id();
- auto it = _unused_rowsets.find(rowset->unique_id());
+
+ auto rowset_id = rowset->rowset_id().to_string();
+
+ MutexLock lock(&_gc_mutex);
+ auto it = _unused_rowsets.find(rowset_id);
if (it == _unused_rowsets.end()) {
rowset->set_need_delete_file();
rowset->close();
- _unused_rowsets[rowset->unique_id()] = rowset;
+ _unused_rowsets[rowset_id] = rowset;
release_rowset_id(rowset->rowset_id());
}
- _gc_mutex.unlock();
}
// TODO(zc): refactor this funciton
@@ -918,15 +923,9 @@ OLAPStatus StorageEngine::execute_task(EngineTask* task) {
// check whether any unused rowsets's id equal to rowset_id
bool StorageEngine::check_rowset_id_in_unused_rowsets(const RowsetId&
rowset_id) {
- _gc_mutex.lock();
- for (auto& _unused_rowset_pair : _unused_rowsets) {
- if (_unused_rowset_pair.second->rowset_id() == rowset_id) {
- _gc_mutex.unlock();
- return true;
- }
- }
- _gc_mutex.unlock();
- return false;
+ MutexLock lock(&_gc_mutex);
+ auto search = _unused_rowsets.find(rowset_id.to_string());
+ return search != _unused_rowsets.end();
}
} // namespace doris
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 3cb5dd0..8d32d9b 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -296,6 +296,7 @@ private:
static StorageEngine* _s_instance;
Mutex _gc_mutex;
+ // map<rowset_id(str), RowsetSharedPtr>, if we use RowsetId as the key, we
need custom hash func
std::unordered_map<std::string, RowsetSharedPtr> _unused_rowsets;
bool _stop_bg_worker = false;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
