This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 014b70909da [fix](segcompaction) fix coredump for inverted index file
writer close in segment compaction #43477 (#43587)
014b70909da is described below
commit 014b70909dacd5283494fbb4dd4b92aa1bf4384d
Author: airborne12 <airborn...@gmail.com>
AuthorDate: Tue Nov 12 06:51:26 2024 +0800
[fix](segcompaction) fix coredump for inverted index file writer close in
segment compaction #43477 (#43587)
cherry pick from #43477
Co-authored-by: airborne12 <jiang...@selectdb.com>
---
be/src/olap/rowset/segcompaction.cpp | 4 ++-
be/src/runtime/exec_env.cpp | 4 +++
be/src/runtime/exec_env.h | 2 ++
be/test/olap/segcompaction_test.cpp | 59 ++++++++++++++++++++++++++++++++++++
4 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/rowset/segcompaction.cpp
b/be/src/olap/rowset/segcompaction.cpp
index 92b903d3a90..8f4184b7175 100644
--- a/be/src/olap/rowset/segcompaction.cpp
+++ b/be/src/olap/rowset/segcompaction.cpp
@@ -310,7 +310,9 @@ Status
SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt
_writer->_num_segcompacted);
}
RETURN_IF_ERROR(_writer->_rename_compacted_segments(begin, end));
-
+ if (_inverted_index_file_writer != nullptr) {
+ _inverted_index_file_writer.reset();
+ }
if (VLOG_DEBUG_IS_ON) {
_writer->vlog_buffer.clear();
for (const auto& entry :
std::filesystem::directory_iterator(ctx.tablet_path)) {
diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp
index e41cc982a74..872069ee70a 100644
--- a/be/src/runtime/exec_env.cpp
+++ b/be/src/runtime/exec_env.cpp
@@ -45,6 +45,10 @@ ExecEnv::~ExecEnv() {
}
#ifdef BE_TEST
+void ExecEnv::set_inverted_index_searcher_cache(
+ segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache)
{
+ _inverted_index_searcher_cache = inverted_index_searcher_cache;
+}
void ExecEnv::set_storage_engine(std::unique_ptr<BaseStorageEngine>&& engine) {
_storage_engine = std::move(engine);
}
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index afd99a624f2..307c2586fc6 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -271,6 +271,8 @@ public:
}
void set_storage_engine(std::unique_ptr<BaseStorageEngine>&& engine);
+ void set_inverted_index_searcher_cache(
+ segment_v2::InvertedIndexSearcherCache*
inverted_index_searcher_cache);
void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; }
void set_tablet_schema_cache(TabletSchemaCache* c) {
this->_tablet_schema_cache = c; }
void set_tablet_column_object_pool(TabletColumnObjectPool* c) {
diff --git a/be/test/olap/segcompaction_test.cpp
b/be/test/olap/segcompaction_test.cpp
index 32d724d246b..2457ff11b83 100644
--- a/be/test/olap/segcompaction_test.cpp
+++ b/be/test/olap/segcompaction_test.cpp
@@ -49,6 +49,7 @@ using namespace ErrorCode;
static const uint32_t MAX_PATH_LEN = 1024;
static StorageEngine* l_engine = nullptr;
static const std::string lTestDir = "./data_test/data/segcompaction_test";
+constexpr static std::string_view tmp_dir = "./data_test/tmp";
class SegCompactionTest : public testing::Test {
public:
@@ -59,6 +60,7 @@ public:
config::tablet_map_shard_size = 1;
config::txn_map_shard_size = 1;
config::txn_shard_size = 1;
+ config::inverted_index_fd_number_limit_percent = 0;
char buffer[MAX_PATH_LEN];
EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
@@ -72,6 +74,23 @@ public:
std::vector<StorePath> paths;
paths.emplace_back(config::storage_root_path, -1);
+ // tmp dir
+
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok());
+
EXPECT_TRUE(io::global_local_filesystem()->create_directory(tmp_dir).ok());
+ paths.emplace_back(std::string(tmp_dir), 1024000000);
+ auto tmp_file_dirs = std::make_unique<segment_v2::TmpFileDirs>(paths);
+ EXPECT_TRUE(tmp_file_dirs->init().ok());
+ ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs));
+
+ // use memory limit
+ int64_t inverted_index_cache_limit = 0;
+ _inverted_index_searcher_cache =
std::unique_ptr<segment_v2::InvertedIndexSearcherCache>(
+
InvertedIndexSearcherCache::create_global_instance(inverted_index_cache_limit,
+ 256));
+
+ ExecEnv::GetInstance()->set_inverted_index_searcher_cache(
+ _inverted_index_searcher_cache.get());
+
doris::EngineOptions options;
options.store_paths = paths;
@@ -99,6 +118,7 @@ public:
ExecEnv* exec_env = doris::ExecEnv::GetInstance();
l_engine = nullptr;
exec_env->set_storage_engine(nullptr);
+ exec_env->set_inverted_index_searcher_cache(nullptr);
}
protected:
@@ -140,6 +160,7 @@ protected:
tablet_schema_pb.set_num_rows_per_row_block(1024);
tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
tablet_schema_pb.set_next_column_unique_id(4);
+
tablet_schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
ColumnPB* column_1 = tablet_schema_pb.add_column();
column_1->set_unique_id(1);
@@ -150,6 +171,11 @@ protected:
column_1->set_index_length(4);
column_1->set_is_nullable(true);
column_1->set_is_bf_column(false);
+ auto tablet_index_1 = tablet_schema_pb.add_index();
+ tablet_index_1->set_index_id(1);
+ tablet_index_1->set_index_name("column_1");
+ tablet_index_1->set_index_type(IndexType::INVERTED);
+ tablet_index_1->add_col_unique_id(1);
ColumnPB* column_2 = tablet_schema_pb.add_column();
column_2->set_unique_id(2);
@@ -162,6 +188,11 @@ protected:
column_2->set_is_key(true);
column_2->set_is_nullable(true);
column_2->set_is_bf_column(false);
+ auto tablet_index_2 = tablet_schema_pb.add_index();
+ tablet_index_2->set_index_id(2);
+ tablet_index_2->set_index_name("column_2");
+ tablet_index_2->set_index_type(IndexType::INVERTED);
+ tablet_index_2->add_col_unique_id(2);
for (int i = 1; i <= num_value_col; i++) {
ColumnPB* v_column = tablet_schema_pb.add_column();
@@ -245,6 +276,7 @@ protected:
private:
std::unique_ptr<DataDir> _data_dir;
+ std::unique_ptr<InvertedIndexSearcherCache> _inverted_index_searcher_cache;
};
TEST_F(SegCompactionTest, SegCompactionThenRead) {
@@ -300,6 +332,13 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) {
ls.push_back("10047_4.dat");
ls.push_back("10047_5.dat");
ls.push_back("10047_6.dat");
+ ls.push_back("10047_0.idx");
+ ls.push_back("10047_1.idx");
+ ls.push_back("10047_2.idx");
+ ls.push_back("10047_3.idx");
+ ls.push_back("10047_4.idx");
+ ls.push_back("10047_5.idx");
+ ls.push_back("10047_6.idx");
EXPECT_TRUE(check_dir(ls));
}
@@ -502,6 +541,13 @@ TEST_F(SegCompactionTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
ls.push_back("10048_4.dat"); // O
ls.push_back("10048_5.dat"); // oooooooo
ls.push_back("10048_6.dat"); // O
+ ls.push_back("10048_0.idx"); // oooo
+ ls.push_back("10048_1.idx"); // O
+ ls.push_back("10048_2.idx"); // O
+ ls.push_back("10048_3.idx"); // o
+ ls.push_back("10048_4.idx"); // O
+ ls.push_back("10048_5.idx"); // oooooooo
+ ls.push_back("10048_6.idx"); // O
EXPECT_TRUE(check_dir(ls));
}
}
@@ -628,6 +674,11 @@ TEST_F(SegCompactionTest,
SegCompactionInterleaveWithBig_OoOoO) {
ls.push_back("10049_2.dat"); // O
ls.push_back("10049_3.dat"); // o
ls.push_back("10049_4.dat"); // O
+ ls.push_back("10049_0.idx"); // O
+ ls.push_back("10049_1.idx"); // o
+ ls.push_back("10049_2.idx"); // O
+ ls.push_back("10049_3.idx"); // o
+ ls.push_back("10049_4.idx"); // O
EXPECT_TRUE(check_dir(ls));
}
}
@@ -789,6 +840,10 @@ TEST_F(SegCompactionTest,
SegCompactionThenReadUniqueTableSmall) {
ls.push_back("10051_1.dat");
ls.push_back("10051_2.dat");
ls.push_back("10051_3.dat");
+ ls.push_back("10051_0.idx");
+ ls.push_back("10051_1.idx");
+ ls.push_back("10051_2.idx");
+ ls.push_back("10051_3.idx");
EXPECT_TRUE(check_dir(ls));
}
@@ -1052,6 +1107,10 @@ TEST_F(SegCompactionTest,
SegCompactionThenReadAggTableSmall) {
ls.push_back("10052_1.dat");
ls.push_back("10052_2.dat");
ls.push_back("10052_3.dat");
+ ls.push_back("10052_0.idx");
+ ls.push_back("10052_1.idx");
+ ls.push_back("10052_2.idx");
+ ls.push_back("10052_3.idx");
EXPECT_TRUE(check_dir(ls));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
