chaoyli closed pull request #493: Make tablet and tablet_meta can be compiled
URL: https://github.com/apache/incubator-doris/pull/493
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index a58bb035..5863e3df 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -584,16 +584,13 @@ void TaskWorkerPool::_alter_tablet(
break;
}
- TTabletId base_tablet_id = alter_tablet_request.base_tablet_id;
- TSchemaHash base_schema_hash = alter_tablet_request.base_schema_hash;
-
// Check last schema change status, if failed delete tablet file
// Do not need to adjust delete success or not
// Because if delete failed create rollup will failed
if (status == DORIS_SUCCESS) {
EngineSchemaChangeTask engine_task(alter_tablet_request, signature);
- OLAPStatus ret = engine_task.execute();
- if (ret != OLAP_SUCCESS) {
+ AgentStatus ret = engine_task.execute();
+ if (ret != DORIS_SUCCESS) {
status = DORIS_ERROR;
} else {
status = DORIS_SUCCESS;
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index 22a29aec..26e32c2e 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -59,7 +59,7 @@ add_library(Olap STATIC
serialize.cpp
storage_engine.cpp
data_dir.cpp
- snapshot_manager.cpp
+ #snapshot_manager.cpp
stream_index_common.cpp
stream_index_reader.cpp
stream_index_writer.cpp
diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h
index c1e746a5..49b5993f 100644
--- a/be/src/olap/olap_define.h
+++ b/be/src/olap/olap_define.h
@@ -173,7 +173,7 @@ enum OLAPStatus {
// Tablet
// [-400, -500)
OLAP_ERR_TABLE_VERSION_DUPLICATE_ERROR = -400,
- OLAP_ERR_TABLE_VERSION_INDEX_MISMATCH_ERROR = -401,
+ OLAP_ERR_TABLE_VERSION_sNDEX_MISMATCH_ERROR = -401,
OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR = -402,
OLAP_ERR_TABLE_INDEX_FIND_ERROR = -403,
OLAP_ERR_TABLE_CREATE_FROM_HEADER_ERROR = -404,
diff --git a/be/src/olap/rowset/alpha_rowset.cpp
b/be/src/olap/rowset/alpha_rowset.cpp
index 6c692852..89b1d190 100644
--- a/be/src/olap/rowset/alpha_rowset.cpp
+++ b/be/src/olap/rowset/alpha_rowset.cpp
@@ -31,7 +31,7 @@ AlphaRowset::AlphaRowset(const RowFields& tablet_schema,
_rowset_meta(rowset_meta),
_segment_group_size(0),
_is_cumulative_rowset(false) {
- Version version = _rowset_meta->get_version();
+ Version version = _rowset_meta->version();
if (version.first == version.second) {
_is_cumulative_rowset = false;
} else {
@@ -39,9 +39,9 @@ AlphaRowset::AlphaRowset(const RowFields& tablet_schema,
}
}
-NewStatus AlphaRowset::init() {
+OLAPStatus AlphaRowset::init() {
_init_segment_groups();
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
std::unique_ptr<RowsetReader> AlphaRowset::create_reader() {
@@ -54,15 +54,15 @@ std::unique_ptr<RowsetReader> AlphaRowset::create_reader() {
_rowset_path, _rowset_meta.get(), _segment_groups));
}
-NewStatus AlphaRowset::copy(RowsetBuilder* dest_rowset_builder) {
- return NewStatus::OK();
+OLAPStatus AlphaRowset::copy(RowsetBuilder* dest_rowset_builder) {
+ return OLAP_SUCCESS;
}
-NewStatus AlphaRowset::remove() {
+OLAPStatus AlphaRowset::remove() {
// TODO(hkp) : add delete code
// delete rowset from meta
// delete segment groups
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
RowsetMetaSharedPtr AlphaRowset::get_meta() {
@@ -73,12 +73,12 @@ void AlphaRowset::set_version(Version version) {
_rowset_meta->set_version(version);
}
-NewStatus AlphaRowset::_init_segment_groups() {
+OLAPStatus AlphaRowset::_init_segment_groups() {
std::vector<SegmentGroupPB> segment_group_metas;
AlphaRowsetMeta* _alpha_rowset_meta = (AlphaRowsetMeta*)_rowset_meta.get();
_alpha_rowset_meta->get_segment_groups(&segment_group_metas);
for (auto& segment_group_meta : segment_group_metas) {
- Version version = _rowset_meta->get_version();
+ Version version = _rowset_meta->version();
int64_t version_hash = _rowset_meta->get_version_hash();
std::shared_ptr<SegmentGroup> segment_group(new
SegmentGroup(_rowset_meta->get_tablet_id(),
_tablet_schema, _num_key_fields, _num_short_key_fields,
@@ -86,8 +86,8 @@ NewStatus AlphaRowset::_init_segment_groups() {
false, segment_group_meta.segment_group_id(),
segment_group_meta.num_segments()));
if (segment_group.get() == nullptr) {
LOG(WARNING) << "fail to create olap segment_group. [version='" <<
version.first
- << "-" << version.second << "' rowset_id='" <<
_rowset_meta->get_rowset_id() << "']";
- return NewStatus::NoSpace("new segment group failed");
+ << "-" << version.second << "' rowset_id='" <<
_rowset_meta->rowset_id() << "']";
+ return OLAP_ERR_CREATE_FILE_ERROR;
}
_segment_groups.push_back(segment_group);
if (segment_group_meta.has_empty()) {
@@ -99,7 +99,7 @@ NewStatus AlphaRowset::_init_segment_groups() {
LOG(WARNING) << "fail to validate segment_group. [version="<<
version.first
<< "-" << version.second << " version_hash=" <<
version_hash;
// if load segment group failed, rowset init failed
- return NewStatus::Corruption("segment group invalidate failed");
+ return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR;
}
if (segment_group_meta.column_pruning_size() != 0) {
@@ -108,7 +108,7 @@ NewStatus AlphaRowset::_init_segment_groups() {
LOG(ERROR) << "column pruning size is error."
<< "column_pruning_size=" << column_pruning_size << ",
"
<< "num_key_fields=" << _num_key_fields;
- return NewStatus::InvalidArgument("invalid segment group
column prunning size");
+ return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR;
}
std::vector<std::pair<std::string, std::string>>
column_statistic_strings(_num_key_fields);
std::vector<bool> null_vec(_num_key_fields);
@@ -124,7 +124,7 @@ NewStatus AlphaRowset::_init_segment_groups() {
}
OLAPStatus status =
segment_group->add_column_statistics(column_statistic_strings, null_vec);
if (status != OLAP_SUCCESS) {
- return NewStatus::NotSupported("add column statistics failed");
+ return status;
}
OLAPStatus res = segment_group->load();
@@ -133,7 +133,7 @@ NewStatus AlphaRowset::_init_segment_groups() {
<< "version=" << version.first << "-"
<< version.second << ", "
<< "version_hash=" << version_hash;
- return NewStatus::Corruption("segment group load failed");
+ return res;
}
}
}
@@ -141,9 +141,9 @@ NewStatus AlphaRowset::_init_segment_groups() {
if (_is_cumulative_rowset && _segment_group_size > 1) {
LOG(WARNING) << "invalid segment group meta for cumulative rowset.
segment group size:"
<< _segment_group_size;
- return NewStatus::InvalidArgument("invalid segment group size:" +
std::to_string(_segment_group_size));
+ return OLAP_ERR_ENGINE_LOAD_INDEX_TABLE_ERROR;
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/alpha_rowset.h
b/be/src/olap/rowset/alpha_rowset.h
index 3486d158..077ce324 100644
--- a/be/src/olap/rowset/alpha_rowset.h
+++ b/be/src/olap/rowset/alpha_rowset.h
@@ -36,20 +36,20 @@ class AlphaRowset : public Rowset {
int num_rows_per_row_block, const std::string rowset_path,
RowsetMetaSharedPtr rowset_meta);
- virtual NewStatus init();
+ virtual OLAPStatus init();
virtual std::unique_ptr<RowsetReader> create_reader();
- virtual NewStatus copy(RowsetBuilder* dest_rowset_builder);
+ virtual OLAPStatus copy(RowsetBuilder* dest_rowset_builder);
- virtual NewStatus remove();
+ virtual OLAPStatus remove();
virtual RowsetMetaSharedPtr get_meta();
virtual void set_version(Version version);
private:
- NewStatus _init_segment_groups();
+ OLAPStatus _init_segment_groups();
private:
RowFields _tablet_schema;
@@ -65,4 +65,4 @@ class AlphaRowset : public Rowset {
} // namespace doris
-#endif // DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_H
\ No newline at end of file
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_H
diff --git a/be/src/olap/rowset/alpha_rowset_builder.cpp
b/be/src/olap/rowset/alpha_rowset_builder.cpp
index cdb75702..a1156346 100644
--- a/be/src/olap/rowset/alpha_rowset_builder.cpp
+++ b/be/src/olap/rowset/alpha_rowset_builder.cpp
@@ -27,7 +27,7 @@ AlphaRowsetBuilder::AlphaRowsetBuilder() :
_segment_group_id(0),
_current_rowset_meta(nullptr) {
}
-NewStatus AlphaRowsetBuilder::init(const RowsetBuilderContext&
rowset_builder_context) {
+OLAPStatus AlphaRowsetBuilder::init(const RowsetBuilderContext&
rowset_builder_context) {
_rowset_builder_context = rowset_builder_context;
_init();
_current_rowset_meta->set_rowset_id(_rowset_builder_context.rowset_id);
@@ -40,24 +40,24 @@ NewStatus AlphaRowsetBuilder::init(const
RowsetBuilderContext& rowset_builder_co
_current_rowset_meta->set_version(_rowset_builder_context.version);
_current_rowset_meta->set_version_hash(_rowset_builder_context.version_hash);
_current_rowset_meta->set_load_id(_rowset_builder_context.load_id);
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-NewStatus AlphaRowsetBuilder::add_row(RowCursor* row) {
+OLAPStatus AlphaRowsetBuilder::add_row(RowCursor* row) {
OLAPStatus status = _column_data_writer->attached_by(row);
if (status != OLAP_SUCCESS) {
std::string error_msg = "add row failed";
LOG(WARNING) << error_msg;
- return NewStatus::Corruption(error_msg);
+ return status;
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-NewStatus AlphaRowsetBuilder::flush() {
+OLAPStatus AlphaRowsetBuilder::flush() {
_column_data_writer->finalize();
SAFE_DELETE(_column_data_writer);
_init();
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
std::shared_ptr<Rowset> AlphaRowsetBuilder::build() {
@@ -110,4 +110,4 @@ void AlphaRowsetBuilder::_init() {
DCHECK(_column_data_writer != nullptr) << "memory error occur when
creating writer";
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/alpha_rowset_builder.h
b/be/src/olap/rowset/alpha_rowset_builder.h
index a8aa2293..6cb431c4 100644
--- a/be/src/olap/rowset/alpha_rowset_builder.h
+++ b/be/src/olap/rowset/alpha_rowset_builder.h
@@ -31,12 +31,12 @@ class AlphaRowsetBuilder : public RowsetBuilder {
public:
AlphaRowsetBuilder();
- virtual NewStatus init(const RowsetBuilderContext& rowset_builder_context);
+ virtual OLAPStatus init(const RowsetBuilderContext&
rowset_builder_context);
// add a row block to rowset
- virtual NewStatus add_row(RowCursor* row);
+ virtual OLAPStatus add_row(RowCursor* row);
- virtual NewStatus flush();
+ virtual OLAPStatus flush();
// get a rowset
virtual std::shared_ptr<Rowset> build();
@@ -55,4 +55,4 @@ class AlphaRowsetBuilder : public RowsetBuilder {
} // namespace doris
-#endif // DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_BUILDER_H
\ No newline at end of file
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_BUILDER_H
diff --git a/be/src/olap/rowset/alpha_rowset_meta.cpp
b/be/src/olap/rowset/alpha_rowset_meta.cpp
index 5d6c886a..1cd12c93 100644
--- a/be/src/olap/rowset/alpha_rowset_meta.cpp
+++ b/be/src/olap/rowset/alpha_rowset_meta.cpp
@@ -51,7 +51,7 @@ void AlphaRowsetMeta::add_pending_segment_group(const
PendingSegmentGroupPB& pen
if (present_segment_group.pending_segment_group_id() ==
pending_segment_group.pending_segment_group_id()) {
LOG(WARNING) << "pending segment_group already exists in meta."
- << "rowset_id:" << get_rowset_id()
+ << "rowset_id:" << rowset_id()
<< ", pending_segment_group_id: " <<
pending_segment_group.pending_segment_group_id();
return;
}
@@ -66,4 +66,4 @@ void AlphaRowsetMeta::_serialize_extra_meta_pb() {
set_extra_properties(extra_properties);
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/alpha_rowset_reader.cpp
b/be/src/olap/rowset/alpha_rowset_reader.cpp
index 0b9d2ad0..67400413 100644
--- a/be/src/olap/rowset/alpha_rowset_reader.cpp
+++ b/be/src/olap/rowset/alpha_rowset_reader.cpp
@@ -33,7 +33,7 @@ AlphaRowsetReader::AlphaRowsetReader(const RowFields&
tablet_schema,
_key_range_size(0),
_key_range_index(0) {
_alpha_rowset_meta = (AlphaRowsetMeta*)rowset_meta;
- Version version = _alpha_rowset_meta->get_version();
+ Version version = _alpha_rowset_meta->version();
if (version.first == version.second) {
_is_cumulative_rowset = false;
} else {
@@ -41,10 +41,10 @@ AlphaRowsetReader::AlphaRowsetReader(const RowFields&
tablet_schema,
}
}
-NewStatus AlphaRowsetReader::init(ReaderContext* read_context) {
+OLAPStatus AlphaRowsetReader::init(ReaderContext* read_context) {
_current_read_context = read_context;
- NewStatus status = _init_segment_groups(read_context);
- if (!status.ok()) {
+ OLAPStatus status = _init_segment_groups(read_context);
+ if (status != OLAP_SUCCESS) {
return status;
}
status = _init_column_datas(read_context);
@@ -62,26 +62,26 @@ bool AlphaRowsetReader::has_next() {
return false;
}
-NewStatus AlphaRowsetReader::_get_next_block(ColumnData* column_data,
RowBlock* row_block) {
+OLAPStatus AlphaRowsetReader::_get_next_block(ColumnData* column_data,
RowBlock* row_block) {
// get next block
OLAPStatus status = column_data->get_next_block(&row_block);
if (status != OLAP_SUCCESS) {
if (status == OLAP_ERR_DATA_EOF) {
// reach the end of one predicate
// refresh the predicate and continue read
- NewStatus status = _refresh_next_block(column_data, row_block);
- if (!status.ok()) {
+ status = _refresh_next_block(column_data, row_block);
+ if (status != OLAP_SUCCESS) {
LOG(WARNING) << "refresh next block failed";
return status;
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
- return NewStatus::Corruption("get next block failed");
+ return status;
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-NewStatus AlphaRowsetReader::_get_next_row_for_singleton_rowset(RowCursor*
row) {
+OLAPStatus AlphaRowsetReader::_get_next_row_for_singleton_rowset(RowCursor*
row) {
RowCursor* min_row = nullptr;
int min_index = -1;
for (int i = 0; i < _row_blocks.size(); i++) {
@@ -95,36 +95,36 @@ NewStatus
AlphaRowsetReader::_get_next_row_for_singleton_rowset(RowCursor* row)
}
}
if (min_row == nullptr || min_index == -1) {
- return NewStatus::Corruption("get next row failed");
+ return OLAP_ERR_READER_READING_ERROR;
}
row = min_row;
RowBlock* row_block = _row_blocks[min_index];
ColumnData* column_data = _column_datas[min_index].get();
row_block->pos_inc();
if (!row_block->has_remaining()) {
- NewStatus status = _get_next_block(column_data, row_block);
- if (!status.ok()) {
+ OLAPStatus status = _get_next_block(column_data, row_block);
+ if (status != OLAP_SUCCESS) {
return status;
}
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-NewStatus AlphaRowsetReader::_get_next_row_for_cumulative_rowset(RowCursor*
row) {
+OLAPStatus AlphaRowsetReader::_get_next_row_for_cumulative_rowset(RowCursor*
row) {
RowBlock* current_row_block = _row_blocks[0];
size_t pos = current_row_block->pos();
current_row_block->get_row(pos, row);
current_row_block->pos_inc();
if (!current_row_block->has_remaining()) {
- NewStatus status = _get_next_block(_column_datas[0].get(),
current_row_block);
- if (!status.ok()) {
+ OLAPStatus status = _get_next_block(_column_datas[0].get(),
current_row_block);
+ if (status != OLAP_SUCCESS) {
return status;
}
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-NewStatus AlphaRowsetReader::next(RowCursor* row) {
+OLAPStatus AlphaRowsetReader::next(RowCursor* row) {
if (_is_cumulative_rowset) {
return _get_next_row_for_cumulative_rowset(row);
} else {
@@ -136,12 +136,12 @@ void AlphaRowsetReader::close() {
_column_datas.clear();
}
-NewStatus AlphaRowsetReader::_init_column_datas(ReaderContext* read_context) {
+OLAPStatus AlphaRowsetReader::_init_column_datas(ReaderContext* read_context) {
for (auto& segment_group : _segment_groups) {
std::unique_ptr<ColumnData>
new_column_data(ColumnData::create(segment_group.get()));
OLAPStatus status = new_column_data->init();
if (status != OLAP_SUCCESS) {
- return NewStatus::Corruption("init column data failed");
+ return OLAP_ERR_READER_READING_ERROR;
}
new_column_data->set_delete_handler(read_context->delete_handler);
new_column_data->set_stats(read_context->stats);
@@ -155,7 +155,7 @@ NewStatus
AlphaRowsetReader::_init_column_datas(ReaderContext* read_context) {
|| read_context->upper_bound_keys.size() !=
read_context->is_upper_keys_included.size()) {
std::string error_msg = "invalid key range arguments";
LOG(WARNING) << error_msg;
- return NewStatus::InvalidArgument(error_msg);
+ return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
new_column_data->set_read_params(read_context->return_columns,
read_context->load_bf_columns,
@@ -196,14 +196,14 @@ NewStatus
AlphaRowsetReader::_init_column_datas(ReaderContext* read_context) {
&row_block);
if (status != OLAP_SUCCESS) {
LOG(WARNING) << "prepare block read failed";
- return NewStatus::Corruption("prepare block read failed");
+ return status;
}
_key_range_index++;
} else {
status = new_column_data->get_first_row_block(&row_block);
if (status != OLAP_SUCCESS) {
LOG(WARNING) << "get first row block failed";
- return NewStatus::Corruption("get first row block failed");
+ return status;
}
}
_row_blocks.push_back(row_block);
@@ -211,12 +211,12 @@ NewStatus
AlphaRowsetReader::_init_column_datas(ReaderContext* read_context) {
if (_is_cumulative_rowset && _column_datas.size() > 1) {
LOG(WARNING) << "invalid column_datas for cumulative rowset.
column_datas size:"
<< _column_datas.size();
- return NewStatus::InvalidArgument("invalid _column_datas size:" +
std::to_string(_column_datas.size()));
+ return OLAP_ERR_READER_READING_ERROR;
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-NewStatus AlphaRowsetReader::_refresh_next_block(ColumnData* column_data,
RowBlock* next_block) {
+OLAPStatus AlphaRowsetReader::_refresh_next_block(ColumnData* column_data,
RowBlock* next_block) {
OLAPStatus status = column_data->get_next_block(&next_block);
if (status == OLAP_ERR_DATA_EOF) {
// currently, SegmentReader can only support filter one key range a
time
@@ -229,19 +229,19 @@ NewStatus
AlphaRowsetReader::_refresh_next_block(ColumnData* column_data, RowBlo
&next_block);
if (status != OLAP_SUCCESS) {
LOG(WARNING) << "prepare block read failed";
- return NewStatus::Corruption("prepare block read failed");
+ return status;
}
_key_range_index++;
} else {
// read finished
next_block = nullptr;
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
} else if (status != OLAP_SUCCESS) {
- return NewStatus::Corruption("get next block failed");
+ return status;
} else {
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/alpha_rowset_reader.h
b/be/src/olap/rowset/alpha_rowset_reader.h
index fc1ec556..336b1497 100644
--- a/be/src/olap/rowset/alpha_rowset_reader.h
+++ b/be/src/olap/rowset/alpha_rowset_reader.h
@@ -35,29 +35,29 @@ class AlphaRowsetReader : public RowsetReader {
RowsetMeta* rowset_meta, std::vector<std::shared_ptr<SegmentGroup>>
segment_groups);
// reader init
- virtual NewStatus init(ReaderContext* read_context);
+ virtual OLAPStatus init(ReaderContext* read_context);
// check whether rowset has more data
virtual bool has_next();
// read next block data
- virtual NewStatus next(RowCursor* row);
+ virtual OLAPStatus next(RowCursor* row);
// close reader
virtual void close();
private:
- NewStatus _init_segment_groups(ReaderContext* read_context);
+ OLAPStatus _init_segment_groups(ReaderContext* read_context);
- NewStatus _init_column_datas(ReaderContext* read_context);
+ OLAPStatus _init_column_datas(ReaderContext* read_context);
- NewStatus _get_next_row_for_singleton_rowset(RowCursor* row);
+ OLAPStatus _get_next_row_for_singleton_rowset(RowCursor* row);
- NewStatus _get_next_row_for_cumulative_rowset(RowCursor* row);
+ OLAPStatus _get_next_row_for_cumulative_rowset(RowCursor* row);
- NewStatus _get_next_block(ColumnData* column_data, RowBlock* row_block);
+ OLAPStatus _get_next_block(ColumnData* column_data, RowBlock* row_block);
- NewStatus _refresh_next_block(ColumnData* column_datam, RowBlock*
row_block);
+ OLAPStatus _refresh_next_block(ColumnData* column_datam, RowBlock*
row_block);
private:
RowFields _tablet_schema;
@@ -77,4 +77,4 @@ class AlphaRowsetReader : public RowsetReader {
}
-#endif // DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_READER_H
\ No newline at end of file
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_READER_H
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 60d5c9b9..b2861414 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -34,18 +34,23 @@ using RowsetSharedPtr = std::shared_ptr<Rowset>;
class Rowset {
public:
+ static Rowset* create();
virtual ~Rowset() { }
- virtual NewStatus init() = 0;
+ virtual OLAPStatus init() = 0;
virtual std::unique_ptr<RowsetReader> create_reader() = 0;
- virtual NewStatus copy(RowsetBuilder* dest_rowset_builder) = 0;
+ virtual OLAPStatus copy(RowsetBuilder* dest_rowset_builder) = 0;
- virtual NewStatus remove() = 0;
+ virtual OLAPStatus remove() = 0;
virtual OLAPStatus to_rowset_pb(const RowsetMetaPB& rs_meta);
- virtual OLAPStatus get_rs_meta();
+ virtual RowsetMetaSharedPtr& get_rs_meta() const;
+ virtual int get_data_disk_size() const;
+ virtual bool empty() const;
+ virtual bool zero_num_rows() const;
+ virtual size_t get_num_rows() const;
};
} // namespace doris
diff --git a/be/src/olap/rowset/rowset_builder.h
b/be/src/olap/rowset/rowset_builder.h
index 4787abd1..ec942e02 100644
--- a/be/src/olap/rowset/rowset_builder.h
+++ b/be/src/olap/rowset/rowset_builder.h
@@ -51,12 +51,12 @@ class RowsetBuilder {
public:
virtual ~RowsetBuilder() { }
- virtual NewStatus init(const RowsetBuilderContext& rowset_builder_context)
= 0;
+ virtual OLAPStatus init(const RowsetBuilderContext&
rowset_builder_context) = 0;
// add a row to rowset
- virtual NewStatus add_row(RowCursor* row_block) = 0;
+ virtual OLAPStatus add_row(RowCursor* row_block) = 0;
- virtual NewStatus flush() = 0;
+ virtual OLAPStatus flush() = 0;
// get a rowset
virtual std::shared_ptr<Rowset> build() = 0;
@@ -64,4 +64,4 @@ class RowsetBuilder {
} // namespace doris
-#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_BUILDER_H
\ No newline at end of file
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_BUILDER_H
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index 50a13c61..13242493 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -64,7 +64,7 @@ class RowsetMeta {
return ret;
}
- virtual int64_t get_rowset_id() {
+ virtual int64_t rowset_id() {
return _rowset_meta_pb.rowset_id();
}
@@ -112,7 +112,7 @@ class RowsetMeta {
_rowset_meta_pb.set_rowset_state(rowset_state);
}
- virtual Version get_version() {
+ virtual Version version() {
Version version;
version.first = _rowset_meta_pb.start_version();
version.second = _rowset_meta_pb.end_version();
@@ -124,7 +124,7 @@ class RowsetMeta {
_rowset_meta_pb.set_end_version(version.second);
}
- virtual int get_start_version() {
+ virtual int start_version() {
return _rowset_meta_pb.start_version();
}
@@ -132,7 +132,7 @@ class RowsetMeta {
_rowset_meta_pb.set_start_version(start_version);
}
- virtual int get_end_version() {
+ virtual int end_version() {
return _rowset_meta_pb.end_version();
}
@@ -241,6 +241,8 @@ class RowsetMeta {
_rowset_meta_pb.set_extra_properties(extra_properties);
}
+ OLAPStatus to_rowset_pb(RowsetMetaPB* rs_meta_pb);
+ virtual int64_t creation_time() const;
private:
bool _deserialize_from_pb(const std::string& value) {
return _rowset_meta_pb.ParseFromString(value);
@@ -259,4 +261,4 @@ class RowsetMeta {
} // namespace doris
-#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H
\ No newline at end of file
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H
diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp
b/be/src/olap/rowset/rowset_meta_manager.cpp
index e0a9a36a..1fec296b 100644
--- a/be/src/olap/rowset/rowset_meta_manager.cpp
+++ b/be/src/olap/rowset/rowset_meta_manager.cpp
@@ -33,65 +33,65 @@ namespace doris {
const std::string ROWSET_PREFIX = "rst_";
-NewStatus convert_meta_status(OLAPStatus status) {
+OLAPStatus convert_meta_status(OLAPStatus status) {
if (status == OLAP_SUCCESS) {
- return NewStatus::OK();
+ return OLAP_SUCCESS;
} else {
std::string error_msg = "meta operation failed";
LOG(WARNING) << error_msg;
- return NewStatus::IOError(error_msg);
+ return OLAP_ERR_IO_ERROR;
}
}
-NewStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, int64_t
rowset_id, RowsetMeta* rowset_meta) {
+OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, int64_t
rowset_id, RowsetMeta* rowset_meta) {
std::string key = ROWSET_PREFIX + std::to_string(rowset_id);
std::string value;
OLAPStatus s = meta->get(META_COLUMN_FAMILY_INDEX, key, value);
if (s == OLAP_ERR_META_KEY_NOT_FOUND) {
std::string error_msg = "rowset id:" + std::to_string(rowset_id) + "
not found.";
LOG(WARNING) << error_msg;
- return NewStatus::NotFound(error_msg);
+ return OLAP_ERR_META_KEY_NOT_FOUND;
} else if (s != OLAP_SUCCESS) {
std::string error_msg = "load rowset id:" + std::to_string(rowset_id)
+ " failed.";
LOG(WARNING) << error_msg;
- return NewStatus::IOError(error_msg);
+ return OLAP_ERR_IO_ERROR;
}
bool ret = rowset_meta->init(value);
if (!ret) {
std::string error_msg = "parse rowset meta failed. rowset id:" +
std::to_string(rowset_id);
- return NewStatus::Corruption(error_msg);
+ return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR;
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-NewStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, int64_t
rowset_id, std::string* json_rowset_meta) {
+OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, int64_t
rowset_id, std::string* json_rowset_meta) {
RowsetMeta rowset_meta;
- NewStatus s = get_rowset_meta(meta, rowset_id, &rowset_meta);
- if (!s.ok()) {
- return s;
+ OLAPStatus status = get_rowset_meta(meta, rowset_id, &rowset_meta);
+ if (status != OLAP_SUCCESS) {
+ return status;
}
bool ret = rowset_meta.get_json_rowset_meta(json_rowset_meta);
if (!ret) {
std::string error_msg = "get json rowset meta failed. rowset id:" +
std::to_string(rowset_id);
- return NewStatus::Corruption(error_msg);
+ return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR;
}
- return NewStatus::OK();
+ return OLAP_SUCCESS;
}
-NewStatus RowsetMetaManager::save(OlapMeta* meta, int64_t rowset_id,
RowsetMeta* rowset_meta) {
+OLAPStatus RowsetMetaManager::save(OlapMeta* meta, int64_t rowset_id,
RowsetMeta* rowset_meta) {
std::string key = ROWSET_PREFIX + std::to_string(rowset_id);
std::string value;
bool ret = rowset_meta->serialize(&value);
if (!ret) {
std::string error_msg = "serialize rowset pb failed. rowset id:" +
std::to_string(rowset_id);
LOG(WARNING) << error_msg;
- return NewStatus::Corruption(error_msg);
+ return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR;
}
OLAPStatus status = meta->put(META_COLUMN_FAMILY_INDEX, key, value);
return convert_meta_status(status);
}
-NewStatus RowsetMetaManager::remove(OlapMeta* meta, int64_t rowset_id) {
+OLAPStatus RowsetMetaManager::remove(OlapMeta* meta, int64_t rowset_id) {
std::string key = ROWSET_PREFIX + std::to_string(rowset_id);
LOG(INFO) << "start to remove rowset, key:" << key;
OLAPStatus status = meta->remove(META_COLUMN_FAMILY_INDEX, key);
@@ -99,7 +99,7 @@ NewStatus RowsetMetaManager::remove(OlapMeta* meta, int64_t
rowset_id) {
return convert_meta_status(status);
}
-NewStatus RowsetMetaManager::traverse_rowset_metas(OlapMeta* meta,
+OLAPStatus RowsetMetaManager::traverse_rowset_metas(OlapMeta* meta,
std::function<bool(uint64_t, const std::string&)> const& func) {
auto traverse_rowset_meta_func = [&func](const std::string& key, const
std::string& value) -> bool {
std::vector<std::string> parts;
@@ -116,7 +116,7 @@ NewStatus
RowsetMetaManager::traverse_rowset_metas(OlapMeta* meta,
return convert_meta_status(status);
}
-NewStatus RowsetMetaManager::load_json_rowset_meta(OlapMeta* meta, const
std::string& rowset_meta_path) {
+OLAPStatus RowsetMetaManager::load_json_rowset_meta(OlapMeta* meta, const
std::string& rowset_meta_path) {
std::ifstream infile(rowset_meta_path);
char buffer[1024];
std::string json_rowset_meta;
@@ -130,11 +130,11 @@ NewStatus
RowsetMetaManager::load_json_rowset_meta(OlapMeta* meta, const std::st
if (!ret) {
std::string error_msg = "parse json rowset meta failed.";
LOG(WARNING) << error_msg;
- return NewStatus::Corruption(error_msg);
+ return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR;
}
- uint64_t rowset_id = rowset_meta.get_rowset_id();
- NewStatus status = save(meta, rowset_id, &rowset_meta);
+ uint64_t rowset_id = rowset_meta.rowset_id();
+ OLAPStatus status = save(meta, rowset_id, &rowset_meta);
return status;
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/rowset_meta_manager.h
b/be/src/olap/rowset/rowset_meta_manager.h
index 40b9fc6c..52b70b37 100644
--- a/be/src/olap/rowset/rowset_meta_manager.h
+++ b/be/src/olap/rowset/rowset_meta_manager.h
@@ -29,20 +29,20 @@ namespace doris {
// Helper class for managing rowset meta of one root path.
class RowsetMetaManager {
public:
- static NewStatus get_rowset_meta(OlapMeta* meta, int64_t rowset_id,
RowsetMeta* rowset_meta);
+ static OLAPStatus get_rowset_meta(OlapMeta* meta, int64_t rowset_id,
RowsetMeta* rowset_meta);
- static NewStatus get_json_rowset_meta(OlapMeta* meta, int64_t rowset_id,
std::string* json_rowset_meta);
+ static OLAPStatus get_json_rowset_meta(OlapMeta* meta, int64_t rowset_id,
std::string* json_rowset_meta);
- static NewStatus save(OlapMeta* meta, int64_t rowset_id, RowsetMeta*
rowset_meta);
+ static OLAPStatus save(OlapMeta* meta, int64_t rowset_id, RowsetMeta*
rowset_meta);
- static NewStatus remove(OlapMeta* meta, int64_t rowset_id);
+ static OLAPStatus remove(OlapMeta* meta, int64_t rowset_id);
- static NewStatus traverse_rowset_metas(OlapMeta* meta,
+ static OLAPStatus traverse_rowset_metas(OlapMeta* meta,
std::function<bool(uint64_t, const std::string&)> const& func);
- static NewStatus load_json_rowset_meta(OlapMeta* meta, const std::string&
rowset_meta_path);
+ static OLAPStatus load_json_rowset_meta(OlapMeta* meta, const std::string&
rowset_meta_path);
};
}
-#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_MANAGER_H
\ No newline at end of file
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_MANAGER_H
diff --git a/be/src/olap/rowset/rowset_reader.h
b/be/src/olap/rowset/rowset_reader.h
index bab4c73b..64f8c1d6 100644
--- a/be/src/olap/rowset/rowset_reader.h
+++ b/be/src/olap/rowset/rowset_reader.h
@@ -58,16 +58,17 @@ struct ReaderContext {
class RowsetReader {
public:
+ static RowsetReader* create();
virtual ~RowsetReader() { }
// reader init
- virtual NewStatus init(ReaderContext* read_context) = 0;
+ virtual OLAPStatus init(ReaderContext* read_context) = 0;
// check whether rowset has more data
virtual bool has_next() = 0;
// read next block data
- virtual NewStatus next(RowCursor* row) = 0;
+ virtual OLAPStatus next(RowCursor* row) = 0;
// close reader
virtual void close() = 0;
@@ -75,4 +76,4 @@ class RowsetReader {
}
-#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_H
\ No newline at end of file
+#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_H
diff --git a/be/src/olap/rowset/segment_reader.cpp
b/be/src/olap/rowset/segment_reader.cpp
index 51051b31..5d319a65 100644
--- a/be/src/olap/rowset/segment_reader.cpp
+++ b/be/src/olap/rowset/segment_reader.cpp
@@ -48,26 +48,26 @@ SegmentReader::SegmentReader(
_file_name(file),
_segment_group(segment_group),
_segment_id(segment_id),
- _used_columns(used_columns),
- _load_bf_columns(load_bf_columns),
_conditions(conditions),
_delete_handler(delete_handler),
_delete_status(delete_status),
- _runtime_state(runtime_state),
- _stats(stats),
- _lru_cache(lru_cache),
_eof(false),
_end_block(-1),
// 确保第一次调用_move_to_next_row，会执行seek_to_block
_block_count(0),
_num_rows_in_block(0),
_null_supported(false),
+ _used_columns(used_columns),
+ _load_bf_columns(load_bf_columns),
_mmap_buffer(NULL),
_include_blocks(NULL),
_is_using_mmap(false),
_is_data_loaded(false),
_buffer_size(0),
- _shared_buffer(NULL) {
+ _lru_cache(lru_cache),
+ _runtime_state(runtime_state),
+ _shared_buffer(NULL),
+ _stats(stats) {
_tracker.reset(new MemTracker(-1));
_mem_pool.reset(new MemPool(_tracker.get()));
}
@@ -947,4 +947,4 @@ OLAPStatus SegmentReader::_load_to_vectorized_row_batch(
return OLAP_SUCCESS;
}
-} //unamespace doris
\ No newline at end of file
+} //unamespace doris
diff --git a/be/src/olap/rowset/segment_reader.h
b/be/src/olap/rowset/segment_reader.h
index 2ce33c6c..d62360fa 100644
--- a/be/src/olap/rowset/segment_reader.h
+++ b/be/src/olap/rowset/segment_reader.h
@@ -374,4 +374,4 @@ class SegmentReader {
} // namespace doris
-#endif // DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_READER_H
\ No newline at end of file
+#endif // DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_READER_H
diff --git a/be/src/olap/rowset_graph.cpp b/be/src/olap/rowset_graph.cpp
index be0d5a8e..40901268 100644
--- a/be/src/olap/rowset_graph.cpp
+++ b/be/src/olap/rowset_graph.cpp
@@ -19,7 +19,7 @@
namespace doris {
-OLAPStatus Tablet::construct_rowset_graph(const std::vector<RowsetMeta>&
rs_metas) {
+OLAPStatus Tablet::construct_rowset_graph(const
std::vector<RowsetMetaSharedPtr>& rs_metas) {
if (rs_metas.size() == 0) {
VLOG(3) << "there is no version in the header.";
return Status::OK();
@@ -30,8 +30,8 @@ OLAPStatus Tablet::construct_rowset_graph(const
std::vector<RowsetMeta>& rs_meta
vertex_values.reserve(2 * rs_metas.size());
for (int i = 0; i < rs_metas.size(); ++i) {
- vertex_values.push_back(rs_metas.Get(i).start_version());
- vertex_values.push_back(rs_metas.Get(i).end_version() + 1);
+ vertex_values.push_back(rs_metas.Get(i)->start_version());
+ vertex_values.push_back(rs_metas.Get(i)->end_version() + 1);
}
sort(vertex_values.begin(), vertex_values.end());
@@ -58,8 +58,8 @@ OLAPStatus Tablet::construct_rowset_graph(const
std::vector<RowsetMeta>& rs_meta
for (int i = 0; i < rs_metas.size(); ++i) {
// Versions in header are unique.
// We ensure _vertex_index_map has its start_version.
- int start_vertex_index =
(*_vertex_index_map)[rs_metas.Get(i).start_version()];
- int end_vertex_index =
(*_vertex_index_map)[rs_metas.Get(i).end_version() + 1];
+ int start_vertex_index =
(*_vertex_index_map)[rs_metas.Get(i)->start_version()];
+ int end_vertex_index =
(*_vertex_index_map)[rs_metas.Get(i)->end_version() + 1];
// Add one edge from start_version to end_version.
list<int>* edges = (*_version_graph)[start_vertex_index].edges;
edges->insert(edges->begin(), end_vertex_index);
diff --git a/be/src/olap/rowset_graph.h b/be/src/olap/rowset_graph.h
index 43e90262..d45d6a46 100644
--- a/be/src/olap/rowset_graph.h
+++ b/be/src/olap/rowset_graph.h
@@ -25,7 +25,7 @@ namespace doris {
class RowsetGraph {
public:
- OLAPStatus construct_rowset_graph(const std::vector<RowsetMeta>& rs_metas);
+ OLAPStatus construct_rowset_graph(const std::vector<RowsetMetaSharedPtr>&
rs_metas);
OLAPStatus add_version_to_graph(const Version& version);
OLAPStatus delete_version_from_graph(const Version& version);
OLAPStatus capture_consistent_versions(const Version& spec_version,
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 001f1731..5eb23759 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -362,7 +362,7 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
// load tablet header, in order to remove versions that not in
shortest version path
DataDir* store = ref_tablet->data_dir();
- new_tablet_meta = new(nothrow) TabletMeta();
+ new_tablet_meta = new(nothrow) TabletMeta(store);
if (new_tablet_meta == NULL) {
OLAP_LOG_WARNING("fail to malloc TabletMeta.");
res = OLAP_ERR_MALLOC_ERROR;
@@ -563,7 +563,7 @@ OLAPStatus SnapshotManager::_append_single_delta(
const TSnapshotRequest& request, DataDir* store) {
OLAPStatus res = OLAP_SUCCESS;
string root_path = store->path();
- TabletMeta* new_tablet_meta = new(nothrow) TabletMeta();
+ TabletMeta* new_tablet_meta = new(nothrow) TabletMeta(store);
if (new_tablet_meta == NULL) {
OLAP_LOG_WARNING("fail to malloc TabletMeta.");
return OLAP_ERR_MALLOC_ERROR;
@@ -664,7 +664,7 @@ OLAPStatus SnapshotManager::storage_medium_migrate(
return OLAP_SUCCESS;
}
- TStorageMedium::type src_storage_medium =
tablet->store()->storage_medium();
+ TStorageMedium::type src_storage_medium =
tablet->data_dir()->storage_medium();
if (src_storage_medium == storage_medium) {
LOG(INFO) << "tablet is already on specified storage medium. "
<< "storage_medium=" << storage_medium;
diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h
index a928950b..20dcb424 100644
--- a/be/src/olap/snapshot_manager.h
+++ b/be/src/olap/snapshot_manager.h
@@ -57,10 +57,10 @@ class SnapshotManager {
// @brief 释放snapshot
// @param snapshot_path [in] 要被释放的snapshot的路径，只包含到ID
OLAPStatus release_snapshot(const std::string& snapshot_path);
-
OLAPStatus storage_medium_migrate(
- TTabletId tablet_id, TSchemaHash schema_hash,
- TStorageMedium::type storage_medium);
+ TTabletId tablet_id,
+ TSchemaHash schema_hash,
+ TStorageMedium::type storage_medium);
static SnapshotManager* instance();
@@ -151,4 +151,4 @@ class SnapshotManager {
} // doris
-#endif
\ No newline at end of file
+#endif
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 24c2ded0..84e17068 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -154,7 +154,7 @@ OLAPStatus StorageEngine::_load_data_dir(DataDir* data_dir)
{
};
OLAPStatus s =
RowsetMetaManager::traverse_rowset_metas(data_dir->get_meta(),
load_rowset_func);
if (has_error) {
- LOG(WARN) << "errors when load rowset meta from meta env, skip
this data dir:" << data_dir_path;
+ LOG(WARNING) << "errors when load rowset meta from meta env, skip
this data dir:" << data_dir_path;
return OLAP_ERR_META_ITERATOR;
}
LOG(INFO) << "load header from meta finished";
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index ca6f83d3..0ae56f73 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -66,12 +66,11 @@ Tablet::~Tablet() {
}
OLAPStatus Tablet::init_once() {
- OLAPStatus res = OLAP_SUCCESS;
ReadLock rdlock(&_meta_lock);
- for (int ser = 0; ser < _tablet_meta.rowset_size(); ++ser) {
- const RowsetMeta* rs_meta = _tablet_meta.get_rs_meta(ser);
- Version version = rs_meta->version();
- RowsetSharedPtr rowset(new Rowset(rs_meta));
+ for (auto& it : _tablet_meta.all_rs_metas()) {
+ Version version = it->version();
+ RowsetSharedPtr rowset(Rowset::create());
+ rowset->init();
_rs_version_map[version] = rowset;
rowset->init();
}
@@ -83,8 +82,8 @@ bool Tablet::can_do_compaction() {
// 如果选路成功，则转换完成，可以进行BE
// 如果选路失败，则转换未完成，不能进行BE
ReadLock rdlock(&_meta_lock);
- const PDelta* lastest_rowset = lastest_version();
- if (lastest_rowset == NULL) {
+ const PDelta* lastest_delta = lastest_version();
+ if (lastest_delta == NULL) {
return false;
}
@@ -109,7 +108,7 @@ bool Tablet::can_do_compaction() {
OLAPStatus Tablet::capture_consistent_versions(
const Version& version, vector<Version>*
span_versions) const {
OLAPStatus status = _rs_graph->capture_consistent_versions(version,
span_versions);
- if (!status.ok()) {
+ if (status != OLAP_SUCCESS) {
LOG(WARNING) << "fail to generate shortest version path. tablet=" <<
full_name()
<< ", version='" << version.first << "-" <<
version.second;
}
@@ -137,29 +136,25 @@ void Tablet::acquire_rs_reader_by_version(const
vector<Version>& version_vec,
return;
}
- std::shared_ptr<RowsetReader> rs_reader(new RowsetReader());
- Status status = rs_reader->init();
- if (!status.ok()) {
+ std::shared_ptr<RowsetReader> rs_reader(RowsetReader::create());
+ OLAPStatus status = rs_reader->init(nullptr);
+ if (status != OLAP_SUCCESS) {
LOG(WARNING) << "fail to init rowset_reader. tablet=" <<
full_name()
<< ", version=" << version.first << "-" <<
version.second;
release_rs_readers(rs_readers);
}
- rs_readers->push_back(std::move(rs_reader)):
+ rs_readers->push_back(std::move(rs_reader));
}
}
OLAPStatus Tablet::release_rs_readers(vector<std::shared_ptr<RowsetReader>>*
rs_readers) const {
DCHECK(rs_readers != nullptr) << "rs_readers is null. tablet=" <<
full_name();
- for (auto data : *rs_readers) {
- delete data;
- }
-
rs_readers->clear();
return OLAP_SUCCESS;
}
OLAPStatus Tablet::add_inc_rowset(const Rowset& rowset) {
- return _table_meta.add_inc_rs_meta(rowset.get_rs_meta());
+ return _tablet_meta.add_inc_rs_meta(rowset.get_rs_meta());
}
OLAPStatus Tablet::delete_expired_inc_rowset() {
@@ -167,7 +162,7 @@ OLAPStatus Tablet::delete_expired_inc_rowset() {
vector<Version> expired_versions;
WriteLock wrlock(&_meta_lock);
for (auto& it : _tablet_meta.all_inc_rs_metas()) {
- double diff = difftime(now, it.creation_time());
+ double diff = difftime(now, it->creation_time());
if (diff >= config::inc_rowset_expired_sec) {
expired_versions.push_back(it->version());
}
@@ -185,18 +180,19 @@ OLAPStatus Tablet::delete_expired_inc_rowset() {
return OLAP_SUCCESS;
}
-void Tablet::delete_inc_rowset_by_version(const Version& version) {
+OLAPStatus Tablet::delete_inc_rowset_by_version(const Version& version) {
_tablet_meta.delete_inc_rs_meta_by_version(version);
VLOG(3) << "delete inc rowset. tablet=" << full_name()
<< ", version=" << version.first << "-" << version.second;
+ return OLAP_SUCCESS;
}
void Tablet::calc_missed_versions(int64_t spec_version,
vector<Version>* missed_versions) const {
DCHECK(spec_version > 0) << "invalid spec_version: " << spec_version;
std::list<Version> existing_versions;
- for (RowsetMeta& rs : _tablet_meta.all_rs_metas()) {
- existing_versions.emplace_back(rs.get_version());
+ for (auto& rs : _tablet_meta.all_rs_metas()) {
+ existing_versions.emplace_back(rs->version());
}
// sort the existing versions in ascending order
@@ -230,15 +226,14 @@ OLAPStatus
Tablet::modify_rowsets(vector<RowsetSharedPtr>& to_add,
RowsetSharedPtr Tablet::rowset_with_largest_size() {
RowsetSharedPtr largest_rowset = nullptr;
- size_t ser = 0;
-
for (auto& it : _rs_version_map) {
// use segment_group of base file as target segment_group when base is
not empty,
// or try to find the biggest segment_group.
if (largest_rowset->empty() || largest_rowset->zero_num_rows()) {
continue;
}
- if (it.second->get_rs_meta().get_index_index_size() >
largest_rowset->get_rs_meta().get_index_disk_size()) {
+ if (it.second->get_rs_meta()->get_index_disk_size()
+ > largest_rowset->get_rs_meta()->get_index_disk_size()) {
largest_rowset = it.second;
}
}
@@ -370,7 +365,7 @@ OLAPStatus Tablet::split_range(
}
cur_start_key.attach(entry.data);
- last_start_key.allocate_memory_for_string_type(_schema);
+ last_start_key.allocate_memory_for_string_type(tablet_schema());
last_start_key.copy_without_pool(cur_start_key);
// start_key是last start_key, 但返回的实际上是查询层给出的key
ranges->emplace_back(start_key.to_tuple());
@@ -416,16 +411,16 @@ void Tablet::list_versions(vector<Version>* versions)
const {
}
void Tablet::list_entities(vector<VersionEntity>* entities) const {
- DCHECK(entities != nullptr && entities.empty());
+ DCHECK(entities != nullptr && entities->empty());
for (auto& it : _rs_version_map) {
- RowsetSharedPtr rowset = it->second;
- VersionEntity entity(it->first, rowset);
+ RowsetSharedPtr rowset = it.second;
+ VersionEntity entity(it.first, 0);
entities->push_back(entity);
}
}
-int32_t Tablet::get_field_index(const string& field_name) const {
+size_t Tablet::get_field_index(const string& field_name) const {
return _schema->get_field_index(field_name);
}
@@ -433,19 +428,20 @@ size_t Tablet::get_row_size() const {
return _schema->get_row_size();
}
-int64_t Tablet::get_data_size() const {
- int64_t total_size = 0;
+size_t Tablet::get_data_size() const {
+ size_t total_size = 0;
for (auto& it : _rs_version_map) {
- total_size += it.second->get_data_size();
+ total_size += it.second->get_data_disk_size();
}
return total_size;
}
-int64_t Tablet::get_num_rows() const {
- int64_t num_rows = 0;
+size_t Tablet::get_num_rows() const {
+ size_t num_rows = 0;
for (auto& it : _rs_version_map) {
- total_size += it.second->get_num_rows();
+ num_rows += it.second->get_num_rows();
}
+ return num_rows;
}
bool Tablet::is_deletion_rowset(const Version& version) {
@@ -454,7 +450,8 @@ bool Tablet::is_deletion_rowset(const Version& version) {
}
for (auto& it : _tablet_meta.delete_predicates()) {
- if (it->version() == version) {
+ if (it.version() == version.first
+ && it.version() == version.second) {
return true;
}
}
@@ -466,7 +463,7 @@ bool Tablet::is_schema_changing() {
bool is_schema_changing = false;
ReadLock rdlock(&_meta_lock);
- if (_tablet_meta.alter_state() != AlterTabletState::NONE) {
+ if (_tablet_meta.alter_state() != AlterTabletState::ALTER_NONE) {
is_schema_changing = true;
}
@@ -476,14 +473,14 @@ bool Tablet::is_schema_changing() {
bool Tablet::get_schema_change_request(int64_t* tablet_id, TSchemaHash*
schema_hash,
vector<Version>* versions_to_alter,
AlterTabletType* alter_tablet_type)
const {
- if (_tablet_meta.alter_state() == AlterTabletState::none) {
+ if (_tablet_meta.alter_state() == AlterTabletState::ALTER_NONE) {
return false;
}
- const AlterTabletTask alter_task = _tablet_meta.alter_task();
+ const AlterTabletTask& alter_task = _tablet_meta.alter_task();
*tablet_id = alter_task.related_tablet_id();
*schema_hash = alter_task.related_schema_hash();
- *alter_tablet_type = alter_task.alter_tablet_type();
+ *alter_tablet_type = alter_task.alter_type();
for (auto& rs : alter_task.rowsets_to_alter()) {
versions_to_alter->push_back(rs->version());
}
@@ -494,18 +491,18 @@ bool Tablet::get_schema_change_request(int64_t*
tablet_id, TSchemaHash* schema_h
void Tablet::set_schema_change_request(int64_t tablet_id,
int64_t schema_hash,
const vector<Version>&
versions_to_alter,
- const AlterTabletType
alter_tablet_type) {
+ const AlterTabletType alter_type) {
clear_schema_change_request();
AlterTabletTask alter_task;
alter_task.set_related_tablet_id(tablet_id);
alter_task.set_related_schema_hash(schema_hash);
- for (Version& version : versions_to_alter) {
+ for (auto& version : versions_to_alter) {
RowsetMeta* rs_meta = new RowsetMeta();
rs_meta->set_version(version);
}
- alter_task->set_alter_tablet_type(alter_tablet_type);
- _tablet_meta.add_alter_task();
+ alter_task.set_alter_type(alter_type);
+ _tablet_meta.add_alter_task(alter_task);
}
bool Tablet::remove_last_schema_change_version(TabletSharedPtr new_tablet) {
@@ -528,7 +525,7 @@ bool Tablet::is_used() {
VersionEntity Tablet::get_version_entity_by_version(const Version& version) {
RowsetSharedPtr rowset = _rs_version_map[version];
- VersionEntity entity(version, rowset);
+ VersionEntity entity(version, 0);
return entity;
}
@@ -537,7 +534,8 @@ size_t Tablet::get_version_data_size(const Version&
version) {
return rowset->get_data_disk_size();
}
-OLAPStatus Tablet::recover_tablet_until_specfic_version(const int64_t&
spec_version) {
+OLAPStatus Tablet::recover_tablet_until_specfic_version(const int64_t&
spec_version,
+ const int64_t&
version_hash) {
return OLAP_SUCCESS;
}
@@ -550,7 +548,7 @@ OLAPStatus Tablet::test_version(const Version& version) {
}
OLAPStatus Tablet::register_tablet_into_dir() {
- return _store->register_tablet(this);
+ return _data_dir->register_tablet(this);
}
OLAPStatus Tablet::clear_schema_change_info(
@@ -558,12 +556,13 @@ OLAPStatus Tablet::clear_schema_change_info(
bool only_one,
bool check_only) {
if (!check_only) {
- WriteLock w(_meta_lock);
+ WriteLock w(&_meta_lock);
_unprotect_clear_schema_change_info(type, only_one, check_only);
} else {
- ReadLock r(_meta_lock);
+ ReadLock r(&_meta_lock);
_unprotect_clear_schema_change_info(type, only_one, check_only);
}
+ return OLAP_SUCCESS;
}
OLAPStatus Tablet::_unprotect_clear_schema_change_info(
@@ -572,10 +571,6 @@ OLAPStatus Tablet::_unprotect_clear_schema_change_info(
bool check_only) {
OLAPStatus res = OLAP_SUCCESS;
- if (NULL == tablet.get()) {
- return res;
- }
-
vector<Version> versions_to_be_changed;
if (this->get_schema_change_request(NULL,
NULL,
@@ -584,14 +579,14 @@ OLAPStatus Tablet::_unprotect_clear_schema_change_info(
if (versions_to_be_changed.size() != 0) {
OLAP_LOG_WARNING("schema change is not allowed now, "
"until previous schema change is done.
[tablet='%s']",
- tablet->full_name().c_str());
+ full_name().c_str());
return OLAP_ERR_PREVIOUS_SCHEMA_CHANGE_NOT_FINISHED;
}
}
if (!check_only) {
VLOG(3) << "broke old schema change chain";
- tablet->clear_schema_change_request();
+ clear_schema_change_request();
}
return res;
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index e5308588..5ae78f9c 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -177,7 +177,7 @@ class Tablet : public std::enable_shared_from_this<Tablet> {
void set_schema_change_request(int64_t tablet_id,
int64_t schema_hash,
const std::vector<Version>&
versions_to_changed,
- const AlterTabletType alter_table_type);
+ const AlterTabletType alter_type);
bool remove_last_schema_change_version(TabletSharedPtr new_olap_table);
void clear_schema_change_request();
SchemaChangeStatus schema_change_status();
@@ -195,17 +195,20 @@ class Tablet : public
std::enable_shared_from_this<Tablet> {
VersionEntity get_version_entity_by_version(const Version& version);
size_t get_version_index_size(const Version& version);
size_t get_version_data_size(const Version& version);
- OLAPStatus recover_tablet_until_specfic_version(const int64_t&
until_version,
+ OLAPStatus recover_tablet_until_specfic_version(const int64_t&
spec_version,
const int64_t&
version_hash);
const std::string& rowset_path_prefix();
void set_id(int64_t id);
OLAPStatus register_tablet_into_dir();
+ void list_entities(vector<VersionEntity>* entities) const;
+
OLAPStatus init_once();
OLAPStatus capture_consistent_rowsets(const Version& spec_version,
vector<std::shared_ptr<RowsetReader>>* rs_readers);
+ OLAPStatus capture_consistent_versions(const Version& version,
vector<Version>* span_versions) const;
void acquire_rs_reader_by_version(const vector<Version>& version_vec,
vector<std::shared_ptr<RowsetReader>>*
rs_readers) const;
OLAPStatus release_rs_readers(vector<std::shared_ptr<RowsetReader>>*
rs_readers) const;
@@ -251,7 +254,7 @@ class Tablet : public std::enable_shared_from_this<Tablet> {
RowsetSharedPtr get_inc_rowset(const Version& version) const;
OLAPStatus delete_inc_rowset_by_version(const Version& version);
OLAPStatus delete_expired_inc_rowset();
- OLAPStatus is_deletion_rowset(const Version& version) const;
+ bool is_deletion_rowset(const Version& version);
OLAPStatus create_snapshot();
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 9cfecd49..315104e4 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -88,10 +88,9 @@ bool _sort_tablet_by_create_time(const TabletSharedPtr& a,
const TabletSharedPtr
}
TabletManager::TabletManager()
- : _available_storage_medium_type_count(0),
- _global_tablet_id(0),
- _tablet_stat_cache_update_time_ms(0) {
-}
+ : _global_tablet_id(0),
+ _tablet_stat_cache_update_time_ms(0),
+ _available_storage_medium_type_count(0) { }
OLAPStatus TabletManager::add_tablet(TTabletId tablet_id, SchemaHash
schema_hash,
const TabletSharedPtr& tablet, bool force) {
@@ -268,7 +267,7 @@ OLAPStatus TabletManager::create_init_version(TTabletId
tablet_id, SchemaHash sc
}
// Create writer, which write nothing to tablet, to generate empty
data file
- writer = ColumnDataWriter::create(tablet, new_segment_group, false);
+ writer = ColumnDataWriter::create(new_segment_group, false,
tablet->compress_kind(), tablet->bloom_filter_fpp());
if (writer == NULL) {
LOG(WARNING) << "fail to create writer. [tablet=" <<
tablet->full_name() << "]";
res = OLAP_ERR_MALLOC_ERROR;
@@ -684,7 +683,7 @@ TabletSharedPtr
TabletManager::find_best_tablet_to_compaction(CompactionType com
OLAPStatus TabletManager::load_tablet_from_header(DataDir* data_dir, TTabletId
tablet_id,
TSchemaHash schema_hash, const std::string& header) {
std::unique_ptr<TabletMeta> tablet_meta(new TabletMeta());
- bool parsed = tablet_meta->ParseFromString(header);
+ bool parsed = tablet_meta->deserialize(header);
if (!parsed) {
LOG(WARNING) << "parse header string failed for tablet_id:" <<
tablet_id << " schema_hash:" << schema_hash;
return OLAP_ERR_HEADER_PB_PARSE_FAILED;
@@ -1115,8 +1114,8 @@ OLAPStatus TabletManager::_create_new_tablet_header(
// set column information
uint32_t i = 0;
- uint32_t key_count = 0;
- bool has_bf_columns = false;
+ //uint32_t key_count = 0;
+ //bool has_bf_columns = false;
uint32_t next_unique_id = 0;
if (true == is_schema_change_tablet) {
next_unique_id = ref_tablet->next_unique_id();
@@ -1141,7 +1140,7 @@ OLAPStatus TabletManager::_create_new_tablet_header(
size_t field_off = 0;
for (field_off = 0; field_off < field_num; ++field_off) {
if (ref_tablet->tablet_schema()[field_off].name ==
column.column_name) {
- uint32_t unique_id =
ref_tablet->tablet_schema()[field_off].unique_id;
+ //uint32_t unique_id =
ref_tablet->tablet_schema()[field_off].unique_id;
//header->mutable_column(i)->set_unique_id(unique_id);
break;
}
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 5c02c723..58b4df5d 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -26,12 +26,12 @@ TabletMeta::TabletMeta(DataDir* data_dir) {
_data_dir = data_dir;
}
-OLAPStatus TabletMeta::serialize(string* meta_binary) {
+OLAPStatus TabletMeta::serialize(string* meta_binary) const {
std::lock_guard<std::mutex> lock(_mutex);
return serialize_unlock(meta_binary);
};
-OLAPStatus TabletMeta::serialize_unlock(string* meta_binary) {
+OLAPStatus TabletMeta::serialize_unlock(string* meta_binary) const {
_tablet_meta_pb.SerializeToString(meta_binary);
return OLAP_SUCCESS;
};
@@ -51,12 +51,12 @@ OLAPStatus TabletMeta::deserialize_unlock(const string&
meta_binary) {
RETURN_NOT_OK(_schema.init_from_pb(_tablet_meta_pb.schema()));
for (auto& it : _tablet_meta_pb.rs_metas()) {
RowsetMetaSharedPtr rs_meta(new RowsetMeta());
- rs_meta.init_from_pb(it);
+ rs_meta->init_from_pb(it);
_rs_metas.push_back(std::move(rs_meta));
}
for (auto& it : _tablet_meta_pb.inc_rs_metas()) {
RowsetMetaSharedPtr rs_meta(new RowsetMeta());
- rs_meta.init_from_pb(it);
+ rs_meta->init_from_pb(it);
_rs_metas.push_back(std::move(rs_meta));
}
@@ -96,7 +96,7 @@ OLAPStatus TabletMeta::save_meta_unlock() {
string meta_binary;
serialize_unlock(&meta_binary);
OLAPStatus status = TabletMetaManager::save(_data_dir, _tablet_id,
_schema_hash, meta_binary);
- if (status != OLAP_SUCCESS)
+ if (status != OLAP_SUCCESS) {
LOG(WARNING) << "fail to save tablet_meta. status=" << status
<< ", tablet_id=" << _tablet_id
<< ", schema_hash=" << _schema_hash;
@@ -116,32 +116,31 @@ OLAPStatus TabletMeta::to_tablet_pb_unlock(TabletMetaPB*
tablet_meta_pb) {
tablet_meta_pb->set_schema_hash(_schema_hash);
tablet_meta_pb->set_shard_id(_shard_id);
- tablet_meta_pb->set_tablet_name(_tablet_name);
- for (auto rs : _rs_metas) {
- rs.to_rowset_pb(tablet_meta_pb->add_rs_meta());
+ for (auto& rs : _rs_metas) {
+ rs->to_rowset_pb(tablet_meta_pb->add_rs_metas());
}
for (auto rs : _inc_rs_metas) {
- rs.to_rowset_pb(tablet_meta_pb->add_inc_rc_meta());
+ rs->to_rowset_pb(tablet_meta_pb->add_inc_rs_metas());
}
_schema.to_schema_pb(tablet_meta_pb->mutable_schema());
return OLAP_SUCCESS;
}
-OLAPStatus TabletMeta::add_inc_rs_meta(const RowsetMeta& rs_meta) {
+OLAPStatus TabletMeta::add_inc_rs_meta(const RowsetMetaSharedPtr& rs_meta) {
std::lock_guard<std::mutex> lock(_mutex);
// check RowsetMeta is valid
for (auto rs : _inc_rs_metas) {
- if (rs.rowset_id() == rs_meta.rowset_id()) {
- LOG(WARNING) << "rowset already exist. rowset_id=" <<
rs.rowset_id();
- return OLAPStatus::AlreadyExist("rowset already exist.");
+ if (rs->rowset_id() == rs_meta->rowset_id()) {
+ LOG(WARNING) << "rowset already exist. rowset_id=" <<
rs->rowset_id();
+ return OLAP_ERR_VERSION_NOT_EXIST;
}
}
_inc_rs_metas.push_back(std::move(rs_meta));
- RowsetMetaPB* rs_meta_pb = _tablet_meta_pb.add_inc_rs_meta();
- RETURN_NOT_OK(rs_meta.to_rowset_pb(rs_meta_pb));
+ RowsetMetaPB* rs_meta_pb = _tablet_meta_pb.add_inc_rs_metas();
+ RETURN_NOT_OK(rs_meta->to_rowset_pb(rs_meta_pb));
RETURN_NOT_OK(save_meta_unlock());
return OLAP_SUCCESS;
@@ -149,9 +148,11 @@ OLAPStatus TabletMeta::add_inc_rs_meta(const RowsetMeta&
rs_meta) {
OLAPStatus TabletMeta::delete_inc_rs_meta_by_version(const Version& version) {
std::lock_guard<std::mutex> lock(_mutex);
- for (auto rs : _inc_rs_metas) {
- if (rs.version() == version) {
- _inc_rs_metas.erase(rs);
+ auto it = _inc_rs_metas.begin();
+ while (it != _inc_rs_metas.end()) {
+ if ((*it)->version().first == version.first
+ && (*it)->version().second == version.second) {
+ _inc_rs_metas.erase(it);
}
}
@@ -165,10 +166,11 @@ OLAPStatus
TabletMeta::delete_inc_rs_meta_by_version(const Version& version) {
const RowsetMeta* TabletMeta::get_inc_rowset(const Version& version) const {
std::lock_guard<std::mutex> lock(_mutex);
- RowsetMeta* rs_meta = nullptr;
+ const RowsetMeta* rs_meta = nullptr;
for (int i = 0; i < _inc_rs_metas.size(); ++i) {
- if (_inc_rs_metas[i].version() == version) {
- rs_meta = &(_inc_rs_metas[i]);
+ if (_inc_rs_metas[i]->version().first == version.first
+ && _inc_rs_metas[i]->version().second == version.second) {
+ rs_meta = _inc_rs_metas[i].get();
break;
}
}
@@ -178,9 +180,13 @@ const RowsetMeta* TabletMeta::get_inc_rowset(const
Version& version) const {
OLAPStatus TabletMeta::modify_rowsets(const vector<RowsetMetaSharedPtr>&
to_add,
const vector<RowsetMetaSharedPtr>&
to_delete) {
std::lock_guard<std::mutex> lock(_mutex);
- for (auto rs : to_delete) {
- if (ContainsKey(rs, _rs_metas)) {
- _rs_metas.erase(rs);
+ for (auto del_rs : to_delete) {
+ auto it = _rs_metas.begin();
+ while (it != _rs_metas.end()) {
+ if (del_rs->version().first == (*it)->version().first
+ && del_rs->version().second == (*it)->version().second) {
+ _rs_metas.erase(it);
+ }
}
}
@@ -199,14 +205,13 @@ OLAPStatus TabletMeta::modify_rowsets(const
vector<RowsetMetaSharedPtr>& to_add,
OLAPStatus TabletMeta::add_alter_task(const AlterTabletTask& alter_task) {
std::lock_guard<std::mutex> lock(_mutex);
_alter_task = alter_task;
-
RETURN_NOT_OK(_alter_task.to_alter_pb(_tablet_meta_pb.mutable_alter_tablet_task));
+
RETURN_NOT_OK(_alter_task.to_alter_pb(_tablet_meta_pb.mutable_alter_tablet_task()));
RETURN_NOT_OK(save_meta_unlock());
return OLAP_SUCCESS;
}
OLAPStatus TabletMeta::delete_alter_task() {
std::lock_guard<std::mutex> lock(_mutex);
- alter_task.clear();
_tablet_meta_pb.clear_alter_tablet_task();
RETURN_NOT_OK(save_meta_unlock());
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index fa6fbe74..67c3515c 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -28,6 +28,7 @@
#include "olap/olap_define.h"
#include "olap/tablet_schema.h"
#include "olap/rowset/rowset.h"
+#include "olap/rowset/rowset_meta.h"
using std::string;
using std::vector;
@@ -59,15 +60,16 @@ class AlterTabletTask {
OLAPStatus to_alter_pb(AlterTabletPB* alter_task);
OLAPStatus clear();
- inline int64_t related_tablet_id() { return _related_tablet_id; }
- inline int64_t related_schema_hash() { return _related_schema_hash; }
- inline int64_t set_related_tablet_id(int64_t related_tablet_id) {
_related_tablet_id = related_tablet_id; }
- inline int64_t set_related_schema_hash(int64_t schema_hash) {
_related_schema_hash = schema_hash; }
+ inline int64_t related_tablet_id() const { return _related_tablet_id; }
+ inline int64_t related_schema_hash() const { return _related_schema_hash; }
+ inline void set_related_tablet_id(int64_t related_tablet_id) {
_related_tablet_id = related_tablet_id; }
+ inline void set_related_schema_hash(int64_t schema_hash) {
_related_schema_hash = schema_hash; }
- vector<RowsetMetaSharedPtr>& rowsets_to_alter() { return
_rowsets_to_alter; }
+ const vector<RowsetMetaSharedPtr>& rowsets_to_alter() const { return
_rowsets_to_alter; }
const AlterTabletState& alter_state() const { return _alter_state; }
const AlterTabletType& alter_type() const { return _alter_type; }
+ void set_alter_type(AlterTabletType alter_type) { _alter_type =
alter_type; }
private:
int64_t _related_tablet_id;
int64_t _related_schema_hash;
@@ -113,8 +115,10 @@ class TabletMeta {
TabletMeta();
TabletMeta(const std::string& file_name);
TabletMeta(DataDir* data_dir);
- OLAPStatus serialize(string* meta_binary);
- OLAPStatus serialize_unlock(string* meta_binary);
+
+ OLAPStatus init_from_pb(const TabletMetaPB& tablet_meta_pb);
+ OLAPStatus serialize(string* meta_binary) const;
+ OLAPStatus serialize_unlock(string* meta_binary) const;
OLAPStatus deserialize(const string& meta_binary);
OLAPStatus deserialize_unlock(const string& meta_binary);
@@ -127,8 +131,10 @@ class TabletMeta {
OLAPStatus add_inc_rs_meta(const RowsetMetaSharedPtr& rs_meta);
OLAPStatus delete_inc_rs_meta_by_version(const Version& version);
+ const RowsetMeta* get_inc_rowset(const Version& version) const;
const RowsetMetaSharedPtr get_inc_rs_meta(const Version& version) const;
DeletePredicatePB* add_delete_predicates();
+ std::vector<DeletePredicatePB>& delete_predicates();
const std::vector<RowsetMetaSharedPtr>& all_inc_rs_metas() const;
const std::vector<RowsetMetaSharedPtr>& all_rs_metas() const;
@@ -167,7 +173,7 @@ class TabletMeta {
TabletMetaPB _tablet_meta_pb;
DataDir* _data_dir;
- std::mutex _mutex;
+ mutable std::mutex _mutex;
};
inline const int64_t TabletMeta::table_id() const {
diff --git a/be/src/olap/tablet_meta_manager.cpp
b/be/src/olap/tablet_meta_manager.cpp
index 20a02fb0..7ef840fd 100755
--- a/be/src/olap/tablet_meta_manager.cpp
+++ b/be/src/olap/tablet_meta_manager.cpp
@@ -61,7 +61,8 @@ OLAPStatus TabletMetaManager::get_header(DataDir* store,
LOG(WARNING) << "load tablet_id:" << tablet_id << ", schema_hash:" <<
schema_hash << " failed.";
return s;
}
- header->ParseFromString(value);
+ //header->ParseFromString(value);
+ header->deserialize(value);
return header->init();
}
@@ -72,25 +73,36 @@ OLAPStatus TabletMetaManager::get_json_header(DataDir*
store,
if (s != OLAP_SUCCESS) {
return s;
}
+ TabletMetaPB tablet_meta_pb;
+ header.to_tablet_pb(&tablet_meta_pb);
json2pb::Pb2JsonOptions json_options;
json_options.pretty_json = true;
- json2pb::ProtoMessageToJson(header, json_header, json_options);
+ json2pb::ProtoMessageToJson(tablet_meta_pb, json_header, json_options);
return OLAP_SUCCESS;
}
-
OLAPStatus TabletMetaManager::save(DataDir* store,
TTabletId tablet_id, TSchemaHash schema_hash, const TabletMeta*
header) {
std::stringstream key_stream;
key_stream << HEADER_PREFIX << tablet_id << "_" << schema_hash;
std::string key = key_stream.str();
std::string value;
- header->SerializeToString(&value);
+ header->serialize(&value);
OlapMeta* meta = store->get_meta();
OLAPStatus s = meta->put(META_COLUMN_FAMILY_INDEX, key, value);
return s;
}
+OLAPStatus TabletMetaManager::save(DataDir* store,
+ TTabletId tablet_id, TSchemaHash schema_hash, const std::string&
meta_binary) {
+ std::stringstream key_stream;
+ key_stream << HEADER_PREFIX << tablet_id << "_" << schema_hash;
+ std::string key = key_stream.str();
+ OlapMeta* meta = store->get_meta();
+ OLAPStatus s = meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary);
+ return s;
+}
+
OLAPStatus TabletMetaManager::remove(DataDir* store, TTabletId tablet_id,
TSchemaHash schema_hash) {
std::stringstream key_stream;
key_stream << HEADER_PREFIX << tablet_id << "_" << schema_hash;
@@ -152,11 +164,13 @@ OLAPStatus TabletMetaManager::load_json_header(DataDir*
store, const std::string
json_header = json_header + buffer;
}
boost::algorithm::trim(json_header);
- TabletMeta header;
- bool ret = json2pb::JsonToProtoMessage(json_header, &header);
+ TabletMetaPB tablet_meta_pb;
+ bool ret = json2pb::JsonToProtoMessage(json_header, &tablet_meta_pb);
if (!ret) {
return OLAP_ERR_HEADER_LOAD_JSON_HEADER;
}
+ TabletMeta header;
+ header.init_from_pb(tablet_meta_pb);
TTabletId tablet_id = header.tablet_id();
TSchemaHash schema_hash = header.schema_hash();
OLAPStatus s = save(store, tablet_id, schema_hash, &header);
diff --git a/be/src/olap/tablet_meta_manager.h
b/be/src/olap/tablet_meta_manager.h
index c06a8526..bf3d7ad9 100644
--- a/be/src/olap/tablet_meta_manager.h
+++ b/be/src/olap/tablet_meta_manager.h
@@ -35,6 +35,7 @@ class TabletMetaManager {
TSchemaHash schema_hash, std::string* json_header);
static OLAPStatus save(DataDir* store, TTabletId tablet_id, TSchemaHash
schema_hash, const TabletMeta* header);
+ static OLAPStatus save(DataDir* store, TTabletId tablet_id, TSchemaHash
schema_hash, const std::string& meta_binary);
static OLAPStatus remove(DataDir* store, TTabletId tablet_id, TSchemaHash
schema_hash);
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 118a50c6..e4d39811 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -47,7 +47,9 @@ class TabletSchema {
public:
TabletSchema();
OLAPStatus init_from_pb(const TabletSchemaPB& schema);
+ OLAPStatus to_schema_pb(TabletSchemaPB* tablet_meta_pb);
size_t get_row_size() const;
+ size_t get_field_index(const std::string& field_name) const;
private:
std::vector<TabletColumnSchema> _cols;
size_t _num_columns;
diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp
index af39e2d6..49f9f54e 100644
--- a/be/src/tools/meta_tool.cpp
+++ b/be/src/tools/meta_tool.cpp
@@ -24,6 +24,7 @@
#include <gflags/gflags.h>
#include "common/status.h"
+#include "gen_cpp/olap_file.pb.h"
#include "olap/data_dir.h"
#include "olap/tablet_meta_manager.h"
#include "olap/olap_define.h"
@@ -130,13 +131,13 @@ int main(int argc, char** argv) {
auto rollback_func = [&root_path](long tablet_id,
long schema_hash, const std::string& value) -> bool {
TabletMeta tablet_meta;
- bool parsed = tablet_meta.ParseFromString(value);
+ bool parsed = tablet_meta.deserialize(value);
if (!parsed) {
std::cout << "parse header failed";
return true;
}
std::string tablet_id_str = std::to_string(tablet_id);
- std::string schema_hash_path = root_path + "/data/" +
std::to_string(tablet_meta.shard())
+ std::string schema_hash_path = root_path + "/data/" +
std::to_string(tablet_meta.shard_id())
+ "/" + tablet_id_str + "/" + std::to_string(schema_hash);
std::string header_file_path = schema_hash_path + "/" +
tablet_id_str + ".hdr";
std::cout << "save header to path:" << header_file_path <<
std::endl;
@@ -157,7 +158,9 @@ int main(int argc, char** argv) {
std::string json_header;
json2pb::Pb2JsonOptions json_options;
json_options.pretty_json = true;
- json2pb::ProtoMessageToJson(header, &json_header, json_options);
+ TabletMetaPB tablet_meta_pb;
+ header.to_tablet_pb(&tablet_meta_pb);
+ json2pb::ProtoMessageToJson(tablet_meta_pb, &json_header,
json_options);
std::cout << "header:" << std::endl;
std::cout << json_header << std::endl;
} else {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
