This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 22207ce5c7f [be](refactor) move rowtype template parameter from key
util (#61226)
22207ce5c7f is described below
commit 22207ce5c7fb2999ce33f4eea963615c558cd273
Author: yiguolei <[email protected]>
AuthorDate: Thu Mar 12 10:24:05 2026 +0800
[be](refactor) move rowtype template parameter from key util (#61226)
### What problem does this PR solve?
1. remove rowtype template parameter from encode key methods
2. move encode key methods from key util to rowcurosr because it only
encode rowcursor.
3. move _key_is_not_in_segment to base tablet
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/common/consts.h | 23 +++++
be/src/service/point_query_executor.cpp | 5 +-
be/src/storage/row_cursor.h | 58 +++++++++++
be/src/storage/segment/segment_iterator.cpp | 9 +-
be/src/storage/segment/segment_writer.cpp | 12 +--
be/src/storage/segment/segment_writer.h | 3 +-
be/src/storage/segment/vertical_segment_writer.cpp | 2 +-
be/src/storage/tablet/base_tablet.cpp | 16 ++-
be/src/storage/tablet/base_tablet.h | 5 +
be/src/util/key_util.cpp | 32 ------
be/src/util/key_util.h | 115 ---------------------
be/test/storage/row_cursor_test.cpp | 65 ++++++++++++
.../segment_writer_full_encode_keys_test.cpp | 1 -
.../segments_key_bounds_truncation_test.cpp | 6 +-
be/test/util/key_util_test.cpp | 104 -------------------
15 files changed, 179 insertions(+), 277 deletions(-)
diff --git a/be/src/common/consts.h b/be/src/common/consts.h
index 07e6b636447..8e67f0c81fd 100644
--- a/be/src/common/consts.h
+++ b/be/src/common/consts.h
@@ -49,4 +49,27 @@ static constexpr int MAX_DECIMALV3_SCALE =
MAX_DECIMALV3_PRECISION;
static constexpr int DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATS_SIZE = 10000;
static constexpr int DEFAULT_VARIANT_SPARSE_HASH_SHARD_COUNT = 1;
} // namespace BeConsts
+
+// In our system, we have more complicated situation.
+// First, our keys can be nullptr.
+// Second, when key columns are not complete we want to distinguish GT and GE.
For example,
+// there are two key columns a and b, we have only one condition a > 1. We can
only encode
+// a prefix key 1, which is less than 1|2. This will make our read more data
than
+// we actually need. So we want to add more marker.
+// a > 1: will be encoded into 1|\xFF
+// a >= 1: will be encoded into 1|\x00
+// a = 1 and b > 1: will be encoded into 1|\x02|1
+// a = 1 and b is null: will be encoded into 1|\x01
+namespace KeyConsts {
+// Used to represent minimal value for that field
+constexpr uint8_t KEY_MINIMAL_MARKER = 0x00;
+// Used to represent a null field, which value is seemed as minimal than other
values
+constexpr uint8_t KEY_NULL_FIRST_MARKER = 0x01;
+// Used to represent a normal field, which content is encoded after this marker
+constexpr uint8_t KEY_NORMAL_MARKER = 0x02;
+// Used to represent maximal value for that field
+constexpr uint8_t KEY_MAXIMAL_MARKER = 0xFF;
+// Used to represent a value greater than the normal marker by 1, using by MoW
+constexpr uint8_t KEY_NORMAL_NEXT_MARKER = 0x03;
+} // namespace KeyConsts
} // namespace doris
diff --git a/be/src/service/point_query_executor.cpp
b/be/src/service/point_query_executor.cpp
index db58a3705dc..3c28227a653 100644
--- a/be/src/service/point_query_executor.cpp
+++ b/be/src/service/point_query_executor.cpp
@@ -55,7 +55,6 @@
#include "storage/tablet/tablet_schema.h"
#include "storage/utils.h"
#include "util/jsonb/serialize.h"
-#include "util/key_util.h"
#include "util/lru_cache.h"
#include "util/simd/bits.h"
#include "util/thrift_util.h"
@@ -402,8 +401,8 @@ Status PointQueryExecutor::_init_keys(const
PTabletKeyLookupRequest* request) {
RowCursor cursor;
RETURN_IF_ERROR(cursor.init_scan_key(_tablet->tablet_schema(),
olap_tuples[i].values()));
RETURN_IF_ERROR(cursor.from_tuple(olap_tuples[i]));
- encode_key_with_padding<RowCursor,
true>(&_row_read_ctxs[i]._primary_key, cursor,
-
_tablet->tablet_schema()->num_key_columns(), true);
+ cursor.encode_key_with_padding<true>(&_row_read_ctxs[i]._primary_key,
+
_tablet->tablet_schema()->num_key_columns(), true);
}
return Status::OK();
}
diff --git a/be/src/storage/row_cursor.h b/be/src/storage/row_cursor.h
index 0cb89891581..e7911fa79c9 100644
--- a/be/src/storage/row_cursor.h
+++ b/be/src/storage/row_cursor.h
@@ -25,6 +25,7 @@
#include <string>
#include <vector>
+#include "common/consts.h"
#include "common/status.h"
#include "storage/olap_tuple.h"
#include "storage/row_cursor_cell.h"
@@ -69,6 +70,63 @@ public:
const Schema* schema() const { return _schema.get(); }
+ // Encode one row into binary according given num_keys.
+ // A cell will be encoded in the format of a marker and encoded content.
+ // When encoding row, if any cell isn't found in row, this function will
+ // fill a marker and return. If padding_minimal is true,
KEY_MINIMAL_MARKER will
+ // be added, if padding_minimal is false, KEY_MAXIMAL_MARKER will be added.
+ // If all num_keys are found in row, no marker will be added.
+ template <bool is_mow = false>
+ void encode_key_with_padding(std::string* buf, size_t num_keys, bool
padding_minimal) const {
+ for (uint32_t cid = 0; cid < num_keys; cid++) {
+ auto field = _schema->column(cid);
+ if (field == nullptr) {
+ if (padding_minimal) {
+ buf->push_back(KeyConsts::KEY_MINIMAL_MARKER);
+ } else {
+ if (is_mow) {
+ buf->push_back(KeyConsts::KEY_NORMAL_NEXT_MARKER);
+ } else {
+ buf->push_back(KeyConsts::KEY_MAXIMAL_MARKER);
+ }
+ }
+ break;
+ }
+
+ auto c = cell(cid);
+ if (c.is_null()) {
+ buf->push_back(KeyConsts::KEY_NULL_FIRST_MARKER);
+ continue;
+ }
+ buf->push_back(KeyConsts::KEY_NORMAL_MARKER);
+ if (is_mow) {
+ field->full_encode_ascending(c.cell_ptr(), buf);
+ } else {
+ field->encode_ascending(c.cell_ptr(), buf);
+ }
+ }
+ }
+
+ // Encode one row into binary according given num_keys.
+ // Client call this function must assure that row contains the first
+ // num_keys columns.
+ template <bool full_encode = false>
+ void encode_key(std::string* buf, size_t num_keys) const {
+ for (uint32_t cid = 0; cid < num_keys; cid++) {
+ auto c = cell(cid);
+ if (c.is_null()) {
+ buf->push_back(KeyConsts::KEY_NULL_FIRST_MARKER);
+ continue;
+ }
+ buf->push_back(KeyConsts::KEY_NORMAL_MARKER);
+ if (full_encode) {
+ _schema->column(cid)->full_encode_ascending(c.cell_ptr(), buf);
+ } else {
+ _schema->column(cid)->encode_ascending(c.cell_ptr(), buf);
+ }
+ }
+ }
+
private:
Status _init(TabletSchemaSPtr schema, uint32_t column_count);
Status _init(const std::vector<uint32_t>& columns);
diff --git a/be/src/storage/segment/segment_iterator.cpp
b/be/src/storage/segment/segment_iterator.cpp
index 5cfeaf43406..29a5881f6c9 100644
--- a/be/src/storage/segment/segment_iterator.cpp
+++ b/be/src/storage/segment/segment_iterator.cpp
@@ -106,7 +106,6 @@
#include "storage/utils.h"
#include "util/concurrency_stats.h"
#include "util/defer_op.h"
-#include "util/key_util.h"
#include "util/simd/bits.h"
namespace doris {
@@ -1566,8 +1565,8 @@ Status
SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool
DCHECK(sk_index_decoder != nullptr);
std::string index_key;
- encode_key_with_padding(&index_key, key,
_segment->_tablet_schema->num_short_key_columns(),
- is_include);
+ key.encode_key_with_padding(&index_key,
_segment->_tablet_schema->num_short_key_columns(),
+ is_include);
const auto& key_col_ids = key.schema()->column_ids();
_convert_rowcursor_to_short_key(key, key_col_ids.size());
@@ -1626,8 +1625,8 @@ Status
SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool
DCHECK(pk_index_reader != nullptr);
std::string index_key;
- encode_key_with_padding<RowCursor, true>(
- &index_key, key, _segment->_tablet_schema->num_key_columns(),
is_include);
+ key.encode_key_with_padding<true>(&index_key,
_segment->_tablet_schema->num_key_columns(),
+ is_include);
if (index_key < _segment->min_key()) {
*rowid = 0;
return Status::OK();
diff --git a/be/src/storage/segment/segment_writer.cpp
b/be/src/storage/segment/segment_writer.cpp
index 48641d67f02..aa010e8ed99 100644
--- a/be/src/storage/segment/segment_writer.cpp
+++ b/be/src/storage/segment/segment_writer.cpp
@@ -73,13 +73,13 @@
#include "util/coding.h"
#include "util/faststring.h"
#include "util/jsonb/serialize.h"
-#include "util/key_util.h"
#include "util/simd/bits.h"
namespace doris {
namespace segment_v2 {
#include "common/compile_check_begin.h"
using namespace ErrorCode;
+using namespace KeyConsts;
const char* k_segment_magic = "D0R1";
const uint32_t k_segment_magic_length = 4;
@@ -890,15 +890,13 @@ std::string SegmentWriter::_encode_keys(const
std::vector<IOlapColumnDataAccesso
}
return encoded_keys;
}
-
-template <typename RowType>
-Status SegmentWriter::append_row(const RowType& row) {
+Status SegmentWriter::append_row(const RowCursor& row) {
for (size_t cid = 0; cid < _column_writers.size(); ++cid) {
auto cell = row.cell(cast_set<uint32_t>(cid));
RETURN_IF_ERROR(_column_writers[cid]->append(cell));
}
std::string full_encoded_key;
- encode_key<RowType, true>(&full_encoded_key, row, _num_sort_key_columns);
+ row.encode_key<true>(&full_encoded_key, _num_sort_key_columns);
if (_tablet_schema->has_sequence_col()) {
full_encoded_key.push_back(KEY_NORMAL_MARKER);
auto cid = _tablet_schema->sequence_col_idx();
@@ -915,7 +913,7 @@ Status SegmentWriter::append_row(const RowType& row) {
// At the beginning of one block, so add a short key index entry
if ((_num_rows_written % _opts.num_rows_per_block) == 0) {
std::string encoded_key;
- encode_key(&encoded_key, row, _num_short_key_columns);
+ row.encode_key(&encoded_key, _num_short_key_columns);
RETURN_IF_ERROR(_short_key_index_builder->add_item(encoded_key));
}
set_min_max_key(full_encoded_key);
@@ -924,8 +922,6 @@ Status SegmentWriter::append_row(const RowType& row) {
return Status::OK();
}
-template Status SegmentWriter::append_row(const RowCursor& row);
-
// TODO(lingbin): Currently this function does not include the size of various
indexes,
// We should make this more precise.
// NOTE: This function will be called when any row of data is added, so we
need to
diff --git a/be/src/storage/segment/segment_writer.h
b/be/src/storage/segment/segment_writer.h
index e9b062d048d..d3c6078704e 100644
--- a/be/src/storage/segment/segment_writer.h
+++ b/be/src/storage/segment/segment_writer.h
@@ -89,8 +89,7 @@ public:
// for vertical compaction
Status init(const std::vector<uint32_t>& col_ids, bool has_key);
- template <typename RowType>
- Status append_row(const RowType& row);
+ Status append_row(const RowCursor& row);
Status append_block(const Block* block, size_t row_pos, size_t num_rows);
Status probe_key_for_mow(std::string key, std::size_t segment_pos, bool
have_input_seq_column,
diff --git a/be/src/storage/segment/vertical_segment_writer.cpp
b/be/src/storage/segment/vertical_segment_writer.cpp
index eaaf789bdc4..215d069425f 100644
--- a/be/src/storage/segment/vertical_segment_writer.cpp
+++ b/be/src/storage/segment/vertical_segment_writer.cpp
@@ -79,12 +79,12 @@
#include "util/faststring.h"
#include "util/json/path_in_data.h"
#include "util/jsonb/serialize.h"
-#include "util/key_util.h"
namespace doris::segment_v2 {
#include "common/compile_check_begin.h"
using namespace ErrorCode;
+using namespace KeyConsts;
static const char* k_segment_magic = "D0R1";
static const uint32_t k_segment_magic_length = 4;
diff --git a/be/src/storage/tablet/base_tablet.cpp
b/be/src/storage/tablet/base_tablet.cpp
index bb20add6896..88e71901312 100644
--- a/be/src/storage/tablet/base_tablet.cpp
+++ b/be/src/storage/tablet/base_tablet.cpp
@@ -55,7 +55,6 @@
#include "util/bvar_helper.h"
#include "util/debug_points.h"
#include "util/jsonb/serialize.h"
-#include "util/key_util.h"
namespace doris {
#include "common/compile_check_begin.h"
@@ -468,8 +467,8 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key,
TabletSchema* latest
DCHECK_EQ(segments_key_bounds.size(), num_segments);
std::vector<uint32_t> picked_segments;
for (int j = num_segments - 1; j >= 0; j--) {
- if (key_is_not_in_segment(key_without_seq, segments_key_bounds[j],
-
rs->rowset_meta()->is_segments_key_bounds_truncated())) {
+ if (_key_is_not_in_segment(key_without_seq, segments_key_bounds[j],
+
rs->rowset_meta()->is_segments_key_bounds_truncated())) {
continue;
}
picked_segments.emplace_back(j);
@@ -2135,4 +2134,15 @@ void
BaseTablet::prefill_dbm_agg_cache_after_compaction(const RowsetSharedPtr& o
}
}
+bool BaseTablet::_key_is_not_in_segment(Slice key, const KeyBoundsPB&
segment_key_bounds,
+ bool is_segments_key_bounds_truncated)
{
+ Slice maybe_truncated_min_key {segment_key_bounds.min_key()};
+ Slice maybe_truncated_max_key {segment_key_bounds.max_key()};
+ bool res1 = Slice::lhs_is_strictly_less_than_rhs(key, false,
maybe_truncated_min_key,
+
is_segments_key_bounds_truncated);
+ bool res2 = Slice::lhs_is_strictly_less_than_rhs(maybe_truncated_max_key,
+
is_segments_key_bounds_truncated, key, false);
+ return res1 || res2;
+}
+
} // namespace doris
diff --git a/be/src/storage/tablet/base_tablet.h
b/be/src/storage/tablet/base_tablet.h
index 8792bfc80ae..b98a89eb734 100644
--- a/be/src/storage/tablet/base_tablet.h
+++ b/be/src/storage/tablet/base_tablet.h
@@ -356,6 +356,11 @@ protected:
const RowsetIdUnorderedSet& pre,
RowsetIdUnorderedSet* to_add,
RowsetIdUnorderedSet* to_del);
+ // We can only know if a key is excluded from the segment
+ // based on strictly order compare result with segments key bounds
+ static bool _key_is_not_in_segment(Slice key, const KeyBoundsPB&
segment_key_bounds,
+ bool is_segments_key_bounds_truncated);
+
Status sort_block(Block& in_block, Block& output_block);
Result<CaptureRowsetResult> _remote_capture_rowsets(const Version&
version_range) const;
diff --git a/be/src/util/key_util.cpp b/be/src/util/key_util.cpp
deleted file mode 100644
index b49639d7075..00000000000
--- a/be/src/util/key_util.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "util/key_util.h"
-
-namespace doris {
-
-bool key_is_not_in_segment(Slice key, const KeyBoundsPB& segment_key_bounds,
- bool is_segments_key_bounds_truncated) {
- Slice maybe_truncated_min_key {segment_key_bounds.min_key()};
- Slice maybe_truncated_max_key {segment_key_bounds.max_key()};
- bool res1 = Slice::lhs_is_strictly_less_than_rhs(key, false,
maybe_truncated_min_key,
-
is_segments_key_bounds_truncated);
- bool res2 = Slice::lhs_is_strictly_less_than_rhs(maybe_truncated_max_key,
-
is_segments_key_bounds_truncated, key, false);
- return res1 || res2;
-}
-} // namespace doris
\ No newline at end of file
diff --git a/be/src/util/key_util.h b/be/src/util/key_util.h
deleted file mode 100644
index 01094905cf5..00000000000
--- a/be/src/util/key_util.h
+++ /dev/null
@@ -1,115 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <gen_cpp/olap_file.pb.h>
-#include <gen_cpp/segment_v2.pb.h>
-
-#include <cstdint>
-#include <string>
-
-#include "util/slice.h"
-
-namespace doris {
-
-// In our system, we have more complicated situation.
-// First, our keys can be nullptr.
-// Second, when key columns are not complete we want to distinguish GT and GE.
For example,
-// there are two key columns a and b, we have only one condition a > 1. We can
only encode
-// a prefix key 1, which is less than 1|2. This will make our read more data
than
-// we actually need. So we want to add more marker.
-// a > 1: will be encoded into 1|\xFF
-// a >= 1: will be encoded into 1|\x00
-// a = 1 and b > 1: will be encoded into 1|\x02|1
-// a = 1 and b is null: will be encoded into 1|\x01
-
-// Used to represent minimal value for that field
-constexpr uint8_t KEY_MINIMAL_MARKER = 0x00;
-// Used to represent a null field, which value is seemed as minimal than other
values
-constexpr uint8_t KEY_NULL_FIRST_MARKER = 0x01;
-// Used to represent a normal field, which content is encoded after this marker
-constexpr uint8_t KEY_NORMAL_MARKER = 0x02;
-// Used to represent maximal value for that field
-constexpr uint8_t KEY_MAXIMAL_MARKER = 0xFF;
-// Used to represent a value greater than the normal marker by 1, using by MoW
-constexpr uint8_t KEY_NORMAL_NEXT_MARKER = 0x03;
-
-// Encode one row into binary according given num_keys.
-// A cell will be encoded in the format of a marker and encoded content.
-// When function encoding row, if any cell isn't found in row, this function
will
-// fill a marker and return. If padding_minimal is true, KEY_MINIMAL_MARKER
will
-// be added, if padding_minimal is false, KEY_MAXIMAL_MARKER will be added.
-// If all num_keys are found in row, no marker will be added.
-template <typename RowType, bool is_mow = false>
-void encode_key_with_padding(std::string* buf, const RowType& row, size_t
num_keys,
- bool padding_minimal) {
- for (auto cid = 0; cid < num_keys; cid++) {
- auto field = row.schema()->column(cid);
- if (field == nullptr) {
- if (padding_minimal) {
- buf->push_back(KEY_MINIMAL_MARKER);
- } else {
- if (is_mow) {
- buf->push_back(KEY_NORMAL_NEXT_MARKER);
- } else {
- buf->push_back(KEY_MAXIMAL_MARKER);
- }
- }
- break;
- }
-
- auto cell = row.cell(cid);
- if (cell.is_null()) {
- buf->push_back(KEY_NULL_FIRST_MARKER);
- continue;
- }
- buf->push_back(KEY_NORMAL_MARKER);
- if (is_mow) {
- field->full_encode_ascending(cell.cell_ptr(), buf);
- } else {
- field->encode_ascending(cell.cell_ptr(), buf);
- }
- }
-}
-
-// Encode one row into binary according given num_keys.
-// Client call this function must assure that row contains the first
-// num_keys columns.
-template <typename RowType, bool full_encode = false>
-void encode_key(std::string* buf, const RowType& row, size_t num_keys) {
- for (auto cid = 0; cid < num_keys; cid++) {
- auto cell = row.cell(cid);
- if (cell.is_null()) {
- buf->push_back(KEY_NULL_FIRST_MARKER);
- continue;
- }
- buf->push_back(KEY_NORMAL_MARKER);
- if (full_encode) {
- row.schema()->column(cid)->full_encode_ascending(cell.cell_ptr(),
buf);
- } else {
- row.schema()->column(cid)->encode_ascending(cell.cell_ptr(), buf);
- }
- }
-}
-
-// we can only know if a key is excluded from the segment
-// based on strictly order compare result with segments key bounds
-bool key_is_not_in_segment(Slice key, const KeyBoundsPB& segment_key_bounds,
- bool is_segments_key_bounds_truncated);
-
-} // namespace doris
diff --git a/be/test/storage/row_cursor_test.cpp
b/be/test/storage/row_cursor_test.cpp
index ff37ee63095..358afa7de32 100644
--- a/be/test/storage/row_cursor_test.cpp
+++ b/be/test/storage/row_cursor_test.cpp
@@ -21,8 +21,11 @@
#include "common/object_pool.h"
#include "core/packed_int128.h"
+#include "storage/row_cursor_cell.h"
#include "storage/schema.h"
#include "storage/tablet/tablet_schema.h"
+#include "storage/tablet/tablet_schema_helper.h"
+#include "util/debug_util.h"
namespace doris {
@@ -289,4 +292,66 @@ TEST_F(TestRowCursor, InitRowCursorWithScanKey) {
EXPECT_EQ(res, Status::OK());
}
+TEST_F(TestRowCursor, encode_key) {
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ tablet_schema->_cols.push_back(create_int_key(0));
+ tablet_schema->_cols.push_back(create_int_key(1));
+ tablet_schema->_cols.push_back(create_int_key(2));
+ tablet_schema->_cols.push_back(create_int_value(3));
+ tablet_schema->_num_columns = 4;
+ tablet_schema->_num_key_columns = 3;
+ tablet_schema->_num_short_key_columns = 3;
+
+ // test encoding with padding
+ {
+ RowCursor row;
+ static_cast<void>(row._init(tablet_schema, 2));
+
+ {
+ // test padding
+ {
+ auto cell = row.cell(0);
+ cell.set_is_null(false);
+ *(int*)cell.mutable_cell_ptr() = 12345;
+ }
+ {
+ auto cell = row.cell(1);
+ cell.set_is_null(false);
+ *(int*)cell.mutable_cell_ptr() = 54321;
+ }
+ std::string buf;
+ row.encode_key_with_padding(&buf, 3, true);
+ // should be \x02\x80\x00\x30\x39\x02\x80\x00\xD4\x31\x00
+ EXPECT_STREQ("0280003039028000D43100", hexdump(buf.c_str(),
buf.size()).c_str());
+ }
+ // test with null
+ {
+ {
+ auto cell = row.cell(0);
+ cell.set_is_null(false);
+ *(int*)cell.mutable_cell_ptr() = 54321;
+ }
+ {
+ auto cell = row.cell(1);
+ cell.set_is_null(true);
+ *(int*)cell.mutable_cell_ptr() = 54321;
+ }
+
+ {
+ std::string buf;
+ row.encode_key_with_padding(&buf, 3, false);
+ // should be \x02\x80\x00\xD4\x31\x01\xff
+ EXPECT_STREQ("028000D43101FF", hexdump(buf.c_str(),
buf.size()).c_str());
+ }
+ // encode key
+ {
+ std::string buf;
+ row.encode_key(&buf, 2);
+ // should be \x02\x80\x00\xD4\x31\x01
+ EXPECT_STREQ("028000D43101", hexdump(buf.c_str(),
buf.size()).c_str());
+ }
+ }
+ }
+}
+
} // namespace doris
diff --git a/be/test/storage/segment/segment_writer_full_encode_keys_test.cpp
b/be/test/storage/segment/segment_writer_full_encode_keys_test.cpp
index 7387dcff677..c2748a43fb9 100644
--- a/be/test/storage/segment/segment_writer_full_encode_keys_test.cpp
+++ b/be/test/storage/segment/segment_writer_full_encode_keys_test.cpp
@@ -25,7 +25,6 @@
#include "storage/key_coder.h"
#include "storage/olap_common.h"
#include "storage/segment/segment_writer.h"
-#include "util/key_util.h"
#include "util/slice.h"
namespace doris {
diff --git a/be/test/storage/segment/segments_key_bounds_truncation_test.cpp
b/be/test/storage/segment/segments_key_bounds_truncation_test.cpp
index c007f8a2f9c..30bab8b93e6 100644
--- a/be/test/storage/segment/segments_key_bounds_truncation_test.cpp
+++ b/be/test/storage/segment/segments_key_bounds_truncation_test.cpp
@@ -26,6 +26,7 @@
#include <vector>
#include "common/config.h"
+#include "common/consts.h"
#include "io/fs/local_file_system.h"
#include "runtime/exec_env.h"
#include "storage/compaction/cumulative_compaction.h"
@@ -37,7 +38,6 @@
#include "storage/tablet/tablet_meta.h"
#include "storage/tablet/tablet_reader.h"
#include "storage/tablet/tablet_schema.h"
-#include "util/key_util.h"
namespace doris {
static std::string kSegmentDir =
"./ut_dir/segments_key_bounds_truncation_test";
@@ -203,8 +203,8 @@ public:
}
// segments key bounds have marker
- min_key = std::string {KEY_NORMAL_MARKER} + min_key;
- max_key = std::string {KEY_NORMAL_MARKER} + max_key;
+ min_key = std::string {KeyConsts::KEY_NORMAL_MARKER} + min_key;
+ max_key = std::string {KeyConsts::KEY_NORMAL_MARKER} + max_key;
cur.emplace_back(do_trunacte(min_key));
cur.emplace_back(do_trunacte(max_key));
diff --git a/be/test/util/key_util_test.cpp b/be/test/util/key_util_test.cpp
deleted file mode 100644
index 439c375a495..00000000000
--- a/be/test/util/key_util_test.cpp
+++ /dev/null
@@ -1,104 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "util/key_util.h"
-
-#include <gtest/gtest-message.h>
-#include <gtest/gtest-test-part.h>
-
-#include <algorithm>
-#include <memory>
-#include <vector>
-
-#include "gtest/gtest_pred_impl.h"
-#include "storage/row_cursor.h"
-#include "storage/row_cursor_cell.h"
-#include "storage/tablet/tablet_schema.h"
-#include "storage/tablet/tablet_schema_helper.h"
-#include "util/debug_util.h"
-
-namespace doris {
-
-class KeyUtilTest : public testing::Test {
-public:
- KeyUtilTest() {}
- virtual ~KeyUtilTest() {}
-};
-
-TEST_F(KeyUtilTest, encode) {
- TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
- tablet_schema->_cols.push_back(create_int_key(0));
- tablet_schema->_cols.push_back(create_int_key(1));
- tablet_schema->_cols.push_back(create_int_key(2));
- tablet_schema->_cols.push_back(create_int_value(3));
- tablet_schema->_num_columns = 4;
- tablet_schema->_num_key_columns = 3;
- tablet_schema->_num_short_key_columns = 3;
-
- // test encoding with padding
- {
- RowCursor row;
- static_cast<void>(row._init(tablet_schema, 2));
-
- {
- // test padding
- {
- auto cell = row.cell(0);
- cell.set_is_null(false);
- *(int*)cell.mutable_cell_ptr() = 12345;
- }
- {
- auto cell = row.cell(1);
- cell.set_is_null(false);
- *(int*)cell.mutable_cell_ptr() = 54321;
- }
- std::string buf;
- encode_key_with_padding(&buf, row, 3, true);
- // should be \x02\x80\x00\x30\x39\x02\x80\x00\xD4\x31\x00
- EXPECT_STREQ("0280003039028000D43100", hexdump(buf.c_str(),
buf.size()).c_str());
- }
- // test with null
- {
- {
- auto cell = row.cell(0);
- cell.set_is_null(false);
- *(int*)cell.mutable_cell_ptr() = 54321;
- }
- {
- auto cell = row.cell(1);
- cell.set_is_null(true);
- *(int*)cell.mutable_cell_ptr() = 54321;
- }
-
- {
- std::string buf;
- encode_key_with_padding(&buf, row, 3, false);
- // should be \x02\x80\x00\xD4\x31\x01\xff
- EXPECT_STREQ("028000D43101FF", hexdump(buf.c_str(),
buf.size()).c_str());
- }
- // encode key
- {
- std::string buf;
- encode_key(&buf, row, 2);
- // should be \x02\x80\x00\xD4\x31\x01
- EXPECT_STREQ("028000D43101", hexdump(buf.c_str(),
buf.size()).c_str());
- }
- }
- }
-}
-
-} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
