imay commented on a change in pull request #1413: Add short key index builder, decoder, iterator URL: https://github.com/apache/incubator-doris/pull/1413#discussion_r298942725
########## File path: be/src/olap/short_key_index.h ########## @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <cstdint> +#include <vector> + +#include "common/status.h" +#include "gen_cpp/column_data_file.pb.h" +#include "gen_cpp/olap_file.pb.h" +#include "olap/field_info.h" +#include "olap/file_helper.h" +#include "olap/row_cursor.h" +#include "olap/olap_index.h" +#include "util/faststring.h" +#include "util/slice.h" + +namespace doris { + +// Used to encode a segment short key indices to +// Usage: +// ShortKeyIndexBuilder builder(schema, segment_id, num_rows, delete_flag); +// builder.init(); +// builder.add_item(key1, block1); +// ... +// builder.add_item(keyN, blockN); +// builder.finalize(segment_size, num_rows, &slices); +class ShortKeyIndexBuilder { +public: + ShortKeyIndexBuilder(const std::vector<FieldInfo>& key_fields, + uint32_t segment_id, + uint32_t num_rows_per_block, + bool delete_flag) + : _key_fields(key_fields), + _segment_id(segment_id), + _num_rows_per_block(num_rows_per_block), + _delete_flag(delete_flag) { } + + Status init(); + // Actually, block_id is useless now, may be removed in some day + Status add_item(const RowCursor& key, uint32_t block_id); + Status finalize(uint32_t segment_size, uint32_t num_rows, std::vector<Slice>* slices); + +private: + std::vector<FieldInfo> _key_fields; + uint32_t _segment_id = 0; + uint32_t _num_rows_per_block; + bool _delete_flag; + + // short key length + size_t _key_length; + + // used to save serialized key temporarily + faststring _tmp_key_buf; + + faststring _header_buf; + faststring _index_buf; + + // NOTE: most of these two fields are useless. + // put here just for compatible. I will remove this one day + // only used to compatible with old version + FileHeader<OLAPIndexHeaderMessage, OLAPIndexFixedHeader> _header; +}; + +// Used to decode short key to header and encoded index data. +// Actually, this class should decode index data to memory format. +// But for compatible with old code, we reuse MemIndex class. +// Usage: +// MemIndex index; +// ShortKeyIndexDecoder decoder(slice) +// decoder.parse(); +// index.load_segment(decoder.header(), decoder.index_data()); +class ShortKeyIndexDecoder { +public: + ShortKeyIndexDecoder(const Slice& data) : _data(data) { } + Status parse(); + const FileHeader<OLAPIndexHeaderMessage, OLAPIndexFixedHeader>& header() const { + return _header; + } + const Slice& index_data() const { return _index_data; } +private: + Slice _data; + + // valid after parse() is called, Header has been removed. + // Only contains index data + Slice _index_data; + + // only used to compatible with old version + FileHeader<OLAPIndexHeaderMessage, OLAPIndexFixedHeader> _header; +}; + +class ShortKeyIndexIterator { +public: + ShortKeyIndexIterator(const MemIndex* index) : _index(index), _segment(0), _block(0) { + _helper.init(_index->short_key_fields()); + } + + // Position at the first key in the index that at or past target + inline void seek_to(const RowCursor& key) { + auto offset = _index->find(key, &_helper, false); + if (offset.offset > 0) { + offset.offset -= 1; + offset = _index->next(offset); + } + _segment = offset.segment; + _block = offset.offset; + } + + // Position at the first key in the index that post target + inline void seek_after(const RowCursor& key) { + auto offset = _index->find(key, &_helper, true); + if (offset.offset > 0) { + offset.offset -= 1; + offset = _index->next(offset); + } + _segment = offset.segment; + _block = offset.offset; + } + + // Position at the first key in the index that before target + inline void seek_before(const RowCursor& key) { + seek_to(key); + if (valid()) { + prev(); + } + } + + inline bool valid() const { + return _segment < _index->segment_count(); + } + + inline void prev() { Review comment: OK, I will add comment ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@doris.apache.org For additional commands, e-mail: dev-h...@doris.apache.org
