xiaokang commented on code in PR #10322: URL: https://github.com/apache/doris/pull/10322#discussion_r928253643
########## be/src/vec/columns/column_json.h: ########## @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <cassert> +#include <cstring> + +#include "vec/columns/column.h" +#include "vec/columns/column_impl.h" +#include "vec/common/assert_cast.h" +#include "vec/common/memcmp_small.h" +#include "vec/common/memcpy_small.h" +#include "vec/common/pod_array.h" +#include "vec/common/sip_hash.h" +#include "vec/core/field.h" + +namespace doris::vectorized { +class ColumnJson final : public COWHelper<IColumn, ColumnJson> { +public: + using Char = UInt8; + using Chars = PaddedPODArray<UInt8>; + +private: + friend class COWHelper<IColumn, ColumnJson>; + + Offsets offsets; + + Chars chars; + + size_t ALWAYS_INLINE offset_at(ssize_t i) const { return offsets[i - 1]; } + + size_t ALWAYS_INLINE size_at(ssize_t i) const { return offsets[i] - offsets[i - 1]; } + + template <bool positive> + struct less; + + template <bool positive> + struct lessWithCollation; + + ColumnJson() = default; + + ColumnJson(const ColumnJson& src) + : offsets(src.offsets.begin(), src.offsets.end()), + chars(src.chars.begin(), src.chars.end()) {} + +public: + const char* get_family_name() const override { return "JSON"; } + + size_t size() const override { return offsets.size(); } + + size_t byte_size() const override { return chars.size() + offsets.size() * sizeof(offsets[0]); } + + size_t allocated_bytes() const override { + return chars.allocated_bytes() + offsets.allocated_bytes(); + } + + void protect() override; + + MutableColumnPtr clone_resized(size_t to_size) const override; + + Field operator[](size_t n) const override { + assert(n < size()); + return Field(&chars[offset_at(n)], size_at(n) - 1); + } + + void get(size_t n, Field& res) const override { + assert(n < size()); + res.assign_json(&chars[offset_at(n)], size_at(n) - 1); + } + + StringRef get_data_at(size_t n) const override { + assert(n < size()); + return StringRef(&chars[offset_at(n)], size_at(n) - 1); + } + +/// Suppress gcc 7.3.1 warning: '*((void*)&<anonymous> +8)' may be used uninitialized in this function +#if !__clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + + void insert(const Field& x) override { + const JsonField& s = doris::vectorized::get<const JsonField&>(x); + + const size_t old_size = chars.size(); + const size_t size_to_append = s.get_size() + 1; + const size_t new_size = old_size + size_to_append; + + chars.resize(new_size); + memcpy(chars.data() + old_size, s.get_value(), size_to_append); + offsets.push_back(new_size); + } + +#if !__clang__ +#pragma GCC diagnostic pop +#endif + + void insert_from(const IColumn& src_, size_t n) override { + const ColumnJson& src = assert_cast<const ColumnJson&>(src_); + const size_t size_to_append = + src.offsets[n] - src.offsets[n - 1]; /// -1th index is Ok, see PaddedPODArray. + + if (size_to_append == 1) { + /// shortcut for empty json + chars.push_back(0); + offsets.push_back(chars.size()); + } else { + const size_t old_size = chars.size(); + const size_t offset = src.offsets[n - 1]; + const size_t new_size = old_size + size_to_append; + + chars.resize(new_size); + memcpy_small_allow_read_write_overflow15(chars.data() + old_size, &src.chars[offset], + size_to_append); + offsets.push_back(new_size); + } + } + + void insert_data(const char* pos, size_t length) override { + const size_t old_size = chars.size(); + const size_t new_size = old_size + length + 1; + + chars.resize(new_size); + if (length) memcpy(chars.data() + old_size, pos, length); + chars[old_size + length] = 0; + offsets.push_back(new_size); + } + + void insert_many_binary_data(char* data_array, uint32_t* len_array, + uint32_t* start_offset_array, size_t num) override { + for (size_t i = 0; i < num; i++) { + uint32_t len = len_array[i]; + uint32_t start_offset = start_offset_array[i]; + insert_data(data_array + start_offset, len); + } + }; + + void insert_many_dict_data(const int32_t* data_array, size_t start_index, const StringRef* dict, + size_t num, uint32_t /*dict_num*/) { + for (size_t end_index = start_index + num; start_index < end_index; ++start_index) { + int32_t codeword = data_array[start_index]; + insert_data(dict[codeword].data, dict[codeword].size); + } + } + + void pop_back(size_t n) override { + size_t nested_n = offsets.back() - offset_at(offsets.size() - n); + chars.resize(chars.size() - nested_n); + offsets.resize_assume_reserved(offsets.size() - n); + } + + StringRef serialize_value_into_arena(size_t n, Arena& arena, char const*& begin) const override; + + const char* deserialize_and_insert_from_arena(const char* pos) override; + + void update_hash_with_value(size_t n, SipHash& hash) const override { + size_t string_size = size_at(n); + size_t offset = offset_at(n); + + hash.update(reinterpret_cast<const char*>(&string_size), sizeof(string_size)); + hash.update(reinterpret_cast<const char*>(&chars[offset]), string_size); + } + + void insert_range_from(const IColumn& src, size_t start, size_t length) override; + + void insert_indices_from(const IColumn& src, const int* indices_begin, + const int* indices_end) override; + + ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const override; + + ColumnPtr permute(const Permutation& perm, size_t limit) const override; + + // ColumnPtr index(const IColumn & indexes, size_t limit) const override; + + template <typename Type> + ColumnPtr index_impl(const PaddedPODArray<Type>& indexes, size_t limit) const; + + void insert_default() override { + chars.push_back(0); + offsets.push_back(offsets.back() + 1); + } + + void insert_many_defaults(size_t length) override { Review Comment: same porblem as insert_default -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
