Gabriel39 commented on code in PR #69672:
URL: https://github.com/apache/doris/pull/69672#discussion_r4409481581
##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -723,15 +801,20 @@ Status ScrollParser::fill_columns(const
std::vector<SlotDescriptor*>& slots,
}
}
- const rapidjson::Value& col = (*line)[col_name];
-
auto type = slot_desc->type()->get_primitive_type();
-
- // when the column value is null, the subsequent type casting will
report an error
- if (col.IsNull() && slot_desc->is_nullable()) {
+ const rapidjson::Value& source_value = (*line)[col_name];
+ // Source fallback changes every sibling's decoder. A scalar
numeric/Boolean/date field
+ // may hold [value], [] or [null]; preserve its single value or
missing-value semantics.
+ // STRING keeps its JSON representation, and declared ARRAY keeps its
container.
+ bool scalar_source_array = !pure_doc_value && source_value.IsArray() &&
+ type != TYPE_ARRAY && !is_string_type(type);
+ const rapidjson::Value& col =
+ scalar_source_array && source_value.Size() == 1 ?
source_value[0] : source_value;
+ bool is_null = col.IsNull() || (scalar_source_array &&
source_value.Empty());
Review Comment:
Fixed in c11be7b2769ebf43424deade2b5df79c8a1a30f8. Empty numeric source
strings now materialize as SQL NULL, including scalar singleton arrays and
declared numeric ARRAY members; Boolean empty strings retain false semantics.
Added integer/FLOAT/DOUBLE parser controls plus live sibling-fallback and
single-column controls across both readers and docvalue/parallel settings.
##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -156,36 +146,153 @@ static const std::string INVALID_NULL_VALUE =
return Status::RuntimeError(ss.str()); \
} while (false)
+// ES integer coercion truncates decimal/scientific notation, e.g. "1.2e3" ->
1200.
+// Shift the decimal point using digits so values near INT64_MAX never pass
through double.
template <typename T>
-Status get_int_value(const rapidjson::Value& col, PrimitiveType type, void*
slot,
- bool pure_doc_value) {
- if (col.IsNumber()) {
- *reinterpret_cast<T*>(slot) = (T)(sizeof(T) < 8 ? col.GetInt() :
col.GetInt64());
- return Status::OK();
+static T parse_es_integer(const char* text, size_t length,
StringParser::ParseResult* result) {
+ text = skip_ascii_whitespaces(text, length);
+ size_t i = 0;
+ bool negative = length > 0 && text[0] == '-';
+ if (length > 0 && (text[0] == '-' || text[0] == '+')) {
+ ++i;
+ }
+ size_t start = i;
+ int64_t digits = 0;
+ int64_t integer_digits = 0;
+ int64_t first_nonzero = -1;
+ bool dot = false;
+ for (; i < length; ++i) {
+ char c = text[i];
+ if (c >= '0' && c <= '9') {
+ if (c != '0' && first_nonzero < 0) {
+ first_nonzero = digits;
+ }
+ ++digits;
+ integer_digits += !dot;
+ } else if (c == '.' && !dot) {
+ dot = true;
+ } else {
+ break;
+ }
}
-
- if (pure_doc_value && col.IsArray() && !col.Empty()) {
- RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col[0], type);
- *reinterpret_cast<T*>(slot) = (T)(sizeof(T) < 8 ? col[0].GetInt() :
col[0].GetInt64());
- return Status::OK();
+ size_t end = i;
+ int64_t exponent = 0;
+ bool negative_exponent = false;
+ if (i < length && (text[i] == 'e' || text[i] == 'E')) {
+ ++i;
+ if (i < length && (text[i] == '+' || text[i] == '-')) {
+ negative_exponent = text[i++] == '-';
+ }
+ size_t exponent_start = i;
+ for (; i < length && text[i] >= '0' && text[i] <= '9'; ++i) {
+ // Saturation avoids overflow and work proportional to an
untrusted exponent.
+ exponent = std::min(exponent * 10 + text[i] - '0',
static_cast<int64_t>(length) + 64);
+ }
+ if (i == exponent_start) {
+ *result = StringParser::PARSE_FAILURE;
+ return 0;
+ }
}
+ if (digits == 0 || i != length) {
+ *result = StringParser::PARSE_FAILURE;
+ return 0;
+ }
+ int64_t count = integer_digits + (negative_exponent ? -exponent :
exponent) - first_nonzero;
+ if (first_nonzero < 0 || count <= 0) {
+ *result = StringParser::PARSE_SUCCESS;
+ return 0;
+ }
+ // 39 digits cover every signed 128-bit value; the final parser checks the
exact type bounds.
+ if (count > 39) {
+ *result = StringParser::PARSE_OVERFLOW;
+ return 0;
+ }
+ char integral[40];
+ size_t size = 0;
+ if (negative) {
+ integral[size++] = '-';
+ }
+ int64_t digit = 0;
+ for (i = start; i < end && count > 0; ++i) {
+ if (text[i] != '.' && digit++ >= first_nonzero) {
+ integral[size++] = text[i];
+ --count;
+ }
+ }
+ while (count-- > 0) {
+ integral[size++] = '0';
+ }
+ return StringParser::string_to_int<T, true>(integral, size, result);
+}
- RETURN_ERROR_IF_COL_IS_ARRAY(col, type, true);
- RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type);
-
- StringParser::ParseResult result;
- const std::string& val = col.GetString();
- size_t len = col.GetStringLength();
- T v = StringParser::string_to_int<T>(val.c_str(), len, &result);
- RETURN_ERROR_IF_PARSING_FAILED(result, col, type);
+template <typename T>
+static T parse_es_narrow_integer(const std::string& text,
StringParser::ParseResult* result) {
+ // Unlike long's BigDecimal conversion, ES parses byte/short/integer
strings through double:
+ // "0.99999999999999999" rounds to 1 before truncation. Keep long's exact
path separate.
+ auto number = StringParser::string_to_float<double>(text.data(),
text.size(), result);
Review Comment:
Fixed in c11be7b2769ebf43424deade2b5df79c8a1a30f8. Shared floating-string
parsing accepts Java hexadecimal exponents, f/F/d/D suffixes and outer ASCII
trim characters while rejecting incomplete hex tokens, junk and whitespace
before a suffix. Narrow integers retain double rounding and checked bounds;
FLOAT strings parse directly to float to avoid double rounding. Added red/green
grammar and exact float-midpoint unit controls and live ES comparisons.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
