Gabriel39 commented on code in PR #69672:
URL: https://github.com/apache/doris/pull/69672#discussion_r4400243399
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMappingUtils.java:
##########
@@ -150,12 +190,155 @@ public static EsFieldContext
resolveFieldContext(List<String> columnNames,
String fieldType = fieldObject.get("type").asText();
fieldContext.getColumn2typeMap().put(colName, fieldType);
resolveDateFields(fieldContext, fieldObject, colName, fieldType);
- resolveKeywordFields(fieldContext, fieldObject, colName,
fieldType);
resolveDocValuesFields(fieldContext, fieldObject, colName,
fieldType);
}
+ resolveSourceValueSafety(fieldContext, indexMapping, mappingType,
properties);
return fieldContext;
}
+ private static void resolveSourceValueSafety(EsFieldContext context,
String indexMapping, String mappingType,
+ ObjectNode schemaProperties) {
+ try {
+ // Mapping-induced source/index differences on any alias backing
index must prevent
+ // filtering or replacing source values through that path.
+ List<ObjectNode> roots = new ArrayList<>();
+ Set<String> columns = new
HashSet<>(context.getColumn2typeMap().keySet());
+ for (JsonNode index : MAPPER.readTree(indexMapping)) {
+ ObjectNode root = getRootSchema((ObjectNode)
index.get("mappings"), mappingType,
+ new ArrayList<>(), true);
+ roots.add(root);
+ }
+ // Use the same populated schema as inference, including
unprojected filter columns.
+ schemaProperties.fieldNames().forEachRemaining(columns::add);
+ // Filters can reference columns absent from the projection.
Resolve their query paths
+ // from the exposed schema too; the projection still controls
docvalue read eligibility.
+ for (String column : columns) {
+ ObjectNode field = (ObjectNode) schemaProperties.get(column);
+ resolveKeywordFields(context, field, column,
field.path("type").asText());
+ }
+ Set<String> unsafeDocValues = new HashSet<>();
+ for (ObjectNode root : roots) {
+ JsonNode properties = root.path("properties");
+ if (isSourceDisabled(root.path("_source"))) {
+ // A source projection fails on disabled backings; full
source preserves the
+ // enabled backings of a mixed alias while disabled hits
remain NULL.
+ context.requireFullSourceRead();
+ }
+ Set<String> copyTargets = new HashSet<>();
+ collectCopyTargets(properties, copyTargets);
+ for (String column : columns) {
+ JsonNode field = properties.path(column);
+ JsonNode queryField = fieldAtPath(field, column,
context.getFetchFieldsContext().get(column));
+ JsonNode readField = fieldAtPath(field, column,
context.getDocValueFieldsContext().get(column));
+ // Disabled root parsing leaves source fields but produces
no index/docvalues,
+ // even if an explicit properties section still describes
those fields.
+ boolean sourceChanged =
!root.path("enabled").asBoolean(true)
+ || filtersSourceColumn(root.path("_source"),
column)
+ || root.path("runtime").has(column) ||
copyTargets.contains(column);
+ // ES mappings do not distinguish arrays from scalars.
Even a short equality can
+ // match one indexed array element while Doris reads the
entire source JSON string.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(queryField)
+ || "text".equals(queryField.path("type").asText())
|| !isSearchable(queryField)) {
+ context.getSourceValueUnsafeFields().add(column);
+ }
+ // Source can omit generated/aliased values, retain
unrounded numbers or decode
+ // default dates in local time. Docvalues remain eligible
for these mappings.
+ if (requiresSourcePredicateEvaluation(field) ||
requiresSourcePredicateEvaluation(queryField)) {
+ context.getSourceModeUnsafeFields().add(column);
+ }
+ // Query and docvalue paths may select different
multi-fields. An unused sibling's
+ // normalizer must not invalidate either path; only the
chosen read value matters.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(readField)
+ || !readField.path("doc_values").asBoolean(true)
+ ||
DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(readField.path("type").asText())) {
+ // One backing without this docvalue path would return
missing values for its hits.
+ unsafeDocValues.add(column);
+ }
+ }
+ }
+
unsafeDocValues.forEach(context.getDocValueFieldsContext()::remove);
Review Comment:
Fixed in bbe765a93d950f19fbd7a73b494ab2917b52783b.
Scalar source arrays are normalized before dispatch to the per-type decoder.
A singleton reads its element; empty and null-only scalar arrays read SQL NULL.
This covers floating-point, Boolean, integer and date siblings through both
readers. STRING retains its JSON representation and declared ARRAY retains its
container; multi-value scalar source arrays are not arbitrarily unwrapped. The
live fixture covers FLOAT/DOUBLE singleton/empty/null/missing values,
corresponding Boolean/date cases, declared ARRAY controls and pure-docvalue
projections across both readers and docvalue/parallel modes.
Validation: 159 FE tests passed (one optional ES9 class skipped), all 17 BE
unit tests passed, and five live ES8 suites passed 1,612 stored checks and
1,516 independent comparisons. All 1,452 previous stored results are unchanged.
The 160 new outputs were generated and independently rerun without generation.
Standard ASAN BE build, FE Checkstyle, clang-format 16, scoped clang-tidy and
build hygiene passed.
##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -156,36 +145,117 @@ static const std::string INVALID_NULL_VALUE =
return Status::RuntimeError(ss.str()); \
} while (false)
+// ES integer coercion truncates decimal/scientific notation, e.g. "1.2e3" ->
1200.
+// Shift the decimal point using digits so values near INT64_MAX never pass
through double.
template <typename T>
-Status get_int_value(const rapidjson::Value& col, PrimitiveType type, void*
slot,
- bool pure_doc_value) {
- if (col.IsNumber()) {
- *reinterpret_cast<T*>(slot) = (T)(sizeof(T) < 8 ? col.GetInt() :
col.GetInt64());
- return Status::OK();
+static T parse_es_integer(const char* text, size_t length,
StringParser::ParseResult* result) {
+ text = skip_ascii_whitespaces(text, length);
+ size_t i = 0;
+ bool negative = length > 0 && text[0] == '-';
+ if (length > 0 && (text[0] == '-' || text[0] == '+')) {
+ ++i;
+ }
+ size_t start = i;
+ int64_t digits = 0;
+ int64_t integer_digits = 0;
+ int64_t first_nonzero = -1;
+ bool dot = false;
+ for (; i < length; ++i) {
+ char c = text[i];
+ if (c >= '0' && c <= '9') {
+ if (c != '0' && first_nonzero < 0) {
+ first_nonzero = digits;
+ }
+ ++digits;
+ integer_digits += !dot;
+ } else if (c == '.' && !dot) {
+ dot = true;
+ } else {
+ break;
+ }
}
-
- if (pure_doc_value && col.IsArray() && !col.Empty()) {
- RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col[0], type);
- *reinterpret_cast<T*>(slot) = (T)(sizeof(T) < 8 ? col[0].GetInt() :
col[0].GetInt64());
- return Status::OK();
+ size_t end = i;
+ int64_t exponent = 0;
+ bool negative_exponent = false;
+ if (i < length && (text[i] == 'e' || text[i] == 'E')) {
+ ++i;
+ if (i < length && (text[i] == '+' || text[i] == '-')) {
+ negative_exponent = text[i++] == '-';
+ }
+ size_t exponent_start = i;
+ for (; i < length && text[i] >= '0' && text[i] <= '9'; ++i) {
+ // Saturation avoids overflow and work proportional to an
untrusted exponent.
+ exponent = std::min(exponent * 10 + text[i] - '0',
static_cast<int64_t>(length) + 64);
+ }
+ if (i == exponent_start) {
+ *result = StringParser::PARSE_FAILURE;
+ return 0;
+ }
}
+ if (digits == 0 || i != length) {
+ *result = StringParser::PARSE_FAILURE;
+ return 0;
+ }
+ int64_t count = integer_digits + (negative_exponent ? -exponent :
exponent) - first_nonzero;
Review Comment:
Fixed in bbe765a93d950f19fbd7a73b494ab2917b52783b.
Narrow integer strings now use double rounding before truncation separately
from long's exact decimal path. Byte follows ES intValue then byte-range
checking; short/integer check their double bounds before narrowing. NaN-to-zero
is handled explicitly and Infinity/overflow cannot reach an undefined C++ cast.
JSON numeric values retain their existing coercion path. The new tests verify
positive/negative 0.99999999999999999 boundaries, byte fractional bounds, a
numeric short boundary, NaN, declared byte ARRAY elements, and 64-bit precision
controls. The previous head returned source zero for the pushed i=1 query on
both readers; the updated source and pure-docvalue results agree.
Validation: 159 FE tests passed (one optional ES9 class skipped), all 17 BE
unit tests passed, and five live ES8 suites passed 1,612 stored checks and
1,516 independent comparisons. All 1,452 previous stored results are unchanged.
The 160 new outputs were generated and independently rerun without generation.
Standard ASAN BE build, FE Checkstyle, clang-format 16, scoped clang-tidy and
build hygiene passed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
