Gabriel39 commented on code in PR #69672:
URL: https://github.com/apache/doris/pull/69672#discussion_r4386557985
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsQueryDslBuilder.java:
##########
@@ -143,18 +168,85 @@ public static EsQueryDslResult
buildQueryDslWithResult(ConnectorExpression filte
return new EsQueryDslResult(queryDsl, notPushedIndices);
}
+ // Dropping a source-value predicate after translating a
transformed/missing index value
+ // cannot be repaired by a residual: matching source documents may
already be absent.
+ if (referencesUnsafeSourceField(filter, sourceValueUnsafeFields,
keywordIgnoreAboveLimits)) {
+ requireLocalEvaluation(filter);
+ return new EsQueryDslResult(matchAllQuery(),
Collections.singletonList(0));
+ }
// Single expression (not AND)
List<ConnectorExpression> notPushDownList = new ArrayList<>();
String dsl = toEsDsl(filter, notPushDownList, fieldsContext,
likePushDown, needCompatDateFields, column2typeMap);
// A nested AND can return a partial DSL while recording an
unsupported child. OR/NOT must
// remain local in that case: e.g. NOT(a = 1 AND local_predicate)
cannot become NOT(a = 1).
if (dsl == null || !notPushDownList.isEmpty()) {
+ requireLocalEvaluation(filter);
return new EsQueryDslResult(matchAllQuery(),
Collections.singletonList(0));
}
return new EsQueryDslResult(dsl, Collections.emptyList());
}
+ private static boolean referencesUnsafeSourceField(ConnectorExpression
expression, Set<String> unsafeFields,
+ Map<String, Integer> keywordIgnoreAboveLimits) {
+ if (isExactKeywordPredicate(expression, keywordIgnoreAboveLimits)) {
+ return false;
+ }
+ if (expression instanceof ConnectorFunctionCall
+ && "esquery".equals(((ConnectorFunctionCall)
expression).getFunctionName())) {
+ // esquery explicitly requests ES index semantics rather than SQL
source-value semantics.
+ return false;
+ }
+ if (expression instanceof ConnectorColumnRef) {
+ return unsafeFields.contains(((ConnectorColumnRef)
expression).getColumnName());
+ }
+ for (ConnectorExpression child : expression.getChildren()) {
+ if (referencesUnsafeSourceField(child, unsafeFields,
keywordIgnoreAboveLimits)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isExactKeywordPredicate(ConnectorExpression
expression, Map<String, Integer> limits) {
+ if (expression instanceof ConnectorComparison) {
+ ConnectorComparison comparison = (ConnectorComparison) expression;
+ if (comparison.getOperator() != ConnectorComparison.Operator.EQ
+ && comparison.getOperator() !=
ConnectorComparison.Operator.EQ_FOR_NULL) {
+ return false;
+ }
+ return isBoundedKeywordLiteral(comparison.getLeft(),
comparison.getRight(), limits)
+ || isBoundedKeywordLiteral(comparison.getRight(),
comparison.getLeft(), limits);
+ }
+ if (expression instanceof ConnectorIn) {
+ ConnectorIn in = (ConnectorIn) expression;
+ return !in.isNegated() && in.getInList().stream()
+ .allMatch(value -> isBoundedKeywordLiteral(in.getValue(),
value, limits));
+ }
+ return false;
+ }
+
+ private static boolean isBoundedKeywordLiteral(ConnectorExpression column,
ConnectorExpression value,
+ Map<String, Integer> limits) {
+ if (!(column instanceof ConnectorColumnRef) || !(value instanceof
ConnectorLiteral)) {
+ return false;
+ }
+ Integer limit = limits.get(((ConnectorColumnRef)
column).getColumnName());
+ Object literal = ((ConnectorLiteral) value).getValue();
+ return limit != null && literal instanceof String && ((String)
literal).length() <= limit;
Review Comment:
Fixed in cff8803bf3. Removed the bounded-literal equality/IN exemption and
its metadata entirely: neither a keyword mapping nor a short literal
establishes scalar source values. Affected SQL predicates stay local without a
narrowing ES query. Native-only calls are never left in an OR/NOT residual;
unsupported mixtures fail during planning, while independent AND conjuncts
remain usable. Live ES8 tests cover multi-value, singleton, empty and
null-element source arrays plus scalar controls across both readers and
docvalue/parallel modes. Ordinary STRING equality returns only the scalar cat
row; standalone esquery deliberately matches array elements and is checked
against native ES. The pre-update connector reproduced the false positives on
both readers.
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMappingUtils.java:
##########
@@ -153,9 +162,101 @@ public static EsFieldContext
resolveFieldContext(List<String> columnNames,
resolveKeywordFields(fieldContext, fieldObject, colName,
fieldType);
resolveDocValuesFields(fieldContext, fieldObject, colName,
fieldType);
}
+ resolveSourceValueSafety(fieldContext, indexMapping, mappingType);
return fieldContext;
}
+ private static void resolveSourceValueSafety(EsFieldContext context,
String indexMapping, String mappingType) {
+ try {
+ // An alias can span mappings. Every backing index must preserve
source semantics before
+ // a predicate may remove rows or doc_values may replace the value
returned to Doris.
+ List<ObjectNode> roots = new ArrayList<>();
+ Set<String> columns = new
HashSet<>(context.getColumn2typeMap().keySet());
+ for (JsonNode index : MAPPER.readTree(indexMapping)) {
+ ObjectNode root = getRootSchema((ObjectNode)
index.get("mappings"), mappingType,
+ new ArrayList<>(), true);
+ roots.add(root);
+ }
+ // Only the first mapping's schema is exposed by Doris, including
unprojected filter columns.
+
roots.get(0).path("properties").fieldNames().forEachRemaining(columns::add);
+ Set<String> inexactKeywords = new HashSet<>();
+ for (ObjectNode root : roots) {
+ JsonNode properties = root.path("properties");
+ JsonNode source = root.path("_source");
+ boolean filteredSource =
!source.path("enabled").asBoolean(true)
+ || source.has("includes") || source.has("excludes");
Review Comment:
Fixed in cff8803bf3. Source filtering is evaluated per column using the
include/exclude '*' patterns, including empty filter arrays, instead of
tainting every field. Preserved columns retain their existing query and
docvalue paths; affected columns still use the fallback. Added positive and
negative FE cases and live native-OR regressions on preserved fields in both
include-filtered and exclude-filtered indices.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
