LinSimon-901101 commented on code in PR #7155:
URL: https://github.com/apache/datafusion-comet/pull/7155#discussion_r4373483604
##########
spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala:
##########
@@ -1176,11 +1174,46 @@ case class CometExecRule(session: SparkSession,
queryStagePrep: Boolean = false)
serde.convert(op, builder)
}
}
- return nativeOp.map { nativeOp =>
- val exec = serde.createExec(nativeOp, op)
- rollUpInfoMessages(op, exec)
- exec
- }
+ return nativeOp
+ .filter { proto =>
+ // A serde may emit several native nodes for one Spark operator.
Inspect those nodes,
+ // but stop at real Spark children, whose consumption context was
tagged independently.
+ def containsOwnAssertion(node: Operator): Boolean =
+
CometAssertNotNull.containsNativeAssertion(node.toBuilder.clearChildren().build())
||
+ node.getChildrenList.asScala.exists { child =>
+ child.getPlanId == proto.getPlanId &&
containsOwnAssertion(child)
+ }
+
+ def hasNativeAssertion: Boolean = op match {
+ // TopK's sink proto omits its runtime projection over the
selected rows.
+ case topK: TakeOrderedAndProjectExec =>
+ QueryPlanSerde.withSharedSubexpressions(topK.projectList) {
+ topK.projectList.exists { expression =>
+ QueryPlanSerde
+ .exprToProto(expression, topK.child.output)
+ .exists(CometAssertNotNull.containsNativeAssertion)
+ }
+ }
+ case _: HashAggregateExec | _: ObjectHashAggregateExec =>
+ // Input/grouping expressions consume all rows; only the result
projection is lazy.
+ proto.hasProjection &&
CometAssertNotNull.containsNativeAssertion(
+ proto.getProjection)
+ case _ => containsOwnAssertion(proto)
+ }
+ op.getTagValue(CometExecRule.PARTIAL_READER) match {
Review Comment:
Fixed in `b5af037`, included in the current head `6e850d7d9`.
In that head, `CometExecRule.scala:1366–1391` keeps semi/anti joins with
native `AssertNotNull` conditions on Spark, regardless of `PARTIAL_READER`, so
assertions on skipped candidates are not evaluated.
Regression tests are in `CometJoinSuite.scala:2083–2139`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
