This is an automated email from the ASF dual-hosted git repository.
rui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b8bc9c07aa [GLUTEN-10392][VL] Fix filter fallback in scan-only
execution (#10505)
b8bc9c07aa is described below
commit b8bc9c07aab12894615ac4fb90f4bea9708c8502
Author: Rui Mo <[email protected]>
AuthorDate: Fri Aug 22 16:26:31 2025 +0100
[GLUTEN-10392][VL] Fix filter fallback in scan-only execution (#10505)
---
.../org/apache/gluten/execution/DeltaSuite.scala | 25 ++++++++++++++++++++++
.../extension/columnar/validator/Validators.scala | 3 ++-
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git
a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
b/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
index e5ee33f8d1..d1304d952f 100644
--- a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
+++ b/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
@@ -315,4 +315,29 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
}
+
+ test("delta: filter should be offloaded with scan") {
+ withSQLConf("spark.gluten.sql.columnar.scanOnly" -> "true") {
+ // TODO: When RAS is enabled, filter is not offloaded along with scan.
+ withSQLConf("spark.gluten.ras.enabled" -> "false") {
+ withTable("delta_pf") {
+ spark.sql(s"""
+ |create table test (id int, name string) using delta
+ |""".stripMargin)
+ spark.sql(s"""
+ |insert into test values (1, "v1"), (2, "v2"), (3,
"v1"), (4, "v2")
+ |""".stripMargin)
+ runQueryAndCompare(
+ "select id from test where name > 'v1'",
+ compareResult = true,
+ noFallBack = false) {
+ df =>
+ val plan = df.queryExecution.executedPlan
+ assert(plan.collect { case node: BasicScanExecTransformer =>
node }.nonEmpty)
+ assert(plan.collect { case node: FilterExecTransformerBase =>
node }.nonEmpty)
+ }
+ }
+ }
+ }
+ }
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
index 33f4927017..8456fd4a70 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
@@ -216,7 +216,8 @@ object Validators {
case p if HiveTableScanExecTransformer.isHiveTableScan(p) => pass()
case filter: FilterExec =>
val childIsScan = filter.child.isInstanceOf[FileSourceScanExec] ||
- filter.child.isInstanceOf[BatchScanExec]
+ filter.child.isInstanceOf[BatchScanExec] || filter.child
+ .isInstanceOf[BasicScanExecTransformer]
if (childIsScan) {
pass()
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
