This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 87b58aa5af [GLUTEN-10356][VL] Support basic arithmetic expressions
with ANSI mode (#10357)
87b58aa5af is described below
commit 87b58aa5af6bff8d1efecffda0ae9a5e8a2676ba
Author: nimesh1601 <[email protected]>
AuthorDate: Mon Aug 18 16:18:16 2025 +0530
[GLUTEN-10356][VL] Support basic arithmetic expressions with ANSI mode
(#10357)
---
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 23 +++----
.../functions/ArithmeticAnsiValidateSuite.scala | 75 ++++++++++++++++++++++
docs/Configuration.md | 1 +
.../org/apache/gluten/config/GlutenConfig.scala | 10 +++
.../gluten/extension/columnar/FallbackRules.scala | 2 +-
5 files changed, 99 insertions(+), 12 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index 2bad45ee2e..18e26f8a25 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -152,19 +152,20 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
ExpressionMappings.expressionsMap(classOf[TryEval]),
Seq(GenericExpressionTransformer(checkArithmeticExprName, Seq(left,
right), original)),
original)
+ } else if (
+ left.dataType.isInstanceOf[DecimalType] &&
+ right.dataType.isInstanceOf[DecimalType] &&
+ !SQLConf.get.decimalOperationsAllowPrecisionLoss
+ ) {
+ if (SparkShimLoader.getSparkShims.withAnsiEvalMode(original)) {
+ throw new GlutenNotSupportException(s"$substraitExprName with ansi
mode is not supported")
+ }
+ val newName = substraitExprName + "_deny_precision_loss"
+ GenericExpressionTransformer(newName, Seq(left, right), original)
} else if (SparkShimLoader.getSparkShims.withAnsiEvalMode(original)) {
- throw new GlutenNotSupportException(s"$substraitExprName with ansi mode
is not supported")
+ GenericExpressionTransformer(checkArithmeticExprName, Seq(left, right),
original)
} else {
- if (
- left.dataType.isInstanceOf[DecimalType] && right.dataType
- .isInstanceOf[DecimalType] &&
!SQLConf.get.decimalOperationsAllowPrecisionLoss
- ) {
- // https://github.com/facebookincubator/velox/pull/10383
- val newName = substraitExprName + "_deny_precision_loss"
- GenericExpressionTransformer(newName, Seq(left, right), original)
- } else {
- GenericExpressionTransformer(substraitExprName, Seq(left, right),
original)
- }
+ GenericExpressionTransformer(substraitExprName, Seq(left, right),
original)
}
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala
new file mode 100644
index 0000000000..b3ee4d4321
--- /dev/null
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.functions
+
+import org.apache.gluten.config.GlutenConfig
+import org.apache.gluten.execution.ProjectExecTransformer
+
+import org.apache.spark.SparkConf
+import org.apache.spark.SparkException
+import org.apache.spark.sql.internal.SQLConf
+
+class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite {
+
+ disableFallbackCheck
+
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key, "false")
+ .set(SQLConf.ANSI_ENABLED.key, "true")
+ }
+
+ test("add") {
+ runQueryAndCompare("SELECT int_field1 + 100 FROM datatab WHERE int_field1
IS NOT NULL") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ intercept[ArithmeticException] {
+ sql("SELECT 2147483647 + 1").collect()
+ }
+ }
+
+ test("subtract") {
+ runQueryAndCompare("SELECT int_field1 - 50 FROM datatab WHERE int_field1
IS NOT NULL") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+
+ test("multiply") {
+ runQueryAndCompare("SELECT int_field1 * 2 FROM datatab WHERE int_field1 IS
NOT NULL") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ intercept[ArithmeticException] {
+ sql("SELECT 2147483647 * 2").collect()
+ }
+ }
+
+ test("divide") {
+ runQueryAndCompare("SELECT int_field1 / 2 FROM datatab WHERE int_field1 IS
NOT NULL") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ if (isSparkVersionGE("3.4")) {
+ // Spark 3.4+ throws exception for division by zero in ANSI mode
+ intercept[SparkException] {
+ sql("SELECT 1 / 0").collect()
+ }
+ } else {
+ // Spark 3.2 and 3.3 don't throw exception for division by zero in ANSI
mode
+ sql("SELECT 1 / 0").collect()
+ }
+ }
+
+}
diff --git a/docs/Configuration.md b/docs/Configuration.md
index 995bdf3886..4b1013f8e2 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -139,6 +139,7 @@ nav_order: 15
| spark.gluten.sql.debug.cudf | false
|
| spark.gluten.sql.debug.keepJniWorkspace | false
|
| spark.gluten.sql.debug.keepJniWorkspaceDir | /tmp
|
+| spark.gluten.sql.ansiFallback.enabled | true
| When true (default), Gluten will fallback to Spark when ANSI mode
is enabled. When false, Gluten will attempt to execute in ANSI mode.
[...]
| spark.gluten.sql.enable.native.validation | true
| This is tmp config to specify whether to enable the native
validation based on Substrait plan. After the validations in all backends are
correctly implemented, this config should be removed.
[...]
| spark.gluten.sql.extendedColumnPruning.enabled | true
| Do extended nested column pruning for cases ignored by vanilla
Spark.
[...]
| spark.gluten.sql.fallbackEncryptedParquet | false
| If enabled, gluten will not offload scan when encrypted parquet
files are detected
[...]
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 4dbdf11a52..c5bd12259b 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -59,6 +59,8 @@ class GlutenConfig(conf: SQLConf) extends
GlutenCoreConfig(conf) {
def enableAnsiMode: Boolean = conf.ansiEnabled
+ def enableAnsiFallback: Boolean = getConf(GLUTEN_ANSI_FALLBACK_ENABLED)
+
def glutenUiEnabled: Boolean = getConf(GLUTEN_UI_ENABLED)
// FIXME the option currently controls both JVM and native validation
against a Substrait plan.
@@ -786,6 +788,14 @@ object GlutenConfig {
.booleanConf
.createWithDefault(true)
+ val GLUTEN_ANSI_FALLBACK_ENABLED =
+ buildConf("spark.gluten.sql.ansiFallback.enabled")
+ .doc(
+ "When true (default), Gluten will fall back to Spark when ANSI mode is
enabled. " +
+ "When false, Gluten will attempt to execute in ANSI mode.")
+ .booleanConf
+ .createWithDefault(true)
+
val COLUMNAR_BATCHSCAN_ENABLED =
buildConf("spark.gluten.sql.columnar.batchscan")
.doc("Enable or disable columnar batchscan.")
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/FallbackRules.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/FallbackRules.scala
index 8c8bcaf128..926708ee33 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/FallbackRules.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/FallbackRules.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.joins._
case class FallbackOnANSIMode(session: SparkSession) extends Rule[SparkPlan] {
override def apply(plan: SparkPlan): SparkPlan = {
- if (GlutenConfig.get.enableAnsiMode) {
+ if (GlutenConfig.get.enableAnsiMode &&
GlutenConfig.get.enableAnsiFallback) {
plan.foreach(FallbackTags.add(_, "does not support ansi mode"))
}
plan
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
