This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/branch-1.5 by this push:
new 564df6f8cf [GLUTEN-10574][Branch-1.5] Back port important fixes to
branch-1.5 (#10576)
564df6f8cf is described below
commit 564df6f8cfdc838b88c649a5690bc3917ec96d06
Author: Yuan <[email protected]>
AuthorDate: Thu Aug 28 19:36:24 2025 +0100
[GLUTEN-10574][Branch-1.5] Back port important fixes to branch-1.5 (#10576)
* [VL][INFRA] Fix docker build error on Centos-7 (#10522)
* [TEST][VL] Reinclude "cast string to timestamp" test (#10532)
* [GLUTEN-10552][VL] Fix openEuler compiling issue (#10564)
This patch adds a extra gcc flag -Wno-restrict to workaround the openeuler
compiling issue
Velox did added this for gcc-12.2.1 however openeuler is using gcc-12.3.1
https://github.com/facebookincubator/velox/blob/main/CMakeLists.txt#L408
fixes: #10552
---------
Signed-off-by: Yuan <[email protected]>
---------
Signed-off-by: Yuan <[email protected]>
Co-authored-by: PHILO-HE <[email protected]>
---
dev/vcpkg/setup-build-depends.sh | 5 +--
ep/build-velox/src/get_velox.sh | 3 ++
.../apache/spark/util/DebuggableThreadUtils.scala | 37 ++++++++++++++++++++++
.../catalyst/expressions/GlutenAnsiCastSuite.scala | 8 +++--
.../sql/catalyst/expressions/GlutenCastSuite.scala | 8 +++--
.../catalyst/expressions/GlutenAnsiCastSuite.scala | 8 +++--
.../sql/catalyst/expressions/GlutenCastSuite.scala | 8 +++--
.../sql/catalyst/expressions/GlutenCastSuite.scala | 8 +++--
.../catalyst/expressions/GlutenTryCastSuite.scala | 8 +++--
.../sql/catalyst/expressions/GlutenCastSuite.scala | 8 +++--
.../catalyst/expressions/GlutenTryCastSuite.scala | 8 +++--
11 files changed, 83 insertions(+), 26 deletions(-)
diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh
index 12a8ee8180..6d4695fb00 100755
--- a/dev/vcpkg/setup-build-depends.sh
+++ b/dev/vcpkg/setup-build-depends.sh
@@ -98,9 +98,10 @@ install_centos_7() {
# Requires git >= 2.7.4
if [[ "$(git --version)" != "git version 2."* ]]; then
- [ -f /etc/yum.repos.d/ius.repo ] || yum -y install
https://repo.ius.io/ius-release-el7.rpm
yum -y remove git
- yum -y install git236
+ # Requires 'centos-release-scl' package to be installed.
+ yum -y install rh-git227
+ source /opt/rh/rh-git227/enable
fi
# flex>=2.6.0
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index ea15759f94..9acc85d458 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -189,6 +189,9 @@ function setup_linux {
;;
esac
elif [[ "$LINUX_DISTRIBUTION" == "openEuler" ]]; then
+ # this is workaround for gcc-12.3.1
+ #
https://github.com/facebookincubator/velox/blob/b263d9dd8b8910dc642d8fdb0c0adee4b2a1fb29/CMakeLists.txt#L433
+ sed -i "s|no-unknown-warning-option|no-unknown-warning-option
-Wno-restrict|g" ../../src/build_velox.sh
case "$LINUX_VERSION_ID" in
24.03) ;;
*)
diff --git
a/gluten-ut/common/src/test/scala/org/apache/spark/util/DebuggableThreadUtils.scala
b/gluten-ut/common/src/test/scala/org/apache/spark/util/DebuggableThreadUtils.scala
new file mode 100644
index 0000000000..294c5df291
--- /dev/null
+++
b/gluten-ut/common/src/test/scala/org/apache/spark/util/DebuggableThreadUtils.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.util
+
+import scala.util.{Failure, Success, Try}
+
+object DebuggableThreadUtils {
+
+ /** Logs message for failure occurring during the execution of
ThreadUtils.parmap. */
+ def parmap[I, O](in: Seq[I], prefix: String, maxThreads: Int)(f: I => O):
Seq[O] = {
+ ThreadUtils.parmap(in, prefix, maxThreads) {
+ i =>
+ Try(f(i)) match {
+ case Success(result) => result
+ case Failure(exception) =>
+ // scalastyle:off println
+ println(s"Test failed for case: ${i.toString}:
${exception.getMessage}")
+ // scalastyle:on println
+ throw exception
+ }
+ }
+ }
+}
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala
index 08c0808e03..687ff84aea 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala
@@ -20,7 +20,7 @@ import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{ALL_TIMEZONES,
UTC}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{DataType, StringType, TimestampType}
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.DebuggableThreadUtils
import java.sql.Timestamp
import java.time.LocalDateTime
@@ -113,13 +113,15 @@ class GlutenTryCastSuite extends TryCastSuite with
GlutenTestsTrait {
}
testGluten("cast string to timestamp") {
- ThreadUtils.parmap(
+ DebuggableThreadUtils.parmap(
ALL_TIMEZONES
.filterNot(_.getId.contains("SystemV"))
.filterNot(_.getId.contains("Europe/Kyiv"))
.filterNot(_.getId.contains("America/Ciudad_Juarez"))
.filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton")),
+ .filterNot(_.getId.contains("Pacific/Kanton"))
+ .filterNot(_.getId.contains("Asia/Tehran"))
+ .filterNot(_.getId.contains("Iran")),
prefix = "CastSuiteBase-cast-string-to-timestamp",
maxThreads = 1
) {
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index cd4351e708..1751252cc7 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -21,7 +21,7 @@ import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.DebuggableThreadUtils
import java.sql.{Date, Timestamp}
import java.util.{Calendar, TimeZone}
@@ -169,13 +169,15 @@ class GlutenCastSuite extends CastSuite with
GlutenTestsTrait {
}
testGluten("cast string to timestamp") {
- ThreadUtils.parmap(
+ DebuggableThreadUtils.parmap(
ALL_TIMEZONES
.filterNot(_.getId.contains("SystemV"))
.filterNot(_.getId.contains("Europe/Kyiv"))
.filterNot(_.getId.contains("America/Ciudad_Juarez"))
.filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton")),
+ .filterNot(_.getId.contains("Pacific/Kanton"))
+ .filterNot(_.getId.contains("Asia/Tehran"))
+ .filterNot(_.getId.contains("Iran")),
prefix = "CastSuiteBase-cast-string-to-timestamp",
maxThreads = 1
) {
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala
index 08c0808e03..687ff84aea 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala
@@ -20,7 +20,7 @@ import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{ALL_TIMEZONES,
UTC}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{DataType, StringType, TimestampType}
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.DebuggableThreadUtils
import java.sql.Timestamp
import java.time.LocalDateTime
@@ -113,13 +113,15 @@ class GlutenTryCastSuite extends TryCastSuite with
GlutenTestsTrait {
}
testGluten("cast string to timestamp") {
- ThreadUtils.parmap(
+ DebuggableThreadUtils.parmap(
ALL_TIMEZONES
.filterNot(_.getId.contains("SystemV"))
.filterNot(_.getId.contains("Europe/Kyiv"))
.filterNot(_.getId.contains("America/Ciudad_Juarez"))
.filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton")),
+ .filterNot(_.getId.contains("Pacific/Kanton"))
+ .filterNot(_.getId.contains("Asia/Tehran"))
+ .filterNot(_.getId.contains("Iran")),
prefix = "CastSuiteBase-cast-string-to-timestamp",
maxThreads = 1
) {
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 9be180930d..edfc182886 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -21,7 +21,7 @@ import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.DebuggableThreadUtils
import java.sql.{Date, Timestamp}
import java.util.{Calendar, TimeZone}
@@ -194,13 +194,15 @@ class GlutenCastSuite extends CastSuite with
GlutenTestsTrait {
}
testGluten("cast string to timestamp") {
- ThreadUtils.parmap(
+ DebuggableThreadUtils.parmap(
ALL_TIMEZONES
.filterNot(_.getId.contains("SystemV"))
.filterNot(_.getId.contains("Europe/Kyiv"))
.filterNot(_.getId.contains("America/Ciudad_Juarez"))
.filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton")),
+ .filterNot(_.getId.contains("Pacific/Kanton"))
+ .filterNot(_.getId.contains("Asia/Tehran"))
+ .filterNot(_.getId.contains("Iran")),
prefix = "CastSuiteBase-cast-string-to-timestamp",
maxThreads = 1
) {
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 64fa85d37f..0e1af4a4e2 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -21,7 +21,7 @@ import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.DebuggableThreadUtils
import java.sql.{Date, Timestamp}
import java.util.{Calendar, TimeZone}
@@ -166,13 +166,15 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with
GlutenTestsTrait {
}
testGluten("cast string to timestamp") {
- ThreadUtils.parmap(
+ DebuggableThreadUtils.parmap(
ALL_TIMEZONES
.filterNot(_.getId.contains("SystemV"))
.filterNot(_.getId.contains("Europe/Kyiv"))
.filterNot(_.getId.contains("America/Ciudad_Juarez"))
.filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton")),
+ .filterNot(_.getId.contains("Pacific/Kanton"))
+ .filterNot(_.getId.contains("Asia/Tehran"))
+ .filterNot(_.getId.contains("Iran")),
prefix = "CastSuiteBase-cast-string-to-timestamp",
maxThreads = 1
) {
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
index 1e1da8b0f1..3b2dcae63b 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
@@ -21,7 +21,7 @@ import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{BinaryType, ByteType, DateType, Decimal,
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType,
StringType, TimestampType}
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.DebuggableThreadUtils
import java.sql.{Date, Timestamp}
import java.util.{Calendar, TimeZone}
@@ -104,13 +104,15 @@ class GlutenTryCastSuite extends TryCastSuite with
GlutenTestsTrait {
}
testGluten("cast string to timestamp") {
- ThreadUtils.parmap(
+ DebuggableThreadUtils.parmap(
ALL_TIMEZONES
.filterNot(_.getId.contains("SystemV"))
.filterNot(_.getId.contains("Europe/Kyiv"))
.filterNot(_.getId.contains("America/Ciudad_Juarez"))
.filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton")),
+ .filterNot(_.getId.contains("Pacific/Kanton"))
+ .filterNot(_.getId.contains("Asia/Tehran"))
+ .filterNot(_.getId.contains("Iran")),
prefix = "CastSuiteBase-cast-string-to-timestamp",
maxThreads = 1
) {
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 7cf89937a6..ef06a98539 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -21,7 +21,7 @@ import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.DebuggableThreadUtils
import java.sql.{Date, Timestamp}
import java.util.{Calendar, TimeZone}
@@ -169,13 +169,15 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with
GlutenTestsTrait {
}
testGluten("cast string to timestamp") {
- ThreadUtils.parmap(
+ DebuggableThreadUtils.parmap(
ALL_TIMEZONES
.filterNot(_.getId.contains("SystemV"))
.filterNot(_.getId.contains("Europe/Kyiv"))
.filterNot(_.getId.contains("America/Ciudad_Juarez"))
.filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton")),
+ .filterNot(_.getId.contains("Pacific/Kanton"))
+ .filterNot(_.getId.contains("Asia/Tehran"))
+ .filterNot(_.getId.contains("Iran")),
prefix = "CastSuiteBase-cast-string-to-timestamp",
maxThreads = 1
) {
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
index 526624be56..063b646981 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
@@ -21,7 +21,7 @@ import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{BinaryType, ByteType, DateType, Decimal,
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType,
StringType, TimestampType}
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.DebuggableThreadUtils
import java.sql.{Date, Timestamp}
import java.util.{Calendar, TimeZone}
@@ -105,13 +105,15 @@ class GlutenTryCastSuite extends TryCastSuite with
GlutenTestsTrait {
}
testGluten("cast string to timestamp") {
- ThreadUtils.parmap(
+ DebuggableThreadUtils.parmap(
ALL_TIMEZONES
.filterNot(_.getId.contains("SystemV"))
.filterNot(_.getId.contains("Europe/Kyiv"))
.filterNot(_.getId.contains("America/Ciudad_Juarez"))
.filterNot(_.getId.contains("Antarctica/Vostok"))
- .filterNot(_.getId.contains("Pacific/Kanton")),
+ .filterNot(_.getId.contains("Pacific/Kanton"))
+ .filterNot(_.getId.contains("Asia/Tehran"))
+ .filterNot(_.getId.contains("Iran")),
prefix = "CastSuiteBase-cast-string-to-timestamp",
maxThreads = 1
) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
