sunchao commented on code in PR #6170:
URL: https://github.com/apache/datafusion-comet/pull/6170#discussion_r4151914759
##########
spark/src/test/scala/org/apache/spark/sql/benchmark/CometArrayExpressionBenchmark.scala:
##########
@@ -146,7 +149,66 @@ object CometArrayExpressionBenchmark extends
CometBenchmarkBase {
}
}
+ def arrayExtremaCollationBenchmark(values: Int, expectNative: Boolean =
true): Unit = {
+ // Repeated values exercise dictionaries; long prefixes and Unicode
exercise comparison work.
+ val shapes = Seq(
+ ("short ASCII, no nulls", "AbC", 1, 0),
+ ("long ASCII, sparse nulls", "AbC", 32, 8),
+ ("Unicode, dense nulls", "ΣİÄ", 4, 50))
+ for ((shape, prefix, repeats, nullPercent) <- shapes) {
+ withTempPath { dir =>
+ withTempTable("parquetV1Table") {
+ val columns = (0 until 8).map { i =>
+ val text = if (i % 2 == 0) prefix else
prefix.toLowerCase(java.util.Locale.ROOT)
+ s"""CASE WHEN PMOD(value + ${i * 13}, 100) < $nullPercent THEN NULL
+ |ELSE CONCAT(REPEAT('$text', $repeats),
+ | CAST(PMOD(value + ${i / 2}, 64) AS STRING), '${" " * (i %
3)}')
+ |END AS c$i""".stripMargin
+ }
+ prepareTable(dir, spark.sql(s"SELECT ${columns.mkString(", ")} FROM
$tbl"))
+ for (collation <- Seq(
+ "UTF8_BINARY",
+ "UTF8_BINARY_RTRIM",
+ "UTF8_LCASE",
+ "UTF8_LCASE_RTRIM");
+ function <- Seq("array_min", "array_max")) {
+ val input = (0 until 8)
+ .map(i => s"CAST(c$i AS STRING COLLATE $collation)")
+ .mkString("array(", ", ", ")")
+ val query = s"SELECT $function($input) FROM parquetV1Table"
+ // The dispatcher baseline uses the same inputs and verifies its
execution path too.
+ withSQLConf(
+ CometConf.COMET_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_ENABLED.key -> "true") {
+ val plan =
stripAQEPlan(spark.sql(query).queryExecution.executedPlan)
+ val explain = new ExtendedExplainInfo()
+ if (expectNative || collation == "UTF8_BINARY") {
+ require(
+ explain.getNativeExpressions(plan).contains(function) &&
+
!explain.getCodegenDispatchExpressions(plan).contains(function),
+ s"$function did not run natively: $plan")
+ } else {
+ require(
+
explain.getCodegenDispatchExpressions(plan).contains(function),
+ s"$function did not use the dispatcher: $plan")
+ }
+ }
+ runExpressionBenchmark(s"$function $collation - $shape", values,
query)
Review Comment:
[P2] Make the cardinality conversion explicit here. `values` is an `Int`,
while `runExpressionBenchmark` takes a `Long`. Running `./mvnw -B test-compile
-Pspark-3.5 -Pstrict-warnings -DskipTests` therefore fails with `implicit
numeric widening`, because this profile treats that warning as an error. This
prevents the required compilation check from passing even though the benchmark
is not executed on Spark 3.5. Pass `values.toLong` at this call.
Evidence: Exact-head CI job
https://github.com/apache/datafusion-comet/actions/runs/36747428085/job/110016920898
ran the stated command and reported `CometArrayExpressionBenchmark.scala:196:
implicit numeric widening`, followed by `BUILD FAILURE`. Its tested merge tree
is identical to the requested head. The receiving parameter is declared
`cardinality: Long` in `CometBenchmarkBase.scala:137`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
