This is an automated email from the ASF dual-hosted git repository.
jmalkin pushed a commit to branch simplify_registration
in repository https://gitbox.apache.org/repos/asf/datasketches-spark.git
The following commit(s) were added to refs/heads/simplify_registration by this
push:
new c689e44 Clean up codegen errors a bit -- still one for pmf/cdf but
fallback works ok
c689e44 is described below
commit c689e44242b87f7573d4d5714dfef754a243f1a7
Author: Jon Malkin <[email protected]>
AuthorDate: Tue Jan 7 14:50:04 2025 -0800
Clean up codegen errors a bit -- still one for pmf/cdf but fallback works ok
---
src/main/scala/org/apache/spark/sql/expressions/KllExpressions.scala | 5 +----
src/main/scala/org/apache/spark/sql/types/KllDoublesSketchType.scala | 4 ++--
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git
a/src/main/scala/org/apache/spark/sql/expressions/KllExpressions.scala
b/src/main/scala/org/apache/spark/sql/expressions/KllExpressions.scala
index 34b6296..9eec024 100644
--- a/src/main/scala/org/apache/spark/sql/expressions/KllExpressions.scala
+++ b/src/main/scala/org/apache/spark/sql/expressions/KllExpressions.scala
@@ -256,15 +256,12 @@ case class KllGetPmfCdf(first: Expression,
s"""
|${sketchEval.code}
|${splitPointsEval.code}
- |if (${sketchEval.isNull} || ${splitPointsEval.isNull}) {
- | boolean ${ev.isNull} = true;
- |} else {
+ |if (!${sketchEval.isNull} && !${splitPointsEval.isNull}) {
| org.apache.datasketches.quantilescommon.QuantileSearchCriteria
searchCriteria = ${if (isInclusive)
"org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE" else
"org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE"};
| final org.apache.datasketches.kll.KllDoublesSketch $sketch =
org.apache.spark.sql.types.KllDoublesSketchType.wrap(${sketchEval.value});
| final double[] splitPoints =
((org.apache.spark.sql.catalyst.util.GenericArrayData)${splitPointsEval.value}).toDoubleArray();
| final double[] result = ${if (isPmf) s"$sketch.getPMF(splitPoints,
searchCriteria)" else s"$sketch.getCDF(splitPoints, searchCriteria)"};
| org.apache.spark.sql.catalyst.util.GenericArrayData ${ev.value} =
new org.apache.spark.sql.catalyst.util.GenericArrayData(result);
- | boolean ${ev.isNull} = false;
|}
""".stripMargin
ev.copy(code = CodeBlock(Seq(code), Seq.empty))
diff --git
a/src/main/scala/org/apache/spark/sql/types/KllDoublesSketchType.scala
b/src/main/scala/org/apache/spark/sql/types/KllDoublesSketchType.scala
index fda471d..2fb01e9 100644
--- a/src/main/scala/org/apache/spark/sql/types/KllDoublesSketchType.scala
+++ b/src/main/scala/org/apache/spark/sql/types/KllDoublesSketchType.scala
@@ -51,11 +51,11 @@ case object KllDoublesSketchType extends
KllDoublesSketchType {
})
// non-udf version
- val wrap = ((bytes: Array[Byte]) => {
+ def wrap(bytes: Array[Byte]): KllDoublesSketch = {
if (bytes == null) {
null
} else {
deserialize(bytes)
}
- })
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
