This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch theta
in repository https://gitbox.apache.org/repos/asf/datasketches-spark.git
The following commit(s) were added to refs/heads/theta by this push:
new 5addc8d fixed union
5addc8d is described below
commit 5addc8ddbf68b2662911aa9d58551490684b8b56
Author: AlexanderSaydakov <[email protected]>
AuthorDate: Mon Jan 6 17:46:00 2025 -0800
fixed union
---
src/main/scala/org/apache/spark/sql/aggregate/ThetaUnion.scala | 4 ++--
src/test/scala/org/apache/spark/sql/ThetaTest.scala | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/main/scala/org/apache/spark/sql/aggregate/ThetaUnion.scala
b/src/main/scala/org/apache/spark/sql/aggregate/ThetaUnion.scala
index 29dd408..f36ad29 100644
--- a/src/main/scala/org/apache/spark/sql/aggregate/ThetaUnion.scala
+++ b/src/main/scala/org/apache/spark/sql/aggregate/ThetaUnion.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes,
Expression, ExpressionDescription, Literal}
import
org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate
import org.apache.spark.sql.catalyst.trees.BinaryLike
-import org.apache.spark.sql.types.{AbstractDataType, DataType,
ThetaSketchWrapper, ThetaSketchType}
+import org.apache.spark.sql.types.{AbstractDataType, DataType, IntegerType,
ThetaSketchWrapper, ThetaSketchType}
import org.apache.spark.SparkUnsupportedOperationException
/**
@@ -87,7 +87,7 @@ case class ThetaUnion(
override def nullable: Boolean = false
// TODO: refine this?
- override def inputTypes: Seq[AbstractDataType] = Seq(ThetaSketchType)
+ override def inputTypes: Seq[AbstractDataType] = Seq(ThetaSketchType,
IntegerType)
override def createAggregationBuffer(): ThetaSketchWrapper = new
ThetaSketchWrapper(union =
Some(SetOperation.builder().setLogNominalEntries(lgk).buildUnion))
diff --git a/src/test/scala/org/apache/spark/sql/ThetaTest.scala
b/src/test/scala/org/apache/spark/sql/ThetaTest.scala
index 18fc021..f7c207a 100644
--- a/src/test/scala/org/apache/spark/sql/ThetaTest.scala
+++ b/src/test/scala/org/apache/spark/sql/ThetaTest.scala
@@ -76,7 +76,7 @@ class ThetaTest extends SparkSessionManager {
assert(result.getAs[Double]("estimate") == numDistinct)
}
-/*
+
test("Theta Union via SQL default lgk") {
val numGroups = 10
val numDistinct = 2000
@@ -128,5 +128,5 @@ class ThetaTest extends SparkSessionManager {
""")
assert(mergedDf.head.getAs[Double]("estimate") == numDistinct)
}
-*/
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
