[
https://issues.apache.org/jira/browse/SPARK-18300?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Emlyn Corrin updated SPARK-18300:
---------------------------------
Component/s: SQL
> ClassCastException during count distinct
> ----------------------------------------
>
> Key: SPARK-18300
> URL: https://issues.apache.org/jira/browse/SPARK-18300
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 2.0.1
> Reporter: Emlyn Corrin
>
> While trying to reproduce SPARK-18172 in SQL, I found the following SQL query
> fails in Spark 2.0.1 (via spark-beeline) with a CassCastException:
> {code}
> select count(distinct b), count(distinct b, c) from (select 1 as a, 2 as b, 3
> as c) group by a;
> {code}
> Selecting either of the two counts on their own runs fine, it is only when
> both are selected in the same query that it fails. I also tried the same
> query in pyspark:
> {code}
> spark.sql('select count(distinct b) as x, count(distinct b, c) as y from
> (select 1 as a, 2 as b, 3 as c) group by a').show()
> {code}
> And get the same error, with a full stack trace:
> {code}
> : java.lang.ClassCastException:
> org.apache.spark.sql.catalyst.expressions.Literal cannot be cast to
> org.apache.spark.sql.catalyst.expressions.Attribute
> at
> org.apache.spark.sql.types.StructType$$anonfun$fromAttributes$1.apply(StructType.scala:388)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.AbstractTraversable.map(Traversable.scala:104)
> at
> org.apache.spark.sql.types.StructType$.fromAttributes(StructType.scala:388)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.schema$lzycompute(QueryPlan.scala:242)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.schema(QueryPlan.scala:242)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$10.apply(WholeStageCodegenExec.scala:454)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$10.apply(WholeStageCodegenExec.scala:454)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.supportCodegen(WholeStageCodegenExec.scala:454)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:482)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:469)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:483)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:469)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:483)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.apply(WholeStageCodegenExec.scala:490)
> at
> org.apache.spark.sql.execution.CollapseCodegenStages.apply(WholeStageCodegenExec.scala:430)
> at
> org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:93)
> at
> org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:93)
> at
> scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:124)
> at scala.collection.immutable.List.foldLeft(List.scala:84)
> at
> org.apache.spark.sql.execution.QueryExecution.prepareForExecution(QueryExecution.scala:93)
> at
> org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:83)
> at
> org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:83)
> at org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2572)
> at org.apache.spark.sql.Dataset.head(Dataset.scala:1934)
> at org.apache.spark.sql.Dataset.take(Dataset.scala:2149)
> at org.apache.spark.sql.Dataset.showString(Dataset.scala:239)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:483)
> at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237)
> at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
> at py4j.Gateway.invoke(Gateway.java:280)
> at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
> at py4j.commands.CallCommand.execute(CallCommand.java:79)
> at py4j.GatewayConnection.run(GatewayConnection.java:214)
> at java.lang.Thread.run(Thread.java:745)
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
