[
https://issues.apache.org/jira/browse/SPARK-20182?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sean Owen resolved SPARK-20182.
-------------------------------
Resolution: Duplicate
> Dot in DataFrame Column title causes errors
> -------------------------------------------
>
> Key: SPARK-20182
> URL: https://issues.apache.org/jira/browse/SPARK-20182
> Project: Spark
> Issue Type: Bug
> Components: PySpark
> Affects Versions: 2.1.0
> Reporter: Evan Zamir
>
> I did a search and saw this issue pop up before, and while it seemed like it
> had been solved before 2.1, I'm still seeing an error.
> ```
> emp = spark.createDataFrame([(["Joe", "Bob", "Mary"],),
> (["Mike", "Matt", "Stacy"],)],
> ["first.names"])
> print(emp.collect())
> emp.select(['first.names']).alias('first')
> ```
> [Row(first.names=['Joe', 'Bob', 'Mary']), Row(first.names=['Mike', 'Matt',
> 'Stacy'])]
> Py4JJavaError Traceback (most recent call last)
> /usr/local/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
> 62 try:
> ---> 63 return f(*a, **kw)
> 64 except py4j.protocol.Py4JJavaError as e:
> /usr/local/spark/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py in
> get_return_value(answer, gateway_client, target_id, name)
> 318 "An error occurred while calling {0}{1}{2}.\n".
> --> 319 format(target_id, ".", name), value)
> 320 else:
> Py4JJavaError: An error occurred while calling o1734.select.
> : org.apache.spark.sql.AnalysisException: cannot resolve '`first.names`'
> given input columns: [first.names];;
> 'Project ['first.names]
> +- LogicalRDD [first.names#466]
> at
> org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:77)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:74)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:310)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:310)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:309)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionUp$1(QueryPlan.scala:282)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:292)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2$1.apply(QueryPlan.scala:296)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.AbstractTraversable.map(Traversable.scala:104)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:296)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$7.apply(QueryPlan.scala:301)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:188)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:301)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:74)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:67)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:128)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:67)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:57)
> at
> org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:48)
> at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:63)
> at
> org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withPlan(Dataset.scala:2822)
> at org.apache.spark.sql.Dataset.select(Dataset.scala:1121)
> at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
> at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
> at py4j.Gateway.invoke(Gateway.java:280)
> at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
> at py4j.commands.CallCommand.execute(CallCommand.java:79)
> at py4j.GatewayConnection.run(GatewayConnection.java:214)
> at java.lang.Thread.run(Thread.java:745)
> During handling of the above exception, another exception occurred:
> AnalysisException Traceback (most recent call last)
> <ipython-input-88-d6007b4ebb58> in <module>()
> 3 ["first.names"])
> 4 print(emp.collect())
> ----> 5 emp.select(['first.names']).alias('first')
> 6 # w2v = Word2Vec(vectorSize=5, seed=42, inputCol="Warriors.on",
> outputCol='model', minCount=1, windowSize=1)
> 7 # model = w2v.fit(gs)
> /usr/local/spark/python/pyspark/sql/dataframe.py in select(self, *cols)
> 982 [Row(name=u'Alice', age=12), Row(name=u'Bob', age=15)]
> 983 """
> --> 984 jdf = self._jdf.select(self._jcols(*cols))
> 985 return DataFrame(jdf, self.sql_ctx)
> 986
> /usr/local/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in
> __call__(self, *args)
> 1131 answer = self.gateway_client.send_command(command)
> 1132 return_value = get_return_value(
> -> 1133 answer, self.gateway_client, self.target_id, self.name)
> 1134
> 1135 for temp_arg in temp_args:
> /usr/local/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
> 67
> e.java_exception.getStackTrace()))
> 68 if s.startswith('org.apache.spark.sql.AnalysisException:
> '):
> ---> 69 raise AnalysisException(s.split(': ', 1)[1],
> stackTrace)
> 70 if s.startswith('org.apache.spark.sql.catalyst.analysis'):
> 71 raise AnalysisException(s.split(': ', 1)[1],
> stackTrace)
> AnalysisException: "cannot resolve '`first.names`' given input columns:
> [first.names];;\n'Project ['first.names]\n+- LogicalRDD [first.names#466]\n"`
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
