Hello,
I am trying to define an external Hive table from Spark HiveContext like the
following:
import org.apache.spark.sql.hive.HiveContext
val hiveCtx = new HiveContext(sc)
hiveCtx.sql(s"""CREATE EXTERNAL TABLE IF NOT EXISTS Rentrak_Ratings (Version
string, Gen_Date string, Market_Number string, Market_Name string, Time_Zone
string, Number_Households string,
| DateTime string, Program_Start_Time string, Program_End_Time string, Station
string, Station_Name string, Call_Sign string, Network_Name string, Program
string,
| Series_Name string, Series_Number string, Episode_Number string,
Episode_Title string, Demographic string, Demographic_Name string, HHUniverse
string,
| Share_15min_Segment string, PHUT_15min_Segment string, Rating_15min_Segment
string, AV_Audience_15min_Segment string)
| PARTITIONED BY (year INT, month INT)
| ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'""".stripMargin)
And I am getting the following error:
org.apache.spark.sql.execution.QueryExecutionException: FAILED: Hive Internal
Error: java.lang.ClassNotFoundException(org.apache.hadoop.hive.ql.hooks.ATSHook)
at org.apache.spark.sql.hive.HiveContext.runHive(HiveContext.scala:324)
at
org.apache.spark.sql.hive.HiveContext.runSqlHive(HiveContext.scala:292)
at
org.apache.spark.sql.hive.execution.HiveNativeCommand.run(HiveNativeCommand.scala:33)
at
org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult$lzycompute(commands.scala:54)
at
org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult(commands.scala:54)
at
org.apache.spark.sql.execution.ExecutedCommand.execute(commands.scala:64)
at
org.apache.spark.sql.SQLContext$QueryExecution.toRdd$lzycompute(SQLContext.scala:1099)
at
org.apache.spark.sql.SQLContext$QueryExecution.toRdd(SQLContext.scala:1099)
at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:147)
at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:130)
at org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:51)
at org.apache.spark.sql.hive.HiveContext.sql(HiveContext.scala:103)
at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:27)
at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:37)
at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:39)
at $iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:41)
at $iwC$$iwC$$iwC$$iwC.<init>(<console>:43)
at $iwC$$iwC$$iwC.<init>(<console>:45)
at $iwC$$iwC.<init>(<console>:47)
at $iwC.<init>(<console>:49)
at <init>(<console>:51)
at .<init>(<console>:55)
at .<clinit>(<console>)
at .<init>(<console>:7)
at .<clinit>(<console>)
Can anybody help please?
Stefan Panayotov, PhD
Home: 610-355-0919
Cell: 610-517-5586
email: [email protected]
[email protected]
[email protected]
