tLogger("org").setLevel(Level.WARN)
>>>> Logger.getLogger("akka").setLevel(Level.WARN)
>>>>
>>>> val conf = new
>>>>
>>>> SparkConf().setMaster("local").setAppName("HiveWordCount").set("
;> object HiveWordCount {
>>>>
>>>> def main(args: Array[String]): Unit =
>>>> {
>>>> Logger.getLogger("org").setLevel(Level.WARN)
>>>> Logger.getLogger("akka").setLevel(Level.WARN)
>>>>
>>
spark.executor.memory",
>>> "1g")
>>> val sc = new SparkContext(conf)
>>> val sqlContext= new SQLContext(sc)
>>>
>>> val hc=new HiveContext(sc)
>>>
>>> hc.sql("CREATE EXTERNAL TABLE IF NOT EXISTS de
ault.TEST (user_name
>> string ,COMMENTS STRING )ROW FORMAT DELIMITED FIELDS TERMINATED BY '001'
>> STORED AS TEXTFILE LOCATION '/data/kali/test' ")
>>
>> val op=hc.sql("select user_name,COLLECT_SET(text) from (select
>> user_name,con
;
> STORED AS TEXTFILE LOCATION '/data/kali/test' ")
>
> val op=hc.sql("select user_name,COLLECT_SET(text) from (select
> user_name,concat(sub,' ',count(comments)) as text from default.test
> LATERAL
> VIEW explode(split(comments,&#x
m (select
user_name,concat(sub,' ',count(comments)) as text from default.test LATERAL
VIEW explode(split(comments,',')) subView AS sub group by user_name,sub)w
group by user_name")
op.collect.foreach(println)
}
Thanks
--
View this message in context:
