Hi,
1) created a table in hive,
hive -e "DROP TABLE login1;"
hive -e "CREATE EXTERNAL TABLE login1 (
et STRING,
uuid STRING,
)
ROW FORMAT SERDE 'com.proofpoint.hive.serde.JsonSerde'
LOCATION 's3n://testing/test';"
creating table worked.
------------------------------------------------------------
2) query,
can run SELECT * from login1;
but when trying SELECT COUNT(*) FROM login1;
got following error,
Diagnostic Messages for this Task:
Error: java.io.IOException: java.lang.reflect.InvocationTargetException
at
org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97)
at
org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57)
at
org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.initNextRecordReader(HadoopShimsSecure.java:312)
at
org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.<init>(HadoopShimsSecure.java:259)
at
org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileInputFormatShim.getRecordReader(HadoopShimsSecure.java:386)
at
org.apache.hadoop.hive.ql.io.CombineHiveInputFormat.getRecordReader(CombineHiveInputFormat.java:652)
at
org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:169)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:429)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at
org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.initNextRecordReader(HadoopShimsSecure.java:298)
... 11 more
Caused by: java.lang.RuntimeException:
java.lang.ClassNotFoundException: Class
org.apache.hadoop.fs.s3native.NativeS3FileSystem not found
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2076)
at
org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2578)
at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2591)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91)
at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2630)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2612)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:296)
at
org.apache.hadoop.mapred.LineRecordReader.<init>(LineRecordReader.java:107)
at
org.apache.hadoop.mapred.TextInputFormat.getRecordReader(TextInputFormat.java:67)
at
org.apache.hadoop.hive.ql.io.CombineHiveRecordReader.<init>(CombineHiveRecordReader.java:65)
... 16 more
Caused by: java.lang.ClassNotFoundException: Class
org.apache.hadoop.fs.s3native.NativeS3FileSystem not found
at
org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1982)
at
org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2074)
... 26 more
added hadoop-aws.jar in hadoop-env.sh
hadoop classpath:
/usr/hdp/2.2.0.0-2041/hadoop/conf:/usr/hdp/2.2.0.0-2041/hadoop/lib/*:/usr/hdp/2.2.0.0-2041/hadoop/.//*:/usr/hdp/2.2.0.0-2041/hadoop-hdfs/./:/usr/hdp/2.2.0.0-2041/hadoop-hdfs/lib/*:/usr/hdp/2.2.0.0-2041/hadoop-hdfs/.//*:/usr/hdp/2.2.0.0-2041/hadoop-yarn/lib/*:/usr/hdp/2.2.0.0-2041/hadoop-yarn/.//*:/usr/hdp/2.2.0.0-2041/hadoop-mapreduce/lib/*:/usr/hdp/2.2.0.0-2041/hadoop-mapreduce/.//*::/usr/share/java/mysql-connector-java-5.1.17.jar:/usr/share/java/mysql-connector-java.jar:/usr/hdp/current/hadoop-mapreduce-client/*:/usr/hdp/2.2.0.0-2041/hadoop-mapreduce/hadoop-aws.jar
can anyone help me with this,
Thanks
