Everything in auxlib is added to HADOOP_CLASSPATH. But, the paths in
HADOOP_CLASSPATH are added to the class path of the Job Client, but they
are not added to the class path of the Task Trackers. Therefore if you put
a JAR called MyJar.jar on the HADOOP_CLASSPATH and don't do anything to
make it available to the Task Trackers as well, calls to MyJar.jar code
from the run() method of your job work, but calls from your Mapper or
Reducer will fail at runtime.
Thanks,
Aniket
On Tue, Jan 17, 2012 at 7:04 AM, Edward Capriolo <edlinuxg...@gmail.com>wrote:
> I generally have to put them in my hadoop classpath or hive-0.8.0/auxlib
> folder. I never quite understood why buy the classpath that reads the data
> seems different from the classpath that processes the data. Hence the
> distrinction between add jar and auxlib.
>
> Edward
>
> On Tue, Jan 17, 2012 at 6:39 AM, Bing Li <smallpu...@gmail.com> wrote:
>
>> My Steps:
>> I define a class "public class myInputFormat extends TextInputFormat
>> implements JobConfigurable" to specify input format.
>>
>> hive> add jar /home/biadmin/hiveudf/myFileFormat.jar;
>> Added /home/biadmin/hiveudf/myFileFormat.jar to class path
>> Added resource: /home/biadmin/hiveudf/myFileFormat.jar
>>
>> hive> list jars;
>> /home/biadmin/hiveudf/myFileFormat.jar
>>
>> hive> create table IOtable(str1 string, str2 string, str3 string) stored
>> as INPUTFORMAT 'com.mytest.fileformat.myInputFormat' OUTPUTFORMAT
>> 'org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat' ;
>> OK
>> Time taken: 0.081 seconds
>>
>> hive> load data local inpath '/home/biadmin/hivetbl/IOtable_data.txt'
>> into table IOtable;
>> Copying data from file:/home/biadmin/hivetbl/IOtable_data.txt
>> Copying file: file:/home/biadmin/hivetbl/IOtable_data.txt
>> Loading data to table default.iotable
>> OK
>> Time taken: 0.147 seconds
>>
>> hive> select * from IOtable;
>> OK
>> Failed with exception java.io.IOException:java.io.IOException: Cannot
>> create an instance of InputFormat class com.mytest.fileformat.myInputFormat
>> as specified in mapredWork!
>> Time taken: 0.059 seconds
>>
>>
>>
>>
>> *Here is my source code :*
>> ===============================
>> package com.mytest.fileformat;
>>
>> import java.io.IOException;
>>
>> import org.apache.hadoop.io.LongWritable;
>> import org.apache.hadoop.io.Text;
>> import org.apache.commons.logging.Log;
>> import org.apache.commons.logging.LogFactory;
>> import org.apache.hadoop.mapred.FileSplit;
>> import org.apache.hadoop.mapred.InputSplit;
>> import org.apache.hadoop.mapred.JobConf;
>> import org.apache.hadoop.mapred.JobConfigurable;
>> import org.apache.hadoop.mapred.LineRecordReader;
>> import org.apache.hadoop.mapred.RecordReader;
>> import org.apache.hadoop.mapred.Reporter;
>> import org.apache.hadoop.mapred.InputFormat;
>> import org.apache.hadoop.mapred.TextInputFormat;
>>
>> @SuppressWarnings("deprecation")
>> public class myInputFormat extends TextInputFormat implements
>> JobConfigurable {
>> TextInputFormat format;
>> JobConf job;
>>
>> public myInputFormat() {
>> format = new TextInputFormat();
>> }
>>
>> @Override
>> public void configure(JobConf job) {
>> this.job = job;
>> format.configure(job);
>> }
>> public RecordReader<LongWritable, Text> getRecordReader(
>> InputSplit genericSplit, JobConf job, Reporter reporter)
>> throws IOException {
>>
>> reporter.setStatus(genericSplit.toString());
>> return new myLineRecordReader(job, (FileSplit) genericSplit);
>> }
>>
>>
>> public static class myLineRecordReader implements
>> RecordReader<LongWritable, Text> {
>> LineRecordReader lineReader;
>> LongWritable lineKey;
>> Text lineValue;
>>
>> public myLineRecordReader(JobConf job, FileSplit split) throws
>> IOException {
>> lineReader = new LineRecordReader(job, split);
>> lineKey = lineReader.createKey();
>> lineValue = lineReader.createValue();
>> }
>>
>> public boolean next(LongWritable key, Text value) throws
>> IOException {
>> while (lineReader.next(lineKey, lineValue)) {
>> String strReplace = lineValue.toString().toLowerCase().replace(
>> "$$$$" , "\001" );
>> Text txtReplace = new Text();
>> txtReplace.set(strReplace);
>> value.set(txtReplace.getBytes(), 0, txtReplace.getLength());
>> return true ;
>> }
>> // no more data
>> return false;
>> } /** end next **/
>>
>>
>> public LongWritable createKey() {
>> return lineReader.createKey();
>> }
>> public Text createValue() {
>> return lineReader.createValue();
>> }
>> public long getPos() throws IOException{
>> return lineReader.getPos();
>> }
>> public float getProgress() throws IOException{
>> return lineReader.getProgress();
>> }
>> public void close() throws IOException{
>> lineReader.close();
>> }
>> } /** end class myLineRecordReader **/
>> }
>>
>
>
--
"...:::Aniket:::... Quetzalco@tl"
