CODEWITHSUNDEEP
javahadoopdictionaryreduce
user3157810
user3157810

Reputation: 21

Error while running hadoop map reduce java program on HDP 2.0

I am running following mapreduce program in java it compile successfully but getting at runtime I am using Hortonworks HDP2.0 on EC2.

    package edu.osu.cse.ysmart.TestQuery;
    import java.io.IOException;
    import java.util.*;
    import java.text.*;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.conf.*;
    import org.apache.hadoop.io.*;
    import org.apache.hadoop.util.Tool;
    import org.apache.hadoop.util.ToolRunner;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.input.FileSplit;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
    import org.apache.hadoop.mapreduce.lib.partition.*;


    public class TestQuery1 extends Configured implements Tool{

            public static class Map extends Mapper<Object, Text,NullWritable,Text>{

                    public void map(Object key, Text value, Context context) throws IOException,InterruptedException{

                            String line = value.toString();
                            String[] line_buf = new String[3];
                            int prev=0,i=0,n=0;
                            for(i=0,n=0,prev=0;i<line.length();i++){

                                    if (line.charAt(i) == '|'){
                                            line_buf[n] = line.substring(prev,i);
                                            n = n+1;
                                            prev = i+1;
                                    }
                                    if(n == 3)
                                            break;
                            }

                            if(n<3)
                                    line_buf[n] = line.substring(prev,i);
                            if(line_buf[2].compareTo("Abid Khan") == 0){

                                    NullWritable key_op = NullWritable.get();
                                    context.write(key_op , new Text(line_buf[1]+ "|" ));
                            }
                    }

            }

            public int run(String[] args) throws Exception{

                    Configuration conf = new Configuration();
                    Job job = new Job(conf,"TestQuery1");
                    job.setJarByClass(TestQuery1.class);
                    job.setMapOutputKeyClass(NullWritable.class);
                    job.setMapOutputValueClass(Text.class);
                    job.setOutputKeyClass(NullWritable.class);
                    job.setOutputValueClass(Text.class);
                    job.setMapperClass(Map.class);
                    FileInputFormat.addInputPath(job, new Path(args[0]));
                    FileOutputFormat.setOutputPath(job, new Path(args[1]));
                    return (job.waitForCompletion(true) ? 0 : 1);
            }

            public static void main(String[] args) throws Exception {

                            int res = ToolRunner.run(new Configuration(), new TestQuery1(), args);
                            System.exit(res);
            }

    }

following is the the error...

hadoop jar /usr/local/Read.jar edu.osu.cse.ysmart.TestQuery.TestQuery1 /user/nitin//Resume.csv /user/nitin/outs 14/01/03 17:12:05 INFO client.RMProxy: Connecting to ResourceManager at ip-10-0-0-219.ec2.internal/10.0.0.219:8050 14/01/03 17:12:07 WARN mapreduce.JobSubmitter: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 14/01/03 17:12:07 INFO input.FileInputFormat: Total input paths to process : 1 14/01/03 17:12:07 INFO mapreduce.JobSubmitter: number of splits:1 14/01/03 17:12:07 INFO Configuration.deprecation: user.name is deprecated. Instead, use mapreduce.job.user.name 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.output.value.class is deprecated. Instead, use mapreduce.job.output.value.class 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.mapoutput.value.class is deprecated. Instead, use mapreduce.map.output.value.class 14/01/03 17:12:07 INFO Configuration.deprecation: mapreduce.map.class is deprecated. Instead, use mapreduce.job.map.class 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.job.name is deprecated. Instead, use mapreduce.job.name 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.input.dir is deprecated. Instead, use mapreduce.input.fileinputformat.inputdir 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.output.dir is deprecated. Instead, use mapreduce.output.fileoutputformat.outputdir 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.output.key.class is deprecated. Instead, use mapreduce.job.output.key.class 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.mapoutput.key.class is deprecated. Instead, use mapreduce.map.output.key.class 14/01/03 17:12:07 INFO Configuration.deprecation: mapred.working.dir is deprecated. Instead, use mapreduce.job.working.dir 14/01/03 17:12:08 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1388768204803_0006 14/01/03 17:12:08 INFO impl.YarnClientImpl: Submitted application application_1388768204803_0006 to ResourceManager at ip-10-0-0-219.ec2.internal/10.0.0.219:8050 14/01/03 17:12:08 INFO mapreduce.Job: The url to track the job: http://ip.ec2.internal:8088/proxy/application_1388768204803_0006/ 14/01/03 17:12:08 INFO mapreduce.Job: Running job: job_1388768204803_0006 14/01/03 17:12:36 INFO mapreduce.Job: Job job_1388768204803_0006 running in uber mode : false 14/01/03 17:12:36 INFO mapreduce.Job: map 0% reduce 0% 14/01/03 17:12:58 INFO mapreduce.Job: Task Id : attempt_1388768204803_0006_m_000000_0, Status : FAILED Error: java.lang.NullPointerException at edu.osu.cse.ysmart.TestQuery.TestQuery1$Map.map(TestQuery1.java:42) at edu.osu.cse.ysmart.TestQuery.TestQuery1$Map.map(TestQuery1.java:22) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:763) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:339) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:162) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:157)

Kindly help

Upvotes: 2

Views: 2664

Answers (1)

Rohit Menon
Rohit Menon

Reputation: 445

From the exception trace it appears that your trying to invoke a method on a null object:

attempt_1388768204803_0006_m_000000_0, Status : FAILED Error: java.lang.NullPointerException at edu.osu.cse.ysmart.TestQuery.TestQuery1$Map.map(TestQuery1.java:42) at

From the source code, the following comparison fails:

if(line_buf[2].compareTo("Abid Khan") == 0)

line_buf[2] is not being populated and hence the null pointer exception. You can print the values for line_buf[2] just before the if condition to see if you are having values. This will help you debug the issue.

Upvotes: 1

Related Questions