CODEWITHSUNDEEP
javaapachehadoopmapreducebigdata
Manvi
Manvi

Reputation: 1156

Map Reduce job generating empty output file

Program is generating empty output file. Can anyone please suggest me where am I going wrong. Any help will be highly appreciated. I tried to put job.setNumReduceTask(0) as I am not using reducer but still output file is empty.

public static class PrizeDisMapper extends Mapper<LongWritable, Text, Text, Pair>{
int rating = 0;
Text CustID;
IntWritable r;
Text MovieID;
public void map(LongWritable key, Text line, Context context
                ) throws IOException, InterruptedException {
        String line1 = line.toString();
        String [] fields = line1.split(":");
        if(fields.length > 1)
             {
             String Movieid = fields[0];
             String line2 = fields[1];
             String [] splitline = line2.split(",");
             String Custid = splitline[0];
             int rate = Integer.parseInt(splitline[1]);
             r = new IntWritable(rate);
             CustID = new Text(Custid);
             MovieID = new Text(Movieid);
             Pair P = new Pair();
             context.write(MovieID,P);
             }
             else
             {
             return;
             }
  }
}

 public static class IntSumReducer extends Reducer<Text,Pair,Text,Pair> {
 private IntWritable result = new IntWritable();
 public void reduce(Text key, Iterable<Pair> values,
                   Context context
                   ) throws IOException, InterruptedException {
  for (Pair val : values) {
    context.write(key, val);
  }
  }

  public class Pair implements Writable
  {
  String key;
  int value;
    public void write(DataOutput out) throws IOException {
     out.writeInt(value);
     out.writeChars(key);
  }
  public void readFields(DataInput in) throws IOException {
     key = in.readUTF();
     value = in.readInt();
  }
  public void setVal(String aKey, int aValue)
  {
     key   = aKey;
      value = aValue;
  }

Main class:

public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
  System.err.println("Usage: wordcount <in> <out>");
  System.exit(2);
}
Job job = new Job(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setInputFormatClass (TextInputFormat.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Pair.class);
System.exit(job.waitForCompletion(true) ? 0 : 1);

Thanks @Pathmanaban Palsamy and @Chris Gerken for your suggestions. I have modified the code as per your suggestions but still getting empty output file. Can anyone please suggest me configurations in my main class for input and output. Do I need to specify Pair class in input to mapper & how?

Upvotes: 1

Views: 1526

Answers (2)

PathmanabanP
PathmanabanP

Reputation: 56

Since no reducer required, I suspect below line

Pair P = new Pair(); context.write(MovieID,P);

empty Pair would be the issue. also pls check your Driver class you have given correct keyclass and valueclass like

job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Pair.class);

Upvotes: 1

Chris Gerken
Chris Gerken

Reputation: 16392

I'm guessing the reduce method should be declared as 

public void reduce(Text key, Iterable<Pair> values,
               Context context
               ) throws IOException, InterruptedException

You get passed an Iterable (an object from which you can get an Iterator) which you use to iterate over all of the values that were mapped to the given key.

Upvotes: 3

Related Questions