减速器没有被调用

时间:2016-07-20 18:08:50

标签: hadoop mapreduce

我正在CHD5.4上执行MR程序。我正在获取Mapper输出但是reducer部分没有被执行。有人可以帮我这个吗?

import java.io.IOException;
import java.util.*;

import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.mapred.MapReduceBase;

import java.util.StringTokenizer;

public class StubDriver extends Configured implements Tool {

// This is main method

public static void main(String[] args) throws Exception {

    // new configuration

    Configuration conf = new Configuration();
    System.exit(ToolRunner.run(conf, new StubDriver(), args));

}

// Run method

public int run (String[] args) throws Exception {

// Job Configuration 

    Job job = new Job(getConf(), "My Program");

    job.setJarByClass(StubDriver.class);
    job.setMapperClass(Map1.class);
    job.setReducerClass(Reduce1.class);

    job.setInputFormatClass (TextInputFormat.class);
    job.setOutputFormatClass (TextOutputFormat.class);
    job.setOutputKeyClass (IntWritable.class);
    job.setOutputValueClass (IntWritable.class);
    job.setMapOutputKeyClass (IntWritable.class);
    job.setMapOutputValueClass (Text.class);

    TextInputFormat.addInputPath(job, new Path (args[0]));
    TextOutputFormat.setOutputPath(job, new Path (args[1]));


    return job.waitForCompletion(true) ? 0 : 1; 
}

public static class Map1 extends Mapper <LongWritable,Text,IntWritable,Text> {

    public void map(LongWritable key, Text value, Context context)
            throws IOException, InterruptedException {

        StringTokenizer tokenizer = new StringTokenizer(value.toString());
        String edge_weight = tokenizer.nextToken();
        String source = tokenizer.nextToken();
        String destination = tokenizer.nextToken();
        context.write(new IntWritable(Integer.parseInt(edge_weight)), new Text(source + ":" + destination));

    }
}   


public static class Reduce1 extends Reducer <IntWritable,Text,IntWritable,IntWritable> {


    public void reducer(IntWritable key, Iterable <Text> values, Context context) throws Exception {

        int a = 1;
        for (Text val: values) {
        context.write(key,new IntWritable(a));
        }   

    }
}
}

Mapper输出:

    1   c:f
    2   g:i
    3   c:e
    3   e:f
    5   d:h
    6   f:h
    7   e:d
    8   d:g
    8   b:d
    9   b:c
    9   g:h
   10   a:b
   11   i:h
   12   a:c

我也尝试了@Override,但它抛出了错误:方法没有覆盖或实现超类型的方法

0 个答案:

没有答案