- combiner是MR程序中Mapper和Reducer之外的一种组件
- combiner组件的父类就是Reducer
- combiner和reducer的区别在于运行的位置:
Combiner是在每一个maptask所在的节点运行
Reducer是接收全局所有Mapper的输出结果;
4 combiner的意义就是对每一个maptask的输出进行局部汇总,以减小网络传输量
具体实现步骤:
- 自定义一个combiner继承Reducer,重写reduce方法
- 在job中设置: job.setCombinerClass(CustomCombiner.class)
5 combiner能够应用的前提是不能影响最终的业务逻辑
而且,combiner的输出kv应该跟reducer的输入kv类型要对应起来
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
public class WordCount {
public static void main(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Uage:<in> <out>");
System.exit(2);
}
String inputPath = args[0];
Path outputPath = new Path(args[1]);
//1.configuration
Configuration conf = new Configuration();
URI uri = new URI("hdfs://192.168.0.200:9000");
FileSystem fileSystem = FileSystem.get(uri, conf);
if (fileSystem.exists(outputPath)) {
boolean b = fileSystem.delete(outputPath, true);
System.out.println("已存在目录删除:"+b);
}
//2.建立job
Job job = Job.getInstance(conf, WordCount.class.getName());
job.setJarByClass(WordCount.class);
//3.输入文件
FileInputFormat.setInputPaths(job, new Path(inputPath));
//4.格式化输入文件
job.setInputFormatClass(TextInputFormat.class);
//5.map
job.setMapperClass(MapWordCountTask.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
//6.reduce
job.setReducerClass(ReduceWordCountTask.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
/**指定本job使用combiner组件,组件所用的类为ReduceWordCountTask**/
job.setCombinerClass(ReduceWordCountTask.class);
//7.输出文件
FileOutputFormat.setOutputPath(job, outputPath);
//8.输出文件格式化
job.setOutputFormatClass(TextOutputFormat.class);
//9.提交给集群执行
job.waitForCompletion(true);
}
public static class MapWordCountTask extends Mapper<LongWritable, Text, Text, LongWritable> {
private Text k2 = new Text();
private LongWritable v2 = new LongWritable();
@Override
protected void map(LongWritable key, Text value, Context context) throws Exception {
String content = value.toString();
StringTokenizer st = new StringTokenizer(content);
while (st.hasMoreElements()) {
k2.set(st.nextToken());
v2.set(1L);
context.write(k2, v2);
}
}
}
public static class ReduceWordCountTask extends Reducer<Text, LongWritable, Text, LongWritable> {
private LongWritable v3 = new LongWritable();
@Override
protected void reduce(Text k2, Iterable<LongWritable> v2s,Context context) throws Exception {
long sum = 0;
for (LongWritable longWritable : v2s) {
sum += longWritable.get();
v3.set(sum);
}
context.write(k2, v3);
}
}
}