String pathInput = "/tq/in";
String pathOutput = "/tq/out";
String jobName = "mrqt";
int intNumReduceTasks = 2;
Configuration conf = null;
// job
conf = new Configuration(true);
Job job = Job.getInstance(conf);
job.setJarByClass(mrTQ.class);
job.setJobName(jobName);
// in out
Path pathOut = new Path(pathOutput);
Path pathIn = new Path(pathInput);
if (!pathIn.getFileSystem(conf).exists(new Path(pathInput + "/tq.txt"))) {
System.out.println("[ERROR]:there is no tq.txt");
} else {
System.out.println("[OK]:there is a tq.txt");
}
if (pathOut.getFileSystem(conf).exists(pathOut)) {
pathOut.getFileSystem(conf).delete(pathOut, true);
}
FileInputFormat.addInputPath(job, pathIn);
FileOutputFormat.setOutputPath(job, pathOut);
// map
job.setMapperClass(mrTQ.MyMapper.class);
//// job.setCombinerClass(cls);//not used
job.setMapOutputKeyClass(TQ.class);
job.setMapOutputValueClass(IntWritable.class);
job.setPartitionerClass(mrTQ.MyPartitioner.class);
job.setSortComparatorClass(mrTQ.MySort.class);
// re
job.setReducerClass(mrTQ.MyReducer.class);
job.setGroupingComparatorClass(mrTQ.MyGroup.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
job.setNumReduceTasks(intNumReduceTasks);
// submit
job.waitForCompletion(true);
System.out.println("[LOG:]" + jobName + "success!!!!YYH-@bj");
MapReduce的Map阶段的指定Combiner
猜你喜欢
转载自my.oschina.net/u/3511143/blog/1606542
今日推荐
周排行