MapReduce的Map阶段的指定Combiner

		String pathInput = "/tq/in";
		String pathOutput = "/tq/out";
		String jobName = "mrqt";
		int intNumReduceTasks = 2;
		Configuration conf = null;
		// job
		conf = new Configuration(true);
		Job job = Job.getInstance(conf);
		job.setJarByClass(mrTQ.class);
		job.setJobName(jobName);
		// in out
		Path pathOut = new Path(pathOutput);
		Path pathIn = new Path(pathInput);
		if (!pathIn.getFileSystem(conf).exists(new Path(pathInput + "/tq.txt"))) {
			System.out.println("[ERROR]:there is no tq.txt");
		} else {
			System.out.println("[OK]:there is a tq.txt");
		}
		if (pathOut.getFileSystem(conf).exists(pathOut)) {
			pathOut.getFileSystem(conf).delete(pathOut, true);
		}
		FileInputFormat.addInputPath(job, pathIn);
		FileOutputFormat.setOutputPath(job, pathOut);
		// map
		job.setMapperClass(mrTQ.MyMapper.class);
		//// job.setCombinerClass(cls);//not used
		job.setMapOutputKeyClass(TQ.class);
		job.setMapOutputValueClass(IntWritable.class);
		job.setPartitionerClass(mrTQ.MyPartitioner.class);
		job.setSortComparatorClass(mrTQ.MySort.class);
		// re
		job.setReducerClass(mrTQ.MyReducer.class);
		job.setGroupingComparatorClass(mrTQ.MyGroup.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(NullWritable.class);
		job.setNumReduceTasks(intNumReduceTasks);
		// submit
		job.waitForCompletion(true);
		System.out.println("[LOG:]" + jobName + "success!!!!YYH-@bj");

猜你喜欢

转载自my.oschina.net/u/3511143/blog/1606542