Map端
package com.yd.CommFriend;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class CommMapper extends Mapper<LongWritable, Text, Text, Text>{
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
//A:N,K,L数据格式 A 和他的朋友
String[] split = value.toString().split(":");
//key表示本人,value是他的朋友们
context.write(new Text(split[0]), new Text(split[1]));
}
}
Reduce端
package com.yd.CommFriend;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import MR.OrderBean;
public class CommReducer extends Reducer<Text, Text, Text, Text>{
List<List<String>> personAndFriends = new ArrayList<>();
@Override
protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
//其实迭代器中只有一个值
for(Text t : values) {
String[] split = t.toString().split(",");
List<String> friends = new ArrayList<>();
//第一个是本人,其余的都是他的好友
friends.add(0,key.toString());
for(String s: split) {
friends.add(s);
}
System.out.println(friends);
personAndFriends.add(friends);
}
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
personAndFriends.sort((ls1,ls2)->{return ls1.get(0).compareTo(ls2.get(0));});
Map<String,String> map = new HashMap<>();
for(int i=0;i< personAndFriends.size();i++) {
//取出一个人和他的好友
List<String> begin = personAndFriends.get(i);
String beginPerson = begin.get(0);// 取出本人
System.out.println(beginPerson+":"+begin);
//循环其他集合,看看是不是有共同好友
for(int j=i+1 ; j< personAndFriends.size();j++) {
List<String> back = personAndFriends.get(j);
String backPerson = back.get(0);
//Comparator<OrderBean> c = (a,b)->{return 0;};
//begin.sort((s1,s2)->{return s1.compareTo(s2);});
//back.sort((s1,s2)->{return s1.compareTo(s2);});
for(String friend : back) {
//A:N,K,L
//N:P,A
//上面这种情况是不算共同好友的 if(begin.contains(friend)&&!friend.equals(beginPerson)&&!friend.equals(backPerson)) {
String pp = beginPerson+"-"+backPerson;
if(map.containsKey(pp)) {
String value = map.get(pp)+","+friend;
map.remove(pp);
map.put(pp, value);
}else {
map.put(pp, friend);
}
}
}
}
}
//把结果输出
Set<Entry<String,String>> entrySet = map.entrySet();
for(Entry<String,String> es: entrySet) {
context.write(new Text(es.getKey()), new Text(es.getValue()));
}
}
}
Driver端
package com.yd.CommFriend;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class CommDriver {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(CommDriver.class);
job.setInputFormatClass(CombineTextInputFormat.class);
job.setMapperClass(CommMapper.class);
job.setReducerClass(CommReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job, new Path("D:\\friend"));
FileOutputFormat.setOutputPath(job, new Path("D:\\outFriends"));
job.waitForCompletion(true);
}
}