package com.qianfeng.friendrecommendation;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
public class ShareFriend {
static class ShareFriendMapper extends Mapper<LongWritable, Text, Text, Text>{
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
//value:input文件的内容
String line = value.toString();
String[] person_friends = line.split(":");
String person = person_friends[0];
String[] friends = person_friends[1].split(",");
for(String friend : friends){
//context是map的一个内部类,记录map执行的上下文,将结果传递到reduce方法中
context.write(new Text(friend), new Text(person));
}
}
}
/**
* @author yaojin
第二阶段的map函数主要完成以下任务
1.将上一阶段reduce输出的<朋友,拥有这名朋友的所有人>信息中的 “拥有这名朋友的所有人”进行排序 ,以防出现B-C C-B这样的重复
2.将 “拥有这名朋友的所有人”进行两两配对,并将配对后的字符串当做键,“朋友”当做值输出,即输出<人-人,共同朋友>
*/
static class ShareFriendMapper2 extends Mapper<LongWritable, Text, Text, Text>{
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] friend_persons = line.split("\t");
String friend = friend_persons[0];
String[] persons = friend_persons[1].split(",");
Arrays.sort(persons); //排序
//两两配对
for(int i=0;i<persons.length-1;i++){
for(int j=i+1;j<persons.length;j++){
context.write(new Text(persons[i]+"-"+persons[j]+":"), new Text(friend));
}
}
}
}
/**
* @author yaojin
<人-人,list(共同朋友)> 中的“共同好友”进行拼接 最后输出<人-人,两人的所有共同好友>
遍历每一个经过map2生成的两两配对,作为key,通过判断是否存在共同好友,存在,将结果进行拼接,写入到output1目录中并生成文件
*/
static class ShareFriendReducer2 extends Reducer<Text, Text, Text, Text>{
@Override
protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
StringBuffer sb = new StringBuffer();
Set<String> set = new HashSet<String>();
for(Text friend : values){
if(!set.contains(friend.toString()))
set.add(friend.toString());
}
for(String friend : set){
sb.append(friend.toString()).append(",");
}
sb.deleteCharAt(sb.length()-1);
context.write(key, new Text(sb.toString()));
}
}
static class ShareFriendReducer extends Reducer<Text, Text, Text, Text>{
@Override
protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
//key是每一个人
//sb最后的拼接结果是每一个人是哪些人的共同好友
//A->C,F,B,D
StringBuffer sb = new StringBuffer();
for(Text friend : values){
sb.append(friend.toString()).append(",");
}
sb.deleteCharAt(sb.length()-1);
context.write(key, new Text(sb.toString()));
}
}
public static void main(String[] args) {
// System.setProperty("hadoop.home.dir", "E:\\hadoop-common-2.2.0-bin-master");
Configuration config = new Configuration();
org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(ShareFriend.class);
try {
//获取job实例
Job job = Job.getInstance(config);
job.setJarByClass(ShareFriend.class);
job.setMapperClass(ShareFriendMapper.class);
job.setReducerClass(ShareFriendReducer.class);
//设置输出的key的类型
job.setOutputKeyClass(Text.class);
job.setOutputFormatClass(TextOutputFormat.class);
//设置输出的value的类型
job.setOutputValueClass(Text.class);
//读取原文件,并调用map方法
FileInputFormat.setInputPaths(job, new Path("E:/qianfeng/bigData/mapreduce查找共同好友/input/input.txt"));
//将map的context结果作为入参传入到reduce中,将结果写入到目标文件中
FileOutputFormat.setOutputPath(job, new Path("E:/qianfeng/bigData/mapreduce查找共同好友/output/output1.txt"));
boolean res = job.waitForCompletion(true);
logger.info(res);
//第二阶段
Job job2 = Job.getInstance(config);
job2.setJarByClass(ShareFriend.class);
job2.setMapperClass(ShareFriendMapper2.class);
job2.setReducerClass(ShareFriendReducer2.class);
job2.setOutputKeyClass(Text.class);
job2.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job2, new Path("E:/qianfeng/bigData/mapreduce查找共同好友/output/output1.txt"));
FileOutputFormat.setOutputPath(job2, new Path("E:/qianfeng/bigData/mapreduce查找共同好友/output/output2.txt"));
boolean res2 = job2.waitForCompletion(true);
System.exit(res2?0:1);
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}