五、MapReduce第五讲:平均值(Avg)
平均值的话就是通过写MapReduce代码来进行数据的平均值
下面我们通过案列来讲解一下吧!
案列:对联通流量数据进行分析,要求编写MapReduce代码对数据处理,输出手机号、上行流量、下行流量、平均流量。
代码如下:
package F;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class Flowavg {
public static void main(String[] args) throws Exception {
Configuration conf=new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(Flowavg.class);
job.setMapperClass(MMapper.class);
job.setReducerClass(MReduce.class);
job.setNumReduceTasks(Integer.parseInt(args[2]));
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}
public static class MMapper extends Mapper<LongWritable, Text, Text, Text>{
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
//转换数据类型
String line = value.toString();
//制表符切分
String[] lines = line.split("\t");
//进行判断,数据中不包含空格并且数组长度等于12并且手机号位数等于11
if (!line.contains(" ")&&lines.length==12&&lines[1].length()==11) {
String phone = lines[1];
String udflow =lines[8]+"\t"+lines[9];
context.write(new Text(phone), new Text(udflow));
}
}
}
public static class MReduce extends Reducer<Text, Text, Text, Text>{
protected void reduce(Text key, Iterable<Text> value, Context context)
throws IOException, InterruptedException {
//定义三个变量
int upflow=0;
int udwnflow=0;
int avg =0;
for (Text v : value) {
String[] line = v.toString().split("\t");
int up =Integer.parseInt(line[0]);
int down =Integer.parseInt(line[1]);
upflow+=up;
udwnflow+=down;
avg =(upflow+udwnflow)/line.length;
}
context.write(key, new Text(upflow+"\t"+udwnflow+"\t"+avg));
}
}
}
运行结果:
数据如下:
1363157993044 18211575960 B1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.ccc.com/baoguangtai/11729.html?src=yw c视频网站 176 179 577 27655 100 1508947200000
1363157993045 18211575961 77-71-AC-CD-E6-18:CMCC-EASY 120.196.100.99 http://www.aaa.com/yaowen/84358.html a视频网站 195 63 13498 46846 100 1508947200000
1363157993046 18211575962 44-71-AC-CD-E6-18:CMCC-EASY 120.196.100.97 http://www.fff.com/category/727339.html f视频网站 187 200 10567 29857 100 1508947200000
1363157993047 18211575963 34-71-AC-CD-E6-18:CMCC-EASY 120.196.100.97 http://www.aaa.com/article/details/8803989 a视频网站 108 176 18147 8007 200 1508947200000
1363157993048 18211575964 BB-71-AC-CD-E6-18:CMCC-EASY 120.196.100.95 http://www.fff.com/commact?name=chaoji f视频网站 17 129
500 1508947200000
1363157993049 18211575965 14-71-AC-CD-E6-18:CMCC-EASY 120.196.100.99 http://iface.qiyi.com/category/727339.html 视频网站 55 138 11159 400 1508947200000
1363157993050 18211575966 A2-71-AC-CD-E6-18:CMCC-EASY 120.196.100.92 http://www.126.com/category/651658.html 邮箱网站 121 80
30984 400 1508947200000
1363157993051 18211575967 B1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.eee.com/details/46292509 e视频网站 29 166 4363 31643 200 1508947200000
1363157993052 18211575968 BB-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.eee.com/shan2liang.html e视频网站 42 72
500 1508947200000
1363157993053 18211575969 77-71-AC-CD-E6-18:CMCC-EASY 120.196.100.98 http://www.126.com/yaowen/84358.html 邮箱网站 58 185 18759 39091 200 1508947200000
1363157993054 18211575970 CC-71-AC-CD-E6-18:CMCC-EASY 120.196.100.90 http://www.eee.com/189984289.html?skiptype=2 e视频网站 138 73 4910 4818 100 1508947200000
1363157993055 18211575971 84-71-AC-CD-E6-18:CMCC-EASY 120.196.100.92 http://www.aaa.com/category/651658.html a视频网站 46 129 17611 7574 200 1508947200000
1363157993056 18211575972 44-71-AC-CD-E6-18:CMCC-EASY 120.196.100.97 http://www.fff.com/yaowen/84358.html f视频网站 102 82 2471 15559 200 1508947200000
1363157993057 18211575973 44-71-AC-CD-E6-18:CMCC-EASY 120.196.100.94 http://www.sina.com.cn/article/details/8803989 门户网站 99 136 18312 43959 100 1508947200000
1363157993058 18211575974 14-71-AC-CD-E6-18:CMCC-EASY 120.196.100.99 http://www.fff.com/details/46292509 f视频网站 120 143 6487 22316 100 1508947200000
1363157993059 18211575975 34-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://iface.qiyi.com/commact?name=chaoji 视频网站 24 147 17840 21882 100 1508947200000
1363157993060 18211575976 54-71-AC-CD-E6-18:CMCC-EASY 120.196.100.95 http://www.ddd.com/details/46292509 d视频网站 18 32 14359 21613 100 1508947200000
1363157993061 18211575977 54-71-AC-CD-E6-18:CMCC-EASY 120.196.100.90 http://www.bbb.com/gaopeng527/p/4332420.html b视频网站 52 146 13963 33015 200 1508947200000
1363157993062
24-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://iface.qiyi.com/article/details/8803989 视频网站 122 103 18571 18996 100 1508947200000
1363157993063 18211575979 77-71-AC-CD-E6-18:CMCC-EASY 120.196.100.97 http://www.ccc.com/yaowen/84358.html c视频网站 7 69 2818 18013 100 1508947200000
1363157993144 18211575960 14-71-AC-CD-E6-18:CMCC-EASY 120.196.100.96 http://www.fff.com/yaowen/84358.html f视频网站 79 162 5381 3706 200 1508947220000
1363157993145 18211575961 64-71-AC-CD-E6-18:CMCC-EASY 120.196.100.98 http://www.ddd.com/article/238687.shtml d视频网站 173 17 9843
400 1508947220000
1363157993146 18211575962 CC-71-AC-CD-E6-18:CMCC-EASY 120.196.100.98 http://www.aaa.com/commact?name=chaoji a视频网站 104 144 770 14244 200 1508947220000
1363157993147 18211575963 B1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.91 http://www.ccc.com/commact?name=chaoji c视频网站 175 142 2957 45115 100 1508947220000
1363157993148 18211575964 64-71-AC-CD-E6-18:CMCC-EASY 120.196.100.95 http://www.ddd.com/yaowen/84358.html d视频网站 130 106 19091 32172 200 1508947220000
1363157993149 18211575965 DE-71-AC-CD-E6-18:CMCC-EASY 120.196.100.90 http://www.sina.com.cn/yaowen/84358.html 门户网站 37 186
500 1508947220000 1363157993150
A1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.97 http://www.eee.com/details/46292509 e视频网站 113 73 19948 885 200 1508947220000
1363157993151 18211575967 A2-71-AC-CD-E6-18:CMCC-EASY 120.196.100.99 http://www.aaa.com/gaopeng527/p/4332420.html a视频网站 173 56 5537 34445 200 1508947220000
1363157993152 18211575968 99-71-AC-CD-E6-18:CMCC-EASY 120.196.100.95 http://www.sina.com.cn/gaopeng527/p/4332420.html 门户网站 171 10 18069 22245 200 1508947220000
1363157993153 18211575969 CC-71-AC-CD-E6-18:CMCC-EASY 120.196.100.99 http://www.sina.com.cn/details/46292509 门户网站 56 60 8006 15781 100 1508947220000
1363157993154 18211575970 34-71-AC-CD-E6-18:CMCC-EASY 120.196.100.96 http://iface.qiyi.com/article/details/8803989 视频网站 125 58 8201 43383 100 1508947220000
1363157993155 18211575971 DE-71-AC-CD-E6-18:CMCC-EASY 120.196.100.98 http://www.bbb.com/yaowen/84358.html b视频网站 31 23 5438 49414 100 1508947220000
1363157993156 18211575972 A1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.92 http://www.ddd.com/baoguangtai/11729.html?src=yw d视频网站 143 100 15720 18953 100 1508947220000
1363157993157
DE-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.baidu.com/category/727339.html 搜索网站 8 180 2285 32234 200 1508947220000
1363157993158 18211575974 B1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.ddd.com/yaowen/84358_2.html d视频网站 176 152 3153 6119 200 1508947220000
1363157993159 18211575975 B1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.92 http://www.ccc.com/baoguangtai/11729.html?src=yw c视频网站 14 91 5652 29945 100 1508947220000
1363157993160 18211575976 24-71-AC-CD-E6-18:CMCC-EASY 120.196.100.92 http://www.ccc.com/yaowen/84358_2.html c视频网站 71 16 8609 23866 100 1508947220000
1363157993161 18211575977 CC-71-AC-CD-E6-18:CMCC-EASY 120.196.100.91 http://www.ccc.com/yaowen/84358.html c视频网站 140 147 6213 34758 200 1508947220000
1363157993162 18211575978 84-71-AC-CD-E6-18:CMCC-EASY 120.196.100.92 http://www.ddd.com/gaopeng527/p/4332420.html d视频网站 198 120 19515 16837 200 1508947220000
1363157993163 18211575979 77-71-AC-CD-E6-18:CMCC-EASY 120.196.100.94 http://iface.qiyi.com/shan2liang.html 视频网站 10 131 8283 19560 200 1508947220000
1363157993244 18211575960 CC-71-AC-CD-E6-18:CMCC-EASY 120.196.100.96 http://www.fff.com/category/727339.html f视频网站 111 105 500 1508947240000
1363157993245 18211575961 BB-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.eee.com/baoguangtai/11729.html?src=yw e视频网站 167 141 13269 35482 100 1508947240000
1363157993246 18211575962 64-71-AC-CD-E6-18:CMCC-EASY 120.196.100.96 http://www.fff.com/article/details/8803989 f视频网站 109 78 7606 4152 100 1508947240000
1363157993247 18211575963 64-71-AC-CD-E6-18:CMCC-EASY 120.196.100.91 http://www.sina.com.cn/189984289.html?skiptype=2 门户网站 57 88 10552 37305 100 1508947240000
1363157993248 18211575964 A2-71-AC-CD-E6-18:CMCC-EASY 120.196.100.98 http://www.126.com/details/46292509 邮箱网站 133 198 7764 38906 200 1508947240000
1363157993249 18211575965 A2-71-AC-CD-E6-18:CMCC-EASY 120.196.100.90 http://www.fff.com/189984289.html?skiptype=2 f视频网站 22 55 5879 13868 100 1508947240000
1363157993250 18211575966 54-71-AC-CD-E6-18:CMCC-EASY 120.196.100.98 http://www.126.com/189984289.html?skiptype=2 邮箱网站 188 113 5260 12177 200 1508947240000
1363157993251 18211575967 B1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.92 http://www.126.com/category/651658.html 邮箱网站 19 66 6874 24656 100 1508947240000
1363157993252 18211575968 84-71-AC-CD-E6-18:CMCC-EASY 120.196.100.97 http://www.126.com/baoguangtai/11729.html?src=yw 邮箱网站 61 124 10321 10246 100 1508947240000
1363157993253
A1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.96 http://www.aaa.com/shan2liang.html a视频网站 14 56 2541 33014 100 1508947240000
1363157993254 18211575970 BB-71-AC-CD-E6-18:CMCC-EASY 120.196.100.94 http://www.126.com/yaowen/84358.html 邮箱网站 61 112 16437 28197 100 1508947240000
1363157993255 18211575971 24-71-AC-CD-E6-18:CMCC-EASY 120.196.100.99 http://www.fff.com/yaowen/84358_2.html f视频网站 140 94 15956
400 1508947240000
1363157993256 18211575972 CC-71-AC-CD-E6-18:CMCC-EASY 120.196.100.95 http://www.eee.com/category/651658.html e视频网站 110 151 1908 25195 200 1508947240000
1363157993257 18211575973 DE-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.126.com/yaowen/84358.html 邮箱网站 12 30 8585 9730 200 1508947240000
1363157993258 18211575974 84-71-AC-CD-E6-18:CMCC-EASY 120.196.100.91 http://www.fff.com/article/details/8803989 f视频网站 158 140 12761 1675 200 1508947240000
1363157993259
B1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.ccc.com/article/details/8803989 c视频网站 138 80 11957 20652 100 1508947240000
1363157993260
DE-71-AC-CD-E6-18:CMCC-EASY 120.196.100.96 http://www.ccc.com/baoguangtai/11729.html?src=yw c视频网站 111 52 17528 47541 200 1508947240000
1363157993261 18211575977 B1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.90 http://www.ccc.com/commact?name=chaoji c视频网站 137 104 7322 26376 100 1508947240000
1363157993262 18211575978 84-71-AC-CD-E6-18:CMCC-EASY 120.196.100.90 http://www.baidu.com/baoguangtai/11729.html?src=yw 搜索网站 81 105 4200 29731 200 1508947240000
1363157993263 18211575979 A1-71-AC-CD-E6-18:CMCC-EASY 120.196.100.93 http://www.fff.com/commact?name=chaoji f视频网站 8 125 13489 400 1508947240000
本次教程到次结束,多多支持博主。有什么问题评论到下方, 还有记住博主每天一更呦