win 10 + maven + idea 15 + Hadoop 2.7.3开发环境配置

前言

今天想在win 10上搭一个Hadoop的开发环境，希望能够直联Hadoop集群并提交MapReduce任务，这里给出相关的关键配置。

步骤

关于maven以及idea的安装这里不再赘述，非常简单。

• 在win 10上配置Hadoop
  将Hadoop 2.7.3直接解压到系统某个位置，以我的文件名称为例，解压到E:\大数据平台\hadoop\hadoop-2.7.3中
  

• 配置HADOOP_HOME以及PATH
  创建名为HADOOP_HOME的环境变量
  
  将bin路径添加到PATH中
  

• 添加Hadoop在win上需要的相关库文件，将其添加到hadoop的bin目录中
  

• 建立maven项目，在pom文件中添加相关的依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>edu.hfut.wls</groupId>
    <artifactId>hadoop</artifactId>
    <version>1.0-SNAPSHOT</version>
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <hadoop.version>2.7.3</hadoop.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
    </dependencies>
</project>

• 将Hadoop的相关配置文件添加到resources文件夹下

- 编写WordCount程序

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;

/**
 * Created by lianbin zhang on 2016/12/16.
 */
public class WordCount extends Configured implements Tool {
    public int run(String[] strings) throws Exception {
        try {
            Configuration conf = getConf();
            conf.set("mapreduce.job.jar", "src/main/wc.jar");
            conf.set("mapreduce.framework.name", "yarn");
            conf.set("yarn.resourcemanager.hostname", "10.20.10.100");
            conf.set("mapreduce.app-submission.cross-platform", "true");

            Job job = Job.getInstance(conf);
            job.setJarByClass(WordCount.class);

            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(LongWritable.class);

            job.setMapperClass(WcMapper.class);
            job.setReducerClass(WcReducer.class);

            job.setInputFormatClass(TextInputFormat.class);
            job.setOutputFormatClass(TextOutputFormat.class);

            FileInputFormat.setInputPaths(job, "hdfs://ns1/myid");
            FileOutputFormat.setOutputPath(job, new Path("hdfs://ns1/out"));

            job.waitForCompletion(true);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return 0;
    }

    public static class WcMapper extends Mapper<LongWritable, Text, Text, LongWritable>{
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String mVal = value.toString();
            context.write(new Text(mVal), new LongWritable(1));
        }
    }
    public static class WcReducer extends Reducer<Text, LongWritable, Text, LongWritable>{
        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
            long sum = 0;
            for(LongWritable lVal : values){
                sum += lVal.get();
            }
            context.write(key, new LongWritable(sum));
        }
    }
    public static void main(String[] args) throws Exception {
        ToolRunner.run(new WordCount(), args);
    }
}

注意：在run方法中有四项配置：

• mapreduce.job.jar：应用程序打包后的jar位置；
• mapreduce.framework.name：使用的mapreduce框架
• yarn.resourcemanager.hostname：rm的主机名，可以在hosts文件中配置对应的主机名
• mapreduce.app-submission.cross-platform：是否跨平台提交mr程序

• 提交程序
  提交程序进行运行时，由于跨平台提交，默认会将当前win的登陆用户作为user去操作hdfs集群，这里会存在权限问题，大多数解决方案中都是对hdfs文件的权限进行修改。本文采用的方案是在提交时添加虚拟机运行参数

    -DHADOOP_USER_NAME=hadoop   // hadoop需要换成你自己的用户名

• 运行结果