Win10下运行Hadoop WordCount实例

Win10下运行Hadoop实例WordCount实例

环境：

• win10
• jdk1.8
• idea2019
• 不需要安装hadoop

技术需求：

• maven
• java基础开发
• idea相关操作

步骤

1. 新建一个maven项目

2. 在pom中添加相关依赖，记得enable import

<repositories>
        <repository>
            <id>apache</id>
            <url>http://maven.apache.org</url>
        </repository>
    </repositories>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.6.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.6.1</version>
        </dependency>
        <dependency>
            <groupId>commons-cli</groupId>
            <artifactId>commons-cli</artifactId>
            <version>1.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>2.6.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
            <version>2.6.1</version>
        </dependency>
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-examples</artifactId>
            <version>2.6.1</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.16.6</version>
        </dependency>
        <dependency>
            <groupId>org.apache.zookeeper</groupId>
            <artifactId>zookeeper</artifactId>
            <version>3.4.8</version>
        </dependency>
    </dependencies>

3. 新建WordCounts类

   import org.apache.hadoop.conf.Configuration;
   import org.apache.hadoop.examples.WordCount;
   import org.apache.hadoop.fs.Path;
   import org.apache.hadoop.io.IntWritable;
   import org.apache.hadoop.io.Text;
   import org.apache.hadoop.mapreduce.Job;
   import org.apache.hadoop.mapreduce.Mapper;
   import org.apache.hadoop.mapreduce.Reducer;
   import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
   import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
   
   import java.io.IOException;
   import java.util.StringTokenizer;
   
   public class WordCounts {
         
         
   
       public static class TokenizerMapper
               extends Mapper<Object, Text, Text, IntWritable> {
         
         
   
           private final static IntWritable one = new IntWritable(1);
           private Text word = new Text();
   
           public void map(Object key, Text value, Context context
           ) throws IOException, InterruptedException {
         
         
               StringTokenizer itr = new StringTokenizer(value.toString());
               while (itr.hasMoreTokens()) {
         
         
                   word.set(itr.nextToken());
                   context.write(word, one);
               }
           }
       }
   
       public static class IntSumReducer
               extends Reducer<Text, IntWritable, Text, IntWritable> {
         
         
           private IntWritable result = new IntWritable();
   
           public void reduce(Text key, Iterable<IntWritable> values,
                              Context context
           ) throws IOException, InterruptedException {
         
         
               int sum = 0;
               for (IntWritable val : values) {
         
         
                   sum += val.get();
               }
               result.set(sum);
               context.write(key, result);
           }
       }
   
       public static void main(String[] args) throws Exception {
         
         
           Configuration conf = new Configuration();
           Job job = Job.getInstance(conf, "word count");
           job.setJarByClass(WordCount.class);
           job.setMapperClass(TokenizerMapper.class);
           job.setCombinerClass(IntSumReducer.class);
           job.setReducerClass(IntSumReducer.class);
           job.setOutputKeyClass(Text.class);
           job.setOutputValueClass(IntWritable.class);
           FileInputFormat.addInputPath(job, new Path(args[0]));
           FileOutputFormat.setOutputPath(job, new Path(args[1]));
           System.exit(job.waitForCompletion(true) ? 0 : 1);
       }
   }
   

4. 在src同级目录下新建input文件夹， 然后在里面创建1.txt文本，里面随便输入几个单词

5. 选择File->Project Structure,在右侧的Project Setting中选择Modules，选择Language Level为8 - Lambdas

6. 选择Run->Run/Debug Configurations->左上角绿色加号->Application，Name填WordCount，MainClass选择WordCount，Program填 input/ output/,分别代表两个输入文件夹位置和输出文件夹位置

8. Run

9. 在src同级目录下会出现output文件夹，part-r-00000里面就是统计结果

10. 项目结构

常见问题：
Programme argument没对，必须是input/空格 output/
每次运行结束，必须删除output文件夹（hadoop不会覆盖），不然会报错（可以自己写代码优化）

参考博客：
http://wintercolor.azurewebsites.net/windows-ideaan-zhuang-hadoopkai-fa-huan-jing/