Mapreduce est soumis localement au cluster pour être exécuté, cas des statistiques de mots

Mapreduce est soumis localement au cluster pour être exécuté, cas des statistiques de mots

L'environnement de cluster à haute disponibilité Yarn est créé et le mode cluster est en cours d'exécution. Le
mode cluster est en cours d'exécution: https://blog.csdn.net/weixin_43614067/article/details/108400938 Le
mode local est en cours d'exécution: https: // blog .csdn.net / weixin_43614067 / article / details / 108386389
modifier le côté Runner

package com.bjsxt.wc;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.awt.*;
public class WCRunner {
    
    
    public static void main(String[] args) throws Exception {
    
    
        //创建配置对象
        Configuration conf = new Configuration();
        //默认文件系统的名称
        conf.set("fs.defaultFS", "hdfs://node001:8020");*/
        //本地运行，读取hdfs数据，并将数据提到hdfs
        conf.set("fs.defaultFS", "hdfs://node001:8020");
        /*本地集群运行map'mapreduce */
        System.setProperty("HADOOP_USER_NAME","hadoop");
        // 项目jar包地址。先编译打包，再运行main方法
        conf.set("mapreduce.job.jar", "E:\\IDEAproject\\wordcount\\classes\\artifacts\\wordcount.jar");
        conf.set("mapreduce.framework.name", "yarn");
        //node002为ResourceManager的active节点
        //命令yarn rmadmin -getServiceState rm1查看状态
        conf.set("yarn.resourcemanager.address", "http://node002:8032");       
        conf.set("yarn.resourcemanager.scheduler.address","http://node002:8030");
        //允许跨平台提交jar包
        conf.set("mapreduce.app-submission.cross-platform", "true");
        //创建Job对象
        Job job = Job.getInstance(conf, "wordCount");
        //设置mapper类
        job.setMapperClass(WCMapper.class);
        //设置 Reduce类
        job.setReducerClass(WCReducer.class);
        //设置运行job类
        job.setJarByClass(WCRunner.class);
        //设置map输出的key,value类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        //设置reduce输出的key,value类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //设置输入路径金额输出路径
       /* FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));*/
        FileInputFormat.setInputPaths(job, new Path("hdfs://node001:8020/wordcount/input"));
        FileOutputFormat.setOutputPath(job, new Path("hdfs://node001:8020/wordcount/output"));
        long startTime = System.currentTimeMillis();
        try {
    
    
            //传参是：是否显示程序运行状态及进度
            boolean b = job.waitForCompletion(true);
            if (b) {
    
    
                System.out.println("单词统计完成!");
            }
        } finally {
    
    
            // 结束的毫秒数
            long endTime = System.currentTimeMillis();
            System.out.println("Job<" + job.getJobName() + ">是否执行成功:" + job.isSuccessful() + "; 开始时间:" + startTime + "; 结束时间:" + endTime + "; 用时:" + (endTime - startTime) + "ms");
        }
    }
}

Fonctionnement en mode cluster: https://blog.csdn.net/weixin_43614067/article/details/108400938
Fonctionnement en mode local: https://blog.csdn.net/weixin_43614067/article/details/108386389