Idea上Hadoop的简单使用--Hadoop学习笔记

集成IDEA

首先配置好环境变量，新建Maven工程。

添加pom依赖，这里注意对应的hadoop版本

<dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13</version>
        </dependency>

        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.12.1</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.16.18</version>
        </dependency>

        <!--hadoop-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.8.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.8.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.8.2</version>
        </dependency>
    </dependencies>

还可以配置个日志属性log4j.properties，在resources下面新建

### 设置###
log4j.rootLogger = debug,stdout,D,E

### 输出信息到控制抬 ###
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n

### 输出DEBUG 级别以上的日志到=E://logs/debug.log ###
log4j.appender.D = org.apache.log4j.DailyRollingFileAppender
log4j.appender.D.File = E://logs/debug.log
log4j.appender.D.Append = true
log4j.appender.D.Threshold = DEBUG
log4j.appender.D.layout = org.apache.log4j.PatternLayout
log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss}  [ %t:%r ] - [ %p ]  %m%n

### 输出ERROR 级别以上的日志到=E://logs/error.log ###
log4j.appender.E = org.apache.log4j.DailyRollingFileAppender
log4j.appender.E.File =E://logs/error.log
log4j.appender.E.Append = true
log4j.appender.E.Threshold = ERROR
log4j.appender.E.layout = org.apache.log4j.PatternLayout
log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss}  [ %t:%r ] - [ %p ]  %m%n

测试一下向hdfs新建目录，类HdfsClient.java

package hdfs;

import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @author Administrator
 */
@Slf4j
public class HdfsClient {
    
    
    public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {
    
    

        Configuration conf = new Configuration();

        //获取hdfs客户端对象
        URI uri = new URI("hdfs://hadoop102:9000");
        FileSystem fileSystem = FileSystem.get(uri, conf, "root");

        //在hdfs上创建路径
        Path path = new Path("/man");
        fileSystem.mkdirs(path);

        //关闭资源
        fileSystem.close();
        log.info("over");
    }
}

此外还有其他的基本文件操作

 /**
     * 创建目录
     */
public static void createDir() throws URISyntaxException, IOException, InterruptedException {
    
    
    Configuration conf = new Configuration();

    //获取hdfs客户端对象
    URI uri = new URI("hdfs://hadoop102:9000");
    FileSystem fileSystem = FileSystem.get(uri, conf, "root");

    //在hdfs上创建路径
    Path path = new Path("/man");
    fileSystem.mkdirs(path);

    //关闭资源
    fileSystem.close();
}

/**
     * 上传文件到HDFS
     */
public static void copyFromLocal() throws URISyntaxException, IOException, InterruptedException {
    
    
    Configuration conf = new Configuration();

    //获取hdfs客户端对象
    URI uri = new URI("hdfs://hadoop102:9000");
    FileSystem fileSystem = FileSystem.get(uri, conf, "root");

    Path localPath = new Path("C:/Users/Administrator/Desktop/songjiang.txt");
    Path hdfsPath = new Path("/man/songjiang.txt");
    fileSystem.copyFromLocalFile(localPath,hdfsPath);
    fileSystem.close();
}

/**
     * 将文件从hdfs拷贝到本地
     */
public static void copyToLocal() throws IOException, InterruptedException, URISyntaxException {
    
    

    Configuration conf = new Configuration();
    //获取hdfs客户端对象
    URI uri = new URI("hdfs://hadoop102:9000");
    FileSystem fileSystem = FileSystem.get(uri, conf, "root");

    Path localPath = new Path("C:/Users/Administrator/Desktop/down.txt");
    Path hdfsPath = new Path("/man/songjiang.txt");
    fileSystem.copyToLocalFile(false,hdfsPath,localPath,true);

    fileSystem.close();
}

/**
     * 文件更名
     */
public static void reName() throws URISyntaxException, IOException, InterruptedException {
    
    
    Configuration conf = new Configuration();
    //获取hdfs客户端对象
    URI uri = new URI("hdfs://hadoop102:9000");
    FileSystem fileSystem = FileSystem.get(uri, conf, "root");

    Path hdfsOldPath = new Path("/man/songjiang.txt");
    Path hdfsNewPath = new Path("/man/shuihu.txt");
    fileSystem.rename(hdfsOldPath,hdfsNewPath);

    fileSystem.close();
}

/**
     * 查看文件详情
     */
public static void listFile() throws URISyntaxException, IOException, InterruptedException {
    
    
    Configuration conf = new Configuration();
    //获取hdfs客户端对象
    URI uri = new URI("hdfs://hadoop102:9000");
    FileSystem fileSystem = FileSystem.get(uri, conf, "root");

    RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), true);
    while (listFiles.hasNext()){
    
    
        LocatedFileStatus fileStatus = listFiles.next();
        System.out.println("============="+fileStatus.getPath().getName()+"=============");
        System.out.println("文件名称："+fileStatus.getPath().getName()+"\n文件路径："+fileStatus.getPath()+"\n文件权限："+fileStatus.getPermission()+"\n文件大小："+fileStatus.getLen()
                           +"\n分区大小："+fileStatus.getBlockSize()+"\n文件分组："+fileStatus.getGroup()+"\n文件所有者："+fileStatus.getOwner());
        BlockLocation[] blockLocations = fileStatus.getBlockLocations();
        for (BlockLocation blockLocation:blockLocations){
    
    
            String[] hosts = blockLocation.getHosts();
            System.out.printf("所在区间：");
            for (String host:hosts){
    
    
                System.out.printf(host+"\t");
            }
            System.out.println();
        }
    }

    fileSystem.close();
}

/**
     * 判断是文件还是文件夹
     */
public static void listStatus() throws URISyntaxException, IOException, InterruptedException {
    
    
    Configuration conf = new Configuration();
    //获取hdfs客户端对象
    URI uri = new URI("hdfs://hadoop102:9000");
    FileSystem fileSystem = FileSystem.get(uri, conf, "root");

    FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/man"));
    for (FileStatus fileStatuse:fileStatuses){
    
    
        if (fileStatuse.isFile()){
    
    
            System.out.println("文件："+fileStatuse.getPath().getName());
        }else {
    
    
            System.out.println("文件夹："+fileStatuse.getPath().getName());
        }
    }

    fileSystem.close();
}