2021-06-01

HDFS JAVA API

配置`maven`

<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
<dependency>
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-client</artifactId>
  <version>3.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<dependency>
  <groupId>junit</groupId>
  <artifactId>junit</artifactId>
  <version>4.13.2</version>
  <scope>test</scope>
</dependency>

API Demo

配置测试的主入口

private static final String HDFS_PATH = "hdfs://localhost:9000";
private static final String HDFS_USER = "hadoop";
private static FileSystem fileSystem;
@BeforeAll
public static void prepare()
{
    try {
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication", "1");
        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, HDFS_USER);
    } catch (IOException e) {
        e.printStackTrace();
    } catch(InterruptedException e){
        e.printStackTrace();
    } catch(URISyntaxException e){
        e.printStackTrace();
    }
}
@AfterAll
public static void destroy(){
    fileSystem = null;
}

API的使用

创建目录

@Test
public void mkDir() throws Exception{
    fileSystem.mkdirs(new Path("/hdfs-api/test/"));
}

创建指定权限的目录

@Test
public void mkDirWithPermission() throws Exception{
    fileSystem.mkdirs(new Path("/hdfs-api/test1"), 
    new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.READ));
}

创建文件，并写入内容

@Test
public void createAndWrite() throws Exception{
    FSDataOutputStream out = fileSystem.create(new Path("/hdfs-api/test/a.txt"), true, 4096);
    out.write("hello hadoop!".getBytes());
    out.flush();
    out.close();
}

判断文件是否存在

@Test
public void checkFileExist() throws Exception{
    boolean exists = fileSystem.exists(new Path("/hdfs-api/test/a.txt"));
    System.out.println(exists);
}

查看文件内容

@Test
public void checkFileContent() throws Exception{
    FSDataInputStream input = fileSystem.open(new Path("/hdfs-api/test/a.txt"));
    String context = inputStreamToString(input, "utf-8");
    System.out.println(context);
}
private static String inputStreamToString(InputStream inputStream, String encode){
    try {
        if(encode == null || "".equals(encode)){
            encode = "utf-8";
        }
        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, encode));
        StringBuilder builder = new StringBuilder();
        String str = "";
        while ((str = reader.readLine())!= null) {
            builder.append(str).append("\n");
            
        }
        return builder.toString();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}

文件重命名

@Test
public void rename() throws Exception{
    Path oldPath = new Path("/hdfs-api/test/a.txt");
    Path newPath = new Path("/hdfs-api/test/c.txt");
    boolean result = fileSystem.rename(oldPath, newPath);
    System.out.println(result);
}

删除目录或文件

@Test
public void delete() throws Exception{
    boolean result = fileSystem.delete(new Path("/hdfs-api/test/c.txt"),true);
    System.out.println(result);
}

上传文件到HDFS

@Test
public void CopyFileFromLocal() throws Exception{
    Path src = new Path("C:/Users/jzhout1/Downloads/CopyTest/a.txt");
    Path dst = new Path("/hdfs-api/test/");
    fileSystem.copyFromLocalFile(src, dst);
}

上传大文件并显示上传进度

@Test
public void copyBigFileFromLocal() throws Exception{
    File file = new File("C:/Users/jzhout1/Downloads/Hadoop权威指南+中文版.pdf");
    final float fileSize = file.length();
    InputStream input = new BufferedInputStream(new FileInputStream(file));
    FSDataOutputStream output = fileSystem.create(new Path("/hdfs-api/test/" + file.getName()),
        new Progressable(){
            long fileCount = 0;
            public void progress(){
                fileCount++;
                System.out.println("上传速度： " + (fileCount*64*1024/fileSize)*100 + " %");
            }
        });
        IOUtils.copyBytes(input, output, 4096);

}

从HDFS上下载文件

@Test
//从HDFS上下载文件
public void copyFileFromHDFS() throws Exception{
    Path src = new Path("/hdfs-api/test/a.txt");
    Path dst = new Path("C:/Users/jzhout1/Downloads/");
    fileSystem.copyToLocalFile(false,src, dst);
}

查看指定路径下文件或文件夹的信息

@Test
//查看指定路径下文件或文件夹的信息
public void listFiles() throws Exception{
    FileStatus[] status = fileSystem.listStatus(new Path("/hdfs-api"));
    for (FileStatus fileStatus : status) {
        System.out.println(fileStatus.toString());
    }
}

递归查看指定目录下所有文件的信息

@Test
//递推查看指定目录下所有文件信息
public void listFilesRecursive() throws Exception{
    RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfs-api"), true);
    while (files.hasNext()) {
        System.out.println(files.next());
    }
}

查看文件的块信息

@Test
//查看文件块的信息
public void getFileBlockLocations() throws Exception{
    FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfs-api/test/a.txt"));
    BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
    for (BlockLocation blockLocation : blocks) {
        System.out.println(blockLocation);
    }
}

引用

HDFS Java API

配置maven

API Demo

配置测试的主入口

API的使用

引用

配置`maven`