【Java-HDFS】使用Java操作HDFS获取HDFS指定目录下的数据量大小

• Maven依赖

<dependencies><dependency><groupId>junit</groupId><artifactId>junit</artifactId><version>RELEASE</version></dependency><dependency><groupId>org.apache.logging.log4j</groupId><artifactId>log4j-core</artifactId><version>2.8.2</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>3.1.1</version></dependency><!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client --><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>3.1.1</version></dependency><!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs --><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-hdfs</artifactId><version>3.1.1</version></dependency>
<dependencies>

• HDFSUtils工具类

package gaei.cn.x5l.x5lhive2cos.utils;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;import java.io.FileNotFoundException;
import java.io.IOException;public class HdfsUtils {private static final Configuration conf = new Configuration();private static FileSystem fs;public static void init() {//本地认证kerberos认证
//        System.setProperty("java.security.krb5.conf", "D:/cyp/Chinasoft/keytab/prd_keytab/krb5.conf");
//        conf.set("hadoop.security.authentication", "kerberos");
//        UserGroupInformation.setConfiguration(conf);
//        try {
//            UserGroupInformation.loginUserFromKeytab("[e-mail]", "D:/cyp/Chinasoft/keytab/prd_keytab/x9e.keytab");
//        } catch (IOException e) {
//            e.printStackTrace();
//        }
//        System.out.println(Thread.currentThread().getContextClassLoader());try {//将core-site.xml和hdfs-site.xml放在resource目录下即可，必须指定，不然获取不到HDFS的nameserver等信息conf.addResource(Thread.currentThread().getContextClassLoader().getResource("./core-site.xml"));conf.addResource(Thread.currentThread().getContextClassLoader().getResource("./hdfs-site.xml"));fs = FileSystem.get(conf);} catch (FileNotFoundException fnfe) {fnfe.printStackTrace();} catch (IOException ioe) {ioe.printStackTrace();}}public static long getHdfsSampleDateSize(String hdfsPath) {long length = 0L;try {length = fs.getContentSummary(new Path(hdfsPath)).getLength();} catch (IOException e) {e.printStackTrace();}return length;}public static void close() {try {fs.close();} catch (IOException ex) {ex.printStackTrace();}}
}