spark遍历hdfs目录下所有文件

﹏ヽ暗。殇╰゛Y 2023-02-21 11:49 32阅读 0赞

**1、查看hdfs /home/data/test 目录下的所有文件**

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxMjYyNTkzNDIx_size_16_color_FFFFFF_t_70][]

**2、使用 org.apache.hadoop.fs.FileSystem 类遍历hdfs文件**

package com.xtd.hdfs
    
    import org.apache.hadoop.conf.Configuration
    import org.apache.hadoop.fs.{FileSystem, FileUtil, Path}
    import scala.collection.mutable.{ArrayBuffer, ListBuffer}
    
    object HDFSUtils {
    
      def main(args: Array[String]): Unit = {
        val files = getHDFSFiles("hdfs://ns1/home/data/test")
        files.foreach(println(_))
      }
    
      /**
       * 给定hdfs目录返回目录下文件名的集合
       * @param hdfsDirectory
       * @return Array[String]
       */
      def getHDFSFiles(hdfsDirectory:String): Array[String] ={
        val configuration:Configuration = new Configuration()
    //    configuration.set("fs.defaultFS", hdfsFileName)
        val fileSystem:FileSystem = FileSystem.get(configuration)
        val fsPath: Path = new Path(hdfsDirectory)
        val iterator = fileSystem.listFiles(fsPath, true)
        val list = new ListBuffer[String]
        while (iterator.hasNext) {
          val pathStatus = iterator.next()
          val hdfsPath = pathStatus.getPath
          val fileName = hdfsPath.getName
          list += fileName // list.append(fileName)
        }
        fileSystem.close()
        list.toArray
      }
    
    }

**3、运行效果**

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxMjYyNTkzNDIx_size_16_color_FFFFFF_t_70 1][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxMjYyNTkzNDIx_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200630124352240.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxMjYyNTkzNDIx,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxMjYyNTkzNDIx_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/20200630125014843.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxMjYyNTkzNDIx,size_16,color_FFFFFF,t_70