Hadoop HDFS文件系统通过java FileSystem 实现上传下载等

今天药忘吃喽~ 2022-08-06 10:26 177阅读 0赞

package linlintest;
    
    import java.io.File;
    import java.io.FileOutputStream;
    import java.io.IOException;
    import java.net.URI;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FSDataInputStream;
    import org.apache.hadoop.fs.FileStatus;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IOUtils;
    
    public class HdfsFileSystem {
    	
    	 public static void main(String[] args) throws Exception  {
        	 //String uri="hdfs://LL-167:8020/";   //hdfs 地址
    		 // String remote="hdfs://LL-167:8020/lin/in1/1.txt"; //hdfs上的路径
    		 String uri="hdfs://192.168.0.151:8020/";   //hdfs 地址
             String local="C:/Users/Administrator/Desktop/a.txt";  //本地路径
             String remote="hdfs://192.168.0.151:8020/Workspace/houlinlin";
             Configuration conf = new Configuration();
             
             //cat(conf ,uri,"hdfs://LL-167:8020/lin/in1/1.txt");
             //download(conf ,uri,remote,local);
            //  delete(conf ,uri,"hdfs://192.168.0.173:8020/Workspace/houlinlin");
             // markDir(conf ,uri,"hdfs://192.168.0.151:8020/Workspace/houlinlin/file/apply/2014/8a8380c7459dd8b90145a1fafb500235");
            
            //  checkDir(uri,"d:/file");
            // getFile(conf ,uri,"","hdfs://192.168.0.151:8020/Workspace/houlinlin/a.txt");
             copyFile(conf,uri,"C:/Users/Administrator/Desktop/8a8380c745d05d370145d06719aa3c89.txt","hdfs://192.168.0.151:8020/Workspace/houlinlin/file/apply/2014/8a8380c7459dd8b90145a1fafb500235");
           //   ls(conf ,"hdfs://fulonghadoop","hdfs://fulonghadoop/");
        }
        /**
         * 上传文件
         * @param conf
         * @param local
         * @param remote
         * @throws IOException
         */
    	public static void copyFile(Configuration conf , String uri , String local, String remote) throws IOException {
    		FileSystem fs = FileSystem.get(URI.create(uri), conf);
    		fs.copyFromLocalFile(new Path(local), new Path(remote));
    		System.out.println("copy from: " + local + " to " + remote);
    		fs.close();
    	}
    	
    	/**
    	 * 获取hdfs上文件流
    	 * @param conf
    	 * @param uri
    	 * @param local
    	 * @param remote
    	 * @throws IOException
    	 */
    	public static  void getFileStream(Configuration conf , String uri , String local, String remote) throws IOException{
    		FileSystem fs = FileSystem.get(URI.create(uri), conf);
    		Path path= new Path(remote);
    		FSDataInputStream in = fs.open(path);//获取文件流
    		FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/b.txt");//输出流
    		int ch = 0;
    		while((ch=in.read()) != -1){  
                fos.write(ch);  
            }  
    		System.out.println("-----");
    		in.close(); 
    		fos.close();  
    	}
    	
    	/**
    	 * 创建文件夹
    	 * @param conf
    	 * @param uri
    	 * @param remoteFile
    	 * @throws IOException
    	 */
    	public static void markDir(Configuration conf , String uri , String remoteFile ) throws IOException{
    		FileSystem fs = FileSystem.get(URI.create(uri), conf);
    		Path path = new Path(remoteFile);
    		
    		fs.mkdirs(path);
    		System.out.println("创建文件夹"+remoteFile);
    		 
    	}
    	/**
    	 * 查看文件
    	 * @param conf
    	 * @param uri
    	 * @param remoteFile
    	 * @throws IOException
    	 */
    	 public static void cat(Configuration conf , String uri ,String remoteFile) throws IOException {
    			Path path = new Path(remoteFile);
    			FileSystem fs = FileSystem.get(URI.create(uri), conf);
    			FSDataInputStream fsdis = null;
    			System.out.println("cat: " + remoteFile);
    			try {
    				fsdis = fs.open(path);
    				IOUtils.copyBytes(fsdis, System.out, 4096, false);
    			} finally {
    				IOUtils.closeStream(fsdis);
    				fs.close();
    			}
    	}
    	 /**
    	  * 下载 hdfs上的文件
    	  * @param conf
    	  * @param uri
    	  * @param remote
    	  * @param local
    	  * @throws IOException
    	  */
    	 public static void download(Configuration conf , String uri ,String remote, String local) throws IOException {
    			Path path = new Path(remote);
    			FileSystem fs = FileSystem.get(URI.create(uri), conf);
    			fs.copyToLocalFile(path, new Path(local));
    			System.out.println("download: from" + remote + " to " + local);
    			fs.close();
    	}
    	 /**
    	  * 删除文件或者文件夹
    	  * @param conf
    	  * @param uri
    	  * @param filePath
    	  * @throws IOException
    	  */
    	 public static void delete(Configuration conf , String uri,String filePath) throws IOException {
    			Path path = new Path(filePath);
    			FileSystem fs = FileSystem.get(URI.create(uri), conf);
    			fs.deleteOnExit(path);
    			System.out.println("Delete: " + filePath);
    			fs.close();
    	}
    	 /**
    	  * 查看目录下面的文件
    	  * @param conf
    	  * @param uri
    	  * @param folder
    	  * @throws IOException
    	  */
    	 public static  void ls(Configuration conf , String uri , String folder) throws IOException {
    			Path path = new Path(folder);
    			FileSystem fs = FileSystem.get(URI.create(uri), conf);
    			FileStatus[] list = fs.listStatus(path);
    			System.out.println("ls: " + folder);
    			System.out.println("==========================================================");
    			for (FileStatus f : list) {
    				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(),f.isDirectory() , f.getLen());
    			}
    			System.out
    					.println("==========================================================");
    			fs.close();
    	}
    
    		/**
    		 * 
    		 * @param parentName 绝对路径地址
    		 * @throws Exception
    		 */
    		public static void checkDir(String uri,String parentName)	throws Exception{
    			//D:\file
    			Configuration conf = new Configuration();
    			File file = new File(parentName);
    			boolean flag = true;
    			while (flag)	{
    				//查出parentName下的所有文件
    				File[] fileNames = file.listFiles(new FileFilter());
    				if(fileNames != null)	{
    					for (int i = 0; i < fileNames.length; i++) {
    						File f = fileNames[i];
    						//System.out.println("parent directory:"+f.getParent()+",file name:"+f.getName());
    						System.out.println("parent directory:"+f.getParent().replace("\\", "/").substring(2)+",file name:"+f.getName());
    						String remoteFolrd= "hdfs://192.168.0.173:8020/Workspace/houlinlin"+f.getParent().replace("\\", "/").substring(2);
    				        markDir(conf ,uri,remoteFolrd);
     				        copyFile(conf ,uri,f.getParent()+"\\"+f.getName(),remoteFolrd);
    					}				
    				}
    				//查出parentName下的所有目录
    				File[] directories = file.listFiles(new DirectortyFilter());
    				if(directories != null)	{
    					for (int i = 0; i < directories.length; i++) {
    						File dir = directories[i];
    						//绝对路径
    						String path =  dir.getAbsolutePath();
    						//递归
    						checkDir(uri,path);
    					}
    				}
    				flag = false;
    			}
    		}
    	 
    }