hdfs文件操作

逃离我推掉我的手 2022-06-07 06:28 256阅读 0赞

### 常用操作命令 ###

查看目录 hadoop fs -ls /

创建文件夹 hadoop fs -mkdir /文件夹名

往里塞文件 hadoop fs -put 本地文件地址 hdfs文件路径

删除文件 hadoop fs -rm /文件名

删除文件夹 hadoop fs -rm -r /文件夹名

查看文件内容 hadoop fs -cat /文件名

将hdfs中的文件下载到本机上 hadoop fs -get hdfs文件路径 本机路径(保存的地址)

### 通过eclipse代码操作hdfs ###

#### 操作步骤 ####

*  需要导入hadoop/share/hadoop/common目录下3个jar和lib目录下全部jar  
    还需要导入hadoop/share/hadoop/hdfs目录下3个jar和lib目录下全部jar  
    
 *  进行Configuration 类的设置  
    
 *  使用FileSystem进行文件操作
 *      import java.io.IOException;
        
        import org.apache.hadoop.conf.Configuration;
        import org.apache.hadoop.fs.FileSystem;
        import org.apache.hadoop.fs.Path;
        
        public class Demo {
        
            Configuration conf ; 
            FileSystem fs ;
            public static void main(String[] args) {
                new Demo();
            }
        
            public Demo(){
                conf = new Configuration();
                conf.set("fs.defaultFS","hdfs://localhost:9000");
                conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
                createFile(); //创建文件
                createDir(); //创建文件
                openwrite(); //打开test.txt文件进行写入
                openread(); //读取test.txt
                updatename(); //更名
                delete(); //删除
            }
                public void delete(){
                try {
                    fs = FileSystem.get(conf);
                    fs.delete(new Path("test.txt"));
                    fs.close();
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
            public void updatename(){
                try {
                    fs = FileSystem.get(conf);
                    fs.rename(new Path("test"), new Path("newtest"));
                    fs.close();
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
            public void openread(){
                try {
                    fs = FileSystem.get(conf);
                    FSDataInputStream input = fs.open(new Path("test.txt"));
                    //System.out.println(input.readLine());
                    BufferedReader buff = new BufferedReader(new InputStreamReader(input));
                    System.out.println(buff.readLine());
        
                    fs.close();
                    System.out.println("create dir ok");
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
        
            public void openwrite(){
                try {
                    fs = FileSystem.get(conf);
                    String str = "hello hadoop";
                    byte[] b = "hello hadoop".getBytes();
                    FSDataOutputStream out = fs.append(new Path("test.txt"));
                    out.write(b, 0, str.length());
                    out.close();
                    fs.close();
                    System.out.println("write ok");
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
            public void createFile(){
                try {
                    fs = FileSystem.get(conf);
                    fs.create(new Path("test.txt"));
                    fs.close();
                    System.out.println("create ok");
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
            public void createDir(){
            try {
                fs = FileSystem.get(conf);
                fs.create(new Path("myDir"));
                fs.close();
                System.out.println("create dir ok");
            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
        }