SpringBoot集成Hadoop系列一 ---- 对HDFS的文件操作

快来打我* 2022-01-28 09:03 1036阅读 0赞

一.对HDFS操作设计以下几个主要的类：

Configuration：封装了客户端或者服务器的配置信息

FileSystem：此类的对象是一个文件系统对象，可以用该对象的一些方法来对文件进行操作通过FileSystem的静态方法get获得该对象，例：FileSystem hdfs = FileSystem.get(conf);

FSDataInputStream：这是HDFS中的输入流，通过由FileSystem的open方法获取

FSDataOutputStream：这是HDFS中的输出流，通过由FileSystem的create方法获取

二.依赖配置

  1. <project xmlns="http://maven.apache.org/POM/4.0.0"
  2.     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  3.     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  4.     <modelVersion>4.0.0</modelVersion>
  6.     <groupId>com.hdfs</groupId>
  7.     <artifactId>HadoopTest</artifactId>
  8.     <version>0.0.1-SNAPSHOT</version>
  9.     <packaging>jar</packaging>
  11.     <name>HadoopTest</name>
  12.     <url>http://maven.apache.org</url>
  14.     <parent>
  15.         <groupId>org.springframework.boot</groupId>
  16.         <artifactId>spring-boot-starter-parent</artifactId>
  17.         <version>2.0.0.RELEASE</version>
  18.         <relativePath />
  19.     </parent>
  21.     <properties>
  22.         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  23.         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
  24.         <java.version>1.8</java.version>
  25.     </properties>
  27.     <dependencies>
  28.         <dependency>
  29.             <groupId>org.springframework.boot</groupId>
  30.             <artifactId>spring-boot-starter</artifactId>
  31.         </dependency>
  32.         <dependency>
  33.             <groupId>org.springframework.boot</groupId>
  34.             <artifactId>spring-boot-starter-test</artifactId>
  35.             <scope>test</scope>
  36.         </dependency>
  37.         <dependency>
  38.             <groupId>org.springframework.boot</groupId>
  39.             <artifactId>spring-boot-starter-web</artifactId>
  40.         </dependency>
  41.         <dependency>
  42.             <groupId>org.apache.hadoop</groupId>
  43.             <artifactId>hadoop-common</artifactId>
  44.             <version>3.1.1</version>
  45.         </dependency>
  46.         <dependency>
  47.             <groupId>org.apache.hadoop</groupId>
  48.             <artifactId>hadoop-hdfs</artifactId>
  49.             <version>3.1.1</version>
  50.         </dependency>
  51.         <dependency>
  52.             <groupId>org.apache.hadoop</groupId>
  53.             <artifactId>hadoop-client</artifactId>
  54.             <version>3.1.1</version>
  55.         </dependency>
  56.         <dependency>
  57.             <groupId>org.apache.hadoop</groupId>
  58.             <artifactId>hadoop-mapreduce-client-core</artifactId>
  59.             <version>3.1.1</version>
  60.         </dependency>
  61.         <dependency>
  62.             <groupId>cn.bestwu</groupId>
  63.             <artifactId>ik-analyzers</artifactId>
  64.             <version>5.1.0</version>
  65.         </dependency>
  66.         <dependency>
  67.             <groupId>jdk.tools</groupId>
  68.             <artifactId>jdk.tools</artifactId>
  69.             <version>1.8</version>
  70.             <scope>system</scope>
  71.             <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
  72.         </dependency>
  73.         <dependency>
  74.             <groupId>junit</groupId>
  75.             <artifactId>junit</artifactId>
  76.             <scope>test</scope>
  77.         </dependency>
  78.     </dependencies>
  80.     <build>
  81.         <plugins>
  82.             <plugin>
  83.                 <groupId>org.springframework.boot</groupId>
  84.                 <artifactId>spring-boot-maven-plugin</artifactId>
  85.             </plugin>
  86.             <plugin>
  87.                 <groupId>org.apache.maven.plugins</groupId>
  88.                 <artifactId>maven-compiler-plugin</artifactId>
  89.                 <configuration>
  90.                     <source>1.8</source>
  91.                     <target>1.8</target>
  92.                 </configuration>
  93.             </plugin>
  94.         </plugins>
  95.     </build>
  96. </project>
  98. # tomcat thread = 200
  99. server.tomcat.max-threads=1000
  101. <!--edit tomcat port-->
  102. server.port=8900
  103. # session time 30
  104. server.session-timeout=60
  106. spring.application.name=hadoop
  107. spring.servlet.multipart.max-file-size=50MB
  108. spring.servlet.multipart.max-request-size=50MB
  110. hdfs.path=hdfs://localhost:9000
  111. hdfs.username=linhaiy
  112. logging.config=classpath:logback.xml

三.HDFS文件操作接口开发

  1. package com.hadoop.config;
  3. import org.springframework.beans.factory.annotation.Value;
  4. import org.springframework.context.annotation.Configuration;
  6. /**
  7.  * HDFS配置类
  8.  * @author linhaiy
  9.  * @date 2019.05.18
  10.  */
  11. @Configuration
  12. public class HdfsConfig {
  13.     @Value("${hdfs.path}")
  14.     private String path;
  16.     public String getPath() {
  17.         return path;
  18.     }
  20.     public void setPath(String path) {
  21.         this.path = path;
  22.     }
  23. }
  25. package com.hadoop.hdfs.entity;
  27. import java.io.DataInput;
  28. import java.io.DataOutput;
  29. import java.io.IOException;
  31. import org.apache.hadoop.io.Writable;
  33. /**
  34.  * 用户实体类
  35.  * @author linhaiy
  36.  * @date 2019.05.18
  37.  */
  38. public class User implements Writable {
  40.     private String username;
  41.     private Integer age;
  42.     private String address;
  44.     public User() {
  45.         super();
  46.         // TODO Auto-generated constructor stub
  47.     }
  49.     public User(String username, Integer age, String address) {
  50.         super();
  51.         this.username = username;
  52.         this.age = age;
  53.         this.address = address;
  54.     }
  56.     @Override
  57.     public void write(DataOutput output) throws IOException {
  58.         // 把对象序列化
  59.         output.writeChars(username);
  60.         output.writeInt(age);
  61.         output.writeChars(address);
  62.     }
  64.     @Override
  65.     public void readFields(DataInput input) throws IOException {
  66.         // 把序列化的对象读取到内存中
  67.         username = input.readUTF();
  68.         age = input.readInt();
  69.         address = input.readUTF();
  70.     }
  72.     public String getUsername() {
  73.         return username;
  74.     }
  76.     public void setUsername(String username) {
  77.         this.username = username;
  78.     }
  80.     public Integer getAge() {
  81.         return age;
  82.     }
  84.     public void setAge(Integer age) {
  85.         this.age = age;
  86.     }
  88.     public String getAddress() {
  89.         return address;
  90.     }
  92.     public void setAddress(String address) {
  93.         this.address = address;
  94.     }
  96.     @Override
  97.     public String toString() {
  98.         return "User [username=" + username + ", age=" + age + ", address=" + address + "]";
  99.     }
  101. }
  103. package com.hadoop.hdfs.service;
  105. import java.io.BufferedReader;
  106. import java.io.InputStreamReader;
  107. import java.net.URI;
  108. import java.util.ArrayList;
  109. import java.util.HashMap;
  110. import java.util.List;
  111. import java.util.Map;
  112. import javax.annotation.PostConstruct;
  113. import org.apache.commons.lang.StringUtils;
  114. import org.apache.hadoop.conf.Configuration;
  115. import org.apache.hadoop.fs.BlockLocation;
  116. import org.apache.hadoop.fs.FSDataInputStream;
  117. import org.apache.hadoop.fs.FSDataOutputStream;
  118. import org.apache.hadoop.fs.FileStatus;
  119. import org.apache.hadoop.fs.FileSystem;
  120. import org.apache.hadoop.fs.LocatedFileStatus;
  121. import org.apache.hadoop.fs.Path;
  122. import org.apache.hadoop.fs.RemoteIterator;
  123. import org.apache.hadoop.io.IOUtils;
  124. import org.springframework.beans.factory.annotation.Value;
  125. import org.springframework.stereotype.Component;
  126. import org.springframework.web.multipart.MultipartFile;
  128. import com.hadoop.util.JsonUtil;
  130. @Component
  131. public class HdfsService {
  133.     @Value("${hdfs.path}")
  134.     private String path;
  135.     @Value("${hdfs.username}")
  136.     private String username;
  138.     private static String hdfsPath;
  139.     private static String hdfsName;
  140.     private static final int bufferSize = 1024 * 1024 * 64;
  142.     /**
  143.      * 获取HDFS配置信息
  144.      * @return
  145.      */
  146.     private static Configuration getConfiguration() {
  147.         Configuration configuration = new Configuration();
  148.         configuration.set("fs.defaultFS", hdfsPath);
  149.         return configuration;
  150.     }
  152.     /**
  153.      * 获取HDFS文件系统对象
  154.      * @return
  155.      * @throws Exception
  156.      */
  157.     public static FileSystem getFileSystem() throws Exception {
  158.         // 客户端去操作hdfs时是有一个用户身份的，默认情况下hdfs客户端api会从jvm中获取一个参数作为自己的用户身份
  159.         // DHADOOP_USER_NAME=hadoop
  160.         // 也可以在构造客户端fs对象时，通过参数传递进去
  161.         FileSystem fileSystem = FileSystem.get(new URI(hdfsPath), getConfiguration(), hdfsName);
  162.         return fileSystem;
  163.     }
  165.     /**
  166.      * 在HDFS创建文件夹
  167.      * @param path
  168.      * @return
  169.      * @throws Exception
  170.      */
  171.     public static boolean mkdir(String path) throws Exception {
  172.         if (StringUtils.isEmpty(path)) {
  173.             return false;
  174.         }
  175.         if (existFile(path)) {
  176.             return true;
  177.         }
  178.         FileSystem fs = getFileSystem();
  179.         // 目标路径
  180.         Path srcPath = new Path(path);
  181.         boolean isOk = fs.mkdirs(srcPath);
  182.         fs.close();
  183.         return isOk;
  184.     }
  186.     /**
  187.      * 判断HDFS文件是否存在
  188.      * @param path
  189.      * @return
  190.      * @throws Exception
  191.      */
  192.     public static boolean existFile(String path) throws Exception {
  193.         if (StringUtils.isEmpty(path)) {
  194.             return false;
  195.         }
  196.         FileSystem fs = getFileSystem();
  197.         Path srcPath = new Path(path);
  198.         boolean isExists = fs.exists(srcPath);
  199.         return isExists;
  200.     }
  202.     /**
  203.      * 读取HDFS目录信息
  204.      * @param path
  205.      * @return
  206.      * @throws Exception
  207.      */
  208.     public static List<Map<String, Object>> readPathInfo(String path) throws Exception {
  209.         if (StringUtils.isEmpty(path)) {
  210.             return null;
  211.         }
  212.         if (!existFile(path)) {
  213.             return null;
  214.         }
  215.         FileSystem fs = getFileSystem();
  216.         // 目标路径
  217.         Path newPath = new Path(path);
  218.         FileStatus[] statusList = fs.listStatus(newPath);
  219.         List<Map<String, Object>> list = new ArrayList<>();
  220.         if (null != statusList && statusList.length > 0) {
  221.             for (FileStatus fileStatus : statusList) {
  222.                 Map<String, Object> map = new HashMap<>();
  223.                 map.put("filePath", fileStatus.getPath());
  224.                 map.put("fileStatus", fileStatus.toString());
  225.                 list.add(map);
  226.             }
  227.             return list;
  228.         } else {
  229.             return null;
  230.         }
  231.     }
  233.     /**
  234.      * HDFS创建文件
  235.      * @param path
  236.      * @param file
  237.      * @throws Exception
  238.      */
  239.     public static void createFile(String path, MultipartFile file) throws Exception {
  240.         if (StringUtils.isEmpty(path) || null == file.getBytes()) {
  241.             return;
  242.         }
  243.         String fileName = file.getOriginalFilename();
  244.         FileSystem fs = getFileSystem();
  245.         // 上传时默认当前目录，后面自动拼接文件的目录
  246.         Path newPath = new Path(path + "/" + fileName);
  247.         // 打开一个输出流
  248.         FSDataOutputStream outputStream = fs.create(newPath);
  249.         outputStream.write(file.getBytes());
  250.         outputStream.close();
  251.         fs.close();
  252.     }
  254.     /**
  255.      * 读取HDFS文件内容
  256.      * @param path
  257.      * @return
  258.      * @throws Exception
  259.      */
  260.     public static String readFile(String path) throws Exception {
  261.         if (StringUtils.isEmpty(path)) {
  262.             return null;
  263.         }
  264.         if (!existFile(path)) {
  265.             return null;
  266.         }
  267.         FileSystem fs = getFileSystem();
  268.         // 目标路径
  269.         Path srcPath = new Path(path);
  270.         FSDataInputStream inputStream = null;
  271.         try {
  272.             inputStream = fs.open(srcPath);
  273.             // 防止中文乱码
  274.             BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
  275.             String lineTxt = "";
  276.             StringBuffer sb = new StringBuffer();
  277.             while ((lineTxt = reader.readLine()) != null) {
  278.                 sb.append(lineTxt);
  279.             }
  280.             return sb.toString();
  281.         } finally {
  282.             inputStream.close();
  283.             fs.close();
  284.         }
  285.     }
  287.     /**
  288.      * 读取HDFS文件列表
  289.      * @param path
  290.      * @return
  291.      * @throws Exception
  292.      */
  293.     public static List<Map<String, String>> listFile(String path) throws Exception {
  294.         if (StringUtils.isEmpty(path)) {
  295.             return null;
  296.         }
  297.         if (!existFile(path)) {
  298.             return null;
  299.         }
  301.         FileSystem fs = getFileSystem();
  302.         // 目标路径
  303.         Path srcPath = new Path(path);
  304.         // 递归找到所有文件
  305.         RemoteIterator<LocatedFileStatus> filesList = fs.listFiles(srcPath, true);
  306.         List<Map<String, String>> returnList = new ArrayList<>();
  307.         while (filesList.hasNext()) {
  308.             LocatedFileStatus next = filesList.next();
  309.             String fileName = next.getPath().getName();
  310.             Path filePath = next.getPath();
  311.             Map<String, String> map = new HashMap<>();
  312.             map.put("fileName", fileName);
  313.             map.put("filePath", filePath.toString());
  314.             returnList.add(map);
  315.         }
  316.         fs.close();
  317.         return returnList;
  318.     }
  320.     /**
  321.      * HDFS重命名文件
  322.      * @param oldName
  323.      * @param newName
  324.      * @return
  325.      * @throws Exception
  326.      */
  327.     public static boolean renameFile(String oldName, String newName) throws Exception {
  328.         if (StringUtils.isEmpty(oldName) || StringUtils.isEmpty(newName)) {
  329.             return false;
  330.         }
  331.         FileSystem fs = getFileSystem();
  332.         // 原文件目标路径
  333.         Path oldPath = new Path(oldName);
  334.         // 重命名目标路径
  335.         Path newPath = new Path(newName);
  336.         boolean isOk = fs.rename(oldPath, newPath);
  337.         fs.close();
  338.         return isOk;
  339.     }
  341.     /**
  342.      * 删除HDFS文件
  343.      * @param path
  344.      * @return
  345.      * @throws Exception
  346.      */
  347.     public static boolean deleteFile(String path) throws Exception {
  348.         if (StringUtils.isEmpty(path)) {
  349.             return false;
  350.         }
  351.         if (!existFile(path)) {
  352.             return false;
  353.         }
  354.         FileSystem fs = getFileSystem();
  355.         Path srcPath = new Path(path);
  356.         boolean isOk = fs.deleteOnExit(srcPath);
  357.         fs.close();
  358.         return isOk;
  359.     }
  361.     /**
  362.      * 上传HDFS文件
  363.      * @param path
  364.      * @param uploadPath
  365.      * @throws Exception
  366.      */
  367.     public static void uploadFile(String path, String uploadPath) throws Exception {
  368.         if (StringUtils.isEmpty(path) || StringUtils.isEmpty(uploadPath)) {
  369.             return;
  370.         }
  371.         FileSystem fs = getFileSystem();
  372.         // 上传路径
  373.         Path clientPath = new Path(path);
  374.         // 目标路径
  375.         Path serverPath = new Path(uploadPath);
  377.         // 调用文件系统的文件复制方法，第一个参数是否删除原文件true为删除，默认为false
  378.         fs.copyFromLocalFile(false, clientPath, serverPath);
  379.         fs.close();
  380.     }
  382.     /**
  383.      * 下载HDFS文件
  384.      * @param path
  385.      * @param downloadPath
  386.      * @throws Exception
  387.      */
  388.     public static void downloadFile(String path, String downloadPath) throws Exception {
  389.         if (StringUtils.isEmpty(path) || StringUtils.isEmpty(downloadPath)) {
  390.             return;
  391.         }
  392.         FileSystem fs = getFileSystem();
  393.         // 上传路径
  394.         Path clientPath = new Path(path);
  395.         // 目标路径
  396.         Path serverPath = new Path(downloadPath);
  398.         // 调用文件系统的文件复制方法，第一个参数是否删除原文件true为删除，默认为false
  399.         fs.copyToLocalFile(false, clientPath, serverPath);
  400.         fs.close();
  401.     }
  403.     /**
  404.      * HDFS文件复制 
  405.      * @param sourcePath
  406.      * @param targetPath
  407.      * @throws Exception
  408.      */
  409.     public static void copyFile(String sourcePath, String targetPath) throws Exception {
  410.         if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) {
  411.             return;
  412.         }
  413.         FileSystem fs = getFileSystem();
  414.         // 原始文件路径
  415.         Path oldPath = new Path(sourcePath);
  416.         // 目标路径
  417.         Path newPath = new Path(targetPath);
  419.         FSDataInputStream inputStream = null;
  420.         FSDataOutputStream outputStream = null;
  421.         try {
  422.             inputStream = fs.open(oldPath);
  423.             outputStream = fs.create(newPath);
  425.             IOUtils.copyBytes(inputStream, outputStream, bufferSize, false);
  426.         } finally {
  427.             inputStream.close();
  428.             outputStream.close();
  429.             fs.close();
  430.         }
  431.     }
  433.     /**
  434.      * 打开HDFS上的文件并返回byte数组
  435.      * @param path
  436.      * @return
  437.      * @throws Exception
  438.      */
  439.     public static byte[] openFileToBytes(String path) throws Exception {
  440.         if (StringUtils.isEmpty(path)) {
  441.             return null;
  442.         }
  443.         if (!existFile(path)) {
  444.             return null;
  445.         }
  446.         FileSystem fs = getFileSystem();
  447.         // 目标路径
  448.         Path srcPath = new Path(path);
  449.         try {
  450.             FSDataInputStream inputStream = fs.open(srcPath);
  451.             return IOUtils.readFullyToByteArray(inputStream);
  452.         } finally {
  453.             fs.close();
  454.         }
  455.     }
  457.     /**
  458.      * 打开HDFS上的文件并返回java对象
  459.      * @param path
  460.      * @return
  461.      * @throws Exception
  462.      */
  463.     public static <T extends Object> T openFileToObject(String path, Class<T> clazz) throws Exception {
  464.         if (StringUtils.isEmpty(path)) {
  465.             return null;
  466.         }
  467.         if (!existFile(path)) {
  468.             return null;
  469.         }
  470.         String jsonStr = readFile(path);
  471.         return JsonUtil.fromObject(jsonStr, clazz);
  472.     }
  474.     /**
  475.      * 获取某个文件在HDFS的集群位置
  476.      * @param path
  477.      * @return
  478.      * @throws Exception
  479.      */
  480.     public static BlockLocation[] getFileBlockLocations(String path) throws Exception {
  481.         if (StringUtils.isEmpty(path)) {
  482.             return null;
  483.         }
  484.         if (!existFile(path)) {
  485.             return null;
  486.         }
  487.         FileSystem fs = getFileSystem();
  488.         // 目标路径
  489.         Path srcPath = new Path(path);
  490.         FileStatus fileStatus = fs.getFileStatus(srcPath);
  491.         return fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
  492.     }
  494.     @PostConstruct
  495.     public void getPath() {
  496.         hdfsPath = this.path;
  497.     }
  499.     @PostConstruct
  500.     public void getName() {
  501.         hdfsName = this.username;
  502.     }
  504.     public static String getHdfsPath() {
  505.         return hdfsPath;
  506.     }
  508.     public String getUsername() {
  509.         return username;
  510.     }
  511. }
  513. package com.hadoop.hdfs.controller;
  515. import java.util.List;
  516. import java.util.Map;
  518. import org.apache.commons.lang.StringUtils;
  519. import org.apache.hadoop.fs.BlockLocation;
  520. import org.slf4j.Logger;
  521. import org.slf4j.LoggerFactory;
  522. import org.springframework.web.bind.annotation.PostMapping;
  523. import org.springframework.web.bind.annotation.RequestMapping;
  524. import org.springframework.web.bind.annotation.RequestMethod;
  525. import org.springframework.web.bind.annotation.RequestParam;
  526. import org.springframework.web.bind.annotation.ResponseBody;
  527. import org.springframework.web.bind.annotation.RestController;
  528. import org.springframework.web.multipart.MultipartFile;
  530. import com.hadoop.hdfs.entity.User;
  531. import com.hadoop.hdfs.service.HdfsService;
  532. import com.hadoop.util.Result;
  534. @RestController
  535. @RequestMapping("/hadoop/hdfs")
  536. public class HdfsAction {
  538.     private static Logger LOGGER = LoggerFactory.getLogger(HdfsAction.class);
  540.     /**
  541.      * 创建文件夹
  542.      * @param path
  543.      * @return
  544.      * @throws Exception
  545.      */
  546.     @RequestMapping(value = "mkdir", method = RequestMethod.POST)
  547.     @ResponseBody
  548.     public Result mkdir(@RequestParam("path") String path) throws Exception {
  549.         if (StringUtils.isEmpty(path)) {
  550.             LOGGER.debug("请求参数为空");
  551.             return new Result(Result.FAILURE, "请求参数为空");
  552.         }
  553.         // 创建空文件夹
  554.         boolean isOk = HdfsService.mkdir(path);
  555.         if (isOk) {
  556.             LOGGER.debug("文件夹创建成功");
  557.             return new Result(Result.SUCCESS, "文件夹创建成功");
  558.         } else {
  559.             LOGGER.debug("文件夹创建失败");
  560.             return new Result(Result.FAILURE, "文件夹创建失败");
  561.         }
  562.     }
  564.     /**
  565.      * 读取HDFS目录信息
  566.      * @param path
  567.      * @return
  568.      * @throws Exception
  569.      */
  570.     @PostMapping("/readPathInfo")
  571.     public Result readPathInfo(@RequestParam("path") String path) throws Exception {
  572.         List<Map<String, Object>> list = HdfsService.readPathInfo(path);
  573.         return new Result(Result.SUCCESS, "读取HDFS目录信息成功", list);
  574.     }
  576.     /**
  577.      * 获取HDFS文件在集群中的位置
  578.      * @param path
  579.      * @return
  580.      * @throws Exception
  581.      */
  582.     @PostMapping("/getFileBlockLocations")
  583.     public Result getFileBlockLocations(@RequestParam("path") String path) throws Exception {
  584.         BlockLocation[] blockLocations = HdfsService.getFileBlockLocations(path);
  585.         return new Result(Result.SUCCESS, "获取HDFS文件在集群中的位置", blockLocations);
  586.     }
  588.     /**
  589.      * 创建文件
  590.      * @param path
  591.      * @return
  592.      * @throws Exception
  593.      */
  594.     @PostMapping("/createFile")
  595.     public Result createFile(@RequestParam("path") String path, @RequestParam("file") MultipartFile file)
  596.             throws Exception {
  597.         if (StringUtils.isEmpty(path) || null == file.getBytes()) {
  598.             return new Result(Result.FAILURE, "请求参数为空");
  599.         }
  600.         HdfsService.createFile(path, file);
  601.         return new Result(Result.SUCCESS, "创建文件成功");
  602.     }
  604.     /**
  605.      * 读取HDFS文件内容
  606.      * @param path
  607.      * @return
  608.      * @throws Exception
  609.      */
  610.     @PostMapping("/readFile")
  611.     public Result readFile(@RequestParam("path") String path) throws Exception {
  612.         String targetPath = HdfsService.readFile(path);
  613.         return new Result(Result.SUCCESS, "读取HDFS文件内容", targetPath);
  614.     }
  616.     /**
  617.      * 读取HDFS文件转换成Byte类型
  618.      * @param path
  619.      * @return
  620.      * @throws Exception
  621.      */
  622.     @PostMapping("/openFileToBytes")
  623.     public Result openFileToBytes(@RequestParam("path") String path) throws Exception {
  624.         byte[] files = HdfsService.openFileToBytes(path);
  625.         return new Result(Result.SUCCESS, "读取HDFS文件转换成Byte类型", files);
  626.     }
  628.     /**
  629.      * 读取HDFS文件装换成User对象
  630.      * @param path
  631.      * @return
  632.      * @throws Exception
  633.      */
  634.     @PostMapping("/openFileToUser")
  635.     public Result openFileToUser(@RequestParam("path") String path) throws Exception {
  636.         User user = HdfsService.openFileToObject(path, User.class);
  637.         return new Result(Result.SUCCESS, "读取HDFS文件装换成User对象", user);
  638.     }
  640.     /**
  641.      * 读取文件列表
  642.      * @param path
  643.      * @return
  644.      * @throws Exception
  645.      */
  646.     @PostMapping("/listFile")
  647.     public Result listFile(@RequestParam("path") String path) throws Exception {
  648.         if (StringUtils.isEmpty(path)) {
  649.             return new Result(Result.FAILURE, "请求参数为空");
  650.         }
  651.         List<Map<String, String>> returnList = HdfsService.listFile(path);
  652.         return new Result(Result.SUCCESS, "读取文件列表成功", returnList);
  653.     }
  655.     /**
  656.      * 重命名文件
  657.      * @param oldName
  658.      * @param newName
  659.      * @return
  660.      * @throws Exception
  661.      */
  662.     @PostMapping("/renameFile")
  663.     public Result renameFile(@RequestParam("oldName") String oldName, @RequestParam("newName") String newName)
  664.             throws Exception {
  665.         if (StringUtils.isEmpty(oldName) || StringUtils.isEmpty(newName)) {
  666.             return new Result(Result.FAILURE, "请求参数为空");
  667.         }
  668.         boolean isOk = HdfsService.renameFile(oldName, newName);
  669.         if (isOk) {
  670.             return new Result(Result.SUCCESS, "文件重命名成功");
  671.         } else {
  672.             return new Result(Result.FAILURE, "文件重命名失败");
  673.         }
  674.     }
  676.     /**
  677.      * 删除文件
  678.      * @param path
  679.      * @return
  680.      * @throws Exception
  681.      */
  682.     @PostMapping("/deleteFile")
  683.     public Result deleteFile(@RequestParam("path") String path) throws Exception {
  684.         boolean isOk = HdfsService.deleteFile(path);
  685.         if (isOk) {
  686.             return new Result(Result.SUCCESS, "delete file success");
  687.         } else {
  688.             return new Result(Result.FAILURE, "delete file fail");
  689.         }
  690.     }
  692.     /**
  693.      * 上传文件
  694.      * @param path
  695.      * @param uploadPath
  696.      * @return
  697.      * @throws Exception
  698.      */
  699.     @PostMapping("/uploadFile")
  700.     public Result uploadFile(@RequestParam("path") String path, @RequestParam("uploadPath") String uploadPath)
  701.             throws Exception {
  702.         HdfsService.uploadFile(path, uploadPath);
  703.         return new Result(Result.SUCCESS, "upload file success");
  704.     }
  706.     /**
  707.      * 下载文件
  708.      * @param path
  709.      * @param downloadPath
  710.      * @return
  711.      * @throws Exception
  712.      */
  713.     @PostMapping("/downloadFile")
  714.     public Result downloadFile(@RequestParam("path") String path, @RequestParam("downloadPath") String downloadPath)
  715.             throws Exception {
  716.         HdfsService.downloadFile(path, downloadPath);
  717.         return new Result(Result.SUCCESS, "download file success");
  718.     }
  720.     /**
  721.      * HDFS文件复制
  722.      * @param sourcePath
  723.      * @param targetPath
  724.      * @return
  725.      * @throws Exception
  726.      */
  727.     @PostMapping("/copyFile")
  728.     public Result copyFile(@RequestParam("sourcePath") String sourcePath, @RequestParam("targetPath") String targetPath)
  729.             throws Exception {
  730.         HdfsService.copyFile(sourcePath, targetPath);
  731.         return new Result(Result.SUCCESS, "copy file success");
  732.     }
  734.     /**
  735.      * 查看文件是否已存在
  736.      * @param path
  737.      * @return
  738.      * @throws Exception
  739.      */
  740.     @PostMapping("/existFile")
  741.     public Result existFile(@RequestParam("path") String path) throws Exception {
  742.         boolean isExist = HdfsService.existFile(path);
  743.         return new Result(Result.SUCCESS, "file isExist: " + isExist);
  744.     }
  745. }

四.一些测试结果截图

20190523193125578.png

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpbmhhaXl1bl95dGR4_size_16_color_FFFFFF_t_70

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpbmhhaXl1bl95dGR4_size_16_color_FFFFFF_t_70 1

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpbmhhaXl1bl95dGR4_size_16_color_FFFFFF_t_70 2

发表评论

表情：
评论列表 （有 0 条评论，1036人围观）

还没有评论，来说两句吧...

相关阅读

    相关 Hadoop HDFS操作

    启动HDFS 首先，您必须格式化配置的HDFS文件系统，打开namenode（HDFS服务器），然后执行以下命令。 $ hadoop namenode -for

    墨蓝墨蓝/ 0 赞/ 337 阅读