java:快速文件分割及合并

浅浅的花香味﹌ 2022-08-01 00:17 178阅读 0赞

文件分割与合并是一个常见需求，比如：上传大文件时，可以先分割成小块，传到服务器后，再进行合并。很多高大上的分布式文件系统（比如：google的GFS、taobao的TFS）里，也是按block为单位，对文件进行分割或合并。

看下基本思路：

![171443005042853.jpg][]

如果有一个大文件，指定分割大小后（比如：按1M切割）

step 1：

先根据原始文件大小、分割大小，算出最终分割的小文件数N

step 2:

在磁盘上创建这N个小文件

step 3:

开多个线程（线程数=分割文件数），每个线程里，利用RandomAccessFile的seek功能，将读取指针定位到原文件里每一段的段首位置，然后向后读取指定大小（即：分割块大小），最终写入对应的分割文件，因为多线程并行处理，各写各的小文件，速度相对还是比较快的。

合并时，把上面的思路逆向处理即可。

核心代码：

分割处理：

![ExpandedBlockStart.gif][]

![复制代码][copycode.gif]

1 /**
     2      * 拆分文件
     3      * @param fileName 待拆分的完整文件名
     4      * @param byteSize 按多少字节大小拆分
     5      * @return 拆分后的文件名列表
     6      * @throws IOException
     7      */
     8     public List<String> splitBySize(String fileName, int byteSize)
     9             throws IOException {
    10         List<String> parts = new ArrayList<String>();
    11         File file = new File(fileName);
    12         int count = (int) Math.ceil(file.length() / (double) byteSize);
    13         int countLen = (count + "").length();
    14         ThreadPoolExecutor threadPool = new ThreadPoolExecutor(count,
    15                 count * 3, 1, TimeUnit.SECONDS,
    16                 new ArrayBlockingQueue<Runnable>(count * 2));
    17 
    18         for (int i = 0; i < count; i++) {
    19             String partFileName = file.getName() + "."
    20                     + leftPad((i + 1) + "", countLen, '0') + ".part";
    21             threadPool.execute(new SplitRunnable(byteSize, i * byteSize,
    22                     partFileName, file));
    23             parts.add(partFileName);
    24         }
    25         return parts;
    26     }

![复制代码][copycode.gif]

![ExpandedBlockStart.gif][]

![复制代码][copycode.gif]

1 private class SplitRunnable implements Runnable {
     2         int byteSize;
     3         String partFileName;
     4         File originFile;
     5         int startPos;
     6 
     7         public SplitRunnable(int byteSize, int startPos, String partFileName,
     8                 File originFile) {
     9             this.startPos = startPos;
    10             this.byteSize = byteSize;
    11             this.partFileName = partFileName;
    12             this.originFile = originFile;
    13         }
    14 
    15         public void run() {
    16             RandomAccessFile rFile;
    17             OutputStream os;
    18             try {
    19                 rFile = new RandomAccessFile(originFile, "r");
    20                 byte[] b = new byte[byteSize];
    21                 rFile.seek(startPos);// 移动指针到每“段”开头
    22                 int s = rFile.read(b);
    23                 os = new FileOutputStream(partFileName);
    24                 os.write(b, 0, s);
    25                 os.flush();
    26                 os.close();
    27             } catch (IOException e) {
    28                 e.printStackTrace();
    29             }
    30         }
    31     }

![复制代码][copycode.gif]

合并处理：

![ExpandedBlockStart.gif][]

![复制代码][copycode.gif]

1 /**
     2      * 合并文件
     3      * 
     4      * @param dirPath 拆分文件所在目录名
     5      * @param partFileSuffix 拆分文件后缀名
     6      * @param partFileSize 拆分文件的字节数大小
     7      * @param mergeFileName 合并后的文件名
     8      * @throws IOException
     9      */
    10     public void mergePartFiles(String dirPath, String partFileSuffix,
    11             int partFileSize, String mergeFileName) throws IOException {
    12         ArrayList<File> partFiles = FileUtil.getDirFiles(dirPath,
    13                 partFileSuffix);
    14         Collections.sort(partFiles, new FileComparator());
    15 
    16         RandomAccessFile randomAccessFile = new RandomAccessFile(mergeFileName,
    17                 "rw");
    18         randomAccessFile.setLength(partFileSize * (partFiles.size() - 1)
    19                 + partFiles.get(partFiles.size() - 1).length());
    20         randomAccessFile.close();
    21 
    22         ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
    23                 partFiles.size(), partFiles.size() * 3, 1, TimeUnit.SECONDS,
    24                 new ArrayBlockingQueue<Runnable>(partFiles.size() * 2));
    25 
    26         for (int i = 0; i < partFiles.size(); i++) {
    27             threadPool.execute(new MergeRunnable(i * partFileSize,
    28                     mergeFileName, partFiles.get(i)));
    29         }
    30 
    31     }

![复制代码][copycode.gif]

![ExpandedBlockStart.gif][]

![复制代码][copycode.gif]

1 private class MergeRunnable implements Runnable {
     2         long startPos;
     3         String mergeFileName;
     4         File partFile;
     5 
     6         public MergeRunnable(long startPos, String mergeFileName, File partFile) {
     7             this.startPos = startPos;
     8             this.mergeFileName = mergeFileName;
     9             this.partFile = partFile;
    10         }
    11 
    12         public void run() {
    13             RandomAccessFile rFile;
    14             try {
    15                 rFile = new RandomAccessFile(mergeFileName, "rw");
    16                 rFile.seek(startPos);
    17                 FileInputStream fs = new FileInputStream(partFile);
    18                 byte[] b = new byte[fs.available()];
    19                 fs.read(b);
    20                 fs.close();
    21                 rFile.write(b);
    22                 rFile.close();
    23             } catch (IOException e) {
    24                 e.printStackTrace();
    25             }
    26         }
    27     }

![复制代码][copycode.gif]

为了方便文件操作，把关于文件读写的功能，全封装到FileUtil类：

![ExpandedBlockStart.gif][]

![复制代码][copycode.gif]

1 package com.cnblogs.yjmyzz;
      2 
      3 import java.io.*;
      4 import java.util.*;
      5 import java.util.concurrent.*;
      6 
      7 /**
      8  * 文件处理辅助类
      9  * 
     10  * @author yjmyzz@126.com
     11  * @version 0.2
     12  * @since 2014-11-17
     13  *
     14  */
     15 public class FileUtil {
     16 
     17     /**
     18      * 当前目录路径
     19      */
     20     public static String currentWorkDir = System.getProperty("user.dir") + "\\";
     21 
     22     /**
     23      * 左填充
     24      * 
     25      * @param str
     26      * @param length
     27      * @param ch
     28      * @return
     29      */
     30     public static String leftPad(String str, int length, char ch) {
     31         if (str.length() >= length) {
     32             return str;
     33         }
     34         char[] chs = new char[length];
     35         Arrays.fill(chs, ch);
     36         char[] src = str.toCharArray();
     37         System.arraycopy(src, 0, chs, length - src.length, src.length);
     38         return new String(chs);
     39 
     40     }
     41 
     42     /**
     43      * 删除文件
     44      * 
     45      * @param fileName
     46      *            待删除的完整文件名
     47      * @return
     48      */
     49     public static boolean delete(String fileName) {
     50         boolean result = false;
     51         File f = new File(fileName);
     52         if (f.exists()) {
     53             result = f.delete();
     54 
     55         } else {
     56             result = true;
     57         }
     58         return result;
     59     }
     60 
     61     /***
     62      * 递归获取指定目录下的所有的文件（不包括文件夹）
     63      * 
     64      * @param obj
     65      * @return
     66      */
     67     public static ArrayList<File> getAllFiles(String dirPath) {
     68         File dir = new File(dirPath);
     69 
     70         ArrayList<File> files = new ArrayList<File>();
     71 
     72         if (dir.isDirectory()) {
     73             File[] fileArr = dir.listFiles();
     74             for (int i = 0; i < fileArr.length; i++) {
     75                 File f = fileArr[i];
     76                 if (f.isFile()) {
     77                     files.add(f);
     78                 } else {
     79                     files.addAll(getAllFiles(f.getPath()));
     80                 }
     81             }
     82         }
     83         return files;
     84     }
     85 
     86     /**
     87      * 获取指定目录下的所有文件(不包括子文件夹)
     88      * 
     89      * @param dirPath
     90      * @return
     91      */
     92     public static ArrayList<File> getDirFiles(String dirPath) {
     93         File path = new File(dirPath);
     94         File[] fileArr = path.listFiles();
     95         ArrayList<File> files = new ArrayList<File>();
     96 
     97         for (File f : fileArr) {
     98             if (f.isFile()) {
     99                 files.add(f);
    100             }
    101         }
    102         return files;
    103     }
    104 
    105     /**
    106      * 获取指定目录下特定文件后缀名的文件列表(不包括子文件夹)
    107      * 
    108      * @param dirPath
    109      *            目录路径
    110      * @param suffix
    111      *            文件后缀
    112      * @return
    113      */
    114     public static ArrayList<File> getDirFiles(String dirPath,
    115             final String suffix) {
    116         File path = new File(dirPath);
    117         File[] fileArr = path.listFiles(new FilenameFilter() {
    118             public boolean accept(File dir, String name) {
    119                 String lowerName = name.toLowerCase();
    120                 String lowerSuffix = suffix.toLowerCase();
    121                 if (lowerName.endsWith(lowerSuffix)) {
    122                     return true;
    123                 }
    124                 return false;
    125             }
    126 
    127         });
    128         ArrayList<File> files = new ArrayList<File>();
    129 
    130         for (File f : fileArr) {
    131             if (f.isFile()) {
    132                 files.add(f);
    133             }
    134         }
    135         return files;
    136     }
    137 
    138     /**
    139      * 读取文件内容
    140      * 
    141      * @param fileName
    142      *            待读取的完整文件名
    143      * @return 文件内容
    144      * @throws IOException
    145      */
    146     public static String read(String fileName) throws IOException {
    147         File f = new File(fileName);
    148         FileInputStream fs = new FileInputStream(f);
    149         String result = null;
    150         byte[] b = new byte[fs.available()];
    151         fs.read(b);
    152         fs.close();
    153         result = new String(b);
    154         return result;
    155     }
    156 
    157     /**
    158      * 写文件
    159      * 
    160      * @param fileName
    161      *            目标文件名
    162      * @param fileContent
    163      *            写入的内容
    164      * @return
    165      * @throws IOException
    166      */
    167     public static boolean write(String fileName, String fileContent)
    168             throws IOException {
    169         boolean result = false;
    170         File f = new File(fileName);
    171         FileOutputStream fs = new FileOutputStream(f);
    172         byte[] b = fileContent.getBytes();
    173         fs.write(b);
    174         fs.flush();
    175         fs.close();
    176         result = true;
    177         return result;
    178     }
    179 
    180     /**
    181      * 追加内容到指定文件
    182      * 
    183      * @param fileName
    184      * @param fileContent
    185      * @return
    186      * @throws IOException
    187      */
    188     public static boolean append(String fileName, String fileContent)
    189             throws IOException {
    190         boolean result = false;
    191         File f = new File(fileName);
    192         if (f.exists()) {
    193             RandomAccessFile rFile = new RandomAccessFile(f, "rw");
    194             byte[] b = fileContent.getBytes();
    195             long originLen = f.length();
    196             rFile.setLength(originLen + b.length);
    197             rFile.seek(originLen);
    198             rFile.write(b);
    199             rFile.close();
    200         }
    201         result = true;
    202         return result;
    203     }
    204 
    205     /**
    206      * 拆分文件
    207      * 
    208      * @param fileName
    209      *            待拆分的完整文件名
    210      * @param byteSize
    211      *            按多少字节大小拆分
    212      * @return 拆分后的文件名列表
    213      * @throws IOException
    214      */
    215     public List<String> splitBySize(String fileName, int byteSize)
    216             throws IOException {
    217         List<String> parts = new ArrayList<String>();
    218         File file = new File(fileName);
    219         int count = (int) Math.ceil(file.length() / (double) byteSize);
    220         int countLen = (count + "").length();
    221         ThreadPoolExecutor threadPool = new ThreadPoolExecutor(count,
    222                 count * 3, 1, TimeUnit.SECONDS,
    223                 new ArrayBlockingQueue<Runnable>(count * 2));
    224 
    225         for (int i = 0; i < count; i++) {
    226             String partFileName = file.getName() + "."
    227                     + leftPad((i + 1) + "", countLen, '0') + ".part";
    228             threadPool.execute(new SplitRunnable(byteSize, i * byteSize,
    229                     partFileName, file));
    230             parts.add(partFileName);
    231         }
    232         return parts;
    233     }
    234 
    235     /**
    236      * 合并文件
    237      * 
    238      * @param dirPath
    239      *            拆分文件所在目录名
    240      * @param partFileSuffix
    241      *            拆分文件后缀名
    242      * @param partFileSize
    243      *            拆分文件的字节数大小
    244      * @param mergeFileName
    245      *            合并后的文件名
    246      * @throws IOException
    247      */
    248     public void mergePartFiles(String dirPath, String partFileSuffix,
    249             int partFileSize, String mergeFileName) throws IOException {
    250         ArrayList<File> partFiles = FileUtil.getDirFiles(dirPath,
    251                 partFileSuffix);
    252         Collections.sort(partFiles, new FileComparator());
    253 
    254         RandomAccessFile randomAccessFile = new RandomAccessFile(mergeFileName,
    255                 "rw");
    256         randomAccessFile.setLength(partFileSize * (partFiles.size() - 1)
    257                 + partFiles.get(partFiles.size() - 1).length());
    258         randomAccessFile.close();
    259 
    260         ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
    261                 partFiles.size(), partFiles.size() * 3, 1, TimeUnit.SECONDS,
    262                 new ArrayBlockingQueue<Runnable>(partFiles.size() * 2));
    263 
    264         for (int i = 0; i < partFiles.size(); i++) {
    265             threadPool.execute(new MergeRunnable(i * partFileSize,
    266                     mergeFileName, partFiles.get(i)));
    267         }
    268 
    269     }
    270 
    271     /**
    272      * 根据文件名，比较文件
    273      * 
    274      * @author yjmyzz@126.com
    275      *
    276      */
    277     private class FileComparator implements Comparator<File> {
    278         public int compare(File o1, File o2) {
    279             return o1.getName().compareToIgnoreCase(o2.getName());
    280         }
    281     }
    282 
    283     /**
    284      * 分割处理Runnable
    285      * 
    286      * @author yjmyzz@126.com
    287      *
    288      */
    289     private class SplitRunnable implements Runnable {
    290         int byteSize;
    291         String partFileName;
    292         File originFile;
    293         int startPos;
    294 
    295         public SplitRunnable(int byteSize, int startPos, String partFileName,
    296                 File originFile) {
    297             this.startPos = startPos;
    298             this.byteSize = byteSize;
    299             this.partFileName = partFileName;
    300             this.originFile = originFile;
    301         }
    302 
    303         public void run() {
    304             RandomAccessFile rFile;
    305             OutputStream os;
    306             try {
    307                 rFile = new RandomAccessFile(originFile, "r");
    308                 byte[] b = new byte[byteSize];
    309                 rFile.seek(startPos);// 移动指针到每“段”开头
    310                 int s = rFile.read(b);
    311                 os = new FileOutputStream(partFileName);
    312                 os.write(b, 0, s);
    313                 os.flush();
    314                 os.close();
    315             } catch (IOException e) {
    316                 e.printStackTrace();
    317             }
    318         }
    319     }
    320 
    321     /**
    322      * 合并处理Runnable
    323      * 
    324      * @author yjmyzz@126.com
    325      *
    326      */
    327     private class MergeRunnable implements Runnable {
    328         long startPos;
    329         String mergeFileName;
    330         File partFile;
    331 
    332         public MergeRunnable(long startPos, String mergeFileName, File partFile) {
    333             this.startPos = startPos;
    334             this.mergeFileName = mergeFileName;
    335             this.partFile = partFile;
    336         }
    337 
    338         public void run() {
    339             RandomAccessFile rFile;
    340             try {
    341                 rFile = new RandomAccessFile(mergeFileName, "rw");
    342                 rFile.seek(startPos);
    343                 FileInputStream fs = new FileInputStream(partFile);
    344                 byte[] b = new byte[fs.available()];
    345                 fs.read(b);
    346                 fs.close();
    347                 rFile.write(b);
    348                 rFile.close();
    349             } catch (IOException e) {
    350                 e.printStackTrace();
    351             }
    352         }
    353     }
    354 
    355 }

![复制代码][copycode.gif]

单元测试：

![ExpandedBlockStart.gif][]

![复制代码][copycode.gif]

1 package com.cnblogs.yjmyzz;
     2 
     3 import java.io.IOException;
     4 
     5 import org.junit.Test;
     6 
     7 public class FileTest {
     8 
     9     @Test
    10     public void writeFile() throws IOException, InterruptedException {
    11 
    12         System.out.println(FileUtil.currentWorkDir);
    13 
    14         StringBuilder sb = new StringBuilder();
    15 
    16         long originFileSize = 1024 * 1024 * 100;// 100M
    17         int blockFileSize = 1024 * 1024 * 15;// 15M
    18 
    19         // 生成一个大文件
    20         for (int i = 0; i < originFileSize; i++) {
    21             sb.append("A");
    22         }
    23 
    24         String fileName = FileUtil.currentWorkDir + "origin.myfile";
    25         System.out.println(fileName);
    26         System.out.println(FileUtil.write(fileName, sb.toString()));
    27 
    28         // 追加内容
    29         sb.setLength(0);
    30         sb.append("0123456789");
    31         FileUtil.append(fileName, sb.toString());
    32 
    33         FileUtil fileUtil = new FileUtil();
    34 
    35         // 将origin.myfile拆分
    36         fileUtil.splitBySize(fileName, blockFileSize);
    37 
    38         Thread.sleep(10000);// 稍等10秒，等前面的小文件全都写完
    39 
    40         // 合并成新文件
    41         fileUtil.mergePartFiles(FileUtil.currentWorkDir, ".part",
    42                 blockFileSize, FileUtil.currentWorkDir + "new.myfile");
    43 
    44     }
    45 }

![复制代码][copycode.gif]

![171507275665789.jpg][]

[171443005042853.jpg]: /images/20220801/b28a7197973b477b90f74be084a91bd3.png
[ExpandedBlockStart.gif]: /images/20220801/d5e42c9911a04a1184766adcff9b81b8.png
[copycode.gif]: /images/20220801/3e6eb798445b4e00b1c62686eb2ca713.png
[171507275665789.jpg]: /images/20220801/d380f890290246eb980427fbb42fa810.png