岩石教程

大文件分片上传

2024-07-14 10:02:28 230
大文件分片上传，附前后端详细代码

整体代码逻辑说明

  1. 前端部分：

    • 文件选择与分片：用户选择文件后，前端将文件按指定大小进行分片。
    • 计算文件哈希值：使用 spark-md5 库逐块计算文件的哈希值，用于唯一标识文件。
    • 检查已上传分片：向服务器请求已上传的分片索引，以便继续上传未完成的分片。
    • 上传分片：依次上传未完成的分片。
    • 合并分片：所有分片上传完成后，通知服务器进行分片合并。

  2. 后端部分：

    • 接收分片：接受上传的文件分片，并将其存储到临时目录中。
    • 记录已上传分片：记录已上传分片的索引，以便断点续传。
    • 合并分片：当所有分片上传完成后，合并所有分片成一个完整文件。

前端实现

<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <title>File Upload</title>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/spark-md5/3.0.0/spark-md5.min.js"></script>
</head>
<body>
  <input type="file" id="file" />
  <button onclick="uploadFile()">Upload</button>

  <script>
    /**
     * 计算文件的哈希值
     * @param {File} file - 需要计算哈希值的文件
     * @param {number} chunkSize - 每次读取的文件块大小，默认为2MB
     * @returns {Promise<string>} - 返回文件的哈希值
     */
    async function calculateHash(file, chunkSize = 2097152) {
      return new Promise((resolve, reject) => {
        const spark = new SparkMD5.ArrayBuffer();
        const fileReader = new FileReader();
        const chunks = Math.ceil(file.size / chunkSize);
        let currentChunk = 0;

        fileReader.onload = (e) => {
          spark.append(e.target.result);
          currentChunk++;

          if (currentChunk < chunks) {
            loadNext();
          } else {
            resolve(spark.end());
          }
        };

        fileReader.onerror = () => {
          reject('File read error');
        };

        function loadNext() {
          const start = currentChunk * chunkSize;
          const end = Math.min(start + chunkSize, file.size);
          fileReader.readAsArrayBuffer(file.slice(start, end));
        }

        loadNext();
      });
    }

    /**
     * 将文件切割成指定大小的分片
     * @param {File} file - 需要切割的文件
     * @param {number} chunkSize - 每个分片的大小，默认为1MB
     * @returns {Array<Object>} - 返回文件分片数组
     */
    function createFileChunks(file, chunkSize = 1024 * 1024) {
      const chunks = [];
      let cur = 0;
      while (cur < file.size) {
        chunks.push({ file: file.slice(cur, cur + chunkSize) });
        cur += chunkSize;
      }
      return chunks;
    }

    /**
     * 检查已上传的分片
     * @param {string} fileHash - 文件的哈希值
     * @param {string} uploadUrl - 上传接口的URL
     * @returns {Promise<Array<number>>} - 返回已上传分片的索引数组
     */
    async function checkUploadedChunks(fileHash, uploadUrl) {
      const response = await fetch(`${uploadUrl}/uploaded/${fileHash}`);
      const uploadedChunks = await response.json();
      return uploadedChunks;
    }

    /**
     * 上传文件分片
     * @param {Array<Object>} chunks - 文件分片数组
     * @param {string} fileHash - 文件的哈希值
     * @param {string} uploadUrl - 上传接口的URL
     * @returns {Promise<void>}
     */
    async function uploadChunks(chunks, fileHash, uploadUrl) {
      const uploadedChunks = await checkUploadedChunks(fileHash, uploadUrl);

      for (let i = 0; i < chunks.length; i++) {
        if (uploadedChunks.includes(i)) continue;

        const formData = new FormData();
        formData.append('chunk', chunks[i].file);
        formData.append('index', i);
        formData.append('total', chunks.length);
        formData.append('fileHash', fileHash);

        await fetch(uploadUrl, {
          method: 'POST',
          body: formData,
        });
      }
    }

    /**
     * 通知服务器合并分片
     * @param {string} uploadUrl - 上传接口的URL
     * @param {string} filename - 文件名
     * @param {string} fileHash - 文件的哈希值
     * @returns {Promise<void>}
     */
    async function mergeChunks(uploadUrl, filename, fileHash) {
      await fetch(`${uploadUrl}/merge`, {
        method: 'POST',
        body: JSON.stringify({ filename, fileHash }),
        headers: {
          'Content-Type': 'application/json'
        },
      });
    }

    /**
     * 文件上传主函数
     * @returns {Promise<void>}
     */
    async function uploadFile() {
      const fileInput = document.getElementById('file');
      const file = fileInput.files[0];
      const uploadUrl = 'http://localhost:3000/upload';

      const fileHash = await calculateHash(file);
      const chunks = createFileChunks(file);

      await uploadChunks(chunks, fileHash, uploadUrl);
      await mergeChunks(uploadUrl, file.name, fileHash);
    }
  </script>
</body>
</html>

Node.js 后端实现

安装依赖

npm init -y
npm install express multer

服务器代码

const express = require('express');
const multer = require('multer');
const fs = require('fs');
const path = require('path');

const app = express();
const upload = multer({ dest: 'uploads/' });

app.use(express.json());

/**
 * 接收文件分片
 */
app.post('/upload', upload.single('chunk'), (req, res) => {
  const { index, total, fileHash } = req.body;
  const { originalname, path: tempPath } = req.file;

  const chunkDir = path.resolve('uploads', fileHash);
  if (!fs.existsSync(chunkDir)) {
    fs.mkdirSync(chunkDir);
  }

  const chunkPath = path.resolve(chunkDir, String(index));
  fs.renameSync(tempPath, chunkPath);

  const uploadedChunksPath = path.resolve(chunkDir, 'uploadedChunks.json');
  let uploadedChunks = [];
  if (fs.existsSync(uploadedChunksPath)) {
    uploadedChunks = JSON.parse(fs.readFileSync(uploadedChunksPath));
  }
  uploadedChunks.push(Number(index));
  fs.writeFileSync(uploadedChunksPath, JSON.stringify(uploadedChunks));

  res.sendStatus(200);
});

/**
 * 获取已上传的分片索引
 */
app.get('/upload/uploaded/:fileHash', (req, res) => {
  const { fileHash } = req.params;
  const chunkDir = path.resolve('uploads', fileHash);
  const uploadedChunksPath = path.resolve(chunkDir, 'uploadedChunks.json');
  if (fs.existsSync(uploadedChunksPath)) {
    const uploadedChunks = JSON.parse(fs.readFileSync(uploadedChunksPath));
    res.json(uploadedChunks);
  } else {
    res.json([]);
  }
});

/**
 * 合并文件分片
 */
app.post('/upload/merge', (req, res) => {
  const { filename, fileHash } = req.body;
  const chunkDir = path.resolve('uploads', fileHash);
  const filePath = path.resolve('uploads', 'final', filename);

  fs.readdir(chunkDir, (err, files) => {
    if (err) {
      return res.sendStatus(500);
    }

    files = files.filter(file => file !== 'uploadedChunks.json').sort((a, b) => Number(a) - Number(b));
    files.forEach(file => {
      fs.appendFileSync(filePath, fs.readFileSync(path.resolve(chunkDir, file)));
      fs.unlinkSync(path.resolve(chunkDir, file));
    });

    fs.rmdirSync(chunkDir);
    res.sendStatus(200);
  });
});

app.listen(3000, () => {
  console.log('Server started on port 3000');
});

Java 后端实现

Maven 配置

在 pom.xml 文件中添加以下依赖：

<dependencies>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-json</artifactId>
    </dependency>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-tomcat</artifactId>
    </dependency>
</dependencies>

服务器代码

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

@SpringBootApplication
@RestController
public class FileUploadApplication {

    private static final String UPLOAD_DIR = "uploads/";

    public static void main(String[] args) {
        SpringApplication.run(FileUploadApplication.class, args);
    }

    @PostMapping("/upload")
    public void uploadChunk(
            @RequestParam("chunk") MultipartFile chunk,
            @RequestParam("index") int index,
            @RequestParam("total") int total,
            @RequestParam("fileHash") String fileHash) throws IOException {
        File chunkDir = new File(UPLOAD_DIR + fileHash);
        if (!chunkDir.exists()) {
            chunkDir.mkdirs();
        }

        File chunkFile = new File(chunkDir, String.valueOf(index));
        chunk.transferTo(chunkFile);

        File uploadedChunksFile = new File(chunkDir, "uploadedChunks.json");
        List<Integer> uploadedChunks = new ArrayList<>();
        if (uploadedChunksFile.exists()) {
            uploadedChunks = Files.readAllLines(uploadedChunksFile.toPath())
                    .stream()
                    .map(Integer::valueOf)
                    .collect(Collectors.toList());
        }
        uploadedChunks.add(index);
        Files.write(uploadedChunksFile.toPath(), uploadedChunks.stream().map(String::valueOf).collect(Collectors.toList()));
    }

    @GetMapping("/upload/uploaded/{fileHash}")
    public List<Integer> getUploadedChunks(@PathVariable String fileHash) throws IOException {
        File chunkDir = new File(UPLOAD_DIR + fileHash);
        File uploadedChunksFile = new File(chunkDir, "uploadedChunks.json");
        if (uploadedChunksFile.exists()) {
            return Files.readAllLines(uploadedChunksFile.toPath())
                    .stream()
                    .map(Integer::valueOf)
                    .collect(Collectors.toList());
        } else {
            return new ArrayList<>();
        }
    }

    @PostMapping("/upload/merge")
    public void mergeChunks(@RequestBody MergeRequest request) throws IOException {
        File chunkDir = new File(UPLOAD_DIR + request.getFileHash());
        File finalDir = new File(UPLOAD_DIR + "final");
        if (!finalDir.exists()) {
            finalDir.mkdirs();
        }

        File finalFile = new File(finalDir, request.getFilename());
        try (FileOutputStream fos = new FileOutputStream(finalFile)) {
            List<File> chunks = Files.list(chunkDir.toPath())
                    .map(java.nio.file.Path::toFile)
                    .filter(f -> !f.getName().equals("uploadedChunks.json"))
                    .sorted((f1, f2) -> Integer.compare(Integer.parseInt(f1.getName()), Integer.parseInt(f2.getName())))
                    .collect(Collectors.toList());

            for (File chunk : chunks) {
                Files.copy(chunk.toPath(), fos);
                chunk.delete();
            }
        }
        Files.deleteIfExists(chunkDir.toPath());
    }

    static class MergeRequest {
        private String filename;
        private String fileHash;

        // Getters and Setters
    }
}