• 技术文章 >web前端 >js教程

    Node.js怎么实现分片上传?方法介绍

    青灯夜游青灯夜游2022-07-29 20:22:17转载215
    Node怎么实现分片上传?下面本篇文章给大家介绍一下Node.js实现分片上传的方法,希望对大家有所帮助!

    大文件上传会消耗大量的时间,而且中途有可能上传失败。这时我们需要前端和后端配合来解决这个问题。

    解决步骤:

    文件的分片与合并

    在JavaScript中,FIle对象是' Blob '对象的子类,该对象包含一个重要的方法slice,通过该方法我们可以这样分割二进制文件:

    <!DOCTYPE html>
    <html>
    <head>
        <meta charset="UTF-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>Document</title>
        <script src="https://cdn.bootcdn.net/ajax/libs/axios/0.24.0/axios.min.js"></script>
    </head>
    <body>
        <input type="file" multiple="multiple" id="fileInput" />
        <button onclick="SliceUpload()">上传</button>  
        <script>
            function SliceUpload() {
                const file = document.getElementById('fileInput').files[0]
                if (!file) return
    
                // 文件分片
                let size = 1024 * 50; //50KB 50KB Section size
                let fileChunks = [];
                let index = 0;        //Section num
                for (let cur = 0; cur < file.size; cur += size) {
                    fileChunks.push({
                        hash: index++,
                        chunk: file.slice(cur, cur + size),
                    });
                }
    
                // 上传分片
                const uploadList = fileChunks.map((item, index) => {
                    let formData = new FormData();
                    formData.append("filename", file.name);
                    formData.append("hash", item.hash);
                    formData.append("chunk", item.chunk);
                    return axios({
                        method: "post",
                        url: "/upload",
                        data: formData,
                    });
                });
                await Promise.all(uploadList);
    
                // 所有分片上传完成,通知服务器合并分片
                await axios({
                    method: "get",
                    url: "/merge",
                    params: {
                        filename: file.name,
                    },
                });
                console.log("Upload to complete");
            }
        </script>
    </body>
    </html>

    并发控制

    如果文件很大,这样切分的分片会很多,浏览器短时间内就会发起大量的请求,可能会导致内存耗尽,所以要进行并发控制。

    这里我们结合Promise.race()方法 控制并发请求的数量,避免浏览器内存溢出。

    // 加入并发控制
    async function SliceUpload() {
        const file = document.getElementById('fileInput').files[0]
        if (!file) return
    
        // 文件分片
        let size = 1024 * 50; //50KB 50KB Section size
        let fileChunks = [];
        let index = 0;        //Section num
        for (let cur = 0; cur < file.size; cur += size) {
            fileChunks.push({
                hash: index++,
                chunk: file.slice(cur, cur + size),
            });
        }
    
        let pool = []; //Concurrent pool
        let max = 3; //Maximum concurrency
        for (let i = 0; i < fileChunks.length; i++) {
            let item = fileChunks[i];
            let formData = new FormData();
            formData.append("filename", file.name);
            formData.append("hash", item.hash);
            formData.append("chunk", item.chunk);
    
            // 上传分片
            let task = axios({
                method: "post",
                url: "/upload",
                data: formData,
            });
            task.then(() => {
            // 从并发池中移除已经完成的请求
            let index = pool.findIndex((t) => t === task);
                pool.splice(index);
            });
    
            // 把请求放入并发池中,如果已经达到最大并发量
            pool.push(task);
            if (pool.length === max) {
                //All requests are requested complete
                await Promise.race(pool);
            }
        }
    
        // 所有分片上传完成,通知服务器合并分片
        await axios({
            method: "get",
            url: "/merge",
            params: {
                filename: file.name,
            },
        });
        console.log("Upload to complete");
    }

    使代码可复用

    function SliceUpload() {
        const file = document.getElementById('fileInput').files[0]
        if (!file) return
    
        // 文件分片
        let size = 1024 * 50; // 分片大小设置
        let fileChunks = [];
        let index = 0;        // 分片序号
        for (let cur = 0; cur < file.size; cur += size) {
            fileChunks.push({
                hash: index++,
                chunk: file.slice(cur, cur + size),
            });
        }
    
        const uploadFileChunks = async function(list){
            if(list.length === 0){
                // 所有分片上传完成,通知如无
                await axios({
                    method: 'get',
                    url: '/merge',
                    params: {
                        filename: file.name
                    }
                });
                console.log('Upload to complete')
                return
            }
    
            let pool = []       // 并发池
            let max = 3         // 最大并发数
            let finish = 0      // 完成数量
            let failList = []   // 失败列表
            for(let i=0;i<list.length;i++){
                let item = list[i]
                let formData = new FormData()
                formData.append('filename', file.name)
                formData.append('hash', item.hash)
                formData.append('chunk', item.chunk)
                
                let task = axios({
                    method: 'post',
                    url: '/upload',
                    data: formData
                })
    
                task.then((data)=>{
                    // 从并发池中移除已经完成的请求
                    let index = pool.findIndex(t=> t===task)
                    pool.splice(index)
                }).catch(()=>{
                    failList.push(item)
                }).finally(()=>{
                    finish++
                    // 如果有失败的重新上传
                    if(finish===list.length){
                        uploadFileChunks(failList)
                    }
                })
                pool.push(task)
                if(pool.length === max){
                    await Promise.race(pool)
                }
            }
        }
    
        uploadFileChunks(fileChunks)
    }

    服务端接口实现

    const express = require('express')
    const multiparty = require('multiparty')
    const fs = require('fs')
    const path = require('path')
    const { Buffer } = require('buffer')
    // file path
    const STATIC_FILES = path.join(__dirname, './static/files')
    // Temporary path to upload files
    const STATIC_TEMPORARY = path.join(__dirname, './static/temporary')
    const server = express()
    // Static file hosting
    server.use(express.static(path.join(__dirname, './dist')))
    // Interface for uploading slices
    server.post('/upload', (req, res) => {
        const form = new multiparty.Form();
        form.parse(req, function(err, fields, files) {
            let filename = fields.filename[0]
            let hash = fields.hash[0]
            let chunk = files.chunk[0]
            let dir = `${STATIC_TEMPORARY}/${filename}`
            // console.log(filename, hash, chunk)
            try {
                if (!fs.existsSync(dir)) fs.mkdirSync(dir)
                const buffer = fs.readFileSync(chunk.path)
                const ws = fs.createWriteStream(`${dir}/${hash}`)
                ws.write(buffer)
                ws.close()
                res.send(`${filename}-${hash} Section uploaded successfully`)
            } catch (error) {
                console.error(error)
                res.status(500).send(`${filename}-${hash} Section uploading failed`)
            }
        })
    })
    
    //Merged slice interface
    server.get('/merge', async (req, res) => {
        const { filename } = req.query
        try {
            let len = 0
            const bufferList = fs.readdirSync(`${STATIC_TEMPORARY}/${filename}`).map((hash,index) => {
                const buffer = fs.readFileSync(`${STATIC_TEMPORARY}/${filename}/${index}`)
                len += buffer.length
                return buffer
            });
            //Merge files
            const buffer = Buffer.concat(bufferList, len);
            const ws = fs.createWriteStream(`${STATIC_FILES}/${filename}`)
            ws.write(buffer);
            ws.close();
            res.send(`Section merge completed`);
        } catch (error) {
            console.error(error);
        }
    })
    
    server.listen(3000, _ => {
        console.log('http://localhost:3000/')
    })

    更多node相关知识,请访问:nodejs 教程

    以上就是Node.js怎么实现分片上传?方法介绍的详细内容,更多请关注php中文网其它相关文章!

    声明:本文转载于:segmentfault,如有侵犯,请联系admin@php.cn删除
    专题推荐:分片上传 nodejs node
    上一篇:一文了解JavaScript栈 下一篇:如何上手Angular,先从 8 个开源项目开始!
    VIP课程(WEB全栈开发)

    相关文章推荐

    • 【活动】充值PHP中文网VIP即送云服务器• node中的stream(流)有几种类型• 火了!新的JavaScript运行时:Bun,性能完爆Node• 聊聊Node.js中的进程、线程、协程与并发模型• 深入聊聊Node 异步和事件循环的底层实现和执行机制• 聊聊Node.js中的多进程和多线程
    1/1

    PHP中文网