Node.JS同时下载数百个文件

Question

我试图在同一时间下载更多的100个文件。但是，当我执行下载功能，我的MacBook冻结（无法执行新的任务）在Windows中也没有下载（但不冻结），并在这两种情况下（空闲网络）没有下载进度。

这里是我的下载模块：

var express = require('express'); 
var router = express.Router(); 
var fs = require('fs'); 
var youtubedl = require('youtube-dl'); 
var links = require('../models/Links'); 

router.get('/', function (req, res, next) { 
    links.find({dlStatus: false}, function (err, docs) { 

     if (err) { 
      console.log(err); 
      res.end(); 
     } else if (!docs) { 
      console.log('No incomplete downloads!'); 
      res.end(); 
     } else { 
      for (var i = 0; i < docs.length; i++) { 

       //todo scraping 


       var video = youtubedl(docs[i].url, [], {cwd: __dirname}); 

       // Will be called when the download starts. 
       video.on('info', function (info) { 
        console.log('Download started'); 
        console.log(info); 
       }); 

       video.pipe(fs.createWriteStream('./downloads/' + docs[i].id + '-' + i + '.mp4')); 
       video.on('complete', function complete(info) { 
        links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) { 
         if (err)console.log(err); 
         else console.log('Download completed!') 
        }); 
       }); 
      } 
     } 
    }); 
}); 

module.exports = router; 

现在，任何人都可以请帮我在这里？我正在使用this模块下载文件。

Answer 1

在这种情况下，解决方案使用的是async。

试试这样....用async.each()

var express = require('express'); 
var router = express.Router(); 
var fs = require('fs'); 
var youtubedl = require('youtube-dl'); 
var links = require('../models/Links'); 
var async = require('async') 

router.get('/', function (req, res, next) { 
    links.find({dlStatus: false}, function (err, docs) { 

     if (err) { 
      console.log(err); 
      res.end(); 
     } else if (!docs) { 
      console.log('No incomplete downloads!'); 
      res.end(); 
     } else { 
      async.each(docs,function(doc,cb){ 
       var video = youtubedl(doc.url, [], {cwd: __dirname}); 

       // Will be called when the download starts. 
       video.on('info', function (info) { 
        console.log('Download started'); 
        console.log(info); 
       }); 

       video.pipe(fs.createWriteStream('./downloads/' + docs.id + '-' + i + '.mp4')); 
       video.on('complete', function complete(info) { 
        links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) { 
         if (err){ 
         console.log(err); 
         cb(err); 
         } 
         else { 
         console.log('Download completed!'); 
         cb() 
         } 
        }); 
       }); 

      },function(err){ 
       if(err) 
       return console.log(err); 
       console.log("Every thing is done,Here!!"); 
      }) 
     } 
    }); 
}); 

module.exports = router; 

而且你可以以批处理过使用async.eachLimits()每一件事情。