node.js - 对将近2000个网页发起同时20个的并发请求，出去request请求失败

代码不长，如下,爬取的是雪中悍刀行的所有章节内容

var http = require('http');
var $ = require('cheerio');
var async = require('async');
var iconv = require('iconv-lite');
var fs = require('fs');
var chapterNo = 1;

var url = 'http://www.biquku.com/0/761/',
  hrefList = {};

var curCount = 0;

var getChapter = function(url, cb) {
  ++ curCount;

  console.log('读取:' + url + '中, 同时有' + curCount + '并发中');

  var req = http.request(url, function(res) {
    var buffer_arr = [];
    var buffer_len = 0;
    if (res.statusCode == 200) {
      res.on('data', function(chunk) {
        buffer_arr.push(chunk);
        buffer_len += chunk.length;
      });
      res.on('end', function() {
        var $content = $(iconv.decode(Buffer.concat(buffer_arr, buffer_len), 'gbk')).find('#content').text();
        -- curCount;
        cb(null , $content);
      })
    } else {
      console.log("status: "  + res.statusCode);
      getChapter(url, cb);
    }
  });

  req.on('error', function(err) {
    console.log('request-err');
    console.error(err);
  })

  req.end();
}

var req = http.request(url, function(res) {
  var buffer_arr = [];
  var buffer_len = 0;
  res.on('data', function(chunk) {
    buffer_arr.push(chunk);
    buffer_len += chunk.length;
  });
  res.on('end', function() {
    var $html = $(iconv.decode(Buffer.concat(buffer_arr, buffer_len), 'gbk'));
    var $urls = $html.find('#list>dl>dd>a');
    var $a = '';

    for (var i = 0; i < $urls.length; i++) {
      $a = $($urls[i]);

      hrefList[$a.text()] = (function(url) {
        return function(cb) {
          setTimeout(function() {
            getChapter(url, cb);
          }, 0)
        }
      })(url.concat($a.attr('href')));
    }

    console.time('novel');

    async.parallelLimit(hrefList, 20, function(err, res) {
      if (err) {
        console.log("parallel-err:");
        console.error(err);
      } else {
        for (var key of Object.keys(res)) {
          var fileName = './' + key + '.txt';
          (function(key){
            fs.writeFile(fileName, res[key], function(err) {
              if (err) {
                console.log('writefile-err:');
                console.error(err);
              } else {
                console.log(key + ': success');
              }
            })
          })(key)
        }
        console.timeEnd('novel');
      }
    })

  })
})

req.on('error', function(e) {
  console.error(e);
})

req.end();

然后下面是请求报错的地方

希望大家能告诉我下，怎么解决呢？还有我这个算并发请求吗，因为我是通过setTimeout(func,0)这样来发起发起http请求的？
经过后来测试发现，200多张能成功读取，到了300就会发生错误了。。

大家讲道理2785 天前520

node.js - 对将近2000个网页发起同时20个的并发请求，出去request请求失败

全部回覆(1)我來回復