(lehu6.vip官网直营平台) 对将近2000个网页发起同时20个的并发请求,出去request请求失败

代码不长,如下,爬取的是 雪中悍刀行的所有章节内容

var http = requirehttp;
var $ = requirecheerio;
var async = requireasync;
var iconv = requireiconv-lite;
var fs = requirefs;
var chapterNo = 1; var url = http://www.biquku.com/0/761/, hrefList = {}; var curCount = 0; var getChapter = functionurl, cb { ++ curCount; console.log读取: + url + 中, 同时有 + curCount + 并发中; var req = http.requesturl, functionres { var buffer_arr = []; var buffer_len = 0; if res.statusCode == 200 { res.ondata, functionchunk { buffer_arr.pushchunk; buffer_len += chunk.length; }; res.onend, function { var $content = $iconv.decodeBuffer.concatbuffer_arr, buffer_len, gbk.find#content.text; -- curCount; cbnull , $content; } } else { console.log"status: " + res.statusCode; getChapterurl, cb; } }; req.onerror, functionerr { console.logrequest-err; console.errorerr; } req.end;
} var req = http.requesturl, functionres { var buffer_arr = []; var buffer_len = 0; res.ondata, functionchunk { buffer_arr.pushchunk; buffer_len += chunk.length; }; res.onend, function { var $html = $iconv.decodeBuffer.concatbuffer_arr, buffer_len, gbk; var $urls = $html.find#list>dl>dd>a; var $a = ; for var i = 0; i < $urls.length; i++ { $a = $$urls[i]; hrefList[$a.text] = functionurl { return functioncb { setTimeoutfunction { getChapterurl, cb; }, 0 } }url.concat$a.attrhref; } console.timenovel; async.parallelLimithrefList, 20, functionerr, res { if err { console.log"parallel-err:"; console.errorerr; } else { for var key of Object.keysres { var fileName = ./ + key + .txt; functionkey{ fs.writeFilefileName, res[key], functionerr { if err { console.logwritefile-err:; console.errorerr; } else { console.logkey + : success; } } }key } console.timeEndnovel; } } }
} req.onerror, functione { console.errore;
} req.end;

然后下面是请求报错的地方

希望大家能告诉我下,怎么解决呢?还有我这个算并发请求吗,因为我是通过setTimeoutfunc,0这样来发起发起http请求的?
经过后来测试发现,200多张能成功读取,到了300就会发生错误了。。

你这个确实是并发请求,20个会不会有点多,可能这个服务器扛不住了吧。 调小一点比如10个或5个并发试试呢

更新:刚才在我电脑上跑了下你的这段代码,没有出现问题描述中的错误。大概也许还有个原因是你的网不行吧。

发表评论

电子邮件地址不会被公开。 必填项已用*标注