刚接触node,想写个爬虫,爬取新浪微博评论,但是发现页面是由JS动态生成的,用http模块爬取不到,然后用phantomjs爬取,(听说会比较慢,运行了近15分钟,这也太慢了,不知是否写错),还是不行,问一下有什么办法能够爬取类似于新浪微博的网页吗?
let page=require("webpage").create();
let url="http://weibo.com/1713926427/Etq2WnSiR?filter=hot&root_comment_id=0&type=comment";
/*page.settings = {
javascriptEnabled: true,
loadImages: false,
webSecurityEnabled: false,
userAgent: 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36 LBBROWSER'
};*/
page.open(url,(status)=>{
console.log("Status:"+status);
if(status=="success"){
let val = page.evaluate(()=>{
var list_box=document.querySelector(".list_box");
console.log(list_box);
return list_box
});
console.log(val)
}else{
console.log("failed")
}
phantom.exit();
});