python中urllib2与BeautifulSoup爬取数据保存MongoDB

python中urllib2与BeautifulSoup爬取数据保存MongoDB

 　　Beautiful Soup是一个用来解析HTML和XML的python库，它可以按照你喜欢的方式去解析文件，查找并修改解析树。它可以很好的处理不规范标记并生成剖析树(parse tree). 它提供简单又常用的导航(navigating)，搜索以及修改剖析树的操作。

　　如图使用urllib2与BS4模块爬取html页面数据，分别为标题、内容、股票名称、股票ID、发布时间、围观人数。

　　Example:

代码如下

##-coding:utf-8-## import time from bs4 import BeautifulSoup import urllib2 import pymongo import re import datetime def update():     datas = {}     connection = pymongo.Connection('192.168.1.2', 27017)  #连接mongodb     db = connection.test_hq  #创建或连接test_hq库     for i in soup.find_all("div", class_="item"):         datas['_id'] = str(i.h2.a['href']).split('/')[-1].split('.')[0]  #获取html页面名称为id号         datas['title'] = i.h2.get_text()  #获取标题         url2 = i.h2.a['href']  #获取标题内容url地址         html2 = urllib2.urlopen(url2)         html_doc2 = html2.read()         soup2 = BeautifulSoup(html_doc2)         datas['content'] = soup2.find(attrs={"name":"description"})['content']  #获取文章内容         stock_name = []         stock_id = []         for name in re.findall(u"[u4e00-u9fa5]+",i.find(class_="stocks").get_text()):             stock_name.append(name)  #获取影响股票名称，已数组方式保存对应股票id号，mongo支持数组插入         datas['stock_name'] = stock_name         for id in re.findall("d+",i.find(class_="stocks").get_text()):             stock_id.append(id)  #获取影响股票id         datas['stock_id'] = stock_id                datas['update_time'] = datetime.datetime.strptime(re.search("w+.*w+", i.find(class_="fl date").span.get_text()).group(), '%Y-%m-%d %H:%M') - datetime.timedelta(hours=8)  #获取发布时间，转换为mongo时间格式         datas['onlooker'] = int(re.search("d+",i.find(class_="icons ic-wg").get_text()).group())  #获取围观数         db.test.save(datas)  #插入数据库 def get_data():          title = str(soup.h2.a['href']).split('/')[-1].split('.')[0]     #获取html页面名称做更新判断     with open('update.txt', 'r') as f:         time = f.readline()     if  title == time:         print 'currently no update', title     else:         with open('update.txt', 'w') as f:             f.write(title)         update()        while True:     if __name__ == '__main__':         url = 'http://www.ipython.me/qingbao/'         html = urllib2.urlopen(url)         html_doc = html.read()         soup = BeautifulSoup(html_doc)         get_data()         time.sleep(30)  #每30秒刷新一次