ブログ記事クローラーの Python 実装例

复制代码 代码如下:

#!/usr/bin/python
#-*-coding:utf-8-*-
# JCrawler
# Author: Jam <810441377@qq.com>

import time
import urllib2
from bs4 import BeautifulSoup

# 目标站点
TargetHost = "http://adirectory.blog.com"
# User Agent
UserAgent  = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36'
# 链接采集规则
# 目录链接采集规则
CategoryFind    = [{'findMode':'find','findTag':'div','rule':{'id':'cat-nav'}},
                   {'findMode':'findAll','findTag':'a','rule':{}}]
# 文章链接采集规则
ArticleListFind = [{'findMode':'find','findTag':'div','rule':{'id':'content'}},
                   {'findMode':'findAll','findTag':'h2','rule':{'class':'title'}},
                   {'findMode':'findAll','findTag':'a','rule':{}}]
# 分页URL规则
PageUrl  = 'page/#page/'
PageStart = 1
PageStep  = 1
PageStopHtml = '404: Page Not Found'

def GetHtmlText(url):
    request  = urllib2.Request(url)
    request.add_header('Accept', "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp")
    request.add_header('Accept-Encoding', "*")
    request.add_header('User-Agent', UserAgent)
    return urllib2.urlopen(request).read()

def ArrToStr(varArr):
    returnStr = ""
    for s in varArr:
        returnStr += str(s)
    return returnStr

def GetHtmlFind(htmltext, findRule):
    findReturn = BeautifulSoup(htmltext)
    returnText = ""
    for f in findRule:
        if returnText != "":
            findReturn = BeautifulSoup(returnText)
        if f['findMode'] == 'find':
            findReturn = findReturn.find(f['findTag'], f['rule'])
        if f['findMode'] == 'findAll':
            findReturn = findReturn.findAll(f['findTag'], f['rule'])
        returnText = ArrToStr(findReturn)
    return findReturn

def GetCategory():
    categorys = [];
    htmltext = GetHtmlText(TargetHost)
    findReturn = GetHtmlFind(htmltext, CategoryFind)

    for tag in findReturn:
        print "[G]->Category:" + tag.string + "|Url:" + tag['href']
        categorys.append({'name': tag.string, 'url': tag['href']})
    return categorys;

def GetArticleList(categoryUrl):
article = []
page = PageStart
#pageUrl = PageUrl
while True:
htmltext = ""
pageUrl = PageUrl.replace ("#page", str(page))
print "[G]->PageUrl:" + categoryUrl + pageUrl
while True:
try:
htmltext = GetHtmlText(categoryUrl +ページURL )
Break
urllib2.HTTPError を除く,e:
print "[E]->HTTP Error:" + str(e.code)
if e.code == 404:
htmltext = PageStopHtml
Break
if e.code == 504:
print "[E]-> HTTP エラー 504: ゲートウェイ タイムアウト、待機"
time.sleep(5)
else:
休憩

if htmltext.find(PageStopHtml) >= 0:
print "終了ページ"
Break
else:

findReturn = GetHt mlFind(htmltext, ArticleListFind)

findReturn のタグの場合:
if tag.string != None and tag['href'].find(TargetHost) >= 0:
print "[G]->Article:" + tag.string + "|Url:" + tag['href']
articles.append({'name': tag.string, 'url': tag['href']})

ページ += 1

記事を返す;

print "[G]->GetCategory"
Mycategorys = GetCategory();
print "[G]->GetCategory->Success."
time.sleep(3)
Mycategorys のカテゴリの場合:
print "[G]->GetArticleList:" + category['name']
GetArticleList(category['url'])