Home >Backend Development >Python Tutorial >Python crawler beta version crawls Zhihu single page

Python crawler beta version crawls Zhihu single page

高洛峰
高洛峰Original
2016-12-02 16:51:451724browse

In view of the fact that I used python to write a crawler before and helped operators crawl JD.com’s product brands and categories, this time I also used python to do a simple crawling of a single page version, and will add more later.

#-*- coding: UTF-8 -*- 
import requests
import sys
from bs4 import BeautifulSoup

#------知乎答案收集----------

#获取网页body里的内容
def get_content(url , data = None):
    header={
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.235'
    }

    req = requests.get(url, headers=header)
    req.encoding = 'utf-8'
    bs = BeautifulSoup(req.text, "html.parser")  # 创建BeautifulSoup对象
    body = bs.body # 获取body部分
    return body

#获取问题标题
def get_title(html_text):
     data = html_text.find('span', {'class': 'zm-editable-content'})
     return data.string.encode('utf-8')

#获取问题内容
def get_question_content(html_text):
     data = html_text.find('div', {'class': 'zm-editable-content'})
     if data.string is None:
         out = '';
         for datastring in data.strings:
             out = out + datastring.encode('utf-8')
         print '内容:\n' + out
     else:
         print '内容:\n' + data.string.encode('utf-8')

#获取点赞数
def get_answer_agree(body):
    agree = body.find('span',{'class': 'count'})
    print '点赞数:' + agree.string.encode('utf-8') + '\n'

#获取答案
def get_response(html_text):
     response = html_text.find_all('div', {'class': 'zh-summary summary clearfix'})
     for index in range(len(response)):
         #获取标签
         answerhref = response[index].find('a', {'class': 'toggle-expand'})
         if not(answerhref['href'].startswith('javascript')):
             url = 'http://www.zhihu.com/' + answerhref['href']
             print url
             body = get_content(url)
             get_answer_agree(body)
             answer = body.find('div', {'class': 'zm-editable-content clearfix'})
             if answer.string is None:
                 out = '';
                 for datastring in answer.strings:
                     out = out + '\n' + datastring.encode('utf-8')
                 print out
             else:
                 print answer.string.encode('utf-8')


html_text = get_content('https://www.zhihu.com/question/43879769')
title = get_title(html_text)
print "标题:\n" + title + '\n'
questiondata = get_question_content(html_text)
print '\n'
data = get_response(html_text)

Output result:

Python crawler beta version crawls Zhihu single page

Statement:
The content of this article is voluntarily contributed by netizens, and the copyright belongs to the original author. This site does not assume corresponding legal responsibility. If you find any content suspected of plagiarism or infringement, please contact admin@php.cn