Python implements video crawling

What can Python be used for? The company mainly crawls data and analyzes and mines the crawled data. However, we can use it to crawl some resources ourselves, such as the dramas we want to watch. In this article, the editor will share the code for crawling videos. Save it and try it!

To download the streaming file, just set the requested stream in the requests library to True. The document is here.

First find a video address to try:

# -*- coding: utf-8 -*-
import requests
def download_file(url, path):
    with requests.get(url, stream=True) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
if __name__ == '__main__':
    url = '就在原帖...'
    path = '想存哪都行'
    download_file(url, path)

Encountered a blow:

AttributeError: __exit__

This Can documents also lie?

It seems that the __exit__ method required by the context is not implemented. Since it is just to ensure that r is finally closed to release the connection pool, then use the closing feature of contextlib:

# -*- coding: utf-8 -*-
import requests
from contextlib import closing
def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)

The program runs normally, but I stare at this Why doesn’t the size of the file change? How complete is it? I still want to save the downloaded content to the hard disk in time, and save some memory, right?

# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import os
def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                f.flush()
                os.fsync(f.fileno())

The file is growing at a speed visible to the naked eye, I really feel sorry for me The hard disk should be written to the hard disk for the last time. Just record the number in the program:

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            n = 1
            for chunk in r.iter_content(chunk_size=chunk_size):
                loaded = n*1024.0/content_size
                f.write(chunk)
                print '已下载{0:%}'.format(loaded)
                n += 1

The result is very intuitive:

已下载2.579129%
已下载2.581255%
已下载2.583382%
已下载2.585508%

How could I, who has lofty ideals, only be satisfied with this one? Let’s write a class and use it together:

# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import time
def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024*10
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            p = ProgressData(size = content_size, unit='Kb', block=chunk_size)
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                p.output()
class ProgressData(object):
    def __init__(self, block,size, unit, file_name='', ):
        self.file_name = file_name
        self.block = block/1000.0
        self.size = size/1000.0
        self.unit = unit
        self.count = 0
        self.start = time.time()
    def output(self):
        self.end = time.time()
        self.count += 1
        speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
        self.start = time.time()
        loaded = self.count*self.block
        progress = round(loaded/self.size, 4)
        if loaded >= self.size:
            print u'%s下载完成\r\n'%self.file_name
        else:
            print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} 下载速度{5:.2%} {6:.2f}{7}/s'.\
                  format(self.file_name, loaded, self.unit,\
                  self.size, self.unit, progress, speed, self.unit)
            print '%50s'%('/'*int((1-progress)*50))

Run:

下载开始
下载进度10.24Kb/120174.05Kb 0.01% 下载速度4.75Kb/s 
///////////////////////////////////////////////// 
下载进度20.48Kb/120174.05Kb 0.02% 下载速度32.93Kb/s 
/////////////////////////////////////////////////

Looks much more comfortable.

The next thing to do is to download multiple threads at the same time. The main thread produces the url and puts it in the queue, and the download thread gets the url:

# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import time
import Queue
import hashlib
import threading
import os
def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024*10
        content_size = int(r.headers['content-length'])
        if os.path.exists(path) and os.path.getsize(path)>=content_size:
            print '已下载'
            return
        print '下载开始'
        with open(path, "wb") as f:
            p = ProgressData(size = content_size, unit='Kb', block=chunk_size, file_name=path)
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                p.output()

class ProgressData(object):
    def __init__(self, block,size, unit, file_name='', ):
        self.file_name = file_name
        self.block = block/1000.0
        self.size = size/1000.0
        self.unit = unit
        self.count = 0
        self.start = time.time()
    def output(self):
        self.end = time.time()
        self.count += 1
        speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
        self.start = time.time()
        loaded = self.count*self.block
        progress = round(loaded/self.size, 4)
        if loaded >= self.size:
            print u'%s下载完成\r\n'%self.file_name
        else:
            print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} {5:.2%} 下载速度{6:.2f}{7}/s'.\
                  format(self.file_name, loaded, self.unit,\
                  self.size, self.unit, progress, speed, self.unit)
            print '%50s'%('/'*int((1-progress)*50))

queue = Queue.Queue()
def run():
    while True:
        url = queue.get(timeout=100)
        if url is None:
            print u'全下完啦'
            break
        h = hashlib.md5()
        h.update(url)
        name = h.hexdigest()
        path = 'e:/download/' + name + '.mp4'
        download_file(url, path)
def get_url():
    queue.put(None)

if __name__ == '__main__':
    get_url()
    for i in xrange(4):
        t = threading.Thread(target=run)
        t.daemon = True
        t.start()

Added repeated downloads Judgment, as for how to continuously produce URLs, you can explore and take care of yourself!

[Recommended course: Python video tutorial]

The above is the detailed content of Python implements video crawling. For more information, please follow other related articles on the PHP Chinese website!