python - requests爬取代理网站ip并验证，验证通过，但是登陆ip网站查询ip并未变化

改写了网上一个爬取ip代理并验证的代码，验证不是去登陆度娘，而是换了ip查询网站，原以为会用爬取的代理IP，实际上ip并没有变化，烦请大牛看下什么原因，附上运行结果：

#

{'https': u'183.221.50.139:8123'} 您的IP:[218.88.XX.XX] 来自:四川省成都市电信操作系统:Unknown浏览器:Unknown 0.0python-requests/2.4.0 CPython/2.7.3 Windows/7
{'https': u'116.236.216.116:8080'} 您的IP:[218.88.XX.XX] 来自:四川省成都市电信操作系统:Unknown浏览器:Unknown 0.0python-requests/2.4.0 CPython/2.7.3 Windows/7
{'https': u'183.221.160.44:8123'} 您的IP:[218.88.XX.XX] 来自:四川省成都市电信操作系统:Unknown浏览器:Unknown 0.0python-requests/2.4.0 CPython/2.7.3 Windows/7

#

代码如下

import requests
from lxml import etree
from bs4 import BeautifulSoup as bs
import Queue
import threading
import time
import datetime
import sys

reload(sys)
sys.setdefaultencoding('utf-8')

# write proxy
def writeproxy(porxyinfo):
    writefile = file('porxyinfo.txt','a+')
    writefile.write(porxyinfo)
    writefile.write('\n')  
    writefile.close()

# return page code
def GetPageText(url):
    r = requests.get(url)
    return r.text

# return post urllist
def GetPostUrl(source):
    posturllist = []

    iplist = bs(source).find("table",{"id":"ip_list"}).findAll("tr")[1:]
    for item in iplist:
        getinfo = item.findAll("td")
        ip      = getinfo[1].get_text(strip='\r\n')
        port    = getinfo[2].get_text(strip='\r\n')
        address = getinfo[3].get_text(strip='\r\n')
        type    = getinfo[5].get_text(strip='\r\n')
        posturllist.append(type.lower()+'#'+ip+':'+port)
    return posturllist

def Checkproxy(porxyinfo):
    proxies = {}
    if porxyinfo.split('#')[0] == 'http':
        proxies['http'] = porxyinfo.split('#')[1]
    else:
        proxies['https'] = porxyinfo.split('#')[1]
    r = requests.get("http://ip.chinaz.com/", proxies=proxies,timeout=3)
    if r:
        print proxies, bs(requests.get('http://ip.chinaz.com/').content).find("span",{"class":"info3"}).get_text(strip='\r\n')
#         writeproxy(porxyinfo)
    else:
        print 'No'
def getproxyid():
    start = time.time()
    queue = Queue.Queue()
    class ThreadUrl(threading.Thread):
        """Threaded Url Grab"""
        def __init__(self, queue):
            threading.Thread.__init__(self)
            self.queue = queue
            global mutex
        def run(self):
            while True:
                porxyinfo = self.queue.get()
                try:
                    mutex.acquire(5)
                    try:
                        Checkproxy(porxyinfo)
                    except:
                        time.sleep(0.15)
                        mutex.release()
                        self.queue.task_done()
                        continue
                    time.sleep(0.15)
                    mutex.release()

                    self.queue.task_done()
                except Exception,e:
                    time.sleep(0.15)
                    self.queue.task_done()       

    pagenum =5
    targets  = ['http://www.xici.net.co/nn/%d'%page for page in range(1,pagenum+1)]
    targets += ['http://www.xici.net.co/wn/%d'%page for page in range(1,pagenum+1)]     
    for proxyurl in targets:
        try:
            PageText = GetPageText(proxyurl)
        except Exception,e:
            print e
            break
        PostUrlList = GetPostUrl(PageText)

        mutex = threading.Lock()
        for i in range(5):
            t = ThreadUrl(queue)
            t.setDaemon(True)
            try:
                t.start()
            except:
                pass

        for host in PostUrlList:
            queue.put(host)
        queue.join()
    print "Elapsed Time: %s" % (time.time() - start)

if __name__ == '__main__':
    getproxyid()

怪我咯2803 天前908

python - requests爬取代理网站ip并验证，验证通过，但是登陆ip网站查询ip并未变化

#

#

全部回覆(1)我來回復