Home  >  Article  >  Backend Development  >  关于抓取网站数据,出现302重定向!

关于抓取网站数据,出现302重定向!

WBOY
WBOYOriginal
2016-06-23 14:00:051472browse

目标网站:http://www.dltm.net/webtmq/free/free_query.php
第一步:输入商标号,提交(post抓起)

第二步:点击商标号

第三步:需要抓取的数据


前两步均已实现抓取,但抓取最后一步时老是出现302重定向,导致数据抓取不到。
相关代码:

//第一步define(TARGET_URL,'http://www.dltm.net/webtmq/free/free_query.php');define(REFFER_URL,'http://www.dltm.net');$url=TARGET_URL;$ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL);$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch);preg_match_all('<input type="hidden" name="ip" value="(.*)">',$result,$rs);//第二步$fields_post = array(	'ip'=>$rs[1][0],	'textarea_explain'=>'%B2%E9%D1%AF%C8%AB%B2%BF%C0%E0%B1%F0',	'tm_lb'=> '0',	'tm_key'=>'8437927',	'tm_key_item'=>'tm_zch',	'query_mode'=>'1');$fields_string='';foreach($fields_post as $key => $value){    $fields_string .= $key . '=' . $value . '&';}$fields_string = rtrim($fields_string,'&');define(TARGET_URL1,'http://www.dltm.net/webtmq/free/free_res.php');define(REFFER_URL1,'http://www.dltm.net/webtmq/free/free_res.php');$url=TARGET_URL1;$ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL1);curl_setopt($ch,CURLOPT_POST,1);//以POST方式提交curl_setopt($ch,CURLOPT_POSTFIELDS,$fields_string);$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch);//第三步preg_match_all('<a href="(.*)" target="detail">',$result,$res);$url = 'http://www.dltm.net/webtmq/free/'.$res[1][0];$ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL1);$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch);print_r($result);exit;//这一步得不到数据

可以自己copy一下代码运行一下,第三步数据得不到,大神能帮我看看么,如果能得到数据,请贴出你的源码,非常感谢!!!


回复讨论(解决方案)

增加
curl_setopt($ch, CURLOPT_COOKIEJAR, "cookie.txt");
curl_setopt($ch, CURLOPT_COOKIEFILE, "cookie.txt");

增加
curl_setopt($ch, CURLOPT_COOKIEJAR, "cookie.txt");
curl_setopt($ch, CURLOPT_COOKIEFILE, "cookie.txt");

不行啊,打印curl_getinfo($ch)

Array ( [url] => http://www.dltm.net/webtmq/free/free_detail.php?ip=7f6179b25de2974b0e0b33d01c2fef18&class_id=25&id=10309774 [content_type] => text/html [http_code] => 302 [header_size] => 440 [request_size] => 193 [filetime] => -1 [ssl_verify_result] => 0 [redirect_count] => 0 [total_time] => 0.094 [namelookup_time] => 0 [connect_time] => 0.047 [pretransfer_time] => 0.047 [size_upload] => 0 [size_download] => 0 [speed_download] => 0 [speed_upload] => 0 [download_content_length] => -1 [upload_content_length] => 0 [starttransfer_time] => 0.094 [redirect_time] => 0 [certinfo] => Array ( ) [redirect_url] => http://www.dltm.net/webtmq/free/free_query.php )

怎么不可以呢?
以注册号 8437927 为例
新建一文件 cookie.txt,执行代码

$cookie = realpath('cookie.txt'); //这是增加的//第一步define('TARGET_URL','http://www.dltm.net/webtmq/free/free_query.php');define('REFFER_URL','http://www.dltm.net');$url=TARGET_URL;$ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL);curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie); //这是增加的curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie); //这是增加的$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch);preg_match_all('<input type="hidden" name="ip" value="(.*)">',$result,$rs); //第二步$fields_post = array(    'ip'=>$rs[1][0],    'textarea_explain'=>'%B2%E9%D1%AF%C8%AB%B2%BF%C0%E0%B1%F0',    'tm_lb'=> '0',    'tm_key'=>'8437927',    'tm_key_item'=>'tm_zch',    'query_mode'=>'1'); $fields_string='';foreach($fields_post as $key => $value){    $fields_string .= $key . '=' . $value . '&';}$fields_string = rtrim($fields_string,'&'); define('TARGET_URL1','http://www.dltm.net/webtmq/free/free_res.php');define('REFFER_URL1','http://www.dltm.net/webtmq/free/free_res.php');$url=TARGET_URL1;$ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL1);curl_setopt($ch,CURLOPT_POST,1);//以POST方式提交curl_setopt($ch,CURLOPT_POSTFIELDS,$fields_string);curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie); //这是增加的curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie); //这是增加的$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch); //第三步preg_match_all('<a href="(.*)" target="detail">',$result,$res);$url = 'http://www.dltm.net/webtmq/free/'.$res[1][0]; $ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL1);curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie); //这是增加的curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie); //这是增加的$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch);print_r($result);exit;

怎么不可以呢?
以注册号 8437927 为例
新建一文件 cookie.txt,执行代码

$cookie = realpath('cookie.txt'); //这是增加的//第一步define('TARGET_URL','http://www.dltm.net/webtmq/free/free_query.php');define('REFFER_URL','http://www.dltm.net');$url=TARGET_URL;$ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL);curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie); //这是增加的curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie); //这是增加的$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch);preg_match_all('<input type="hidden" name="ip" value="(.*)">',$result,$rs); //第二步$fields_post = array(    'ip'=>$rs[1][0],    'textarea_explain'=>'%B2%E9%D1%AF%C8%AB%B2%BF%C0%E0%B1%F0',    'tm_lb'=> '0',    'tm_key'=>'8437927',    'tm_key_item'=>'tm_zch',    'query_mode'=>'1'); $fields_string='';foreach($fields_post as $key => $value){    $fields_string .= $key . '=' . $value . '&';}$fields_string = rtrim($fields_string,'&'); define('TARGET_URL1','http://www.dltm.net/webtmq/free/free_res.php');define('REFFER_URL1','http://www.dltm.net/webtmq/free/free_res.php');$url=TARGET_URL1;$ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL1);curl_setopt($ch,CURLOPT_POST,1);//以POST方式提交curl_setopt($ch,CURLOPT_POSTFIELDS,$fields_string);curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie); //这是增加的curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie); //这是增加的$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch); //第三步preg_match_all('<a href="(.*)" target="detail">',$result,$res);$url = 'http://www.dltm.net/webtmq/free/'.$res[1][0]; $ch=curl_init($url);curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//返回结果存放在变量中,而不是默认的直接输出curl_setopt($ch, CURLOPT_HEADER, 0);curl_setopt($ch, CURLOPT_REFERER, REFFER_URL1);curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie); //这是增加的curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie); //这是增加的$result= curl_exec($ch);//保存输出的页面到$result中curl_close($ch);print_r($result);exit;


谢谢,刚接触curl,cookie位置少放了
问题已解决,结贴!!!!

Statement:
The content of this article is voluntarily contributed by netizens, and the copyright belongs to the original author. This site does not assume corresponding legal responsibility. If you find any content suspected of plagiarism or infringement, please contact admin@php.cn