首页 > 文章 > php教程 > 提取google,baidu等搜索网页内容

提取google,baidu等搜索网页内容

PHP中文网原创: 2016-05-25 17:11:053249浏览

在本地上没什么问题,测试结果在附图里.

1. [代码][PHP]代码

<html>
<head>
<title>Search test</title>

</head>
<body>
<p>
<center>
<form method="get" action="index.php" enctype="utf-8">
<input type="text" name="keywords" size="30">
<input type="hidden" name="pn" value="0"/>
<input type="submit" value="Search"><br/>
<input type="radio" name="engine" value="google" />Google
<input type="radio" name="engine" value="baidu"/>Baidu
<input type="radio" name="engine" value="php"/>php
<input type="radio" name="engine" value="mysql"/>mysql
<input type="radio" name="engine" value="java"/>Java
<input type="radio" name="engine" value="html"/>html
</form>
</center>
</p>


<?php
$keywords=$_GET[&#39;keywords&#39;];
$pn=$_GET[&#39;pn&#39;];
$engine=$_GET[&#39;engine&#39;];
if(empty($keywords))
	die(&#39;Sorry,no keywords.&#39;);
//添加$keyword中加号
$keywords=str_replace(&#39; &#39;,&#39;+&#39;,$keywords);
//支持特殊字符
$keywords=urlencode($keywords);
/**
 *切换搜索引擎
 */
if(($engine==&#39;google&#39;)||(empty($engine))||($engine==&#39;html&#39;)){

header("Content-type:text/html;charset=big5");//加个头
require(&#39;google.sty&#39;);
//temp variables
//$keywords=&#39;google pig&#39;;
//$pn=20;
if(($engine==&#39;html&#39;)){
  $keywords=$keywords.&#39;+site:www.w3schools.com&#39;;
}

$url=&#39;http://www.google.com.hk/search?q=&#39;.$keywords.&#39;&start=&#39;.$pn;
//echo $url;

//$contents=file_get_contents($url);
//try fopen() with google
//$fp=fopen($url,&#39;r&#39;);
//while(!feof($fp)){
//$contents=$contents.fgets($fp,999);}

//try with curl
$contents=get_data($url);

$start=strpos($contents,&#39;<p id="ires">&#39;);
//echo $start.&#39;start<br/>&#39;;
$end=strpos($contents,&#39;<p id="foot"&#39;);
//echo $end.&#39;end&#39;;

$len=$end-$start;
//echo $len.&#39;lenght<br/>&#39;;
$inf=substr($contents,$start,$len);
//过滤掉google伪链接,替换搜索链接,尾部链接
$inf=str_replace(&#39;/url?q=&#39;,&#39;&#39;,$inf);
$inf=str_replace(&#39;search?q=&#39;,&#39;index.php?engine=google&keywords=&#39;,$inf);
//过滤google js带来的无用后缀
//$pattern=&#39;&sa=([a-z0-9_\-]){1}&ei=([a-z0-9_\-]){18,22}&ved=([a-z0-9_\-]){9,13}&usg=([a-z0-9_\-]){34}&#39;;
$pattern2=&#39;&amp;sa=([a-z0-9_\-]){1}&amp;ei=([a-z0-9_\-]){22}&amp;ved=([a-z0-9_\-]){9,13}&amp;usg=([a-z0-9_\-]){34}&#39;;
//test regular expression below
$inf=eregi_replace($pattern2,&#39; &#39;,$inf);
//$write=&#39;test writing ability&#39;;
//file_put_contents(&#39;/home/mageia/search_test&#39;,$inf);
//替换类似内容链接无法打开链接
$inf=str_replace(&#39;/search?&#39;,&#39;http://www.google.com/search?&#39;,$inf);
//替换google对链接的转换
$inf=urldecode($inf);
echo $inf;
$guid=&#39;<p id="page"><a href="?keywords=&#39;.$keywords.&#39;&amp;pn=0&engine=google">1</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=10&engine=google">2</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=20&engine=google">3</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=30&engine=google">4</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=40&engine=google">5</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=50&engine=google">6</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=60&engine=google">7</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=70&engine=google">8</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=80&engine=google">9</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=90&engine=google">10</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=100&engine=google">11</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=110&engine=google">12</a>
        </p>&#39;;
//$currentpage当前所在页
$currentpage=$pn/10+1;
$currentpos=strpos($guid,$currentpage.&#39;<&#39;);
//$guid为最低下的导航页内容
$guid=substr_replace($guid,&#39;O&#39;,$currentpos,($currentpage<10)?1:2);
echo $guid;
}
elseif($engine==&#39;baidu&#39;){

header("Content-type:text/html;charset=utf-8");//加个头
require(&#39;baidu.sty&#39;);
$url=&#39;http://www.baidu.com/s?wd=&#39;.$keywords.&#39;&pn=&#39;.$pn;
//please check the url
//echo $url;
//$contents=file_get_contents($url);

//try with curl
$contents=get_data($url);
//echo $contents;
$start=strpos($contents,&#39;<p id="container">&#39;);
//echo $start.&#39;<br/>&#39;;
//real start position
$start=strpos($contents,&#39;</table>&#39;,$start);
//real end position
$end=strpos($contents,&#39;<p id="page">&#39;,$start);
$len=$end-$start;
//echo $end.&#39;<br/>&#39;;
$inf=substr($contents,$start,$len);
//替换一些失效链接
$inf=str_replace(&#39;/s?wd=&#39;,&#39;index.php?engine=baidu&keywords=&#39;,$inf);
echo $inf;

$guid=&#39;<p id="page"><a href="?keywords=&#39;.$keywords.&#39;&amp;pn=0&engine=baidu">1</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=10&engine=baidu">2</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=20&engine=baidu">3</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=30&engine=baidu">4</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=40&engine=baidu">5</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=50&engine=baidu">6</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=60&engine=baidu">7</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=70&engine=baidu">8</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=80&engine=baidu">9</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=90&engine=baidu">10</a>
        </p>&#39;;
//$currentpage当前所在页
$currentpage=$pn/10+1;
$currentpos=strpos($guid,$currentpage.&#39;<&#39;);
//$guid为最低下的导航页内容
$guid=substr_replace($guid,&#39;O&#39;,$currentpos,($currentpage<10)?1:2);
echo $guid;
}
elseif($engine==&#39;php&#39;){

/*it will only search php functions within http://www.php.cn/ . If the function name you *give does not exist, it returns nothing but a empty page.
*/
require(&#39;php_print.css&#39;);
require(&#39;php_site.css&#39;);
require(&#39;php_mirror.css&#39;);
$url=&#39;http://php.net/search.php?&pattern=&#39;.$keywords.&#39;&show=quickref&#39;;

//try with file_get_contents,failed remote in us server
$phpcontents=file_get_contents($url);

//try with curl,failed local
//$phpcontents=get_data($url);
//echo $phpcontents;
$start=strpos($phpcontents,&#39;<p id="content"&#39;);
//echo $start.&#39;<br/>&#39;;
//actual start position
$start+=8;
//echo $start.&#39;<br/>&#39;;
$start=strpos($phpcontents,&#39;<p id=&#39;,$start);
//echo $start;
$end=strpos($phpcontents,&#39;<p id="footnav">&#39;);
$phpinf=substr($phpcontents,$start,$end-$start);
echo $phpinf;
}
elseif(($engine==&#39;mysql&#39;)||($engine==&#39;java&#39;)){

header("Content-type:text/html;charset=utf-8");//加个头
require(&#39;search-core.sty&#39;);
require(&#39;search.sty&#39;);
require(&#39;results.sty&#39;);
require(&#39;grids-ses.sty&#39;);
require(&#39;tabview-ses.sty&#39;);
require(&#39;treeview-ses.sty&#39;);

//enable page links to work
$pgstart=$pn+1;
$pgend=$start+9;

//url中不能有回车符
if($engine==&#39;mysql&#39;){
$url=&#39;http://search.oracle.com/search/search?&q=&#39;.$keywords.&#39;&group=MySQL&search_startnum=&#39;.$pgstart.&#39;&search_endnum=&#39;.$pgend.&#39;&num=10&#39;;}
else{
$url=&#39;http://search.oracle.com/search/search?search_p_main_operator=all&group=Documentation&q=&#39;.$keywords.&#39;+url:/javase&#39;.&#39;&search_startnum=&#39;.$pgstart.&#39;&search_endnum=&#39;.$pgend.&#39;&num=10&#39;;
}
//echo $url;
//$mysqlcontents=file_get_contents($url);

//try with curl
$mysqlcontents=get_data($url);

$start=strpos($mysqlcontents,&#39;<p id="bd">&#39;);
$end=strpos($mysqlcontents,&#39;<p id="ft">&#39;);
$mysqlinf=substr($mysqlcontents,$start,$end-$start);
echo $mysqlinf;
if($engine==&#39;mysql&#39;){
$guid=&#39;<p id="page"><a href="?keywords=&#39;.$keywords.&#39;&amp;pn=0&engine=mysql">1</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=10&engine=mysql">2</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=20&engine=mysql">3</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=30&engine=mysql">4</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=40&engine=mysql">5</a>
        </p>&#39;;
}else{
$guid=&#39;<p id="page"><a href="?keywords=&#39;.$keywords.&#39;&amp;pn=0&engine=java">1</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=10&engine=java">2</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=20&engine=java">3</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=30&engine=java">4</a>
        <a href="?keywords=&#39;.$keywords.&#39;&amp;pn=40&engine=java">5</a>
        </p>&#39;;
}
 
$currentpage=$pn/10+1;
$currentpos=strpos($guid,$currentpage.&#39;<&#39;);
//$guid为最低下的导航页内容
$guid=substr_replace($guid,&#39;O&#39;,$currentpos,($currentpage<10)?1:2);
echo $guid;
}
else{
echo &#39;Sorry,This search engine will be supported soon.&#39;;
}


/* gets the data from a URL */
function get_data($url) 
{   
  $ch = curl_init();
  $timeout = 5; 
  curl_setopt($ch,CURLOPT_URL,$url);
  curl_setopt($ch,CURLOPT_RETURNTRANSFER,1);
  curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,$timeout);
  $data = curl_exec($ch);
  curl_close($ch);
  return $data;
} 
?>
</body>
</html>