页面抓取程序

跳至

_init_request($request);
		$this->_init_curl();
	}
	protected function _init_var() {
		$this->pageCount = 0;
	}
	protected function _init_request(request $request) {
		$this->request = $request;
		// $this->request->cache_path = __DIR__ . '/sjm_cache/';
		// $this->request->fetch_item_query = '#J_posts_list .subject .title a';
		// $this->request->fetch_page_current = '.J_page_wrap .pages strong';
		// $this->request->base_url = 'http://bbs.sijiaomao.com/index.php?m=bbs&c=thread&fid=10&page=%d';

	}
	protected function _init_curl() {
		$this->curl = new CurlMulti();
		$this->cacheDir = $this->request->cache_path . 'cache';
		if (! is_dir($this->cacheDir)) {
			mkdir($this->cacheDir, 777, true);
		}
		$this->cacheDataDir =  $this->request->cache_path . 'data';
		if (! is_dir($this->cacheDataDir)) {
			mkdir($this->cacheDataDir, 777, true);
		}
		$this->curl->cache = array(
			'dir' => $this->cacheDir,
			'on' => true,
			'expire' => 3600 * 24
		);
		$this->curl->maxThread = 10;
		$this->curl->opt[CURLOPT_CONNECTTIMEOUT] = 10;
	}

	public function fetch_list(){
		$this->_add_fetch_list_url();
		$this->curl->start();
		$this->_save_article_list();
	}
	public function fetch_article() {
		foreach ($this->article_list as $k => $v) {
			$this->curl->add(array(
				'url' => $v['href']
			), array($this, '_success_article'));
		}
		$this->curl->start();
	}
	public function display() {
		printf(
			"\n共抓取%d个页面\n文章列表%d篇\n相关文章%d篇\n文章目录存放在%s\n",
			$this->pageCount + $this->articleCount,
			$this->pageCount,
			count($this->article_list),
			$this->cacheDataDir . '/list.php'
		);
	}
	public function fetch() {
		return sprintf(
			"\n共抓取%d个页面\n文章列表%d篇\n相关文章%d篇\n文章目录存放在%s\n",
			$this->pageCount + $this->articleCount,
			$this->pageCount,
			count($this->article_list),
			$this->cacheDataDir . '/list.php'
		);
	}
	public function _add_fetch_list_url($page = 1){
		$this->curl->add(
			array(
				'url' => sprintf($this->request->base_url, $page),
				'args' => array('page' => $page)
			), 
			array($this, '_success_list')
		);
	}
	protected function _save_article_list() {
		$res = file_put_contents(
			$this->cacheDataDir . '/list.php',
			sprintf("<?php\n return\t%s;",
			var_export($this->article_list, true))
		);
		// 相关性排序整理
		/*uasort($this->article_list, function ($a, $b){
			preg_match_all(&#39;#([a-zA-Z]+)#is&#39;, $a[&#39;title&#39;], $match);
			$a_title = strtoupper(implode("", $match[0]));
			
			preg_match_all(&#39;#([a-zA-Z]+)#is&#39;, $b[&#39;title&#39;], $match);
			$b_title = strtoupper(implode("", $match[0]));
			return $a_title > $b_title;
		});*/
		$res = file_put_contents(
			$this->cacheDataDir . &#39;/list.txt&#39;,
			array_map(function($a_list){
				$str = sprintf(
					"标题：%s\t超链接:%s \n",
					str_replace(" ", "", $a_list[&#39;title&#39;]),
					$a_list[&#39;href&#39;]
				);
				return $str;
			}, $this->article_list)
		);
		return $res;
	}
	public function _success_article($r, $param){
		++$this->articleCount;
	}
	public function _success_list($r, $param){
		++$this->pageCount;
		$html = phpQuery::newDocumentHTML($r[&#39;content&#39;]);
		$list = $html[$this->request->fetch_item_query];
		foreach ($list as $v) {
			$v = pq($v);

			$item = array(
				"title" => $v->attr(&#39;title&#39;) ? $v->attr(&#39;title&#39;) : $v->text(),
				"href" => real_url($v->attr(&#39;href&#39;), $this->request->base_url)
			);
			$this->article_list[md5($item[&#39;href&#39;])] = $item;
		}
		$page_current = $html[$this->request->fetch_page_current];
		if ($page_current->next()->text()) {
			$page = ++ $param[&#39;page&#39;];
			$this->_add_fetch_list_url($page);
		}
		
		phpQuery::unloadDocuments();
	}
}
class request{
	/*url*/
	public $base_url; 
	/*缓存文件路径*/
	public $cache_path;
	/*获取元素的CSS选择器*/
	public $fetch_item_query;
	/*分页当前页面元素的CSS选择器*/
	public $fetch_page_current;

	static $instance;
	static public function getInstance() {
		if (empty(self::$instance)) {
			self::$instance = new self;
		}

		return self::$instance;
	}
	private function __construct() {
		$this->_init_base();
	}
	function _init_base() {
		$this->cache_path = __DIR__ . &#39;/&#39;. trim($_POST[&#39;cache_path&#39;], &#39;/&#39;) .&#39;/&#39;;
		$this->fetch_item_query = $_POST[&#39;fetch_item_query&#39;];
		$this->fetch_page_current = $_POST[&#39;fetch_page_current&#39;];
		$this->base_url = $_POST[&#39;url&#39;];
	}
	function request() {
		if (strstr($_POST[&#39;url&#39;], &#39;?&#39;)) {
			$url = sprintf("%s&auth=%s", $_POST[&#39;url&#39;], $auth);
		} else {
			$url = sprintf("%s?auth=%s", $_POST[&#39;url&#39;], $auth);
		}
		$param = array();
		if (isset($_POST[&#39;param&#39;])) {
			foreach($_POST[&#39;param&#39;] as $k => $item) {
				if (!empty($item[&#39;method&#39;]) && !empty($item[&#39;name&#39;])) {
					$param[$item[&#39;method&#39;]][$item[&#39;name&#39;]] = $item[&#39;value&#39;];
				}
			}
		}
		if (isset($param[&#39;get&#39;]) && !empty($param[&#39;get&#39;])) {
			foreach ($param[&#39;get&#39;] as $name => $value) {
				$url = sprintf("%s&%s=%s", $url, $name, $value);
			}
		}
		$post_data = null;
		if (isset($param[&#39;post&#39;]) && !empty($param[&#39;post&#39;])) {
			$post_data = $param[&#39;post&#39;];
		}
	}
}
?>fetch_list();
	
	//$myCurl->fetch_article();
	myDebug::set_end();
} else {
	$_POST[&#39;url&#39;] = &#39;http://www.oschina.net/code/tag/php?show=time&lang=&catalog=&p=%d&#39;;
	$_POST[&#39;cache_path&#39;] = &#39;oschina&#39;;
	$_POST[&#39;fetch_item_query&#39;] = &#39;.code_list ul li .code_title > a&#39;;
	$_POST[&#39;fetch_page_current&#39;] = &#39;.pager li.current&#39;;
}
?>页面爬虫