search

Home >Backend Development >PHP Tutorial >Multi-threaded cURL

Multi-threaded cURL

WBOY
WBOYOriginal
2016-07-25 08:49:341003browse
This is a flexible multi-threaded call to cURL.
This is different from the sample provided in the PHP manual http://us2.php.net/manual/zh/function.curl-multi-select.php. The code execution efficiency is much higher
There are two files here, one It is a file of muti_curl, which contains two classes
一个是运用的方法，这里是批量检查代理ip是否可用
  2. class request_setting {
  3. public $url = false;
  4. public $method = 'GET';
  5. public $post_data = null;
  6. public $headers = null;
  7. public $options = null;
  8. function __construct($url, $method = "GET", $post_data = null, $headers = null, $options = null) {
  9. $this->url = $url;
  10. $this->method = $method;
  11. $this->post_data = $post_data;
  12. $this->headers = $headers;
  13. $this->options = $options;
  14. }
  15. public function __destruct() {
  16. unset($this->url, $this->method, $this->post_data, $this->headers, $this->options);
  17. }
  18. }
  19. /***************************************************** *******************************************
  20. Classes for batch operations
  21. * *************************************************** ****************************************/
  22. class muti_curl {
  23. protected $thread_size = 100;
  24. protected $timeout = 30;
  25. private $callback;
  26. protected $options = array(
  27. CURLOPT_SSL_VERIFYPEER => false,//禁用后cURL将终止从服务端进行验证。使用CURLOPT_CAINFO选项设置证书使用CURLOPT_CAPATH选项设置证书目录 如果CURLOPT_SSL_VERIFYPEER(默认值为2)被启用，CURLOPT_SSL_VERIFYHOST需要被设置成TRUE否则设置为FALSE。 自cURL 7.10开始默认为TRUE。从cURL 7.10开始默认绑定安装。
  28. CURLOPT_RETURNTRANSFER => true, //将 curl_exec()获取的信息以文件流的形式返回，而不是直接输
  29. CURLOPT_CONNECTTIMEOUT => 15,
  30. CURLOPT_TIMEOUT => 30,
  31. // CURLOPT_HTTP_VERSION=>CURL_HTTP_VERSION_1_0, //使用代理的时候用这个去抓取数据，更爽
  32. // CURLOPT_AUTOREFERER=>false,// 当根据Location:重定向时，自动设置header中的Referer:信息。
  33. // CURLOPT_BINARYTRANSFER=>false, //在启用CURLOPT_RETURNTRANSFER的时候，返回原生的（Raw）输出这个不用设置。
  34. // CURLOPT_COOKIESESSION=>true,// 启用时curl会仅仅传递一个session cookie，忽略其他的cookie，默认状况下cURL会将所有的cookie返回
  35. // CURLOPT_CRLF=>false,// 启用时将Unix的换行符转换成回车换行符。
  36. // CURLOPT_DNS_USE_GLOBAL_CACHE=>false, // 启用时会启用一个全局的DNS缓存，此项为线程安全的，并且默认启用。
  37. // CURLOPT_FAILONERROR=>false, // 显示HTTP状态码，默认行为是忽略编号小于等于400的HTTP信息。
  38. // CURLOPT_FILETIME=>true, //启用时会尝试修改远程文档中的信息。结果信息会通过 curl_getinfo()函数的CURLINFO_FILETIME选项返回。 curl_getinfo().
  39. // CURLOPT_FOLLOWLOCATION=>false, // 启用时会将服务器服务器返回的"Location: "放在header中递归的返回给服务器，使用CURLOPT_MAXREDIRS可以限定递归返回的数量。
  40. // CURLOPT_FORBID_REUSE=>true, //在完成交互以后强迫断开连接，不能重用。
  41. // CURLOPT_FRESH_CONNECT=>true,// 强制获取一个新的连接，替代缓存中的连接。
  42. // CURLOPT_FTP_USE_EPRT=>false,// 启用时当FTP下载时，使用EPRT (或 LPRT)命令。设置为FALSE时禁用EPRT和LPRT，使用PORT命令 only.
  43. // CURLOPT_FTP_USE_EPSV=>false,// 启用时，在FTP传输过程中回复到PASV模式前首先尝试EPSV命令。设置为FALSE时禁用EPSV命令。
  44. // CURLOPT_FTPAPPEND=>false,// 启用时追加写入文件而不是覆盖它。
  45. // CURLOPT_FTPASCII=>false,// CURLOPT_TRANSFERTEXT的别名。
  46. // CURLOPT_FTPLISTONLY=>false,// 启用时只列出FTP目录的名字。
  47. // CURLOPT_HEADER=>true,// 启用时会将头文件的信息作为数据流输出。
  48. // CURLINFO_HEADER_OUT=>false, //启用时追踪句柄的请求字符串。
  49. // CURLOPT_HTTPGET=>true,// 启用时会设置HTTP的method为GET，因为GET是默认是，所以只在被修改的情况下使用。
  50. // CURLOPT_HTTPPROXYTUNNEL =>true,// 启用时会通过HTTP代理来传输。
  51. // CURLOPT_MUTE=>true,// 启用时将cURL函数中所有修改过的参数恢复默认值。
  52. // CURLOPT_NETRC=>false,// 在连接建立以后，访问~/.netrc文件获取用户名和密码信息连接远程站点。
  53. // CURLOPT_NOBODY=>true, 启用时将不对HTML中的BODY部分进行输出。
  54. // CURLOPT_NOPROGRESS=>false,//启用时关闭curl传输的进度条，此项的默认设置为启用。
  55. // CURLOPT_NOSIGNAL=>false,// 启用时忽略所有的curl传递给php进行的信号。This item is enabled by default during SAPI multi-threaded transmission. Added in cURL 7.10.
  56. // CURLOPT_POST=>false,// When enabled, a regular POST request will be sent, type: application/x-www-form-urlencoded, just like form submission.
  57. // CURLOPT_PUT=>false,// Allow HTTP to send files when enabled, CURLOPT_INFILE and CURLOPT_INFILESIZE must be set at the same time.
  58. // CURLOPT_TRANSFERTEXT=>false,// When enabled, use ASCII mode for FTP transmission. For LDAP, it retrieves plain text information rather than HTML. On Windows systems, the system does not set STDOUT to binary mode.
  59. // CURLOPT_UNRESTRICTED_AUTH=>true,// Continuously append username and password information to multiple locations in the header generated using CURLOPT_FOLLOWLOCATION, even if the domain name has changed.
  60. // CURLOPT_UPLOAD=>false,// Allow file upload when enabled.
  61. // CURLOPT_VERBOSE =>true,// When enabled, all information will be reported and stored in STDERR or the specified CURLOPT_STDERR.
  62. );
  63. private $headers = array();
  64. private $requests = array();
  65. private $requestMap = array();
  66. /*********************
  67. Construct a callback function
  68. ********************/
  69. function __construct($callback = null) {
  70. $this->callback = $callback;
  71. }
  73. /***************************************************** ******************
  74. Overloading the __get method
  75. **************************** *******************************************/
  76. public function __get($name) {
  77. return (isset($this->{$name})) ? $this->{$name} : null;
  78. }
  80. /***************************************************** *******************
  81. Overloading the __set method
  82. ************************* ******************************/
  83. public function __set($name, $value) {
  84. // 增加一个设置到headers
  85. if ($name == "options" || $name == "headers") {
  86. $this->{$name} = $value + $this->{$name};
  87. } else {
  88. $this->{$name} = $value;
  89. }
  90. return true;
  91. }
  92. //增加一个请求
  93. public function add($request) {
  94. $this->requests[] = $request;
  95. return true;
  96. }
  99. public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null) {
  100. $this->requests[] = new request_setting($url, $method, $post_data, $headers, $options);
  101. return true;
  102. }
  104. public function get($url, $headers = null, $options = null) {
  105. return $this->request($url, "GET", null, $headers, $options);
  106. }
  108. public function post($url, $post_data = null, $headers = null, $options = null) {
  109. return $this->request($url, "POST", $post_data, $headers, $options);
  110. }
  112. private function single_curl() {
  113. $ch = curl_init(); //初始化
  114. $request = array_shift($this->requests);//把第一个单元移出并作为结果
  115. $options = $this->get_options($request);//获得该单元的设置
  116. curl_setopt_array($ch, $options);//批设置
  117. $output = curl_exec($ch);
  118. $curl_info = curl_getinfo($ch);
  120. if ($this->callback) {
  121. $callback = $this->callback;
  122. if (is_callable($this->callback)) {
  123. call_user_func($callback, $output, $curl_info, $request);
  124. }
  125. }
  126. else
  127. return $output;
  128. return true;
  129. }
  132. private function rolling_curl($thread_size = null) {
  133. if ($thread_size){
  134. $this->thread_size = $thread_size;
  135. }
  137. if (count($this->requests) < $this->thread_size){
  138. $this->thread_size = count($this->requests);
  139. }
  140. if ($this->thread_size < 2) {
  141. $errorinfo = '线程大小必须大于 1!!!!';
  142. throw new Exception($errorinfo);
  143. }
  144. $queue = curl_multi_init();
  146. //在线程里开始增加任务队列
  147. for ($i = 0; $i < $this->thread_size; $i++) {
  148. $ch = curl_init();
  149. $options = $this->get_options($this->requests[$i]);
  150. curl_setopt_array($ch, $options);//获得设置
  151. curl_multi_add_handle($queue, $ch);//添加进去
  152. $key = (string) $ch;
  153. $this->requestMap[$key] = $i;
  154. }
  155. do {
  156. while (($statu_run_muti_exec = curl_multi_exec($queue, $active)) == CURLM_CALL_MULTI_PERFORM) ;
  157. if ($statu_run_muti_exec != CURLM_OK){ break; }
  158. // 发现完成的一个请求，进行处理
  159. while ($done = curl_multi_info_read($queue)) {
  160. $curl_info = curl_getinfo($done['handle']);
  161. $output = curl_multi_getcontent($done['handle']);
  162. $callback = $this->callback;
  163. if (is_callable($callback)){
  164. $key = (string) $done['handle'];
  165. $request = $this->requests[$this->requestMap[$key]];
  166. unset($this->requestMap[$key]);//这个销毁变量用得很帅
  167. call_user_func($callback, $output, $curl_info, $request);
  168. }
  169. //增加一个未处理的请求加入到一个已经完成的队列中
  170. if ($i < count($this->requests) && isset($this->requests[$i]) && $i < count($this->requests)) {
  171. $ch = curl_init();
  172. $options = $this->get_options($this->requests[$i]);
  173. curl_setopt_array($ch, $options);
  174. curl_multi_add_handle($queue, $ch);
  175. $key = (string) $ch;
  176. $this->requestMap[$key] = $i;
  177. $i++;
  178. }
  179. curl_multi_remove_handle($queue, $done['handle']);
  180. echo "done ";
  181. print_r($queue);
  182. print_r ($done);
  183. }
  184. // This step is very, very important if there is a completion , to reset the timeout setting
  185. //The key point here is to ensure that at least one request in all threads is valid for the first time, otherwise all the requests for the first time will have no effect, resulting in $ active=0, so the following is not executed
  186. if ($active >0){
  187. curl_multi_select($queue, $this->timeout);
  188. }
  190. } while ($active);
  191. curl_multi_close($queue );
  192. return true;
  193. }
  194. public function execute($thread_size = null) {
  195. //Judge the size of thread_size. If there is only one request, use single-threaded mode
  196. if (count($this->requests) == 1) {
  197. return $this->single_curl();
  198. } else {
  199. return $this->rolling_curl($thread_size);
  200. }
  201. }
  202. private function get_options($request) {
  203. $options = $this ->__get('options');
  204. if (ini_get('safe_mode') == 'Off' || !ini_get('safe_mode')) {
  205. // $options[CURLOPT_FOLLOWLOCATION] = 1;
  206. // $ options[CURLOPT_MAXREDIRS] = 5;
  207. }
  208. $headers = $this->__get('headers');
  210. if ($request->options) {
  211. $options = $request->options + $options ;
  212. }
  213. $options[CURLOPT_URL] = $request->url;
  214. //Set the post options and header options respectively below
  215. if ($request->post_data){
  216. $options[CURLOPT_POST] = 1;
  217. $options[CURLOPT_POSTFIELDS] = $request->post_data;
  218. }
  219. if ($headers) {
  220. $options[CURLOPT_HEADER] = 0;
  221. $options[CURLOPT_HTTPHEADER] = $headers;
  222. }
  223. return $options;
  224. }
  226. public function __destruct() {
  227. unset($this->thread_size, $this->callback, $this->options, $this->headers, $this->requests);
  228. }
  229. }
  230. ?>
Copy code
  1. header("content-type:text/html; charset=utf-8");
  2. require("muti_curl_class.php");
  3. set_time_limit(0);
  4. $sucesesnum=0;
  5. $good_proxy=array();
  6. function request_callback($response, $info, $request) {
  7. global $sucesesnum,$good_proxy;
  8. // The following regular expression can selectively display the returned results
  9. /* if ( preg_match("~(.*?)~i", $response, $out)) {
  10. $title = $out[1];
  11. }*/
  12. // echo '< ;br>'.$response .'
    ';
  13. echo '
    ';
  14. //Check the response, which is $response, to determine whether there are set characters in it, and if so, use the agent Success
  15. if( $response !== false && substr_count($response, 'User-agent: Baiduspider') >=1 ) {
  16. // $result = true;
  17. echo "true
    ";
  18. // echo $request[options][10004];
  19. // print_r ($request->options);
  20. echo $request->options[CURLOPT_PROXY];
  21. $good_proxy[]=$request->options[CURLOPT_PROXY] ;
  22. }
  23. echo '
    the-->'. $sucesesnum.'<---use:'. $info['total_time'];
  24. // print_r ($request);
  25. // echo $request->url;
  26. $sucesesnum++;
  27. echo "
    ";
  28. }
  29. $params = array_merge($_GET, $_POST); //Get the address of the passed proxy ip here
  30. $ result = $proxy_ip = trim($params['ip']);
  31. $timeout=intval(trim($params['timeout']));
  32. if($timeout<3 ){$timeout=3;}
  33. if($timeout>300){$timeout=300;}
  34. $thread_size=intval(trim($params['thread_size']));
  35. if($thread_size<5){$thread_size =5;}
  36. if( $thread_size>300){$thread_size =300;}
  38. if($proxy_ip == '') {
  39. echo 'Please enter IP!!';
  40. return;
  41. }
  42. $replace_arr1 = array(' ', 'qq proxy:', 'dn28.com', 'qqip', 'qq proxy', 'qq proxy ip', 'Agent ip:', 'ip:', 'Agent ip','"',"'" ,'\','/',' ');
  43. $result = str_replace($replace_arr1, array(''), $result);
  44. $result = str_replace(",", "n", $result);
  45. $resArr = explode("n", $result);
  46. foreach($resArr as $k => $v) {
  47. $posProxy = getPos($v, '@');
  48. if($posProxy== =false){
  49. if (!empty($v)){$proxyip_and_port = $v; }
  50. }else{
  51. $proxyip_and_port = substr($v, 0, $posProxy);
  52. }
  53. $newRes[] =trim ($proxyip_and_port);
  54. }
  55. print_r($newRes);
  56. //die();
  57. $option_setting = array(
  58. CURLOPT_SSL_VERIFYPEER => 0,
  59. CURLOPT_RETURNTRANSFER => true,
  60. CURLOPT_CONNECTTIMEOUT => 5,
  61. CURLOPT_TIMEOUT => 30,
  62. CURLOPT_HEADER=>false,
  63. CURLOPT_PROXY=>'',//This is where the proxy is set
  64. );
  65. $url= 'http://www.baidu.com/robots.txt' ;
  66. $btime=time();
  67. $rc = new muti_curl("request_callback");
  68. $rc->timeout = $timeout;
  69. $rc->thread_size = $thread_size;
  70. foreach ($newRes as $ v) {
  71. $option_setting[CURLOPT_PROXY]=$v;
  72. $request = new request_setting($url, $method = "GET", $post_data = null,$header= null, $option_setting);
  73. $rc-> add($request);
  74. }
  75. $rc->execute();
  76. $etime=time();
  77. $usedtime=$etime-$btime;
  78. echo 'all'. $sucesesnum.'use'. $ usedtime;
  79. echo '
    ';
  80. $good_proxy= array_unique($good_proxy);
  81. $str='';
  82. foreach ($good_proxy as $v){
  83. $str.="'".trim($ v)."',";
  84. }
  85. $str= str_replace ( ' ' , '' ,$str );
  86. $str = preg_replace('/s+/', ' ', $str);
  87. echo $str. '
    ';
  88. var_export ($good_proxy);
  89. //var_dump ($good_proxy);
  103. //************* *************************************************** *************************************
  104. //********** *********************Only one function is used
  106. function parseProxyInfo ( $proxyStr ) {
  107. //$proxyStr = '202.115.207.25:80@HTTP; Sichuan Normal University, Chengdu, Sichuan Province';
  108. $posIp = getPos($proxyStr, ':');
  109. $ip = substr($proxyStr, 0, $posIp);
  110. $posPort = getPos($proxyStr, '@' );
  111. $port = substr($proxyStr, $posIp+1, $posPort-$posIp-1);
  112. $posType = getPos($proxyStr, ';');
  113. $type = substr($proxyStr, $posPort +1, $posType-$posPort-1);
  114. $location = substr(strstr($proxyStr, ';'), 1);
  115. return array(
  116. 'ip' => $ip,
  117. 'port' = > $port,
  118. 'type' => $type,
  119. 'location' => $location
  120. );
  121. }
  123. function getPos($haystack, $needle){
  124. return strpos($haystack, $needle );
  125. }
  127. function check_proxy_is_useful($model, $proxy_info_arr = array()) {
  128. global $params, $config;
  129. if($model == 'single') {
  130. $proxy_port = intval(trim($params['port']));
  131. $check_proxy_url = $config['verify_url'];
  132. $proxy_time_out = intval(trim($params['timeout']));
  133. $retry = intval(trim($params['retry']));
  134. $proxy_ip = trim($params['ip']);
  135. $proxy = new proxy( $proxy_ip, $proxy_port, $check_proxy_url, $proxy_time_out, $retry );
  136. //成功返回string success, 失败返回boolean false
  137. $result = $proxy -> check_proxy();
  138. //var_dump($result);
  139. $proxy_str_success = ''.$proxy_ip.':'.$proxy_port.'@'.'HTTP 代理验证成功!';
  140. $proxy_str_failed = ''.$proxy_ip.':'.$proxy_port.'@'.'HTTP 代理验证失败!';
  141. return $result !== false ? $proxy_str_success : $proxy_str_failed;
  142. } elseif ($model == 'collect') {
  143. $proxy_port = intval(trim($proxy_info_arr['port']));
  144. $check_proxy_url = $config['verify_url'];
  145. $proxy_time_out = intval(trim($params['timeout']));
  146. $retry = intval(trim($params['retry']));
  147. $proxy_ip = trim($proxy_info_arr['ip']);
  148. /*echo $proxy_ip.'
    ';
  149. echo $proxy_port.'
    ';
  150. echo $check_proxy_url.'
    ';
  151. echo $proxy_time_out.'
    ';
  152. echo $retry.'
    ';*/
  153. if(!isset($proxy)) {
  154. $proxy = new proxy( $proxy_ip, $proxy_port, $check_proxy_url, $proxy_time_out, $retry );
  155. }
  156. //成功返回string success, 失败返回boolean false
  157. $result = $proxy -> check_proxy();
  158. return $result;
  159. }
  160. }
  162. function get_single(){
  163. global $params, $config;
  164. $proxy_ip = trim($params['ip']);
  165. if($proxy_ip == '') {
  166. echo '请输入IP!!';
  167. return;
  168. }
  169. echo check_proxy_is_useful('single');
  170. }
  172. function get_proxy_by_collect(){
  173. global $params, $config;
  174. $params['url'] = trim($params['url']);
  175. if($params['url'] == '') {
  176. echo '请输入url!';
  177. return;
  178. }
  179. //$url = 'http://www.dn28.com/html/75/n-5175.html';
  180. $con = iconv('GBK', 'UTF-8', file_get_contents($params['url']));
  181. preg_match ('/
(.*)
/s', $con, $arr);
  • $result = strip_tags ($arr[1], '
    ');
  • $replace_arr1 = array(' ', 'qq代理：', 'dn28.com', 'qqip', 'qq代理', 'qq代理ip', '代理ip：', 'ip：', '代理ip');
  • $result = str_replace($replace_arr1, array(''), $result);
  • //print_r($arr);
  • $resArr = explode('
    ', $result);
  • //print_r($resArr);
  • echo '

    代理开始批量验证中，整个过程将会花费您几分钟时间。

    ';
  • unset($_SESSION['success_arr']);
  • foreach($resArr as $k => $v) {
  • $newRes[$k] = parseProxyInfo($v);
  • //print_r($newRes[$k]);
  • /*return;*/
  • $result = check_proxy_is_useful('collect', $newRes[$k]);
  • $proxy_str_success = ''.$newRes[$k]['ip'].':'.$newRes[$k]['port'].'@'.$newRes[$k]['type'].' 代理验证成功！   IP地址:'.$newRes[$k]['location'].'';
  • $proxy_str_failed = ''.$newRes[$k]['ip'].':'.$newRes[$k]['port'].'@'.$newRes[$k]['type'].' 代理验证失败！   IP地址:'.$newRes[$k]['location'].'';
  • if($result !== false ){
  • echo $proxy_str_success;
  • $_SESSION['success_arr'][] = $success_arr[] = $newRes[$k];
  • } else {
  • echo $proxy_str_failed;
  • }
  • echo '
    ';
  • }
  • if(isset($success_arr) && count($success_arr) > 0 ) {
  • save_success_proxy($success_arr);
  • echo '

    [保存验证成功的代理到本地电脑]   [我要看看历史数据]

    ';
  • } else {
  • echo '

    [我要看看历史数据]

    ';
  • }
  • //print_r($success_arr);
  • }
  • function get_proxy_by_rule(){
  • global $params, $config;
  • $result = $proxy_ip = trim($params['ip']);
  • if($proxy_ip == '') {
  • echo '请输入IP!!';
  • return;
  • }
  • $replace_arr1 = array(' ', 'qq代理：', 'dn28.com', 'qqip', 'qq代理', 'qq代理ip', '代理ip：', 'ip：', '代理ip');
  • $result = str_replace($replace_arr1, array(''), $result);
  • $resArr = explode("n", $result);
  • //print_r($resArr);
  • echo '

    代理开始批量验证中，整个过程将会花费您几分钟时间。

    ';
  • unset($_SESSION['success_arr']);
  • foreach($resArr as $k => $v) {
  • $newRes[$k] = parseProxyInfo($v);
  • //print_r($newRes[$k]);
  • /*return;*/
  • $result = check_proxy_is_useful('collect', $newRes[$k]);
  • //var_dump($result);
  • $proxy_str_success = ''.$newRes[$k]['ip'].':'.$newRes[$k]['port'].'@'.$newRes[$k]['type'].' 代理验证成功！   IP地址:'.$newRes[$k]['location'].'';
  • $proxy_str_failed = ''.$newRes[$k]['ip'].':'.$newRes[$k]['port'].'@'.$newRes[$k]['type'].' 代理验证失败！   IP地址:'.$newRes[$k]['location'].'';
  • if($result !== false ){
  • echo $proxy_str_success;
  • $_SESSION['success_arr'][] = $success_arr[] = $newRes[$k];
  • } else {
  • echo $proxy_str_failed;
  • }
  • echo '
    ';
  • }
  • if(isset($success_arr) && count($success_arr) > 0 ) {
  • save_success_proxy($success_arr);
  • echo '

    [保存到php格式文件]   [保存验证成功的代理到本地电脑]   [我要看看历史数据]

    ';
  • } else {
  • echo '

    [我要看看历史数据]

    ';
  • }
  • }
  • function save_success_proxy($success_arr){
  • global $config;
  • date_default_timezone_set('PRC');
  • $str = '';
  • foreach($success_arr as $k => $v) {
  • $str .= $v['ip'].':'.$v['port'].'@'.$v['type'].';'.$v['location']."n";
  • }
  • $fp = fopen($config['root_path'].'/success_proxy/'.date('YmdHi').'.log', 'a+');
  • fwrite($fp, $str);
  • fclose($fp);
  • unset($str);
  • }
  • ?>
    • 复制代码


    Statement：
    The content of this article is voluntarily contributed by netizens, and the copyright belongs to the original author. This site does not assume corresponding legal responsibility. If you find any content suspected of plagiarism or infringement, please contact admin@php.cn
    Previous article：Automatic file backup softwareNext article：Automatic file backup software

    Related articles

    See more