search

Home  >  Article  >  Backend Development  >  URL crawler

URL crawler

WBOY
WBOYOriginal
2016-07-25 08:48:111240browse
If you need points-free download of csdn, point-free download of pudn, or point-free 51cto, please go to http://www.itziy.com/
and execute it from the command line. Direct php call will display the usage method
Function description
1. Support agent
2. Supports setting the number of recursive checks
3. Supports output type control and check content control

Function:
主要代替肉眼尽量多的抓取可能的请求包及url地址等，方便渗透测试
  1. error_reporting(E_ERROR | E_WARNING | E_PARSE);
  2. ini_set('memory_limit','1024M');
  3. set_time_limit(0);
  4. define('CHECK_A_TAG', false);
  5. define('CHECK_JS_TAG', true);
  6. define('CHECK_URL', true);
  7. define('SAVE_ERROR', true);
  9. $checkArr = array(
  10. '$.load',
  11. '.ajax',
  12. '$.post',
  13. '$.get',
  14. '.getJSON'
  15. );
  17. if ($argc < 2)
  18. die(showerror('sorry, parameter error', array('example: php debug.php url num filename header proxy', 'detail information:', 'url: target url address which you want to check it', 'num: The number of pages of recursive,default 3', 'filename: output filename default name ret.txt', 'header: The request header file default null', 'proxy: if you want to use proxy set it here default no use proxy')));
  20. if (!check_extension())
  21. die(showerror('extension curl not support', 'please open php curl extension support'));
  23. //global variable
  24. $url = trim($argv[1]);
  25. if (stripos($url, 'http') === false)
  26. $url = 'http://'.$url;
  27. $num = isset($argv[2]) ? intval($argv[2]) : 3;
  28. $output = isset($argv[3]) ? trim(str_replace("\", '/', $argv[3])) : str_replace("\", '/', dirname(__FILE__)).'/ret.txt';
  29. $header = null;
  30. $proxy = null;
  31. $host = null;
  33. if (isset($argv[4]))
  34. {
  35. $header = trim(str_replace("\", '/', $argv[4]));
  36. if (file_exists($header))
  37. $header = array_filter(explode("n", str_replace("r", '', file_get_contents($header))));
  38. else
  39. {
  40. $file = str_replace("\", '/', dirname(__FILE__)).'/'.$header;
  41. if (file_exists($file))
  42. $header = array_filter(explode("n", str_replace("r", '', file_get_contents($file))));
  43. else
  44. $header = null;
  45. }
  46. }
  48. if (isset($argv[5]))
  49. $proxy = trim($argv[5]);
  51. if (!is_array($header) || empty($header))
  52. $header = null;
  53. $result = check_valid_url($url);
  54. $outputArr = array();
  56. if (!empty($result))
  57. {
  58. $result = str_replace("r", '', $result);
  59. $result = str_replace("n", '', $result);
  60. $tmpArr = parse_url($url);
  61. if (!isset($tmpArr['host']))
  62. die(showerror('parse url error', 'can not get host form url: '.$url));
  63. $host = $tmpArr['host'];
  64. if (stripos($host, 'http') === false)
  65. $host = 'http://'.$host;
  66. unset($tmpArr);
  68. //check for current page
  69. if (!isset($outputArr[md5($url)]))
  70. {
  71. $outputArr[md5($url)] = $url;
  72. file_put_contents($output, $url."n", FILE_APPEND);
  73. echo 'url: ',$url,' find ajax require so save it',PHP_EOL;
  74. }
  75. work($result);
  76. }
  77. echo 'run finish',PHP_EOL;
  79. function work($result, $reverse = false)
  80. {
  81. global $num, $host, $outputArr, $checkArr, $output;
  83. if (!$result)
  84. return;
  85. $result = str_replace("r", '', $result);
  86. $result = str_replace("n", '', $result);
  88. while ($num > 0)
  89. {
  90. echo 'remain: ',$num,' now start to check for url address',PHP_EOL,PHP_EOL;
  91. preg_match_all('//i', $result, $match);
  92. if (CHECK_A_TAG && isset($match[2]) && !empty($match[2]))
  93. {
  94. foreach ($match[2] as $mc)
  95. {
  96. $mc = trim($mc);
  97. if ($mc == '#')
  98. continue;
  99. if (stripos($mc, 'http') === false)
  100. $mc = $host.$mc;
  102. if (($ret = check_valid_url($mc)))
  103. {
  104. if (!isset($outputArr[md5($mc)]))
  105. {
  106. $outputArr[md5($mc)] = $mc;
  107. file_put_contents($output, $mc."n", FILE_APPEND);
  108. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  109. }
  110. }
  111. }
  112. }
  114. //check for page url
  115. echo 'remain: ',$num,' now start to check for page url',PHP_EOL,PHP_EOL;
  116. preg_match_all('/(https?|ftp|mms)://([A-z0-9]+[_-]?[A-z0-9]+.)*[A-z0-9]+-?[A-z0-9]+.[A-z]{2,}(/.*)*/?/i', $result, $match);
  117. if (CHECK_URL && isset($match[2]) && !empty($match[2]))
  118. {
  119. foreach ($match[2] as $mc)
  120. {
  121. $mc = trim($mc);
  122. if ($mc == '#')
  123. continue;
  124. if (stripos($mc, 'http') === false)
  125. $mc = $host.$mc;
  127. if (($ret = check_valid_url($mc)))
  128. {
  129. if (!isset($outputArr[md5($mc)]))
  130. {
  131. $outputArr[md5($mc)] = $mc;
  132. file_put_contents($output, $mc."n", FILE_APPEND);
  133. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  134. }
  135. }
  136. }
  137. }
  139. //check for javascript ajax require
  140. echo 'remain: ',$num,' now start to check for javascript ajax require',PHP_EOL,PHP_EOL;
  141. preg_match_all('//i', $result, $match);
  142. if (CHECK_JS_TAG && isset($match[2]) && !empty($match[2]))
  143. {
  144. foreach ($match[2] as $mc)
  145. {
  146. $mc = trim($mc);
  147. if ($mc == '#')
  148. continue;
  149. if (stripos($mc, 'http') === false)
  150. $mc = $host.$mc;
  152. if (($ret = check_valid_url($mc)))
  153. {
  154. //check for current page
  155. foreach ($checkArr as $ck)
  156. {
  157. if (!isset($outputArr[md5($mc)]) && strpos($ret, $ck) !== false)
  158. {
  159. $outputArr[md5($mc)] = $mc;
  160. file_put_contents($output, $mc."n", FILE_APPEND);
  161. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  162. break;
  163. }
  164. }
  165. }
  166. }
  167. }
  169. if ($reverse)
  170. return;
  172. //check for next page
  173. preg_match_all('//i', $result, $match);
  174. if (isset($match[2]) && !empty($match[2]))
  175. {
  176. echo 'check for next page, remain page counts: ',$num,PHP_EOL;
  177. foreach ($match[2] as $mc)
  178. {
  179. $mc = trim($mc);
  180. if ($mc == '#')
  181. continue;
  182. if (stripos($mc, 'http') === false)
  183. $mc = $host.$mc;
  184. echo 'check for next page: ',$mc,PHP_EOL;
  185. work(check_valid_url($mc), true);
  186. }
  187. }
  188. $num--;
  189. sleep(3);
  190. }
  191. }
  193. function check_valid_url($url)
  194. {
  195. if (stripos($url, 'http') === false)
  196. $url = 'http://'.$url;
  197. $ch = curl_init();
  198. curl_setopt($ch, CURLOPT_URL, $url);
  199. curl_setopt($ch, CURLOPT_HEADER, true);
  200. curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
  201. curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  202. curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
  203. if (!is_null($header))
  204. curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
  205. if (!is_null($proxy))
  206. curl_setopt($ch, CURLOPT_PROXY, $proxy);
  207. $ret = curl_exec($ch);
  208. $errinfo = curl_error($ch);
  209. curl_close($ch);
  210. unset($ch);
  211. if (!empty($errinfo) || ((strpos($ret, '200 OK') === false) && (strpos($ret, '302 Moved') === false)) || strpos($ret, '114so.cn') !== false)
  212. {
  213. showerror('check url: '.$url. ' find some errors', array($errinfo, $ret));
  214. if (SAVE_ERROR)
  215. file_put_contents(dirname(__FILE__).'/error.txt', $url."n", FILE_APPEND);
  216. return false;
  217. }
  218. return $ret;
  219. }
  221. function check_extension()
  222. {
  223. if (!function_exists('curl_init') || !extension_loaded('curl'))
  224. return false;
  225. return true;
  226. }
  228. function showerror($t, $c)
  229. {
  230. $str = "#########################################################################n";
  231. $str .= "# ".$t."n";
  232. if (is_string($c))
  233. $str .= "# ".$c;
  234. elseif (is_array($c) && !empty($c))
  235. {
  236. foreach ($c as $c1)
  237. $str .= "# ".$c1."n";
  238. }
  239. $str .= "n#########################################################################n";
  240. echo $str;
  241. unset($str);
  242. }
复制代码


Statement：
The content of this article is voluntarily contributed by netizens, and the copyright belongs to the original author. This site does not assume corresponding legal responsibility. If you find any content suspected of plagiarism or infringement, please contact admin@php.cn
Previous article：Some good programming styles that every programmer should followNext article：Some good programming styles that every programmer should follow

Related articles

See more