cari

Rumah >pembangunan bahagian belakang >tutorial php >URL抓取工具

URL抓取工具

WBOY
WBOYasal
2016-07-25 08:48:111249semak imbas
有需要csdn免积分下载、pudn免积分下载、51cto免积分，请到http://www.itziy.com/
命令行下执行，直接php调用将显示使用方式
功能说明
1.支持代理
2.支持设置递归检查次数
3.支持输出类型控制、检查内容控制

作用：
主要代替肉眼尽量多的抓取可能的请求包及url地址等，方便渗透测试
  1. error_reporting(E_ERROR | E_WARNING | E_PARSE);
  2. ini_set('memory_limit','1024M');
  3. set_time_limit(0);
  4. define('CHECK_A_TAG', false);
  5. define('CHECK_JS_TAG', true);
  6. define('CHECK_URL', true);
  7. define('SAVE_ERROR', true);
  9. $checkArr = array(
  10. '$.load',
  11. '.ajax',
  12. '$.post',
  13. '$.get',
  14. '.getJSON'
  15. );
  17. if ($argc die(showerror('sorry, parameter error', array('example: php debug.php url num filename header proxy', 'detail information:', 'url: target url address which you want to check it', 'num: The number of pages of recursive,default 3', 'filename: output filename default name ret.txt', 'header: The request header file default null', 'proxy: if you want to use proxy set it here default no use proxy')));
  19. if (!check_extension())
  20. die(showerror('extension curl not support', 'please open php curl extension support'));
  22. //global variable
  23. $url = trim($argv[1]);
  24. if (stripos($url, 'http') === false)
  25. $url = 'http://'.$url;
  26. $num = isset($argv[2]) ? intval($argv[2]) : 3;
  27. $output = isset($argv[3]) ? trim(str_replace("\\", '/', $argv[3])) : str_replace("\\", '/', dirname(__FILE__)).'/ret.txt';
  28. $header = null;
  29. $proxy = null;
  30. $host = null;
  32. if (isset($argv[4]))
  33. {
  34. $header = trim(str_replace("\\", '/', $argv[4]));
  35. if (file_exists($header))
  36. $header = array_filter(explode("\n", str_replace("\r", '', file_get_contents($header))));
  37. else
  38. {
  39. $file = str_replace("\\", '/', dirname(__FILE__)).'/'.$header;
  40. if (file_exists($file))
  41. $header = array_filter(explode("\n", str_replace("\r", '', file_get_contents($file))));
  42. else
  43. $header = null;
  44. }
  45. }
  47. if (isset($argv[5]))
  48. $proxy = trim($argv[5]);
  50. if (!is_array($header) || empty($header))
  51. $header = null;
  52. $result = check_valid_url($url);
  53. $outputArr = array();
  55. if (!empty($result))
  56. {
  57. $result = str_replace("\r", '', $result);
  58. $result = str_replace("\n", '', $result);
  59. $tmpArr = parse_url($url);
  60. if (!isset($tmpArr['host']))
  61. die(showerror('parse url error', 'can not get host form url: '.$url));
  62. $host = $tmpArr['host'];
  63. if (stripos($host, 'http') === false)
  64. $host = 'http://'.$host;
  65. unset($tmpArr);
  67. //check for current page
  68. if (!isset($outputArr[md5($url)]))
  69. {
  70. $outputArr[md5($url)] = $url;
  71. file_put_contents($output, $url."\n", FILE_APPEND);
  72. echo 'url: ',$url,' find ajax require so save it',PHP_EOL;
  73. }
  74. work($result);
  75. }
  76. echo 'run finish',PHP_EOL;
  78. function work($result, $reverse = false)
  79. {
  80. global $num, $host, $outputArr, $checkArr, $output;
  82. if (!$result)
  83. return;
  84. $result = str_replace("\r", '', $result);
  85. $result = str_replace("\n", '', $result);
  87. while ($num > 0)
  88. {
  89. echo 'remain: ',$num,' now start to check for url address',PHP_EOL,PHP_EOL;
  90. preg_match_all('//i', $result, $match);
  91. if (CHECK_A_TAG && isset($match[2]) && !empty($match[2]))
  92. {
  93. foreach ($match[2] as $mc)
  94. {
  95. $mc = trim($mc);
  96. if ($mc == '#')
  97. continue;
  98. if (stripos($mc, 'http') === false)
  99. $mc = $host.$mc;
  101. if (($ret = check_valid_url($mc)))
  102. {
  103. if (!isset($outputArr[md5($mc)]))
  104. {
  105. $outputArr[md5($mc)] = $mc;
  106. file_put_contents($output, $mc."\n", FILE_APPEND);
  107. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  108. }
  109. }
  110. }
  111. }
  113. //check for page url
  114. echo 'remain: ',$num,' now start to check for page url',PHP_EOL,PHP_EOL;
  115. preg_match_all('/(https?|ftp|mms):\/\/([A-z0-9]+[_\-]?[A-z0-9]+\.)*[A-z0-9]+\-?[A-z0-9]+\.[A-z]{2,}(\/.*)*\/?/i', $result, $match);
  116. if (CHECK_URL && isset($match[2]) && !empty($match[2]))
  117. {
  118. foreach ($match[2] as $mc)
  119. {
  120. $mc = trim($mc);
  121. if ($mc == '#')
  122. continue;
  123. if (stripos($mc, 'http') === false)
  124. $mc = $host.$mc;
  126. if (($ret = check_valid_url($mc)))
  127. {
  128. if (!isset($outputArr[md5($mc)]))
  129. {
  130. $outputArr[md5($mc)] = $mc;
  131. file_put_contents($output, $mc."\n", FILE_APPEND);
  132. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  133. }
  134. }
  135. }
  136. }
  138. //check for javascript ajax require
  139. echo 'remain: ',$num,' now start to check for javascript ajax require',PHP_EOL,PHP_EOL;
  140. preg_match_all('//i', $result, $match);
  141. if (CHECK_JS_TAG && isset($match[2]) && !empty($match[2]))
  142. {
  143. foreach ($match[2] as $mc)
  144. {
  145. $mc = trim($mc);
  146. if ($mc == '#')
  147. continue;
  148. if (stripos($mc, 'http') === false)
  149. $mc = $host.$mc;
  151. if (($ret = check_valid_url($mc)))
  152. {
  153. //check for current page
  154. foreach ($checkArr as $ck)
  155. {
  156. if (!isset($outputArr[md5($mc)]) && strpos($ret, $ck) !== false)
  157. {
  158. $outputArr[md5($mc)] = $mc;
  159. file_put_contents($output, $mc."\n", FILE_APPEND);
  160. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  161. break;
  162. }
  163. }
  164. }
  165. }
  166. }
  168. if ($reverse)
  169. return;
  171. //check for next page
  172. preg_match_all('//i', $result, $match);
  173. if (isset($match[2]) && !empty($match[2]))
  174. {
  175. echo 'check for next page, remain page counts: ',$num,PHP_EOL;
  176. foreach ($match[2] as $mc)
  177. {
  178. $mc = trim($mc);
  179. if ($mc == '#')
  180. continue;
  181. if (stripos($mc, 'http') === false)
  182. $mc = $host.$mc;
  183. echo 'check for next page: ',$mc,PHP_EOL;
  184. work(check_valid_url($mc), true);
  185. }
  186. }
  187. $num--;
  188. sleep(3);
  189. }
  190. }
  192. function check_valid_url($url)
  193. {
  194. if (stripos($url, 'http') === false)
  195. $url = 'http://'.$url;
  196. $ch = curl_init();
  197. curl_setopt($ch, CURLOPT_URL, $url);
  198. curl_setopt($ch, CURLOPT_HEADER, true);
  199. curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
  200. curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  201. curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
  202. if (!is_null($header))
  203. curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
  204. if (!is_null($proxy))
  205. curl_setopt($ch, CURLOPT_PROXY, $proxy);
  206. $ret = curl_exec($ch);
  207. $errinfo = curl_error($ch);
  208. curl_close($ch);
  209. unset($ch);
  210. if (!empty($errinfo) || ((strpos($ret, '200 OK') === false) && (strpos($ret, '302 Moved') === false)) || strpos($ret, '114so.cn') !== false)
  211. {
  212. showerror('check url: '.$url. ' find some errors', array($errinfo, $ret));
  213. if (SAVE_ERROR)
  214. file_put_contents(dirname(__FILE__).'/error.txt', $url."\n", FILE_APPEND);
  215. return false;
  216. }
  217. return $ret;
  218. }
  220. function check_extension()
  221. {
  222. if (!function_exists('curl_init') || !extension_loaded('curl'))
  223. return false;
  224. return true;
  225. }
  227. function showerror($t, $c)
  228. {
  229. $str = "#########################################################################\n";
  230. $str .= "# ".$t."\n";
  231. if (is_string($c))
  232. $str .= "# ".$c;
  233. elseif (is_array($c) && !empty($c))
  234. {
  235. foreach ($c as $c1)
  236. $str .= "# ".$c1."\n";
  237. }
  238. $str .= "\n#########################################################################\n";
  239. echo $str;
  240. unset($str);
  241. }
复制代码


Kenyataan：
Kandungan artikel ini disumbangkan secara sukarela oleh netizen, dan hak cipta adalah milik pengarang asal. Laman web ini tidak memikul tanggungjawab undang-undang yang sepadan. Jika anda menemui sebarang kandungan yang disyaki plagiarisme atau pelanggaran, sila hubungi admin@php.cn
Artikel sebelumnya：每一个程序员要遵守的一些优秀编程风格 Artikel seterusnya：PHP计算当前程序执行时间

Artikel berkaitan

Lihat lagi