URL crawler

If you need points-free download of csdn, point-free download of pudn, or point-free 51cto, please go to http://www.itziy.com/
and execute it from the command line. Direct php call will display the usage method
Function description
1. Support agent
2. Supports setting the number of recursive checks
3. Supports output type control and check content control

Function:
主要代替肉眼尽量多的抓取可能的请求包及url地址等，方便渗透测试

error_reporting(E_ERROR | E_WARNING | E_PARSE);
ini_set('memory_limit','1024M');
set_time_limit(0);
define('CHECK_A_TAG', false);
define('CHECK_JS_TAG', true);
define('CHECK_URL', true);
define('SAVE_ERROR', true);

$checkArr = array(
    '$.load',
    '.ajax',
    '$.post',
    '$.get',
    '.getJSON'
);

if ($argc     die(showerror('sorry, parameter error', array('example: php debug.php url num filename header proxy', 'detail information:', 'url: target url address which you want to check it', 'num: The number of pages of recursive,default 3', 'filename: output filename default name ret.txt', 'header: The request header file default null', 'proxy: if you want to use proxy set it here default no use proxy')));

if (!check_extension())
    die(showerror('extension curl not support', 'please open php curl extension support'));

//global variable
$url = trim($argv[1]);
if (stripos($url, 'http') === false)
    $url = 'http://'.$url;
$num = isset($argv[2]) ? intval($argv[2]) : 3;
$output = isset($argv[3]) ? trim(str_replace("\", '/', $argv[3])) : str_replace("\", '/', dirname(__FILE__)).'/ret.txt';
$header = null;
$proxy = null;
$host = null;

if (isset($argv[4]))
{
    $header = trim(str_replace("\", '/', $argv[4]));
    if (file_exists($header))
        $header = array_filter(explode("n", str_replace("r", '', file_get_contents($header))));
    else
    {
        $file =  str_replace("\", '/', dirname(__FILE__)).'/'.$header;
        if (file_exists($file))
            $header = array_filter(explode("n", str_replace("r", '', file_get_contents($file))));
        else
            $header = null;
    }
}

if (isset($argv[5]))
    $proxy = trim($argv[5]);

if (!is_array($header) || empty($header))
    $header = null;
$result = check_valid_url($url);
$outputArr = array();

if (!empty($result))
{
    $result = str_replace("r", '', $result);
    $result = str_replace("n", '', $result);
    $tmpArr = parse_url($url);
    if (!isset($tmpArr['host']))
        die(showerror('parse url error', 'can not get host form url: '.$url));
    $host = $tmpArr['host'];
    if (stripos($host, 'http') === false)
        $host = 'http://'.$host;
    unset($tmpArr);

    //check for current page
    if (!isset($outputArr[md5($url)]))
    {
        $outputArr[md5($url)] = $url;
        file_put_contents($output, $url."n", FILE_APPEND);
        echo 'url: ',$url,' find ajax require so save it',PHP_EOL;
    }
    work($result);
}
echo 'run finish',PHP_EOL;

function work($result, $reverse = false)
{
    global $num, $host, $outputArr, $checkArr, $output;

    if (!$result)
        return;
    $result = str_replace("r", '', $result);
    $result = str_replace("n", '', $result);

    while ($num > 0)
    {
        echo 'remain: ',$num,' now start to check for url address',PHP_EOL,PHP_EOL;
        preg_match_all('//i', $result, $match);
        if (CHECK_A_TAG && isset($match[2]) && !empty($match[2]))
        {
            foreach ($match[2] as $mc)
            {
                $mc = trim($mc);
                if ($mc == '#')
                    continue;
                if (stripos($mc, 'http') === false)
                    $mc = $host.$mc;

                if (($ret = check_valid_url($mc)))
                {
                    if (!isset($outputArr[md5($mc)]))
                    {
                        $outputArr[md5($mc)] = $mc;
                        file_put_contents($output, $mc."n", FILE_APPEND);
                        echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
                    }
                }
            }
        }

        //check for page url
        echo 'remain: ',$num,' now start to check for page url',PHP_EOL,PHP_EOL;
        preg_match_all('/(https?|ftp|mms)://([A-z0-9]+[_-]?[A-z0-9]+.)*[A-z0-9]+-?[A-z0-9]+.[A-z]{2,}(/.*)*/?/i', $result, $match);
        if (CHECK_URL && isset($match[2]) && !empty($match[2]))
        {
            foreach ($match[2] as $mc)
            {
                $mc = trim($mc);
                if ($mc == '#')
                    continue;
                if (stripos($mc, 'http') === false)
                    $mc = $host.$mc;

                if (($ret = check_valid_url($mc)))
                {
                    if (!isset($outputArr[md5($mc)]))
                    {
                        $outputArr[md5($mc)] = $mc;
                        file_put_contents($output, $mc."n", FILE_APPEND);
                        echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
                    }
                }
            }
        }

        //check for javascript ajax require
        echo 'remain: ',$num,' now start to check for javascript ajax require',PHP_EOL,PHP_EOL;
        preg_match_all('//i', $result, $match);
        if (CHECK_JS_TAG && isset($match[2]) && !empty($match[2]))
        {
            foreach ($match[2] as $mc)
            {
                $mc = trim($mc);
                if ($mc == '#')
                    continue;
                if (stripos($mc, 'http') === false)
                    $mc = $host.$mc;

                if (($ret = check_valid_url($mc)))
                {
                    //check for current page
                    foreach ($checkArr as $ck)
                    {
                        if (!isset($outputArr[md5($mc)]) && strpos($ret, $ck) !== false)
                        {
                            $outputArr[md5($mc)] = $mc;
                            file_put_contents($output, $mc."n", FILE_APPEND);
                            echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
                            break;
                        }
                    }
                }
            }
        }

        if ($reverse)
            return;

        //check for next page
        preg_match_all('//i', $result, $match);
        if (isset($match[2]) && !empty($match[2]))
        {
            echo 'check for next page, remain page counts: ',$num,PHP_EOL;
            foreach ($match[2] as $mc)
            {
                $mc = trim($mc);
                if ($mc == '#')
                    continue;
                if (stripos($mc, 'http') === false)
                    $mc = $host.$mc;
                echo 'check for next page: ',$mc,PHP_EOL;
                work(check_valid_url($mc), true);
            }
        }
        $num--;
        sleep(3);
    }
}

function check_valid_url($url)
{
    if (stripos($url, 'http') === false)
        $url = 'http://'.$url;
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_HEADER, true);
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
    if (!is_null($header))
        curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
    if (!is_null($proxy))
        curl_setopt($ch, CURLOPT_PROXY, $proxy);
    $ret = curl_exec($ch);
    $errinfo = curl_error($ch);
    curl_close($ch);
    unset($ch);
    if (!empty($errinfo) || ((strpos($ret, '200 OK') === false) && (strpos($ret, '302 Moved') === false)) || strpos($ret, '114so.cn') !== false)
    {
        showerror('check url: '.$url. ' find some errors', array($errinfo, $ret));
        if (SAVE_ERROR)
            file_put_contents(dirname(__FILE__).'/error.txt', $url."n", FILE_APPEND);
        return false;
    }
    return $ret;
}

function check_extension()
{
    if (!function_exists('curl_init') || !extension_loaded('curl'))
        return false;
    return true;
}

function showerror($t, $c)
{
    $str = "#########################################################################n";
    $str .= "#  ".$t."n";
    if (is_string($c))
        $str .= "#  ".$c;
    elseif (is_array($c) && !empty($c))
    {
        foreach ($c as $c1)
            $str .= "#  ".$c1."n";
    }
    $str .= "n#########################################################################n";
    echo $str;
    unset($str);
}