URL抓取工具-php教程-PHP中文網

有需要csdn免積分下載、pudn免積分下載、51cto免積分，請到http://www.itziy.com/
命令列下執行，直接php呼叫將顯示使用方式
功能說明
1.支援代理
2.支援設定遞歸檢查次數
3.支援輸出類型控制、檢查內容控制

作用：
主要代替肉眼盡量多的抓取可能的請求包及url位址等，方便滲透測試

error_reporting(E_ERROR | E_WARNING | E_PARSE);
ini_set('memory_limit','1024M');ini_set('memory_limit','1024M');
define('CHECK_A_TAG', false);
define('CHECK_JS_TAG', true);
define('CHECK_URL', true);
define('SAVE_ERROR', );
$checkArr = array(
'$.load',
'.ajax',
'$.post',
'$.get',
'.getJSON'
);
if ($argc die(showerror('sorry, parameter error', array('example: php debug.php url num filename header proxy', 'detail information:', 'url: target url address which you want to check it', 'num: The number of pages of recursive,default 3', 'filename: output filename default name ret.txt', 'header: The request header file default null', 'proxy: if you want to use proxy set it here default no use proxy')));
if (!check_extension())
die(showerror('extension!check_extension())
die(showerror('extension | support', 'please open php curl extension support'));
//global variable
$url = trim($argv[1]);
if (stripos($url, ' http') === false)
$url = 'http://'.$url;
$num = isset($argv[2]) ? intval($argv[2]) : 3;
$output = isset($argv[3]) ? trim(str_replace("\", '/', $argv[3])) : str_replace("\", '/', dirname(__FILE__)) .'/ret.txt';
$header = null;
$proxy = null;
$host = null;
if (isset($argv[4]))
{
$header = trim(str_replace("\", '/', $argv[4]));
if (file_exists($header))
$header = array_filter(explode( "n", str_replace("r", '', file_get_contents($header)))));
else
{
$file = str_replace("\", '/', dirname(__FILE__) ).'/'.$header;
if (file_exists($file))
$header = array_filter(explode("n", str_replace("r", '', file_get_contents($file))) );
else
$header = null;
}
}
if (isset($argv[5]))
$proxy = trim($argv[ 5]);
if (!is_array($header) || empty($header))
$header = null;
$result = check_valid_url($url);
$$ outputArr = array();
if (!empty($result))
{
$result = str_replace("r", '', $result);
$result = str_replace("n", '', $result);
$tmpArr = parse_url($url);
if (!isset($tmpArr['host']))
die(showerror(' parse url error', 'can not get host form url: '.$url));
$host = $tmpArr['host'];
if (stripos($host, 'http') == = false)
$host = 'http://'.$host;
unset($tmpArr);
//check for current page
if (!isset($outputArr [md5($url)]))
{
$outputArr[md5($url)] = $url;
file_put_contents($output, $url."n", FILE_APPEND);
echo 'url: ',$url,' find ajax require so save it',PHP_EOL;
}
work($result);
}
echo 'run finish',PHP_EOL;
}
echo 'run finish',PHP_EOL;
function work($result, $reverse = false)
{
global $num, $host, $outputArr, $checkArr, $output;
if (!$result)
return;
$result = str_replace("r", '', $result);
$result = str_replace("n", '', $result);
while ($num > 0)
{
echo 'remain: ',$num,' now start to check for url address',PHP_EOL,PHP_EOL;
preg_match_all('//i', $result, $match);
if (CHECK_A_TAG && isset($match[2 ]) && !empty($match[2]))
{
foreach ($match[2] as $mc)
{
$mc = trim($mc);
if ($mc == '#')
continue;
if (stripos($mc, 'http') === false)
$mc = $host.$mc;
if (($ret = check_valid_url($mc)))
{
if (!isset($outputArr[md5($mc)]))
{
$outputArr[md5( $mc)] = $mc;
file_put_contents($output, $mc."n", FILE_APPEND);
echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
}
}
}
}
//check for page url
echo 'remain: ',$num,' now start to check for page url',PHP_EOL ,PHP_EOL; preg_match_all('/(https?|ftp|mms)://([A-z0-9] [_-]?[A-z0-9] .)*[A-z0-9 ] -?[A-z0-9] .[A-z]{2,}(/.*)*/?/i', $result, $match);
if (CHECK_URL && isset($match[2]) && !empty($match[2]))
{
foreach ($match[2] as $mc)
{
$mc = trim($mc);
if ($mc == '#')
continue;
if (stripos($mc, 'http') === false)
$mc = $host.$mc;
if (($ret = check_valid_url($mc)))
{
if (!isset($outputArr[md5($mc)]))
{
$outputArr[md5($mc)] = $mc;
file_put_contents($output, $mc ) ) ."n", FILE_APPEND);
echo 'url: ',$mc,' 找 ajax 需要，保存因此',PHP_EOL;
}
}
}
}
//檢查javascript ajax require
echo 'remain: ',$num,' 現在開始檢查javascript ajax require',PHP_EOL,PHP_EOL;
preg_match_all('//i', $result, $match);
if (CHECK_JS_TAG && isset($ match[2 ]) && !empty($match[2]))
{
foreach ($match[2] as $mc)
{
$mc = trim($mc);
if ($mc == '#')
continue;
if (stripos($mc, 'http') === false)
$mc = $host.$mc;
if (($ret = check_valid_url($mc)))
{
//檢查目前頁
foreach ($checkArr as $ck)
{
if ( !isset($outputArr[md5($mc)]) && strpos($ret, $ck) !== false)
{
$outputArr[md5($mc)] = $mc;
file_put_contents($output, $mc."n", FILE_APPEND);
echo 'url: ',$mc,' 找到ajax require 所以保存它',PHP_EOL;
break;
}
}
}
}
}
if ($reverse)
return;
//檢查下一頁
preg_match_all('//i', $結果, $match);
if ( isset( $match[2]) && !empty($match[2]))
{
echo '檢查下一頁，剩餘頁數：',$num,PHP_EOL;
foreach ($ match[ 2] as $mc)
{
$mc = trim($mc);
if ($mc == '#')
continue;
if (stripos($ mc, 'http') === false)
$mc = $host.$mc;
echo '檢查下一頁：',$mc,PHP_EOL;
work(check_valid_url($mc) , true );
}
}
$num--;
sleep(3);
}
}
函數check_valid_url($url)
{
if (stripos($url, 'http') === false)
$url = 'http://'.$url;
$ch = curl_init();
curl_setopt( $ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_togffchUR CURS,$
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (相容;Google 機器人/2.1;http://www.google.com/bot.html)');
if (!is_null($header) )
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
if (!is_null ($proxy))
curl_setopt($ch, CURLOPT_PROXY, $proxy);
$pret. ch);
$errinfo = curl_error($ch);
curl_close($ch) );
unset($ch);
if (!empty($errinfo) || ((strpos ) ($ret, '200 OK') === false) && (strpos($ret, '302)移動') ===已已)) || strpos($ret, '114so.cn') !== false)
{
showerror('檢查 url: '.$url.' 發現一些錯誤' , array($errinfo, $ret));
if (SAVE_ERROR)
file_put_contents(dirname(__FILE__).'/error.txt', $url."n", FILE_APPEND);
return false ;
}
}
return false ;
}
}
return $ret ;
}
function check_extension()
{
if (!function_exists('curl_init') || !extension_loaded('curl' )))
返回false>回傳true;
}
function showerror($t, $c)
{
$str = "####### ############ ########################################################### #######n";
$str .= "# ".$t."n";
if (is_string($c))
$str .= "# "。 $c;
elseif (is_array($c) && !empty($c))
{
foreach ($c as $c1)
$ str .= "# ".$c1. "n";
}
$str .= "n################################ # ##########################################n";

echo $str ;

unset($str);}

複製程式碼