php implements recursive crawling of web page class instances
The details are as follows:
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
class crawler{
private $_depth=5;
private $_urls=array();
function extract_links($url)
{
if(!$this->_started){
$this->_started=1;
$curr_depth=0;
}else{
$curr_depth ;
}
if($curr_depth<$this->_depth)
{
$data=file_get_contents($url);
if(preg_match_all('/((?:http|https)://(?:www.)*(?:[a-zA-Z0-9_-]{1,15}. [a-zA-Z0-9_]{1,}){1,}(?:[a-zA-Z0-9_/.-?&:%,!;]*))/',$data,$urls12))
{
foreach($urls12[0] as $k=>$v){
$check=get_headers($v,1);
if(strstr($v,$url) && $check[0]=='HTTP/1.1 200 OK' && !array_search($v,$this->_urls) && $curr_depth<$this->_depth){
$this->_urls[]=$v;
$this->extract_links($v);
}
}
}
}
return $this->_urls;
}
}
?>
|
1
2
3
4
5
6
7
8
9
10
11
1213
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
<🎜>class crawler{<🎜>
<🎜>private $_depth=5;<🎜>
<🎜>private $_urls=array();<🎜>
<🎜>function extract_links($url)<🎜>
<🎜>{<🎜>
<🎜>if(!$this->_started){
$this->_started=1;
$curr_depth=0;
}else{
$curr_depth ;
}
if($curr_depth<$this->_depth)
{
$data=file_get_contents($url);
if(preg_match_all('/((?:http|https)://(?:www.)*(?:[a-zA-Z0-9_-]{1,15}. [a-zA -Z0-9_]{1,}){1,}(?:[a-zA-Z0-9_/.-?&:%,!;]*))/',$data,$urls12))
{
foreach($urls12[0] as $k=>$v){
$check=get_headers($v,1);
if(strstr($v,$url) && $check[0]=='HTTP/1.1 200 OK' && !array_search($v,$this->_urls) && $curr_depth<$this-> ;_depth){<🎜>
<🎜>$this->_urls[]=$v;
$this->extract_links($v);
}
}
}
}
return $this->_urls;
}
}
?>
|
http://www.bkjia.com/PHPjc/978265.htmlwww.bkjia.comtruehttp: //www.bkjia.com/PHPjc/978265.htmlTechArticlePHP implementation of recursive crawling web page class examples are as follows: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 ?php class crawler{ private $_depth=5; private...
![[Web front-end] Node.js quick start](https://img.php.cn/upload/course/000/000/067/662b5d34ba7c0227.png)
