PHP가 웹 페이지 크롤링 기능을 구현하는 방법

<?php
//定义url
$url=&#39;[http://t.qq.com](http://t.qq.com/)&#39;;//fiel函数读取内容数组
$lines_array=file($url);//拆分数组为字符串
$lines_string=implode(&#39;&#39;,$lines_array);//输出内容
echo $lines_string;　

$url="[http://news.sina.com.cn/c/nd/2016-10-23/doc-ifxwztru6951143.shtml](http://news.sina.com.cn/c/nd/2016-10-23/doc-ifxwztru6951143.shtml)";
$html=file_get_contents($url);
//如果出现中文乱码使用下面代码`
//$getcontent = iconv("gb2312", "utf-8",$html);
echo"<textarea style=&#39;width:800px;height:600px;&#39;>".$html."</textarea>";

<?php
//定义url
$url=&#39;[http://t.qq.com](http://t.qq.com/)&#39;;//fopen以二进制方式打开 
$handle=fopen($url,"rb");//变量初始化
$lines_string="";//循环读取数据
do{
$data=fread($handle,1024);  
if(strlen($data)==0) {`
break; 
} 
$lines_string.=$data;
}while(true);//关闭fopen句柄，释放资源
fclose($handle);//输出内容
echo $lines_string;

<?php
header("Content-Type: text/html;charset=utf-8");
date_default_timezone_set(&#39;PRC&#39;);
$url = "https://***********ycare";//要爬取的网址
$res = curl_get_contents($url);//curl封装方法
preg_match_all(&#39;/<script>(.*?)<\/script>/&#39;,$res,$arr_all);//这个网页中数据通过js包过来，所以直接抓js就可以
preg_match_all(&#39;/"id"\:"(.*?)",/&#39;,$arr_all[1][1],$arr1);//从js块中匹配要的数据
$list = array_unique($arr1[1]);//（可省）保证不重复
//以下则是同理，循环则可
for($i=0;$i<=6;$i=$i+2){
  $detail_url = &#39;ht*****em/&#39;.$list[$i];
  $detail_res = curl_get_contents($detail_url);
  preg_match_all(&#39;/<script>(.*?)<\/script>/&#39;,$detail_res,$arr_detail);
  preg_match(&#39;/"desc"\:"(.*?)",/&#39;,$arr_detail[1][1],$arr_content);
  ***
    ***
    ***
  $ret=curl_post(&#39;http://**********cms.php&#39;,$result);//此脚本未放在服务器上，原因大家懂就好哈。
}
function curl_get_contents($url,$cookie=&#39;&#39;,$referer=&#39;&#39;,$timeout=300,$ishead=0) {
  $curl = curl_init();
  curl_setopt($curl, CURLOPT_RETURNTRANSFER,1);
  curl_setopt($curl, CURLOPT_FOLLOWLOCATION,1);
  curl_setopt($curl, CURLOPT_URL,$url);
  curl_setopt($curl, CURLOPT_TIMEOUT,$timeout);
  curl_setopt($curl, CURLOPT_USERAGENT,&#39;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36&#39;);
  if($cookie)
  {
    curl_setopt( $curl, CURLOPT_COOKIE,$cookie);
  }
  if($referer)
  {
    curl_setopt ($curl,CURLOPT_REFERER,$referer);
  }
  $ssl = substr($url, 0, 8) == "https://" ? TRUE : FALSE;
  if ($ssl)
  {
    curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, false);
    curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
  }
  $res = curl_exec($curl);
  return $res;
  curl_close($curl);
}
//curl post数据到服务器
function curl_post($url,$data){
  $ch = curl_init();
  curl_setopt($ch,CURLOPT_RETURNTRANSFER,1);
  //curl_setopt($ch,CURLOPT_FOLLOWLOCATION, 1);
  curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
  curl_setopt($ch,CURLOPT_USERAGENT,&#39;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36&#39;);
  curl_setopt($ch,CURLOPT_URL,$url);
  curl_setopt($ch,CURLOPT_POST,true);
  curl_setopt($ch,CURLOPT_POSTFIELDS,$data);
  $output = curl_exec($ch);
  curl_close($ch);
  return $output; 
}
?>

<?php
$fp = fsockopen("t.qq.com", 80, $errno, $errstr, 30);
if (!$fp) {
  echo "$errstr ($errno)<br />\n";
} else {
  $out = "GET / HTTP/1.1\r\n";
  $out .= "Host: t.qq.com\r\n";
  $out .= "Connection: Close\r\n\r\n";
  fwrite($fp, $out);
  while (!feof($fp)) {
    echo fgets($fp, 128);
  }
  fclose($fp);
}