Implementation code for PHP multi-threaded batch collection and downloading of images-PHP Tutorial-php.cn

Implementation code for PHP multi-threaded batch collection and downloading of images

WBOY

Release： 2016-07-25 08:44:17

Original

1081 people have browsed it

Use curl's multi-threading. In addition, curl can set the request time. If you encounter a very slow URL resource, you can give up decisively, so there is no blocking. In addition, with multi-threaded requests, the efficiency should be relatively high.

/**
* curl multi-threading
* @param array $array parallel URL
* @param int $timeout timeout time
* @return mix
*/
public function Curl_http($array,$timeout='15'){
$res = array();
$mh = curl_multi_init();//Create multiple curl handle
foreach($array as $k=>$url){
$conn[$k]=curl_init($url);//Initialization
curl_setopt($conn[$k], CURLOPT_TIMEOUT, $timeout);//Set the timeout period
curl_setopt($conn[$k], CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; MSIE 5.01; Windows NT 5.0)');
curl_setopt($conn[$k], CURLOPT_MAXREDIRS, 7);//HTTp orientation level, 7 is the highest
curl_setopt($conn[$k], CURLOPT_HEADER, false);//No header here, increase block efficiency
curl_setopt($conn[$k], CURLOPT_FOLLOWLOCATION, 1); // 302 redirect
curl_setopt($conn[$k], CURLOPT_RETURNTRANSFER,1); //Require the result to be a string and output it to the screen
curl_setopt($conn[$k], CURLOPT_HTTPGET, true);
curl_multi_add_handle ( $mh,$conn[$k]);
}
//Prevent the endless loop from consuming the CPU. This paragraph is based on the writing method on the Internet
do {
$mrc = curl_multi_exec($mh,$active);//When there is no data , active=true
} while ($mrc == CURLM_CALL_MULTI_PERFORM);//When data is being received
while ($active and $mrc == CURLM_OK) {//When there is no data or the request is suspended, active=true
if (curl_multi_select($mh) != -1) {
do {
$mrc = curl_multi_exec($mh, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}
foreach ($array as $k => $url) {
if(!curl_errno($conn[$k])){
$data[$k]=curl_multi_getcontent($conn[$k]);//Convert data to array
$header[$k]=curl_getinfo($conn[$k]);//Return http header information
curl_close($conn[$k]);//Close the handle
curl_multi_remove_handle($mh, $conn[$k] ]); //Release resources
}else{
unset($k,$url);
}
}
curl_multi_close($mh);
return $data;
}
//Parameter reception
$ callback = $_GET['callback'];
$hrefs = $_GET['hrefs'];
$urlarray = explode(',',trim($hrefs,','));
$date = date(' Ymd',time());
//Instantiation
$img = new HttpImg();
$stime = $img->getMicrotime();//Start time
$data = $img->Curl_http ($urlarray,'20');//List data
mkdir('./img/'.$date,0777);
foreach ((array)$data as $k=>$v){
preg_match_all( "/(href|src)=(["|']?)([^ "'>]+.(jpg|png|PNG|JPG|gif))\2/i", $v, $matches[ $k]);
if(count($matches[$k][3])>0){
$dataimg = $img->Curl_http($matches[$k][3],'20' );//All image data binary
$j = 0;
foreach ((array)$dataimg as $kk=>$vv){
if($vv !=''){
$rand = rand(1000 ,9999);
$basename = time()."_".$rand.".".jpg;//Save as a jpg format file
$fname = './img/'.$date."/" ."$basename";
file_put_contents($fname, $vv);
$j++;
echo "Create the ".$j."th picture"."$fname"."
";
}else{
unset($kk,$vv);
}
}
}else{
unset($matches);
}
}
$etime = $img->getMicrotime();//End time
echo "Time".($etime-$stime)."Seconds";
exit;

Copy code

Multi-threading, download pictures, PHP