Detailed explanation of CURL for PHP web crawler-PHP Tutorial-php.cn

php's curl can simulate various HTTP requests. This is also the basis for PHP to do web crawlers. It is also mostly used for calling interface APIs. At this time, someone will ask: Why don't you use file_get_contents?

curl's performance is better than it, and it can complete more complex operations, not just getting page data.

The following introduces some commonly used functions.

curl_initInitialize a curl conversation
##curl_setoptSet curl parameters, that is, transmission options
curl_execExecute the request
##curl_close
Close a curl conversation

curl_errno returns the last error code, PHP has defined many error enumeration codes

1. Download a web page on the Internet and put the content After replacing "Baidu" in "Diansi", the output is

Copy after login

2. By calling WebService queries the current weather in Beijing

Copy after login

3. Simulate the URL that requires login and crawl it The content of the web page

 'promonkey', 'password' => '1q2w3e', 'remember'=>1);$data='username=zjzhoufy@126.com&password=1q2w3e&remember=1';$curlobj = curl_init(); // 初始化curl_setopt($curlobj, CURLOPT_URL, "http://www.imooc.com/user/login"); // 设置访问网页的URLcurl_setopt($curlobj, CURLOPT_RETURNTRANSFER, true); // 执行之后不直接打印出来// Cookie相关设置，这部分设置需要在所有会话开始之前设置date_default_timezone_set('PRC'); // 使用Cookie时，必须先设置时区curl_setopt($curlobj, CURLOPT_COOKIESESSION, TRUE); curl_setopt($curlobj, CURLOPT_HEADER, 0); curl_setopt($curlobj, CURLOPT_FOLLOWLOCATION, 1); // 这样能够让cURL支持页面链接跳转curl_setopt($curlobj, CURLOPT_POST, 1); curl_setopt($curlobj, CURLOPT_POSTFIELDS, $data); curl_setopt($curlobj, CURLOPT_HTTPHEADER, array("application/x-www-form-urlencoded; charset=utf-8", "Content-length: ".strlen($data) )); curl_exec($curlobj); // 执行curl_setopt($curlobj, CURLOPT_URL, "http://www.imooc.com/space/index"); curl_setopt($curlobj, CURLOPT_POST, 0); curl_setopt($curlobj, CURLOPT_HTTPHEADER, array("Content-type: text/xml" )); $output=curl_exec($curlobj); // 执行curl_close($curlobj); // 关闭cURLecho $output;?>

Copy after login

= $curl_max_loops) { $curl_loops = 0; return FALSE; } curl_setopt($ch, CURLOPT_HEADER, true); // 开启header才能够抓取到重定向到的新URL curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); $data = curl_exec($ch); // 分割返回的内容 $h_len = curl_getinfo($ch, CURLINFO_HEADER_SIZE); $header = substr($data,0,$h_len); $data = substr($data,$h_len - 1); $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($http_code == 301 || $http_code == 302) { $matches = array(); preg_match('/Location:(.*?)\n/', $header, $matches); $url = @parse_url(trim(array_pop($matches))); // print_r($url); if (!$url) { //couldn't process the url to redirect to $curl_loops = 0; return $data; } $last_url = parse_url(curl_getinfo($ch, CURLINFO_EFFECTIVE_URL)); if (!isset($url['scheme'])) $url['scheme'] = $last_url['scheme']; if (!isset($url['host'])) $url['host'] = $last_url['host']; if (!isset($url['path'])) $url['path'] = $last_url['path']; $new_url = $url['scheme'] . '://' . $url['host'] . $url['path'] . (isset($url['query'])?'?'.$url['query']:''); curl_setopt($ch, CURLOPT_URL, $new_url); return curl_redir_exec($ch); } else { $curl_loops=0; return $data; } } ?>