PHP爬虫之百万级别知乎用户数据爬取与分析，php爬虫

$url = 'http://www.zhihu.com/people/mora-hu/about'; //此处mora-hu代表用户ID
$ch = curl_init($url); //初始化会话
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_COOKIE, $this->config_arr['user_cookie']); //设置请求COOKIE
curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); //将curl_exec()获取的信息以文件流的形式返回，而不是直接输出。
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); 
$result = curl_exec($ch);
return $result; //抓取的结果

function getImg($url, $u_id)
{
  if (file_exists('./images/' . $u_id . ".jpg"))
  {
    return "images/$u_id" . '.jpg';
  }
  if (empty($url))
  {
    return '';
  }
  $context_options = array( 
    'http' => 
    array(
      'header' => "Referer:http://www.zhihu.com"//带上referer参数
  　　)
　　);

  $context = stream_context_create($context_options); 
  $img = file_get_contents('http:' . $url, FALSE, $context);
  file_put_contents('./images/' . $u_id . ".jpg", $img);
  return "images/$u_id" . '.jpg';
}

ls -l | grep "^-" | wc -l

  $mh = curl_multi_init(); //返回一个新cURL批处理句柄
  for ($i = 0; $i < $max_size; $i++)
  {
    $ch = curl_init(); //初始化单个cURL会话
    curl_setopt($ch, CURLOPT_HEADER, 0);
    curl_setopt($ch, CURLOPT_URL, 'http://www.zhihu.com/people/' . $user_list[$i] . '/about');
    curl_setopt($ch, CURLOPT_COOKIE, self::$user_cookie);
    curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.130 Safari/537.36');
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
    $requestMap[$i] = $ch;
    curl_multi_add_handle($mh, $ch); //向curl批处理会话中添加单独的curl句柄
  }

  $user_arr = array();
  do {
          //运行当前 cURL 句柄的子连接
    while (($cme = curl_multi_exec($mh, $active)) == CURLM_CALL_MULTI_PERFORM);

    if ($cme != CURLM_OK) {break;}
          //获取当前解析的cURL的相关传输信息
    while ($done = curl_multi_info_read($mh))
    {
      $info = curl_getinfo($done['handle']);
      $tmp_result = curl_multi_getcontent($done['handle']);
      $error = curl_error($done['handle']);

      $user_arr[] = array_values(getUserInfo($tmp_result));

      //保证同时有$max_size个请求在处理
      if ($i < sizeof($user_list) && isset($user_list[$i]) && $i < count($user_list))
      {
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_HEADER, 0);
        curl_setopt($ch, CURLOPT_URL, 'http://www.zhihu.com/people/' . $user_list[$i] . '/about');
        curl_setopt($ch, CURLOPT_COOKIE, self::$user_cookie);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.130 Safari/537.36');
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
        $requestMap[$i] = $ch;
        curl_multi_add_handle($mh, $ch);

        $i++;
      }

      curl_multi_remove_handle($mh, $done['handle']);
    }

    if ($active)
      curl_multi_select($mh, 10);
  } while ($active);

  curl_multi_close($mh);
  return $user_arr;

<&#63;php
  $redis = new Redis();
  $redis->connect('127.0.0.1', '6379');
  $redis->set('tmp', 'value');
  if ($redis->exists('tmp'))
  {
    echo $redis->get('tmp') . "\n";
  }

//PHP多进程demo
//fork10个进程
for ($i = 0; $i < 10; $i++) {
  $pid = pcntl_fork();
  if ($pid == -1) {
    echo "Could not fork!\n";
    exit(1);
  }
  if (!$pid) {
    echo "child process $i running\n";
    //子进程执行完毕之后就退出，以免继续fork出新的子进程
    exit($i);
  }
}

//等待子进程执行完毕，避免出现僵尸进程
while (pcntl_waitpid(0, $status) != -1) {
  $status = pcntl_wexitstatus($status);
  echo "Child $status completed\n";
}

cat /proc/cpuinfo

<&#63;php
   for ($i = 0; $i < 10; $i++) {
     $pid = pcntl_fork();
     if ($pid == -1) {
        echo "Could not fork!\n";
        exit(1);
     }
     if (!$pid) {
        $redis = PRedis::getInstance();
        // do something   
        exit;
     }
   }

<&#63;php
   public static function getInstance() {
     static $instances = array();
     $key = getmypid();//获取当前进程ID
     if ($empty($instances[$key])) {
        $inctances[$key] = new self();
     }

     return $instances[$key];
   }

function microtime_float()
{
   list($u_sec, $sec) = explode(' ', microtime());
   return (floatval($u_sec) + floatval($sec));
}

$start_time = microtime_float();

//do something
usleep(100);

$end_time = microtime_float();
$total_time = $end_time - $start_time;

$time_cost = sprintf("%.10f", $total_time);

echo "program cost total " . $time_cost . "s\n";