页面抓取程序

_init_request($request); $this->_init_curl(); } protected function _init_var() { $this->pageCount = 0; } protected function _init_request(request $request) { $this->request = $request; // $this->request->cache_path = __DIR__ . '/sjm_cache/'; // $this->request->fetch_item_query = '#J_posts_list .subject .title a'; // $this->request->fetch_page_current = '.J_page_wrap .pages strong'; // $this->request->base_url = 'http://bbs.sijiaomao.com/index.php?m=bbs&c=thread&fid=10&page=%d'; } protected function _init_curl() { $this->curl = new CurlMulti(); $this->cacheDir = $this->request->cache_path . 'cache'; if (! is_dir($this->cacheDir)) { mkdir($this->cacheDir, 777, true); } $this->cacheDataDir = $this->request->cache_path . 'data'; if (! is_dir($this->cacheDataDir)) { mkdir($this->cacheDataDir, 777, true); } $this->curl->cache = array( 'dir' => $this->cacheDir, 'on' => true, 'expire' => 3600 * 24 ); $this->curl->maxThread = 10; $this->curl->opt[CURLOPT_CONNECTTIMEOUT] = 10; } public function fetch_list(){ $this->_add_fetch_list_url(); $this->curl->start(); $this->_save_article_list(); } public function fetch_article() { foreach ($this->article_list as $k => $v) { $this->curl->add(array( 'url' => $v['href'] ), array($this, '_success_article')); } $this->curl->start(); } public function display() { printf( "\n共抓取%d个页面\n文章列表%d篇\n相关文章%d篇\n文章目录存放在%s\n", $this->pageCount + $this->articleCount, $this->pageCount, count($this->article_list), $this->cacheDataDir . '/list.php' ); } public function fetch() { return sprintf( "\n共抓取%d个页面\n文章列表%d篇\n相关文章%d篇\n文章目录存放在%s\n", $this->pageCount + $this->articleCount, $this->pageCount, count($this->article_list), $this->cacheDataDir . '/list.php' ); } public function _add_fetch_list_url($page = 1){ $this->curl->add( array( 'url' => sprintf($this->request->base_url, $page), 'args' => array('page' => $page) ), array($this, '_success_list') ); } protected function _save_article_list() { $res = file_put_contents( $this->cacheDataDir . '/list.php', sprintf("article_list, true)) ); // 相关性排序整理 /*uasort($this->article_list, function ($a, $b){ preg_match_all('#([a-zA-Z]+)#is', $a['title'], $match); $a_title = strtoupper(implode("", $match[0])); preg_match_all('#([a-zA-Z]+)#is', $b['title'], $match); $b_title = strtoupper(implode("", $match[0])); return $a_title > $b_title; });*/ $res = file_put_contents( $this->cacheDataDir . '/list.txt', array_map(function($a_list){ $str = sprintf( "标题：%s\t超链接:%s \n", str_replace(" ", "", $a_list['title']), $a_list['href'] ); return $str; }, $this->article_list) ); return $res; } public function _success_article($r, $param){ ++$this->articleCount; } public function _success_list($r, $param){ ++$this->pageCount; $html = phpQuery::newDocumentHTML($r['content']); $list = $html[$this->request->fetch_item_query]; foreach ($list as $v) { $v = pq($v); $item = array( "title" => $v->attr('title') ? $v->attr('title') : $v->text(), "href" => real_url($v->attr('href'), $this->request->base_url) ); $this->article_list[md5($item['href'])] = $item; } $page_current = $html[$this->request->fetch_page_current]; if ($page_current->next()->text()) { $page = ++ $param['page']; $this->_add_fetch_list_url($page); } phpQuery::unloadDocuments(); } } class request{ /*url*/ public $base_url; /*缓存文件路径*/ public $cache_path; /*获取元素的CSS选择器*/ public $fetch_item_query; /*分页当前页面元素的CSS选择器*/ public $fetch_page_current; static $instance; static public function getInstance() { if (empty(self::$instance)) { self::$instance = new self; } return self::$instance; } private function __construct() { $this->_init_base(); } function _init_base() { $this->cache_path = __DIR__ . '/'. trim($_POST['cache_path'], '/') .'/'; $this->fetch_item_query = $_POST['fetch_item_query']; $this->fetch_page_current = $_POST['fetch_page_current']; $this->base_url = $_POST['url']; } function request() { if (strstr($_POST['url'], '?')) { $url = sprintf("%s&auth=%s", $_POST['url'], $auth); } else { $url = sprintf("%s?auth=%s", $_POST['url'], $auth); } $param = array(); if (isset($_POST['param'])) { foreach($_POST['param'] as $k => $item) { if (!empty($item['method']) && !empty($item['name'])) { $param[$item['method']][$item['name']] = $item['value']; } } } if (isset($param['get']) && !empty($param['get'])) { foreach ($param['get'] as $name => $value) { $url = sprintf("%s&%s=%s", $url, $name, $value); } } $post_data = null; if (isset($param['post']) && !empty($param['post'])) { $post_data = $param['post']; } } } ?>fetch_list(); //$myCurl->fetch_article(); myDebug::set_end(); } else { $_POST['url'] = 'http://www.oschina.net/code/tag/php?show=time&lang=&catalog=&p=%d'; $_POST['cache_path'] = 'oschina'; $_POST['fetch_item_query'] = '.code_list ul li .code_title > a'; $_POST['fetch_page_current'] = '.pager li.current'; } ?>页面爬虫