抓取“维库电子市场”供应商程序

/**
* 抓取“维库电子市场”供应商主程序
* author Lee.
* Last modify $Date: 2012-2-3 9:30:21 $
* 注：本程序按照编码 GB2312 执行，因为“维库电子市场”网站是GB2312编码，数据库也得保持一致
*/
class weiku {
private $key; // 型号
private $pageNum; // 页码

/**
* 入口程序
*/
public function go($key) {
$this->key = $key;
$this->pageNum = $this->getPageNum();
$this->getInfo();
}

/**
* 获取供应商 url 链接数组
* @return ArrayObject
*/
private function getInfo() {
if ($this->pageNum==1) { # 处理只有一页的情况
$arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent()));
$this->isAddSuccess($arr);
} elseif ($this->pageNum>1) { # 多页
for ($i=1; $ipageNum; $i++) {
$arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent($i)));
$this->isAddSuccess($arr);
}
}
}

/**
* 打印是否添加成功
* @param ArrayObject $arr
* @return string
*/
private function isAddSuccess($arr) {
foreach ($arr as $k=>$v) {
if ($this->execAdd($this->getInfoByShopUrl($v))) {
echo 'Add Success!!';
} else {
echo 'Add Faild!!';
}
}
}

/**
* 执行添加到数据库
* @param ArrayObject $infoArr
* @return Number 受影响的行数
*/
private function execAdd($infoArr) {
$mysqli = $this->getDb();
if (!emptyempty($infoArr['company'])) {
if (!$this->isExists($mysqli, $infoArr)) {
$num = $mysqli->query("INSERT INTO weiku(company,person,phone,fax,mobile,qq,msn,email,address,shopUrl) VALUES ('{$infoArr['company']}','{$infoArr['person']}','{$infoArr['phone']}','{$infoArr['fax']}','{$infoArr['mobile']}','{$infoArr['qq']}','{$infoArr['msn']}','{$infoArr['email']}','{$infoArr['address']}','{$infoArr['shopUrl']}')");
return $num;
} else {
return false; # 表示数据已经存在
}
} else {
return false;
}
}

/**
* 连接数据库
*/
private function getDb() {
$mysqli = new mysqli('localhost', 'root', '1715544', 'weiku');
$mysqli->query('SET NAMES GB2312');
return $mysqli;
}

/**
* 检查公司是否已经存在
* @param Resource $mysqli
* @param ArrayObject $infoArr
* @return bool
*/
private function isExists($mysqli, $infoArr) {
$mysqli->query("SELECT company FROM weiku WHERE company = '{$infoArr['company']}'");
if ($mysqli->affected_rows) {
return true;
} else {
return false;
}
}

/**
* 抓取信息
* @param $url
* @return ArrayObject
*/
private function getInfoByShopUrl($url) {
$re = $this->getUrlInfo($url);
preg_match_all('/公司名称：(.*)/Usi', $re, $companyArr);
preg_match_all('/联系人：(.*)/Usi', $re, $personArr);
preg_match_all('/电话：(.*)/Usi', $re, $phoneArr);
preg_match_all('/传真：(.*)/Usi', $re, $faxArr);
preg_match_all('/手机：(.*)/Usi', $re, $mobileArr);
preg_match_all('/QQ：(.*)/Usi', $re, $qqArr);
preg_match_all('/MSN：(.*)/Usi', $re, $msnArr);
preg_match_all('/E-Mail：(.*)/Usi', $re, $emailArr);
preg_match_all('/公司地址：(.*)/Usi', $re, $addressArr);
$infoArr = array(
'company'=>$this->stripATags($companyArr[1][0]),
'person'=>trim($personArr[1][0]),
'phone'=>trim($phoneArr[1][0]),
'fax'=>trim($faxArr[1][0]),
'mobile'=>trim($mobileArr[1][0]),
'qq'=>$this->formatQqMsn($qqArr[1][0]),
'msn'=>$this->formatQqMsn($msnArr[1][0], 'MSN'),
'email'=>$this->stripATags($emailArr[1][0]),
'address'=>trim($addressArr[1][0]),
'shopUrl'=>$url
);
return $infoArr;
}

/**
* 根据页面获取供应商 url 数组
* @param string $re
* @return ArrayObject
*/
private function shopUrlMatchReArr($re) {
$re = preg_replace('//', '', $re);
$re = preg_replace('//', '', $re);
$re = preg_replace('/[A-Z]/', '', $re);
$re = preg_replace('/[0-9]/', '', $re);
$re = preg_replace('/.*/', '', $re);
$re = preg_replace('/.*/', '', $re);
$re = preg_replace('/营业执照/', '', $re);
$re = preg_replace('/该企业更多资质>>/', '', $re);
$re = preg_replace('/点此反馈/', '', $re);
$re = preg_replace('/首页/', '', $re);
$re = preg_replace('/IC/', '', $re);
$re = preg_replace('/简洁/', '', $re);
$re = preg_replace('/信用/', '', $re);
$re = preg_replace('/.*更多报价信息>>/', '', $re);
$re = preg_replace('/.*/', '', $re);
$re = preg_replace('/