Example of extracting the body content of a web page with php_PHP tutorial

class Readability {
//The name of the mark bit that saves the judgment result
const ATTR_CONTENT_SCORE = "contentScore";

//The DOM parsing class currently only supports UTF-8 encoding
const DOM_DEFAULT_CHARSET = "utf-8";

//The content displayed when the judgment fails
const MESSAGE_CAN_NOT_GET = "Readability was unable to parse this page for content.";

// DOM parsing class (built-in in PHP5)
protected $DOM = null;

//Source code that needs to be parsed
protected $source = "";

//List of parent elements of chapters
private $parentNodes = array();

// Tags that need to be deleted
// Note: added extra tags from http://www.45it.net

private $junkTags = Array("style", "form", "iframe", "script", "button", "input", "textarea",
"noscript", "select", "option", "object", "applet", "basefont",
"bgsound", "blink", "canvas", "command", "menu", "nav", "datalist",
"embed", "frame", "frameset", "keygen", "label", "marquee", "link");

//Attributes that need to be deleted
private $junkAttrs = Array("style", "class", "onclick", "onmouseover", "align", "border", "margin");

/**
* Constructor
* @param $input_char encoding of string. Default utf-8, can be omitted
*/
function __construct($source, $input_char = "utf-8") {
$this->source = $source;

// The DOM parsing class can only handle characters in UTF-8 format
$source = mb_convert_encoding($source, 'HTML-ENTITIES', $input_char);

// Preprocess HTML tags, remove redundant tags, etc.
$source = $this->preparSource($source);

// 生成 DOM 解析类
$this->DOM = new DOMDocument('1.0', $input_char);
try {
//libxml_use_internal_errors(true);
// 会有些错误信息，不过不要紧 :^)
if (!@$this->DOM->loadHTML(''.$source)) {
throw new Exception("Parse HTML Error!");
}

foreach ($this->DOM->childNodes as $item) {
if ($item->nodeType == XML_PI_NODE) {
$this->DOM->removeChild($item); // remove hack
}
}

// insert proper
$this->DOM->encoding = Readability::DOM_DEFAULT_CHARSET;
} catch (Exception $e) {
// ...
}
}

/**
* Preprocess HTML tags so that they can be accurately processed by DOM parsing classes
*
* @return String
*/
private function preparSource($string) {
// 剔除多余的 HTML 编码标记，避免解析出错
preg_match("/charset=([＼w|＼-]+);?/", $string, $match);
if (isset($match[1])) {
$string = preg_replace("/charset=([＼w|＼-]+);?/", "", $string, 1);
}

// Replace all doubled-up
tags with

tags, and remove fonts.
$string = preg_replace("/[ ＼r＼n＼s]*/i", "

", $string);
$string = preg_replace("/<＼/?font[^>]*>/i", "", $string);

// @see https://github.com/feelinglucky/php-readability/issues/7
// - from http://stackoverflow.com/questions/7130867/remove-script-tag-from-html-content
$string = preg_replace("#(.*?)#is", "", $string);

return trim($string);
}

/**
* Remove all $TagName tags in DOM elements
*
* @return DOMDocument
*/
private function removeJunkTag($RootNode, $TagName) {

$Tags = $RootNode->getElementsByTagName($TagName);

//Note: always index 0, because removing a tag removes it from the results as well.
while($Tag = $Tags->item(0)){
$parentNode = $Tag->parentNode;
$parentNode->removeChild($Tag);
}

return $RootNode;

}

/**
* Remove all unnecessary attributes from the element
*/
private function removeJunkAttr($RootNode, $Attr) {
$Tags = $RootNode->getElementsByTagName("*");

$i = 0;
while($Tag = $Tags->item($i++)) {
$Tag->removeAttribute($Attr);
}

return $RootNode;
}

/**
* Get the box model of the main content of the page based on the rating
* The determination algorithm comes from: http://code.google.com/p/arc90labs-readability/
* This is forwarded by Zheng Xiao’s blog
* @return DOMNode
*/
private function getTopBox() {
// 获得页面所有的章节
$allParagraphs = $this->DOM->getElementsByTagName("p");

// Study all the paragraphs and find the chunk that has the best score.
// A score is determined by things like: Number of

's, commas, special classes, etc.
$i = 0;
while($paragraph = $allParagraphs->item($i++)) {
$parentNode = $paragraph->parentNode;
$contentScore = intval($parentNode->getAttribute(Readability::ATTR_CONTENT_SCORE));
$className = $parentNode->getAttribute("class");
$id = $parentNode->getAttribute("id");

// Look for a special classname
if (preg_match("/(comment|meta|footer|footnote)/i", $className)) {
$contentScore -= 50;
} else if(preg_match(
"/((^|＼＼s)(post|hentry|entry[-]?(content|text|body)?|article[-]?(content|text|body)?)(＼＼s|$))/i",
$className)) {
$contentScore += 25;
}

// Look for a special ID
if (preg_match("/(comment|meta|footer|footnote)/i", $id)) {
$contentScore -= 50;
} else if (preg_match(
"/^(post|hentry|entry[-]?(content|text|body)?|article[-]?(content|text|body)?)$/i",
$id)) {
$contentScore += 25;
}

// Add a point for the paragraph found
// Add points for any commas within this paragraph
if (strlen($paragraph->nodeValue) > 10) {
$contentScore += strlen($paragraph->nodeValue);
}

// 保存父元素的判定得分
$parentNode->setAttribute(Readability::ATTR_CONTENT_SCORE, $contentScore);

// 保存章节的父元素，以便下次快速获取
array_push($this->parentNodes, $parentNode);
}

$topBox = null;

// Assignment from index for performance.
// See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
for ($i = 0, $len = sizeof($this->parentNodes); $i < $len; $i++) {
$parentNode = $this->parentNodes[$i];
$contentScore = intval($parentNode->getAttribute(Readability::ATTR_CONTENT_SCORE));
$orgContentScore = intval($topBox ? $topBox->getAttribute(Readability::ATTR_CONTENT_SCORE) : 0);

if ($contentScore && $contentScore > $orgContentScore) {
$topBox = $parentNode;
}
}

// At this time, $topBox should be the main element of the page content that has been determined
return $topBox;
}

/**
* Get HTML page title
*
* @return String
*/
public function getTitle() {
$split_point = ' - ';
$titleNodes = $this->DOM->getElementsByTagName("title");

if ($titleNodes->length
&& $titleNode = $titleNodes->item(0)) {
// @see http://stackoverflow.com/questions/717328/how-to-explode-string-right-to-left
$title = trim($titleNode->nodeValue);
$result = array_map('strrev', explode($split_point, strrev($title)));
return sizeof($result) > 1 ? array_pop($result) : $title;
}

return null;
}

/**
* Get Leading Image Url
*
* @return String
*/
public function getLeadImageUrl($node) {
$images = $node->getElementsByTagName("img");

if ($images->length && $leadImage = $images->item(0)) {
return $leadImage->getAttribute("src");
}

return null;
}

/**
* Get the main content of the page (content after Readability)
*
* @return Array
*/
public function getContent() {
if (!$this->DOM) return false;

//Get page title
$ContentTitle = $this->getTitle();

// Get the main content of the page
$ContentBox = $this->getTopBox();

//Check if we found a suitable top-box.
if($ContentBox === null)
throw new RuntimeException(Readability::MESSAGE_CAN_NOT_GET);

// Copy content to new DOMDocument
$Target = new DOMDocument;
$Target->appendChild($Target->importNode($ContentBox, true));

//Delete unnecessary tags
foreach ($this->junkTags as $tag) {
$Target = $this->removeJunkTag($Target, $tag);
}

//Delete unnecessary attributes
foreach ($this->junkAttrs as $attr) {
$Target = $this->removeJunkAttr($Target, $attr);
}

$content = mb_convert_encoding($Target->saveHTML(), Readability::DOM_DEFAULT_CHARSET, "HTML-ENTITIES");

//Multiple data, returned in the form of array
return Array(
'lead_image_url' => $this->getLeadImageUrl($Target),
'word_count' => mb_strlen(strip_tags($content), Readability::DOM_DEFAULT_CHARSET),
'title' => $ContentTitle ? $ContentTitle : null,
'content' => $content
);
}

function __destruct() { }
}