HttpClient crawls web page source code

Package UTIL;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util .Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.zip.GZIPInputStream;

import org.apache.commons.httpclient.Header;
import org.apache .commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons .httpclient.NameValuePair;
import org.apache.commons.httpclient.SimpleHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache .commons.httpclient.params.HttpConnectionManagerParams;
import org.apache.commons.httpclient.params.HttpMethodParams;

/ **
* @author Liuwei
* Date: December 18, 2009
*
* TODO
* HttpClient’s auxiliary class
* /
public class HttpClientHelper
{

/ **
* HttpClient’s connection timeout , read data timeout setting (unit: milliseconds)
* /
Public static final int HTTPCLIENT_CONNECTION_TIMEOUT = 30000;
Public static final int HTTPCLIENT_SO_TIMEOUT = 120000;
Public static final int HTTPMETHOD_SO_TIMEOUT = 5000;

//Let the ConnectionMan ager management Whether to close the connection when httpclientconnection
private static boolean alwaysClose = FALSE;
private static string defaultEncode = "UTF-8";

private static last DateFormat DATE_FORMAT = new SimpleDateFormat("YYYY-MM-DD HH:MM:SS ");

/ **
* Get the HttpClient connection and set the relevant parameters
*
* @return
* /
public static HttpClient's getHttpClient()
{
HttpClient client = new HttpClient (new SimpleHttpConnectionManager (alwaysClose));
HttpConnectionManagerParams managerParams = client.getHttpConnectionManager() getParams() method.
//Set the connection timeout (in milliseconds)
managerParams.setConnectionTimeout (HTTPCLIENT_CONNECTION_TIMEOUT);
//Set the read data timeout (in milliseconds)
managerParams.setSoTimeout (HTTPCLIENT_SO_TIMEOUT);
Return to the client;
}

/ * *
* Get the HttpClient connection and set the relevant parameters
*
* @parameter logonSite
* @parameter logonPort
* @parameter protocol
* @return
* /
public static HttpClient’s getHttpClient (last string logonSite, final interpretation logonPort, last string protocol)
{
HttpClient client = new HttpClient(new SimpleHttpConnectionManager(alwaysClose));
client.getHostConfiguration() setHost(logonSite, logonPort, protocol).
HttpConnectionManagerParams managerParams = client.getHttpConnectionManager() getParams() method.
//Set the connection timeout (in milliseconds)
managerParams.setConnectionTimeout (HTTPCLIENT_CONNECTION_TIMEOUT);
//Set the read data timeout (in milliseconds)
managerParams.setSoTimeout (HTTPCLIENT_SO_TIMEOUT);
Return to the client;
}

Private static List getHeaders(MapHeaders)
{
List = ArrayList of Headers new ();
Boolean includeUserAgent = FALSE;
if ( empty = header&& false == header.isEmpty() ! )
{
set> = entrySet header.entrySet();
for (enter entry: entrySet)
{
if (false == includeUserAgent
&& "UserAgent".equals(entry.getKey()))
{
includeUserAgent = TRUE;
}
headers.add(new headers() entry.getKey(), entry.getValue()));
}
}

if (false == includeUserAgent)
{
headers.add(new headers(
"UserAgent",
"Mozilla/4.0( Compatible with; MSIE 7.0; Windows NT 5.1; GTB5; .NET CLR 1.1.4322; .NET CLR 2.0 0.50727; Alexa Toolbar; MAXTHON 2.0)"));
}
Return Header;
}

Private static NameValuePair [] getPairs(Map POSTDATA)
{
if (null == || POSTDATA postData.isEmpty())
{
return NULL;
}

set> = entrySet postData.entrySet();
INT DATALENGTH = entrySet.size();
NameValuePair[] = new NameValuePair[DATALENGTH]
INT I = 0;
For (entry< ;String, string>entrySet)
{
double[i++] = new NameValuePair(entry.getKey(), entry.getValue());
}
return pair;
}

/ **
*Request web content information
*
* @parameter HttpClient
* @parameter reqUrl
*parameter title
* @parameter POSTDATA
*parameter encoding
* @return
* /
public static string doRequest(HttpClient HttpClient, String reqUrl,
Map header, Map POSTDATA, String encoding)
{
String htmlContent = NULL;
if (null == HttpClient )
{
Return htmlContent;
}

//Request encoding settings
encoding = (null == encoding defaultEncode: encoding);

//Header request information
List