자바 크롤러 도구: 웹 페이지 데이터를 크롤링하는 실용적인 도구인 네트워크 데이터 수집의 비밀을 밝힙니다.

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class HttpURLConnectionExample {

    public static void main(String[] args) throws IOException {
        // 设置需要爬取的URL
        String url = "http://example.com";
    
        // 创建URL对象
        URL obj = new URL(url);
        // 打开连接
        HttpURLConnection con = (HttpURLConnection) obj.openConnection();
    
        // 获取响应码
        int responseCode = con.getResponseCode();
        System.out.println("Response Code: " + responseCode);
    
        // 创建BufferedReader对象，读取网页内容
        BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
        String inputLine;
        StringBuilder content = new StringBuilder();
        while ((inputLine = in.readLine()) != null) {
            content.append(inputLine);
        }
        in.close();
    
        // 输出网页内容
        System.out.println(content);
    }
}

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;

public class JsoupExample {

    public static void main(String[] args) throws IOException {
        // 设置需要爬取的URL
        String url = "http://example.com";
    
        // 使用Jsoup连接到网页
        Document doc = Jsoup.connect(url).get();
    
        // 获取所有的a标签
        Elements links = doc.getElementsByTag("a");
        for (Element link : links) {
            // 输出a标签的href属性值和文本内容
            System.out.println("Link: " + link.attr("href") + ", Text: " + link.text());
        }
    }
}

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpClientExample {

    public static void main(String[] args) throws IOException {
        // 设置需要爬取的URL
        String url = "http://example.com";
    
        // 创建HttpClient对象
        HttpClient client = new DefaultHttpClient();
        // 创建HttpGet对象，设置URL
        HttpGet request = new HttpGet(url);
    
        // 发送HTTP请求
        HttpResponse response = client.execute(request);
    
        // 获取响应实体
        HttpEntity entity = response.getEntity();
    
        // 将实体转为字符串
        String content = EntityUtils.toString(entity);
    
        // 输出网页内容
        System.out.println(content);
    }
}