Java クローラー フレームワーク対決: 最良の選択は誰ですか?

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class JsoupExample {

    public static void main(String[] args) throws Exception {
        String url = "https://example.com";
        Document doc = Jsoup.connect(url).get();
        
        // 获取所有标题
        Elements titles = doc.select("h1");
        for (Element title : titles) {
            System.out.println(title.text());
        }
        
        // 获取所有链接
        Elements links = doc.select("a[href]");
        for (Element link : links) {
            System.out.println(link.attr("href"));
        }
        
        // 获取页面内容
        System.out.println(doc.html());
    }
}

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class HttpClientExample {

    public static void main(String[] args) throws Exception {
        String url = "https://example.com";
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(url);
        
        try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
            HttpEntity entity = response.getEntity();
            String html = EntityUtils.toString(entity);
            System.out.println(html);
        }
    }
}

import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class WebMagicExample {

    public static void main(String[] args) {
        Spider.create(new MyPageProcessor())
                .addUrl("https://example.com")
                .addPipeline(new ConsolePipeline())
                .run();
    }

    static class MyPageProcessor implements PageProcessor {
        @Override
        public void process(Page page) {
            // 提取标题
            String title = page.getHtml().$("h1").get();
            System.out.println(title);
            
            // 提取链接
            page.addTargetRequests(page.getHtml().links().regex(".*").all());
        }
        
        @Override
        public Site getSite() {
            return Site.me().setRetryTimes(3).setSleepTime(1000);
        }
    }
}