Java 크롤러 프레임워크 대결: 최선의 선택은 누구입니까?

import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; public class JsoupExample { public static void main(String[] args) throws Exception { String url = "https://example.com"; Document doc = Jsoup.connect(url).get(); // 获取所有标题 Elements titles = doc.select("h1"); for (Element title : titles) { System.out.println(title.text()); } // 获取所有链接 Elements links = doc.select("a[href]"); for (Element link : links) { System.out.println(link.attr("href")); } // 获取页面内容 System.out.println(doc.html()); } }

import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; public class HttpClientExample { public static void main(String[] args) throws Exception { String url = "https://example.com"; CloseableHttpClient httpClient = HttpClients.createDefault(); HttpGet httpGet = new HttpGet(url); try (CloseableHttpResponse response = httpClient.execute(httpGet)) { HttpEntity entity = response.getEntity(); String html = EntityUtils.toString(entity); System.out.println(html); } } }

import us.codecraft.webmagic.*; import us.codecraft.webmagic.pipeline.ConsolePipeline; import us.codecraft.webmagic.processor.PageProcessor; public class WebMagicExample { public static void main(String[] args) { Spider.create(new MyPageProcessor()) .addUrl("https://example.com") .addPipeline(new ConsolePipeline()) .run(); } static class MyPageProcessor implements PageProcessor { @Override public void process(Page page) { // 提取标题 String title = page.getHtml().$("h1").get(); System.out.println(title); // 提取链接 page.addTargetRequests(page.getHtml().links().regex(".*").all()); } @Override public Site getSite() { return Site.me().setRetryTimes(3).setSleepTime(1000); } } }