효율적인 데이터 크롤링 기술 익히기: 강력한 Java 크롤러 구축

// 导入Jsoup库 import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; public class JsoupExample { public static void main(String[] args) throws Exception { // 发送HTTP请求获取网页内容 Document doc = Jsoup.connect("http://example.com").get(); // 解析页面，提取需要的数据 Elements elements = doc.select("h1"); // 使用选择器选择需要的元素 for (Element element : elements) { System.out.println(element.text()); } } }

// 导入HttpClient库 import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; public class HttpClientExample { public static void main(String[] args) throws Exception { // 创建HttpClient实例 HttpClient httpClient = new DefaultHttpClient(); // 创建HttpGet请求 HttpGet httpGet = new HttpGet("http://example.com"); // 发送HTTP请求并获取服务器的响应 HttpResponse response = httpClient.execute(httpGet); // 解析响应，提取需要的数据 HttpEntity entity = response.getEntity(); String content = EntityUtils.toString(entity); System.out.println(content); } }

import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class MultiThreadSpider { private static final int THREAD_POOL_SIZE = 10; public static void main(String[] args) throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE); for (int i = 1; i <= 10; i++) { final int page = i; executorService.execute(() -> { try { // 发送HTTP请求获取网页内容 Document doc = Jsoup.connect("http://example.com/page=" + page).get(); // 解析页面，提取需要的数据 Elements elements = doc.select("h1"); // 使用选择器选择需要的元素 for (Element element : elements) { System.out.println(element.text()); } } catch (Exception e) { e.printStackTrace(); } }); } executorService.shutdown(); } }

import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import java.net.InetSocketAddress; import java.net.Proxy; public class ProxyIPSpider { public static void main(String[] args) throws Exception { // 创建代理IP Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 8080)); // 发送HTTP请求并使用代理IP Document doc = Jsoup.connect("http://example.com").proxy(proxy).get(); // 解析页面，提取需要的数据 Elements elements = doc.select("h1"); // 使用选择器选择需要的元素 for (Element element : elements) { System.out.println(element.text()); } } }