Java爬虫的核心技术从基础到高级全面探讨

// 创建 HttpClient 对象
CloseableHttpClient httpClient = HttpClients.createDefault();

// 创建 HttpGet 对象
HttpGet httpGet = new HttpGet("http://www.example.com");

// 发送 GET 请求
CloseableHttpResponse response = httpClient.execute(httpGet);

// 获取响应内容
String html = EntityUtils.toString(response.getEntity(), "UTF-8");

// 关闭 HttpClient 和响应对象
response.close();
httpClient.close();

// 发送 GET 请求，获取 Document 对象
Document doc = Jsoup.connect("http://www.example.com").get();

// 通过 CSS 选择器提取需要的信息
Element titleElement = doc.select("title").first();
String title = titleElement.text();

// 解析 HTML 字符串
Document doc = Jsoup.parse(html);

// 通过标签名提取需要的信息
Elements elements = doc.getElementsByTag("a");
for (Element element : elements) {
    String href = element.attr("href");
    String text = element.text();
    System.out.println(href + " - " + text);
}

// 解析 HTML 字符串
Document doc = Jsoup.parse(html);

// 使用 XPath 定位元素
XPath xpath = XPathFactory.newInstance().newXPath();
XPathExpression expr = xpath.compile("//a[contains(text(),'click here')]");
NodeList nodeList = (NodeList) expr.evaluate(doc, XPathConstants.NODESET);

// 遍历节点列表，提取需要的信息
for (int i = 0; i < nodeList.getLength(); i++) {
    Node node = nodeList.item(i);
    String href = node.getAttributes().getNamedItem("href").getNodeValue();
    String text = node.getTextContent();
    System.out.println(href + " - " + text);
}

// 创建 BufferedWriter 对象
BufferedWriter writer = new BufferedWriter(new FileWriter("data.txt"));

// 写入数据
writer.write("Data 1");
writer.newLine();
writer.write("Data 2");

// 关闭 BufferedWriter
writer.close();

// 加载数据库驱动
Class.forName("com.mysql.jdbc.Driver");

// 连接数据库
Connection conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/test", "root", "password");

// 创建 PreparedStatement 对象
PreparedStatement ps = conn.prepareStatement("INSERT INTO data VALUES (?, ?)");

// 设置参数
ps.setString(1, "Data 1");
ps.setString(2, "Data 2");

// 执行插入操作
ps.executeUpdate();

// 关闭 PreparedStatement 和连接
ps.close();
conn.close();