효율적인 Java 크롤러 실습: 웹 데이터 크롤링 기술 공유

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class HttpClientExample {
    public static void main(String[] args) {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet("https://movie.douban.com/top250");
        
        try (CloseableHttpResponse response = httpClient.execute(httpGet)){
            HttpEntity entity = response.getEntity();
            String result = EntityUtils.toString(entity);
            System.out.println(result);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class JsoupExample {
    public static void main(String[] args) {
        try {
            Document document = Jsoup.connect("https://movie.douban.com/top250").get();
            Elements elements = document.select("ol.grid_view li");
            
            for (Element element : elements) {
                String title = element.select(".title").text();
                String rating = element.select(".rating_num").text();
                String votes = element.select(".star span:nth-child(4)").text();
                
                System.out.println("电影名称：" + title);
                System.out.println("评分：" + rating);
                System.out.println("评价人数：" + votes);
                System.out.println("-------------------------");
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;

public class DataProcessingExample {
    public static void main(String[] args) {
        String jdbcUrl = "jdbc:mysql://localhost:3306/spider";
        String username = "root";
        String password = "password";
        
        try (Connection conn = DriverManager.getConnection(jdbcUrl, username, password)) {
            String sql = "INSERT INTO movie (title, rating, votes) VALUES (?, ?, ?)";
            PreparedStatement statement = conn.prepareStatement(sql);
            
            // 假设从网页中获取到了以下数据
            String title = "肖申克的救赎";
            String rating = "9.7";
            String votes = "2404447";
            
            statement.setString(1, title);
            statement.setString(2, rating);
            statement.setString(3, votes);
            
            int rowsAffected = statement.executeUpdate();
            System.out.println("插入了 " + rowsAffected + " 条数据");
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }
}