심층실습: 실무에 적용할 수 있는 주요 Java 크롤러 기술 및 경험 공유

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class Spider {
    public static void main(String[] args) throws IOException {
        String url = "https://www.example.com";
        HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
        connection.setRequestMethod("GET");
        connection.setConnectTimeout(5000);
        connection.setReadTimeout(5000);
        
        BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
        String line;
        StringBuilder response = new StringBuilder();
        while ((line = reader.readLine()) != null) {
            response.append(line);
        }
        
        reader.close();
        connection.disconnect();
        
        System.out.println(response.toString());
    }
}

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Spider {
    public static void main(String[] args) throws IOException {
        String url = "https://www.example.com";
        Document document = Jsoup.connect(url).get();
        
        Elements elements = document.select(".class-name");
        for (Element element : elements) {
            String content = element.text();
            System.out.println(content);
        }
    }
}

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;

public class Spider {
    public static void main(String[] args) throws SQLException {
        Connection connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/database", "username", "password");
        PreparedStatement statement = connection.prepareStatement("INSERT INTO table_name (column1, column2) VALUES (?, ?)");
        
        // 假设从网页中获取到的数据存储在dataList中
        for (Data data : dataList) {
            statement.setString(1, data.getField1());
            statement.setString(2, data.getField2());
            statement.executeUpdate();
        }
        
        statement.close();
        connection.close();
    }
}