Python의 실제 크롤러 전투: Baidu는 크롤러를 알고 있습니다.

import requests
from bs4 import BeautifulSoup

# 网页地址
url = "https://zhidao.baidu.com/question/2031956566959407839.html"

# 发送请求
r = requests.get(url)

# 解析网页
soup = BeautifulSoup(r.text, "html.parser")

# 获取问题
question = soup.find("span", class_="ask-title").text
print("问题： ", question)

# 获取最佳答案
answer = soup.find("pre", class_="best-text mb-10").text
print("最佳答案： ", answer)

import requests
from bs4 import BeautifulSoup

# 创建问题列表
questions = [
    "2031956566959407839", 
    "785436012916117832", 
    "1265757662946113922", 
    "455270192556513192", 
    "842556478655981450"
]

# 循环爬取问题和最佳答案
for q in questions:
    # 根据问题ID拼接URL
    url = f"https://zhidao.baidu.com/question/{q}.html"

    # 发送请求
    r = requests.get(url)

    # 解析网页
    soup = BeautifulSoup(r.text, "html.parser")

    # 获取问题
    try:
        question = soup.find("span", class_="ask-title").text
    except:
        question = ""

    # 获取最佳答案
    try:
        answer = soup.find("pre", class_="best-text mb-10").text
    except:
        answer = ""

    # 打印问题和答案
    print("问题： ", question)
    print("最佳答案： ", answer)
    print("----------------------")

import requests
from bs4 import BeautifulSoup
import csv
import codecs

# 创建问题列表
questions = [
    "2031956566959407839", 
    "785436012916117832", 
    "1265757662946113922", 
    "455270192556513192", 
    "842556478655981450"
]

# 创建文件
with open("questions.csv", "w", newline='', encoding='utf-8-sig') as file:
    writer = csv.writer(file)
    writer.writerow(['问题', '最佳答案'])

    # 循环爬取问题和最佳答案
    for q in questions:
        # 根据问题ID拼接URL
        url = f"https://zhidao.baidu.com/question/{q}.html"

        # 发送请求
        r = requests.get(url)

        # 解析网页
        soup = BeautifulSoup(r.text, "html.parser")

        # 获取问题
        try:
            question = soup.find("span", class_="ask-title").text
        except:
            question = ""

        # 获取最佳答案
        try:
            answer = soup.find("pre", class_="best-text mb-10").text
        except:
            answer = ""

        # 保存到csv文件
        writer.writerow([question, answer])