Python での実践的なクローラー戦闘: WeChat パブリック アカウント クローラー

pip install beautifulsoup4

pip install robotframework
pip install robotframework-seleniumlibrary
pip install selenium

import requests
from bs4 import BeautifulSoup
import time

url = "http://weixin.sogou.com/weixin?type=1&query={}".format("Python之禅")

# 使用Selenium工具来模拟浏览器操作
from selenium import webdriver

driver = webdriver.Chrome()
driver.get(url)

# 执行搜索动作
search_box = driver.find_element_by_xpath('//*[@id="query"]')
search_box.send_keys(u"Python之禅")
search_box.submit()

# 点击搜索结果中的公众号
element = driver.find_element_by_xpath('//div[@class="news-box"]/ul/li[2]/div[2]/h3/a')
element.click()

# 等待页面加载
time.sleep(3)

# 点击“历史消息”链接
element = driver.find_element_by_xpath('//a[@title="历史消息"]')
element.click()

# 等待页面加载
time.sleep(3)

# 获取文章链接
soup = BeautifulSoup(driver.page_source, 'html.parser')
urls = []
for tag in soup.find_all("a", href=True):
    url = tag["href"]
    if "mp.weixin.qq.com" in url:
        urls.append(url)

# 获取每篇文章的标题、发布时间、阅读量和点赞数
for url in urls:
    response = requests.get(url)
    response.encoding = 'utf-8'
    soup = BeautifulSoup(response.text, 'html.parser')

    title = soup.find('h2', {'class': 'rich_media_title'}).text.strip()
    date = soup.find('em', {'id': 'post-date'}).text.strip()
    readnum = soup.find('span', {'class': 'read_num'}).text.strip()
    likenum = soup.find('span', {'class': 'like_num'}).text.strip()
    print(title, date, readnum, likenum)