动态网站图片抓取进阶：利用Google CSE API绕过前端渲染限制

import requests
import json
import re # For parsing JSONP

def fetch_dermnet_images_from_api(query_term, start_index=1, num_results=16):
    """
    从Dermnet网站的Google CSE API抓取图片信息。
    :param query_term: 搜索关键词。
    :param start_index: 结果的起始索引（用于分页）。
    :param num_results: 每页返回结果的数量。
    :return: 包含图片信息的列表。
    """
    base_url = "https://cse.google.com/cse/element/v1"

    # 根据网络请求观察到的参数构建
    params = {
        "rsz": "large",
        "num": num_results,
        "hl": "en",
        "source": "gcsc",
        "gss": ".com",
        "cselibv": "8e77c7877b8339e2", # 注意: 此参数可能随时间变化，需定期检查
        "searchtype": "image",
        "cx": "015036873904746004277:nz7deehiccq",
        "q": query_term,
        "safe": "off",
        "start": start_index, # 分页参数
        "exp": "csqr,cc,bf",
        "callback": "google.search.cse.api10440" # 保持此参数以获取JSONP格式
    }

    # 模拟浏览器User-Agent，避免被识别为爬虫
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }

    images_data = []
    try:
        response = requests.get(base_url, params=params, headers=headers, timeout=10)
        response.raise_for_status() # 如果请求失败（HTTP状态码非200），则抛出异常

        # 解析JSONP响应
        # 响应格式通常为 google.search.cse.apiXXXXX({...JSON_CONTENT...})
        jsonp_content = response.text
        match = re.search(r'^\w+\((.*)\)$', jsonp_content, re.DOTALL)
        if match:
            json_str = match.group(1)
            data = json.loads(json_str)

            if "results" in data:
                for item in data["results"]:
                    if "image" in item:
                        images_data.append({
                            "title": item.get("title", "No Title"),
                            "url": item["image"].get("src", ""),
                            "thumbnail_url": item["image"].get("thumbnailSrc", ""),
                            "width": item["image"].get("width", 0),
                            "height": item["image"].get("height", 0)
                        })
        else:
            print(f"Error: Could not parse JSONP response for query '{query_term}'.")
            print(f"Raw response start: {jsonp_content[:200]}...")

    except requests.exceptions.RequestException as e:
        print(f"Network or HTTP error for query '{query_term}': {e}")
    except json.JSONDecodeError as e:
        print(f"JSON decoding error for query '{query_term}': {e}")
    except Exception as e:
        print(f"An unexpected error occurred for query '{query_term}': {e}")

    return images_data

# 示例用法：
if __name__ == "__main__":
    search_query = "basal cell carcinoma dermoscopy"

    print(f"--- 抓取第一页 '{search_query}' 的图片 ---")
    first_page_images = fetch_dermnet_images_from_api(search_query, start_index=1, num_results=16)
    if first_page_images:
        print(f"第一页共找到 {len(first_page_images)} 张图片。")
        for i, img in enumerate(first_page_images[:3]): # 只打印前3张示例
            print(f"  {i+1}. 标题: {img['title']}, URL: {img['url']}")
    else:
        print("未找到第一页图片或发生错误。")

    print(f"\n--- 抓取第二页 '{search_query}' 的图片 ---")
    # 假设每页16张，第二页从索引17开始
    second_page_images = fetch_dermnet_images_from_api(search_query, start_index=17, num_results=16)
    if second_page_images:
        print(f"第二页共找到 {len(second_page_images)} 张图片。")
        for i, img in enumerate(second_page_images[:3]): # 只打印前3张示例
            print(f"  {i+1}. 标题: {img['title']}, URL: {img['url']}")
    else:
        print("未找到第二页图片或发生错误。")