Scrapy 크롤러는 Qianku.com의 아름다운 사진 데이터 수집 및 인기 공유 순위를 실현합니다.

ITEM_PIPELINES = {'qkspider.pipelines.QkspiderPipeline':100,}

import pymongo

class QkspiderPipeline(object):
    def __init__(self):
        client = pymongo.MongoClient("mongodb://localhost:27017/")
        db = client['qkdb']
        self.collection = db['qkpic']

    def process_item(self, item, spider):
        self.collection.insert(dict(item))
        return item

import scrapy

class QkspiderItem(scrapy.Item):
    title = scrapy.Field()
    img_url = scrapy.Field()
    share_num = scrapy.Field()

import scrapy
from qkspider.items import QkspiderItem

class QkSpider(scrapy.Spider):
    name = "qk"
    allowed_domains = ["qkpic.com"]
    start_urls = ["http://www.qkpic.com/"]

    def parse(self, response):
        items = []
        pic_lists = response.xpath('//div[@class="index_mianpic"]/ul/li')
        for i, pic in enumerate(pic_lists):
            item = QkspiderItem()
            item['title'] = pic.xpath('./a/@title').extract_first()
            item['img_url'] = pic.xpath('./a/img/@src').extract_first()
            item['share_num'] = int(pic.xpath('./span/em/text()').extract_first())
            items.append(item)
        return items

class QkSpider(scrapy.Spider):
    name = "qk"
    allowed_domains = ["qkpic.com"]
    start_urls = ["http://www.qkpic.com/", "http://www.qkpic.com/top/"]

    def parse(self, response):
        if response.url.startswith('http://www.qkpic.com/top/'):
            items = self.parse_rank_list(response)
        else:
            items = self.parse_pic_info(response)
        return items

    # 爬取热门榜单信息
    def parse_rank_list(self, response):
        items = []
        pic_lists = response.xpath('//div[@class="topcont"]/ul/li')
        for i, pic in enumerate(pic_lists):
            item = QkspiderItem()
            item['title'] = pic.xpath('./a/@title').extract_first()
            item['img_url'] = pic.xpath('./a/img/@src').extract_first()
            item['share_num'] = int(pic.xpath('./div[1]/i[2]/text()').extract_first())
            items.append(item)
        return items