Bringen Sie Ihnen Schritt für Schritt bei, wie Sie mit Flask eine ES-Suchmaschine erstellen (vorbereitender Teil)

pip install elasticsearch

def __init__(self, index_type: str, index_name: str, ip="127.0.0.1"): # self.es = Elasticsearch([ip], http_auth=('username', 'password'), port=9200) self.es = Elasticsearch("localhost:9200") self.index_type = index_type self.index_name = index_name

def get_doc(self, uid): return self.es.get(index=self.index_name, id=uid)

def insert_one(self, doc: dict): self.es.index(index=self.index_name, doc_type=self.index_type, body=doc) def insert_array(self, docs: list): for doc in docs: self.es.index(index=self.index_name, doc_type=self.index_type, body=doc)

def search(self, query, count: int = 30): dsl = { "query": { "multi_match": { "query": query, "fields": ["title", "content", "link"] } }, "highlight": { "fields": { "title": {} } } } match_data = self.es.search(index=self.index_name, body=dsl, size=count) return match_data def __search(self, query: dict, count: int = 20): # count: 返回的数据大小 results = [] params = { 'size': count } match_data = self.es.search(index=self.index_name, body=query, params=params) for hit in match_data['hits']['hits']: results.append(hit['_source']) return results

def delete_index(self): try: self.es.indices.delete(index=self.index_name) except: pass

from elasticsearch import Elasticsearch class elasticSearch(): def __init__(self, index_type: str, index_name: str, ip="127.0.0.1"): # self.es = Elasticsearch([ip], http_auth=('elastic', 'password'), port=9200) self.es = Elasticsearch("localhost:9200") self.index_type = index_type self.index_name = index_name def create_index(self): if self.es.indices.exists(index=self.index_name) is True: self.es.indices.delete(index=self.index_name) self.es.indices.create(index=self.index_name, ignore=400) def delete_index(self): try: self.es.indices.delete(index=self.index_name) except: pass def get_doc(self, uid): return self.es.get(index=self.index_name, id=uid) def insert_one(self, doc: dict): self.es.index(index=self.index_name, doc_type=self.index_type, body=doc) def insert_array(self, docs: list): for doc in docs: self.es.index(index=self.index_name, doc_type=self.index_type, body=doc) def search(self, query, count: int = 30): dsl = { "query": { "multi_match": { "query": query, "fields": ["title", "content", "link"] } }, "highlight": { "fields": { "title": {} } } } match_data = self.es.search(index=self.index_name, body=dsl, size=count) return match_data

import json from datetime import datetime import pymongo from app.elasticsearchClass import elasticSearch client = pymongo.MongoClient('127.0.0.1', 27017) db = client['spider'] sheet = db.get_collection('Spider').find({}, {'_id': 0, }) es = elasticSearch(index_type="spider_data",index_name="spider") es.create_index() for i in sheet: data = { 'title': i["title"], 'content':i["data"], 'link': i["link"], 'create_time':datetime.now() } es.insert_one(doc=data)

import requests import re import time exist_urls = [] headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36', } def get_link(url): try: response = requests.get(url=url, headers=headers) response.encoding = 'UTF-8' html = response.text link_lists = re.findall('.*?', html) return link_lists except Exception as e: pass finally: exist_urls.append(url) # 当爬取深度小于10层时，递归调用主函数，继续爬取第二层的所有链接 def main(start_url, depth=1): link_lists = get_link(start_url) if link_lists: unique_lists = list(set(link_lists) - set(exist_urls)) for unique_url in unique_lists: unique_url = 'https://baike.baidu.com/item/' + unique_url with open('url.txt', 'a+') as f: f.write(unique_url + '\n') f.close() if depth < 10: main(unique_url, depth + 1) if __name__ == '__main__': start_url = 'https://baike.baidu.com/item/%E7%99%BE%E5%BA%A6%E7%99%BE%E7%A7%91' main(start_url)