Saya ingin merangkum kumpulan proses dalam penghias, tetapi ia tidak berkuat kuasa mahupun melaporkan ralat

# coding:utf-8 import multiprocessing import tornado from tornado.httpclient import AsyncHTTPClient process_num = 20 # 进程数 url = "https://www.baidu.com" def handle_request(response): print str(response) def run_in_process(process_num): def _run_in_process(f): def __run_in_process(*args, **kwargs): pool = multiprocessing.Pool(processes=process_num) for i in range(process_num): pool.apply_async(f, args=args, kwds=kwargs, callback=kwargs.get("callback")) pool.close() pool.join() return __run_in_process return _run_in_process @run_in_process(process_num) def main(): http_client = AsyncHTTPClient() http_client.fetch(url, callback=handle_request) global loop loop = tornado.ioloop.IOLoop.instance() if loop._running is False: loop.start() if __name__ == '__main__': main()

Hasilnya adalah seperti berikut

/usr/bin/python2.7 /home/xxx/workspace/py_project/crawler/center/sample.py Process finished with exit code 0

Tetapi yang peliknya ialah saya menulis semula dalam mod berbilang proses dan mendapati ia boleh berkuat kuasa

# coding:utf-8 import multiprocessing import tornado from tornado.httpclient import AsyncHTTPClient process_num = 20 # 进程数 url = "https://www.baidu.com" def handle_request(response): print str(response) def run_in_process(process_num): def _run_in_process(f): def __run_in_process(*args, **kwargs): _processes = [] for i in xrange(process_num): p = multiprocessing.Process(target=f, args=args, kwargs=kwargs) p.start() _processes.append(p) for p in _processes: p.join() return __run_in_process return _run_in_process @run_in_process(process_num) def main(): http_client = AsyncHTTPClient() http_client.fetch(url, callback=handle_request) global loop loop = tornado.ioloop.IOLoop.instance() if loop._running is False: loop.start() if __name__ == '__main__': main()

Log adalah seperti berikut

/usr/bin/python2.7 /home/shufeng/workspace/private_project/jobscrawler/center/sample.py HTTPResponse(_body=None,buffer=<_io.BytesIO object at 0x7f2fdaa21ef0>,code=200,effective_url='http://www.baidu.com',error=None,headers=,reason='OK',request=,request_time=0.014312028884887695,time_info={}) HTTPResponse(_body=None,buffer=<_io.BytesIO object at 0x7f2fdaa21ef0>,code=200,effective_url='http://www.baidu.com',error=None,headers=,reason='OK',request=,request_time=0.02327895164489746,time_info={}) HTTPResponse(_body=None,buffer=<_io.BytesIO object at 0x7f2fdaa21ef0>,code=200,effective_url='http://www.baidu.com',error=None,headers=,reason='OK',request=,request_time=0.026951074600219727,time_info={}) HTTPResponse(_body=None,buffer=<_io.BytesIO object at 0x7f2fdaa21ef0>,code=200,effective_url='http://www.baidu.com',error=None,headers=,reason='OK',request=,request_time=0.0552978515625,time_info={}) HTTPResponse(_body=None,buffer=<_io.BytesIO object at 0x7f2fdaa24ef0>,code=200,effective_url='http://www.baidu.com',error=None,headers=,reason='OK',request=,request_time=0.05612993240356445,time_info={})

Situasi yang sama juga akan berlaku apabila menggunakan kumpulan benang dan coroutine Adakah sesiapa tahu apa yang sedang berlaku?