多线程爬取都挺好链接并保存到mongodb

• 一个比较简单，python3多线程使用requests库爬取都挺好，并使用正则提取下载链接，保存到mongodb

#!/usr/bin/env python# -*- coding:utf-8 -*-"""@author:Aiker Zhao@file:doutinghao.py@time:下午8:18"""import requestsimport reimport pymongofrom multiprocessing import PoolMONGO_URL = 'localhost:27017'MONGO_DB = 'doutinghao'MONGO_TABLE = 'doutinghao'client = pymongo.MongoClient(MONGO_URL, connect=False)db = client[MONGO_DB]def get_result(url):    response = requests.get(url).text    # print(reponse.text)    pattern = re.compile('', re.S)    result = re.findall(pattern, response)    if result:        for i in result:            url, name = i            yield {                "name": name,                'url': url            }def save_to_mongo(result):    if db[MONGO_TABLE].insert(result):        print('存储到MongoDB成功', result)        return True    return Falsedef main(result):    # result = get_result(url)    save_to_mongo(result)if __name__ == '__main__':    pool = Pool()    url = "https://www.xl720.com/thunder/34283.html"    item = [item for item in get_result(url)]    # print(item)    pool.map(main, item)    pool.close()    pool.join()