怎么用Scrapy构建一个网络爬虫

$ xcode-select --install

# -*- coding: utf-8 -*-

import scrapy

class MyCrawlerItem(scrapy.Item):

title = scrapy.Field() # 文章标题

url = scrapy.Field() # 文章地址

summary = scrapy.Field() # 文章摘要

pass

# -*- coding: utf-8 -*-

import pymongo

from scrapy.conf import settings

from scrapy.exceptions import DropItem

class MyCrawlerPipeline(object):

def __init__(self):

# 设置MongoDB连接

connection = pymongo.Connection(

settings['MONGO_SERVER'],

settings['MONGO_PORT']

)

db = connection[settings['MONGO_DB']]

self.collection = db[settings['MONGO_COLLECTION']]

# 处理每个被抓取的MyCrawlerItem项

def process_item(self, item, spider):

valid = True

for data in item:

if not data: # 过滤掉存在空字段的项

valid = False

raise DropItem("Missing {0}!".format(data))

if valid:

# 也可以用self.collection.insert(dict(item))，使用upsert可以防止重复项

self.collection.update({'url': item['url']}, dict(item), upsert=True)

return item

ITEM_PIPELINES = {

'my_crawler.pipelines.MyCrawlerPipeline': 300, # 设置Pipeline，可以多个，值为执行优先级

}

# MongoDB连接信息

MONGO_SERVER = 'localhost'

MONGO_PORT = 27017

MONGO_DB = 'bjhee'

MONGO_COLLECTION = 'articles'

DOWNLOAD_DELAY=2 # 如果网络慢，可以适当加些延迟，单位是秒