scrapy基础知识之将item 通过pipeline保存数据到mysql mongoDB：
pipelines.pyclass xxPipeline(object):
def process_item(self, item, spider):
con=pymysql.connect(host='localhost,user='',passwd='',db='',charset='utf8')
cur=con.cursor()
sql=("insert into 表名(字段)”"values(%s)")
lis=(item['字段'])
cur.execute(sql,lis)
con.commit()
cur.close()
con.close()
return item
在settings.py中配置
ITEM_PIPELINES = {
’MySpider.pipelines.xxPipeline': 300,
}
需要在mysql中创建数据库以及数据表字段
pipelines.py
import pymongo
class MyspiderPipeline(object): def __init__(self):
# pymongo.MongoClient(host, port) 创建MongoDB链接
client = pymongo.MongoClient(host="localhost",port=27017)
# 指向指定的数据库
db_name = client["dbname"]
table_name=db_name["tablename"]
# 获取数据库里存放数据的表名
self.post = table_name
def process_item(self, item, spider):
data = dict(item)
# 向指定的表里添加数据
self.post.insert(data)
return item
settings.py配置
DEFAULT_REQUEST_HEADERS = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0'}
页:
[1]