怎么在Python中使用Scrapy爬取网页内容-创新互联
本篇文章给大家分享的是有关怎么在Python中使用Scrapy爬取网页内容,小编觉得挺实用的,因此分享给大家学习,希望大家阅读完这篇文章后可以有所收获,话不多说,跟着小编一起来看看吧。
成都创新互联公司主要从事成都网站建设、网站制作、网页设计、企业做网站、公司建网站等业务。立足成都服务龙湾,10余年网站建设经验,价格优惠、服务专业,欢迎来电咨询建站服务:18982081108爬虫主程序:
# -*- coding: utf-8 -*- import scrapy from scrapy.http import Request from zjf.FsmzItems import FsmzItem from scrapy.selector import Selector # 圈圈:情感生活 class MySpider(scrapy.Spider): #爬虫名 name = "MySpider" #设定域名 allowed_domains = ["nvsheng.com"] #爬取地址 start_urls = [] #flag x = 0 #爬取方法 def parse(self, response): item = FsmzItem() sel = Selector(response) item['title'] = sel.xpath('//h2/text()').extract() item['text'] = sel.xpath('//*[@class="content"]/p/text()').extract() item['imags'] = sel.xpath('//div[@id="content"]/p/a/img/@src|//div[@id="content"]/p/img/@src').extract() if MySpider.x == 0: page_list = MySpider.getUrl(self,response) for page_single in page_list: yield Request(page_single) MySpider.x += 1 yield item #init: 动态传入参数 #命令行传参写法: scrapy crawl MySpider -a start_url="http://some_url" def __init__(self,*args,**kwargs): super(MySpider,self).__init__(*args,**kwargs) self.start_urls = [kwargs.get('start_url')] def getUrl(self, response): url_list = [] select = Selector(response) page_list_tmp = select.xpath('//div[@class="viewnewpages"]/a[not(@class="next")]/@href').extract() for page_tmp in page_list_tmp: if page_tmp not in url_list: url_list.append("http://www.nvsheng.com/emotion/px/" + page_tmp) return url_list
PipeLines类
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html from zjf import settings import json,os,re,random import urllib.request import requests, json from requests_toolbelt.multipart.encoder import MultipartEncoder class MyPipeline(object): flag = 1 post_title = '' post_text = [] post_text_imageUrl_list = [] cs = [] user_id= '' def __init__(self): MyPipeline.user_id = MyPipeline.getRandomUser('37619,18441390,18441391') #process the data def process_item(self, item, spider): #获取随机user_id,模拟发帖 user_id = MyPipeline.user_id #获取正文text_str_tmp text = item['text'] text_str_tmp = "" for str in text: text_str_tmp = text_str_tmp + str # print(text_str_tmp) #获取标题 if MyPipeline.flag == 1: title = item['title'] MyPipeline.post_title = MyPipeline.post_title + title[0] #保存并上传图片 text_insert_pic = '' text_insert_pic_w = '' text_insert_pic_h = '' for imag_url in item['imags']: img_name = imag_url.replace('/','').replace('.','').replace('|','').replace(':','') pic_dir = settings.IMAGES_STORE + '%s.jpg' %(img_name) urllib.request.urlretrieve(imag_url,pic_dir) #图片上传,返回json upload_img_result = MyPipeline.uploadImage(pic_dir,'image/jpeg') #获取json中保存图片路径 text_insert_pic = upload_img_result['result']['image_url'] text_insert_pic_w = upload_img_result['result']['w'] text_insert_pic_h = upload_img_result['result']['h'] #拼接json if MyPipeline.flag == 1: cs_json = {"c":text_str_tmp,"i":"","w":text_insert_pic_w,"h":text_insert_pic_h} else: cs_json = {"c":text_str_tmp,"i":text_insert_pic,"w":text_insert_pic_w,"h":text_insert_pic_h} MyPipeline.cs.append(cs_json) MyPipeline.flag += 1 return item #spider开启时被调用 def open_spider(self,spider): pass #sipder 关闭时被调用 def close_spider(self,spider): strcs = json.dumps(MyPipeline.cs) jsonData = {"apisign":"99ea3eda4b45549162c4a741d58baa60","user_id":MyPipeline.user_id,"gid":30,"t":MyPipeline.post_title,"cs":strcs} MyPipeline.uploadPost(jsonData) #上传图片 def uploadImage(img_path,content_type): "uploadImage functions" #UPLOAD_IMG_URL = "http://api.qa.douguo.net/robot/uploadpostimage" UPLOAD_IMG_URL = "http://api.douguo.net/robot/uploadpostimage" # 传图片 #imgPath = 'D:\pics\http___img_nvsheng_com_uploads_allimg_170119_18-1f1191g440_jpg.jpg' m = MultipartEncoder( # fields={'user_id': '192323', # 'images': ('filename', open(imgPath, 'rb'), 'image/JPEG')} fields={'user_id': MyPipeline.user_id, 'apisign':'99ea3eda4b45549162c4a741d58baa60', 'image': ('filename', open(img_path , 'rb'),'image/jpeg')} ) r = requests.post(UPLOAD_IMG_URL,data=m,headers={'Content-Type': m.content_type}) return r.json() def uploadPost(jsonData): CREATE_POST_URL = http://api.douguo.net/robot/uploadimagespost
reqPost = requests.post(CREATE_POST_URL,data=jsonData)
def getRandomUser(userStr): user_list = [] user_chooesd = '' for user_id in str(userStr).split(','): user_list.append(user_id) userId_idx = random.randint(1,len(user_list)) user_chooesd = user_list[userId_idx-1] return user_chooesd
字段保存Items类
# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # http://doc.scrapy.org/en/latest/topics/items.html import scrapy class FsmzItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() title = scrapy.Field() #tutor = scrapy.Field() #strongText = scrapy.Field() text = scrapy.Field() imags = scrapy.Field()
在命令行里键入
scrapy crawl MySpider -a start_url=www.aaa.com
以上就是怎么在Python中使用Scrapy爬取网页内容,小编相信有部分知识点可能是我们日常工作会见到或用到的。希望你能通过这篇文章学到更多知识。更多详情敬请关注创新互联成都网站设计公司行业资讯频道。
另外有需要云服务器可以了解下创新互联scvps.cn,海内外云服务器15元起步,三天无理由+7*72小时售后在线,公司持有idc许可证,提供“云服务器、裸金属服务器、高防服务器、香港服务器、美国服务器、虚拟主机、免备案服务器”等云主机租用服务以及企业上云的综合解决方案,具有“安全稳定、简单易用、服务可用性高、性价比高”等特点与优势,专为企业上云打造定制,能够满足用户丰富、多元化的应用场景需求。
当前文章:怎么在Python中使用Scrapy爬取网页内容-创新互联
标题路径:http://pwwzsj.com/article/coojji.html