import scrapyclass DouyuspiderItem(scrapy.Item):# 存储照片的名字
nickName = scrapy.Field()# 照片的url路径
imageUrl = scrapy.Field()# 照片保存在本地的路径
imagePath = scrapy.Field()

# -*- coding: utf-8 -*-import scrapyfrom douyuSpider.items import DouyuspiderItemimport jsonclass DouyuSpider(scrapy.Spider):name = 'douyu'allowed_domains = ['capi.douyucdn.cn']offset = 0url = "http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&offset="start_urls = [url + str(offset)]def parse(self, response):data = json.loads(response.body)["data"]for each in data:item = DouyuspiderItem()item["nickName"] = each["nickname"]item["imageUrl"] = each["vertical_src"]yield itemself.offset += 20nextUrl = re.sub('offset=\d+', 'offset='+str(self.offset), response.url)yield scrapy.Request(nextUrl, callback=self.parse)

DEFAULT_REQUEST_HEADERS = {'User-Agent': 'News 6.6.5 rv:6.6.5.03 (iPhone; iOS 11.2.6; zh_CN) Cronet'}ITEM_PIPELINES = {'douyuSpider.pipelines.DouyuspiderPipeline': 300,}IMAGES_STORE = 'E:\Python\Spider\day05\douyuSpider\images'# 日志文件名和处理等级

LOG_FILE = "douyu.log"LOG_LEVEL = "DEBUG"

# -*- coding: utf-8 -*-import scrapyfrom scrapy.exceptions import DropItemfrom scrapy.utils.project import get_project_settingsfrom scrapy.pipelines.images import ImagesPipelineimport osclass DouyuspiderPipeline(ImagesPipeline):IMAGES_STORE = get_project_settings().get("IMAGES_STORE")# 根据图片的url生成图片的Requestdef get_media_requests(self, item, info):image_url = item["imageUrl"]yield scrapy.Request(image_url)# 图片下载完毕之后，处理结果会以二元组的方式作为results参数传递给item_completed函数# 这个二元组定义如下：(success, image_info_or_failure)，第一个元素表示图片是否下载成功，第二个元素是一个字典# 字典的含义如下：{'url': 图片的url, 'path': 图片的存储地址，跟IMAGE_STORE相关, 'checksum': 图片的内容hash}def item_completed(self, results, item, info):# 如果图片下载成功，则获取图片的存储地址
image_path = [x["path"] for ok, x in results if ok]#print image_pathif not image_path:raise DropItem("Item contains no images")# 修改图片的存储地址：
os.rename(self.IMAGES_STORE + "\\" + image_path[0], self.IMAGES_STORE + "\\" + item["nickName"] + ".jpg")item["imagePath"] = self.IMAGES_STORE + "\\" + item["nickName"]#print item["imagePath"]return item

import scrapyclass DongguanspiderItem(scrapy.Item):# 每个帖子的标题
title = scrapy.Field()# 每个帖子的编号
number = scrapy.Field()# 每个帖子的文字内容
content = scrapy.Field()# 每个帖子的url
url = scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from dongguanSpider.items import DongguanspiderItemclass DongguanSpider(scrapy.Spider):name = 'dongguan'allowed_domains = ['sun0769.com']url = 'http://wz.sun0769.com/index.php/question/questionType?type=4&page='headers = {"User-Agent": "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}offset = 0start_urls = [url + str(offset)]# 解析每个页面的响应def parse(self, response):# 获取每页帖子的链接列表links = response.xpath('//div[@class="greyframe"]/table//td/a[2]/@href').extract()for link in links:# 获取每个帖子的Request请求yield scrapy.Request(link, headers=self.headers, callback=self.parse_item)# 设置页码终止条件为最后一页的page值，if self.offset <= 88560:self.offset += 30# 获取每页的Request请求yield scrapy.Request(self.url + str(self.offset), headers=self.headers, callback=self.parse)# 解析页面里的每个帖子的响应def parse_item(self, response):item = DongguanspiderItem()titleList = response.xpath('//div[contains(@class, "pagecenter p3")]//strong/text()')[0].extract().strip().split()# 帖子的标题item['title'] = titleList[0][3:]# 帖子的编号item['number'] = titleList[1][3:]# 帖子的内容，先取出有图片帖子的内容，再取出没有图片帖子的内容content = response.xpath('//div[@class="contentext"]/text()').extract()if len(content) == 0:content = response.xpath('//div[@class="c1 text14_2"]/text()').extract()item['content'] = "".join(content).strip()else:item['content'] = "".join(content).strip()# 帖子的链接地址item['url'] = response.urlyield item

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from dongguanSpider.items import DongguanspiderItemclass SunSpider(CrawlSpider):name = 'sun'allowed_domains = ['sun0769.com']start_urls = ['http://wz.sun0769.com/index.php/question/questionType?type=4&page=']# 每页链接的匹配规则pageLink = LinkExtractor(allow=('type=4'))# 每个帖子链接的匹配规则contentLink = LinkExtractor(allow=r'/html/question/\d+/\d+.shtml')rules = (Rule(pageLink),Rule(contentLink, callback='parse_item'))def parse_item(self, response): item = DongguanspiderItem()titleList = response.xpath('//div[contains(@class, "pagecenter p3")]//strong/text()')[0].extract().strip().split()# 帖子的标题item['title'] = titleList[0][3:]# 帖子的编号item['number'] = titleList[1][3:]# 帖子的内容，先取出有图片帖子的内容，再取出没有图片帖子的内容content = response.xpath('//div[@class="contentext"]/text()').extract()if len(content) == 0:content = response.xpath('//div[@class="c1 text14_2"]/text()').extract() # content为列表，通过join方法拼接字符串，并去除首尾空格item['content'] = "".join(content).strip()else:item['content'] = "".join(content).strip()# 帖子的链接地址item['url'] = response.urlyield item

# -*- coding: utf-8 -*-# 文件处理类库，可以指定编码格式
import codecs
import jsonclass DongguanspiderPipeline(object):def __init__(self):# 创建一个可写文件，指定编码格式为utf-8self.filename = codecs.open('dongguan.json', 'w', encoding='utf-8')def process_item(self, item, spider):content = json.dumps(dict(item), ensure_ascii=False) + '\n'self.filename.write(content)return itemdef spider_closed(self, spider):self.filename.close()

ITEM_PIPELINES = {'dongguanSpider.pipelines.DongguanspiderPipeline': 300,
}# 日志文件名和处理等级
LOG_FILE = "dg.log"
LOG_LEVEL = "DEBUG"

import scrapyclass SinaspiderItem(scrapy.Item):# 大类的标题
parentTitle = scrapy.Field()# 大类的url
parentUrl = scrapy.Field()# 小类的标题
subTitle = scrapy.Field()# 小类的url
subUrl = scrapy.Field()# 小类的存储路径
subDir = scrapy.Field()# 文章的url
fileUrl = scrapy.Field()# 文章的标题
title = scrapy.Field()# 文章的内容
content = scrapy.Field()
spiders/sina.py
# -*- coding: utf-8 -*-import scrapyfrom sinaSpider.items import SinaspiderItemimport osclass SinaSpider(scrapy.Spider):name = 'sina'allowed_domains = ['sina.com.cn']start_urls = ['http://news.sina.com.cn/guide/']# 解析首页def parse(self, response):# 获取大类的标题列表
parentTitleList = response.xpath('//div[@id="tab01"]//h3/a/text()').extract()# 获取大类的url列表
parentUrlList = response.xpath('//div[@id="tab01"]//h3/a/@href').extract()# 遍历大类列表for i in range(len(parentTitleList)):# 根据大类的标题名新建目录
parentDir = '.\\Data\\' + parentTitleList[i]if not os.path.exists(parentDir):os.makedirs(parentDir)# 获取每个大类下的小类的标题列表
subTitleList = response.xpath('//div[@id="tab01"]/div[{}]//li/a/text()'.format(i+1)).extract()# 获取每个大类下的小类的url列表
subUrlList = response.xpath('//div[@id="tab01"]/div[{}]//li/a/@href'.format(i+1)).extract()# 遍历某一个大类下的小类列表for j in range(len(subTitleList)):# 根据小类的标题名新建目录
subDir = parentDir + '\\'+ subTitleList[j]if not os.path.exists(subDir):os.makedirs(subDir)item = SinaspiderItem()item['parentTitle'] = parentTitleList[i]item['parentUrl'] = parentUrlList[i]item['subTitle'] = subTitleList[j]item['subUrl'] = subUrlList[j]item['subDir'] = subDir# 发送每个小类的Request请求，在Request中加入meta，即可将meta传递给response作为参数给回调函数使用yield scrapy.Request(item['subUrl'], meta={'meta_1': item}, callback=self.parse_news)# 解析每个小类的url，爬取每个小类下的文章标题和链接def parse_news(self, response):# 获取Request请求发送的meta_1参数
meta_1 = response.meta['meta_1']fileUrlList = response.xpath('//a/@href').re(r'.*\d+\.shtml')for i in range(len(fileUrlList)):item = SinaspiderItem()item['parentTitle'] = meta_1['parentTitle']item['parentUrl'] = meta_1['parentUrl']item['subTitle'] = meta_1['subTitle']item['subUrl'] = meta_1['subUrl']item['subDir'] = meta_1['subDir']item['fileUrl'] = fileUrlList[i]# 发送每篇新闻的Request请求，在Request中加入meta，向回调函数parse_content传递参数meta2yield scrapy.Request(item['fileUrl'], meta={'meta_2': item}, callback=self.parse_content)# 解析每个新闻页，获取新闻标题和内容def parse_content(self, response):# 获取Request请求发送的meta_2参数
item = response.meta['meta_2']# 获取新闻的标题
title = response.xpath('//h1[@class="main-title"]/text()')[0].extract()content = ''contentList = response.xpath('//div[@class="article"]/p/text()').extract()# 获取新闻的内容for content_one in contentList:content += content_oneitem['title'] = titleitem['content'] = contentyield item

class SinaspiderPipeline(object):def process_item(self, item, spider):fileUrl = item['fileUrl']# 根据新闻链接地址命名存储新闻的文件名
fileName = item['subDir'] + '\\' + fileUrl[7:-6].replace('/', '_') + '.txt'with open(fileName, 'w') as f:f.write(item['content'].encode('utf-8'))return item

USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"ITEM_PIPELINES = {'sinaSpider.pipelines.SinaspiderPipeline': 300,}# 日志文件名和处理等级

LOG_FILE = "sina.log"LOG_LEVEL = "DEBUG"