1.先安装ｐｉｐsudo apt install python3-pip2.安装依耐sudo apt install python-dev3.安装非Python的依赖 sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev4.用ｐｉｐ安装ｓｃｒａｐｙsudo pip install scrapy

scrapy startproject mySpider

scrapy genspider itcast 'itcast.cn'

 1 import scrapy
 2
 3
 4 class MyspiderItem(scrapy.Item):
 5     # define the fields for your item here like:
 6     # name = scrapy.Field()
 7
 8     name = scrapy.Field()
 9     title = scrapy.Field()
10     info= scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from mySpider.items import MyspiderItemclass ChuanzhiSpider(scrapy.Spider):name = 'chuanzhi'allowed_domains = ['itcast.cn']start_urls = ['http://www.itcast.cn/channel/teacher.shtml#apython']def parse(self, response):# with open("teacher.html", "w") as f:#    f.write(response.body)# 通过scrapy自带的xpath匹配出所有老师的根节点列表集合teacher_list = response.xpath('//div[@class="li_txt"]')# 所有老师信息的列表集合teacherItem = []# 遍历根节点集合for each in teacher_list:# Item对象用来保存数据的item = MyspiderItem()# name, extract() 将匹配出来的结果转换为Unicode字符串# 不加extract() 结果为xpath匹配对象#extract_first 直接提取第一个不用加下标取值name = each.xpath('./h3/text()').extract_first()# titletitle = each.xpath('./h4/text()').extract()# infoinfo = each.xpath('./p/text()').extract()item['name'] = name# item['title'] = title[0]# item['info'] = info[0]# item['name'] = name[0]# item['title'] = title[0]# item['info'] = info[0]#yield item
teacherItem.append(item)# print name[0]# print title[0]# print info[0]return teacherItem

1 import scrapy
2
3 class cqitItme(scrapy.Item):
4     pageText = scrapy.Field()
5     hrefText = scrapy.Field()

 1 import scrapy
 2 from scrapy.selector import Selector, XmlXPathSelector
 3 from scrapy.http import Request
 4 from ..items import cqitItme
 5
 6
 7 class CqitSpider(scrapy.Spider):
 8     name = 'cqit'
 9     allowed_domains = ['cqsq.com'] #注意这个域名范围　　不能写成　'www.cqsq.com' 或者'cqsq.com/'
10     start_urls = ['https://www.cqsq.com/list/7']
11     set_href = []
12
13     def parse(self, response):
14         a = Selector(response=response).xpath("//div[@class= 'pages']/a")
15 　　#实例化items里面创建的类　以便下面赋值并返回数据到pipelines文件中去做数据处理
16         itme = cqitItme()
17         for i in a:
18
19             i1 = i.xpath('.//@href').extract_first()
20             itext = i.xpath('.//text()').extract_first()
21
22             if i1 in self.set_href or i1 == "//www.cqsq.com/list/7/50":
23                 pass
24             else:
25
26                 self.set_href.append(i1)
27                 url1 = 'https:' + i1
28                 itme['pageText'] =itext
29                 itme['hrefText'] =url1
30
31                 print(url1)
32         　　＃这里是返回实例数据并提交到pipelines文件中去处理
33                 yield itme
34                 #发送新的ｕｒｌ到控制的阵列里去等去等待下次扏行　　注意dont_filter去重参数　true是代表不去重　默认为false
35                 yield Request(url=url1, callback=self.parse, dont_filter=True)

1 class dyspiderPipeline(object):　　＃注意在settings文件中去设置该类的权重
2
3     def __init__(self):
4         self.f = open('test1','w')
5     def process_item(self, item, spider):＃这里的ｉｔｍ就是爬虫文件k yield回来的数据
6         import json
7         content = json.dumps(dict(item),ensure_ascii=False)+'\n'
8         self.f.write(content)
9         return item

ITEM_PIPELINES = {'mySpider.pipelines.MyspiderPipeline': 300,'mySpider.pipelines.dyspiderPipeline': 300, #PIPELINENS　注册的位置　300代表的是权重
}DEPTH_LIMIT = 2 #深度为2　如果为0的话就是代表没有深度

DUPEFILTER_CLASS = "mySpider.spiders.repeatfilter.Repeat"　#自定义去重的路径

 1 class Repeat(object):
 2     def __init__(self):　　　　　　　'''第二步执行　构造函数'''
 3         self.visited_set = []
 4     @classmethod
 5     def from_settings(cls, settings):　　　　　　'''这里是第一步执行　相当于是obj = Repeat.from_settings'''
 6         return cls()
 7
 8     def request_seen(self, request):　　　　'''第里是第四步执行　主要是判断罗辑的'''
 9
10         if request.url in self.visited_set or request.url == "https://www.cqsq.com/list/7/50":
11
12             return True
13         else:
14             self.visited_set.append(request.url)
15
16             return False
17
18
19
20     def open(self):  # can return deferred
21         # print('open')　第三步执行
22         pass
23
24     def close(self, reason):  # can return a deferred
25         # print('close')　第５步执行
26         pass
27     def log(self, request, spider):  # log that a request has been filtered
28         # print('log....')　最后执行的日志
29         pass

 1 from scrapy.exceptions import DropItem
 2
 3 class Day96Pipeline(object):
 4
 5     def __init__(self,conn_str):
 6         '''第二步'''
 7         self.conn_str = conn_str
 8
 9     @classmethod
10     def from_crawler(cls, crawler):
11         """
12         初始化时候，用于创建pipeline对象
13         :param crawler:
14         :return:
15         第一步
16         """
17         conn_str = crawler.settings.get('DB')
18         return cls(conn_str)
19
20     def open_spider(self,spider):
21         """
22         第三步
23         爬虫开始执行时，调用
24         :param spider:
25         :return:
26         """
27         self.conn = open(self.conn_str, 'a')
28
29     def close_spider(self,spider):
30         """
31         第五步
32         爬虫关闭时，被调用
33         :param spider:
34         :return:
35         """
36         self.conn.close()
37
38     def process_item(self, item, spider):
39         """
40         第四步
41         每当数据需要持久化时，就会被调用
42         :param item:
43         :param spider:
44         :return:
45         """
46         # if spider.name == 'chouti'
47         tpl = "%s\n%s\n\n" %(item['title'],item['href'])
48         self.conn.write(tpl)
49         # 交给下一个pipeline处理
50         return item
51         # 丢弃item，不交给
52         # raise DropItem()
53
54 class Day97Pipeline(object):
55
56     def __init__(self,conn_str):
57         self.conn_str = conn_str
58
59     @classmethod
60     def from_crawler(cls, crawler):
61         """
62         初始化时候，用于创建pipeline对象
63         :param crawler:
64         :return:
65         """
66         conn_str = crawler.settings.get('DB')
67         return cls(conn_str)
68
69     def open_spider(self,spider):
70         """
71         爬虫开始执行时，调用
72         :param spider:
73         :return:
74         """
75         self.conn = open(self.conn_str, 'a')
76
77     def close_spider(self,spider):
78         """
79         爬虫关闭时，被调用
80         :param spider:
81         :return:
82         """
83         self.conn.close()
84
85     def process_item(self, item, spider):
86         """
87         每当数据需要持久化时，就会被调用
88         :param item:
89         :param spider:
90         :return:
91         """
92         # if spider.name == 'chouti'
93         tpl = "%s\n%s\n\n" %(item['title'],item['href'])
94         self.conn.write(tpl)
95
96         #这几个方法的套路和上面自定义去重是一样的执行套路　　注意process_itme里面return itme　代表交给下个权重执行　如果是return　Dropｉtme()　则代表抛异常　不交给下个执行