import requests
import os
from lxml import html
import timedef get_title_url(tree):'''一级  获取标题'''# 史书典籍# 格式：/book/sanguoyanyi.htmlHistory_book_url_list = tree.xpath("//div[@class='index-li'][3]/ul/li/a/@href")# 格式：三国演义History_book_name_list = tree.xpath("//div[@class='index-li'][3]/ul/li/a/text()")return History_book_url_list,History_book_name_listdef get_article_url(tree):'''二级  获取文章标题'''# 三国演义典籍# 格式：/book/sanguoyanyi/1.htmlbook_url_list = tree.xpath("//div[@class='book-mulu']/ul/li/a/@href")# 格式：第一回·宴桃园豪杰三结义  斩黄巾英雄首立功book_name_list = tree.xpath("//div[@class='book-mulu']/ul/li/a/text()")return book_url_list,book_name_listdef get_article(tree):'''三级  获取文章内容'''# 第一回·宴桃园豪杰三结义  斩黄巾英雄首立功# 格式：/book/sanguoyanyi/1.htmlarticle_list = tree.xpath("//div[@class='chapter_content']/p/text()")return ''.join(article_list)def get_request(url,headers):'''获取页面'''response = requests.get(url=url,headers=headers)tree = html.fromstring(response.text)return treedef save_mkdir(two):'''三级  保存文章夹'''# 一级文件夹if os.path.exists('史书典籍'):passelse:os.mkdir('史书典籍')# 二级文件夹if os.path.exists('史书典籍/'+ two):passelse:os.mkdir('史书典籍/'+ two)def police_2(a):'''二级中断检测'''b = Noneif os.path.exists('史书典籍/police_2.txt'):with open('史书典籍/police_2.txt', 'r') as f:b = f.read()f.close()if b is None:return Trueelif b is '':return Trueif a < int(b):return False# 写入并返回Truewith open('史书典籍/police_2.txt', 'w') as f:f.write(str(a))f.close()return Truedef police_3(a):'''三级中断检测'''b = Noneif os.path.exists('史书典籍/police_3.txt'):with open('史书典籍/police_3.txt', 'r') as f:b = f.read()f.close()if b is None:return Trueelif b is '':return Trueif a < int(b):return False# 写入并返回Truewith open('史书典籍/police_3.txt', 'w') as f:f.write(str(a))f.close()return Truedef main():'''主函数'''# 根路由root = 'http://www.shicimingju.com'# 头部headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36'}# 获取root页面tree1 = get_request(root,headers)# 获取一级名字和路由History_book_url_list, History_book_name_list = get_title_url(tree1)# 获取二级页面for i in range(len(History_book_url_list)):if police_2(i) is False:continue# 二级路由url2 = root + History_book_url_list[i]print("爬取>>>"+History_book_name_list[i]+'开始')tree2 = get_request(url2,headers)# 获取二级名字和路由book_url_list,book_name_list = get_article_url(tree2)# 文章夹保存
        save_mkdir(History_book_name_list[i])# 下载文章for j in range(len(book_url_list)):if police_3(j) is False:continuetime.sleep(1)# 三级路由url3 = root + book_url_list[j]print("爬取:" + book_name_list[j])# 文章tree3 = get_request(url3, headers)txt = get_article(tree3)# 文章标题txt_name = book_name_list[j]# 文章保存file_path = '史书典籍/{}/{}.txt'.format(History_book_name_list[i],(txt_name.replace(' ','')).replace('·',''))with open(file_path,'w',encoding='utf-8') as f:f.write(txt)f.close()print("爬取>>>" + History_book_name_list[i] + '结束')if __name__ == '__main__':main()