//div[contains(@class,"tab-content zj_list_con ")]/ul[contains(@class,"list_con_li")]/li/a/@href

referer: https://www.dmzj.com/view/jinglingzhidan/60957.html

from selenium import webdriver
from selenium.webdriver.common.by import By
import requests
from lxml import etree
import os
import time
from random import uniform

def main():url = "https://www.dmzj.com/info/jinglingzhidan.html"html = getMainPage(url)page = getChapters(html)image = getImage(page)saveImage(image)if __name__ == "__main__":main()

def getMainPage(url):'''获取章节页面'''headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36',}'''异常管理'''try:response = requests.get(url, headers = headers)if response.status_code == 200:return response.textexcept:print(response.status_code)

def getChapters(html):'''获取章节URL地址'''html = etree.HTML(html)chapters = html.xpath('//div[contains(@class,"tab-content zj_list_con ")]/ul[contains(@class,"list_con_li")]/li/a/@href')return chapters

def getImage(pages):'''获得章节图片页数'''browser = webdriver.Chrome()for page in pages:# print(str(page))browser.get(str(page))page_num = browser.find_elements_by_xpath('//div[@class="btmBtnBox"]/select[@id="page_select"]/option')# wait = WebDriverWait(browser,10)# wait.until(EC.presence_of_element_located((By.XPATH,'//div[@class="btmBtnBox"]/select[@id="page_select"]')))print(len(page_num)) #获取页面数用来计数'''使用for循环遍历所有的图片'''for num in range(1, len(page_num)+1):'''拼接图片URL地址，获取书名，章节名'''image_page = str(page)+ "#@page={num}".format(num=num)browser.get(image_page)image_url = browser.find_element_by_xpath('//div[contains(@class,"comic_wraCon")]/img')book_title = browser.find_element_by_xpath('//div[@class="head_title"]/h1/a')chapter_title = browser.find_element_by_xpath('//div[@class="head_title"]/h2')page_image = image_url.get_attribute('src')book_name = book_title.textchapter_name = chapter_title.textprint("+" *60)'''因为是犯懒所以没有使用代理池，为了不被封IP地址，所以使用了随机的0到3秒的休眠'''t = uniform(0,3)print(t)time.sleep(t)# print(page_image)# print(book_name)# print(chapter_name)yield page_image, book_name, chapter_name, num, page# breakbrowser.close()

def saveImage(image):for im, name, title, page, referer_page in image:'''注意referer，动漫之家的图片是不能直接访问图片的，需要验证来源页'''headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36','referer': referer_page,}image_response = requests.get(im, headers=headers)'''把获取到的文件写入文件夹'''if not os.path.exists('/Users/admin/Desktop/HOMEWORK/python3教程/spider/images/{name}'.format(name=name)):os.mkdir('/Users/admin/Desktop/HOMEWORK/python3教程/spider/images/{name}'.format(name=name))if not os.path.exists('/Users/admin/Desktop/HOMEWORK/python3教程/spider/images/{name}/{title}'.format(name=name, title=title)):os.mkdir('/Users/admin/Desktop/HOMEWORK/python3教程/spider/images/{name}/{title}'.format(name=name, title=title))if image_response.status_code == 200:file_path = '/Users/admin/Desktop/HOMEWORK/python3教程/spider/images/{name}/{title}/{page}.{jpg}'.format(name=name, title=title, page=page, jpg='jpg')if not os.path.exists(file_path):with open(file_path,"wb") as f:f.write(image_response.content)else:print(image_response.status_code)

python3 dmzj_selenium.py