#-*- coding:utf-8 -*-
import requests, threading, time
from lxml import etree
from bs4 import BeautifulSoup# 获取源码
def get_html(url):# url = 'http://www.doutula.com/article/list/?page=1'headers = {'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'}request = requests.get(url=url, headers=headers) # 网址发送get请求response = request.content.decode('utf-8') # 获取源码# print(response)return response# 匹配图片url
def get_img_html(html):# soup = BeautifulSoup(html,'html.parser')soup = BeautifulSoup(html,'lxml') # 解析网页all_a = soup.find_all('a',class_='list-group-item') # 获取a标签，如果有class或id来命名，一定要加上名字# class="list-group-item"是a标签的名字# <a class="list-group-item" href="http://www.doutula.com/article/detail/7536783"># print(type(all_a)) # <class 'bs4.element.ResultSet'># print(all_a)for i in all_a:# print(i['href'])img_html = get_html(i['href']) # 获取内页源码，i['href']表示获取属性值# print(img_html)get_img(img_html)# 获取图片url
def get_img(html):# soup = etree.HTML(html) # 初始化源码# items = soup.xpath('//div[@class="artile_des"]') # //表示某个目录下,从匹配选择的当前节点选择文档中的节点，而不考虑它们的位置。# # []表示过滤条件# for item in items:#     imgurl_list = item.xpath('table/tbody/tr/td/a/img/@onerror')#     # print(imgurl_list)#     # start_save_img(imgurl_list)soup = BeautifulSoup(html, 'lxml')items = soup.find('div',class_='swiper-slide').find_all('div',class_='artile_des')# 不能写成这样：find_all后面不能跟find，因为find是找一个，find_all是找多个，从多个中找一个是不对的# items = soup.find('div',class_='swiper-slide').find_all('div',class_='artile_des').find('img')['src']# print(items)imgurl_list = []for i in items:imgurl = i.find('img')['src'] # img标签下的src属性# print(type(imgurl)) # <class 'str'># print(imgurl)imgurl_list.append(imgurl)start_save_img(imgurl_list) # 这里是对每一组套图做多线程# 下载图片
x = 1
def save_img(img_url):# global x # 全局变量# x +=1# img_url = img_url.split('=')[-1][1:-2].replace('jp','jpg') # 以=分割# print('正在下载'+'http:'+img_url)# img_content = requests.get('http:'+img_url).content# with open('doutu/%s.jpg'%x, 'wb')  as f:# urllib下的retrieve也可以下载#     f.write(img_content)global x # 全局变量x +=1print('正在下载：'+img_url)geshi = img_url.split('.')[-1]  # 因为图片格式不一样，所以切片，把链接中图片后缀获取到，用于下面拼接文件名img_content = requests.get(img_url).contentwith open('doutu/%s.%s'%(x,geshi), 'wb')  as f: # urllib下的retrieve也可以下载f.write(img_content)def start_save_img(imgurl_list):for i in imgurl_list:# print(i)th = threading.Thread(target=save_img,args=(i,)) # i后面加逗号表示args是一个元组# target是可调用对象，是一个函数名，线程启动后执行，th.start()th.join()# 主函数
def main():start_url = 'http://www.doutula.com/article/list/?page={}'for i in range(1,2):# print(start_url.format(i))start_html = get_html(start_url.format(i))get_img_html(start_html) # 获取内页图片的urlif __name__ == '__main__': # 判断文件入口start_time = time.time()main()end_time = time.time()print(start_time)print(end_time)print(end_time-start_time)