如何用python写一个爬虫，下载某论坛的动漫图片

实现过程

获取网页内容

import urllib.request

import re

import json

def getHttpStatusCode(tempUrl):

"""检验url是否可以正常访问"""

opener = urllib.request.build_opener()

opener.addheaders = [("User-agent", "Mozilla/49.0.2")]

try:

opener.open(tempUrl)

# print(tempUrl+'没问题')

return 0

except urllib.error.HTTPError:

print(tempUrl + "=访问页面出错")

return 1

except urllib.error.URLError:

print(tempUrl + "=访问页面出错")

return 2

def getcontent(url):

"""获取链接html内容"""

headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"}

request = urllib.request.Request(url, headers=headers)

response = urllib.request.urlopen(request)

data = response.read().decode("utf-8")

return data

分析内容中的信息，提取需要的链接

def getcatalogarray(content):

"""获取4chan - catalog数据 ---var catalog = {......};var style_group---"""

pattern = r"var\scatalog\s=(.*).var\sstyle_group"

res = re.search(pattern, content, re.M | re.S)

return res.group(1)

获取网页中的图片下载链接

def getimageurls(content):

"""获取网页中的图片链接

pattern = r"a\shref=\"(//i.4cdn.org/.*?\.[j|p|g][p|n|i][g|f])\""

res = re.findall(pattern, content)

return res

下载图片

urllib.request.urlretrieve(

image_download_url, img_save_path

)

完整代码

tools.py

# -*- coding=utf-8 -*-

# author vvyun

import urllib.request

import re

import json

"""

http工具

"""

def getcontent(url):

"""获取链接html内容"""

headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"}

request = urllib.request.Request(url, headers=headers)

response = urllib.request.urlopen(request)

data = response.read().decode("utf-8")

return data

def savestr2file(filename, content):

"""存储字符串到文件"""

output = open(filename, "w+", encoding="utf8")

output.write(content)

output.close()

def getcatalogarray(content):

"""获取4chan - catalog数据 ---var catalog = {......};var style_group---"""

pattern = r"var\scatalog\s=(.*).var\sstyle_group"

res = re.search(pattern, content, re.M | re.S)

return res.group(1)

def getimageurls(content):

"""获取网页中的图片链接