python课程设计----简单爬虫

import GetHtml as g,SaveData as sif __name__ == '__main__':# 入口地址address = ['http://news.zzu.edu.cn/mtzd.htm']# 用来存储已经爬取过的地址，防止重复爬取bin = []  # 队列 存放的是爬取过的url地址while len(address) != 0:get = g.GetHtml()htmls = []for url in address:bin.append(url)htmls.append(get.gethtml(url))for html in htmls:save = s.SaveData(html)save.save()address.extend(save.getOtherUrl())address = list(set(address) - set(bin))

from bs4 import BeautifulSoup# 该类主要解析数据
class SaveData:html = ""  #网页源代码def __init__(self, html):  #构造方法，需要传入html数据print("构造SaveData类")self.html = htmldef save(self):bs = BeautifulSoup(self.html, "html.parser")  #声明bs对象data_title_name = bs.select("div[class='new-center']>h3>a")  #文章标题data_title_detailtime = bs.select("div[class='new-date']")  #日期for i in range(0, len(data_title_name)):print("发布时间：", data_title_detailtime[i].get_text(), end="\t")file = open("output.html", "a+", encoding="utf-8")print("文章标题：", data_title_name[i].get_text(), end="\n")file.write("<p>文章标题：" + data_title_name[i].get_text() + "    " + "发布时间：" + data_title_detailtime[i].get_text() + "\n</p>")file.close()# 用来获取下面爬取页面的连接def getOtherUrl(self):bs = BeautifulSoup(self.html, "html.parser")  # 声明bs对象nexturl = []tem = bs.select("span[class='p_no']>a")print(tem)for url in tem:a = url['href']if "../" in a:a = str(a)[3:]elif "mtzd" in a:a = a[5:]if a == "mtzd.htm":print("http://news.zzu.edu.cn/" + a)nexturl.append("http://news.zzu.edu.cn/" + a)else:nexturl.append("http://news.zzu.edu.cn/mtzd/" + a)print("http://news.zzu.edu.cn/mtzd/" + a)return nexturl

import urllib.request, urllib.response, urllib.error, urllib.parse# 该类主要获取数据
class GetHtml:def __init__(self):print("GetHtml构造")def gethtml(self, url): #发送请求方法html = ""  # 爬取到的网页源代码URL = urllib.parse.quote(url, safe=':/.')  # 爬取的网页urlheaders = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ""Chrome/87.0.4280.88 Safari/537.36 "}  # 请求头，防止爬虫拦截request = urllib.request.Request(URL, headers=headers, method="GET")  # 封装请求对象try:response = urllib.request.urlopen(request)html = response.read().decode()except urllib.error.HTTPError as e:print("超时")return html

下面是清华大学新闻爬虫保存.docx文件

# -*- coding:utf-8 -*-from bs4 import BeautifulSoup
import urllib.request, urllib.response, urllib.error, urllib.parse# 爬取网页函数
def request(url):html = ""  # 爬取到的网页源代码URL = urllib.parse.quote(url, safe=':/.')  # 爬取的网页urlheaders = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"}  # 请求头，防止爬虫拦截request = urllib.request.Request(URL, headers=headers, method="GET")  # 封装请求对象try:response = urllib.request.urlopen(request)html = response.read().decode()except urllib.error.HTTPError as e:print("超时")return html# 获取数据函数
def getData(urllist, urleds):# 需要爬取的是urllist和urleds的差集newurl = list(set(urllist) - set(urleds))# 退出条件，urllist中没有新的urlif len(newurl) == 0:printok()returndata = []  # 存储每个页面的html数据for url in newurl:data.append(request(url))  # 讲爬取到数据存入列表中，这样先爬取到的页面下标越小# 爬取之后讲地址放入列表urleds.append(url)# 处理data列表for dataone in data:# 使用正则分别拿到想要的数据bs = BeautifulSoup(dataone, "html.parser")data_title_name = bs.select("p[class='bt']")  # 用来获取文章标题,存到列表中title_url = bs.select("div[class='news_months']>ul>li>a")data_title_detailtime_day = bs.select("div[class='sj']>p")  # 用来获取文章发布时间data_title_detailtime_mouth = bs.select("div[class='sj']>span")for i in range(0, len(data_title_name)):print("发布时间：", data_title_detailtime_mouth[i].get_text() + "." + data_title_detailtime_day[i].get_text(),end="\t")file = open("output.html","a+",encoding="utf-8")print("文章标题：", data_title_name[i].get_text(), end="\n")file.write("<p>文章标题："+data_title_name[i].get_text()+"\t"+"发布时间："+data_title_detailtime_mouth[i].get_text() + "." + data_title_detailtime_day[i].get_text()+"\n</p>")file.close()# 1:获取其他页面的urlnexturl = []tem = bs.select("span[class='p_no']>a")print(tem)for url in tem:a = url['href']if "../" in a:a = str(a)[3:]elif "rcpy" in a:a = a[5:]if a == "rcpy.htm":print("https://www.tsinghua.edu.cn/news/" + a)nexturl.append("https://www.tsinghua.edu.cn/news/" + a)else:nexturl.append("https://www.tsinghua.edu.cn/news/rcpy/" + a)print("https://www.tsinghua.edu.cn/news/rcpy/" + a)# 最后递归调用getData(nexturl, urleds)def printok():print("--------------------------------------------------------------", end="\n")print("##############################################################", end="\n")print("                                                              ", end="\n")print("           $$$$$$$$$$      ###     ###                        ", end="\n")print("          $$$$$$$$$$$$     ###    ###                         ", end="\n")print("         $############$    ###  ####                          ", end="\n")print("         $$$$$$$$$$$$$$    ########                           ", end="\n")print("         $$$$$$$$$$$$$$    ###  ###                           ", end="\n")print("          $$$$$$$$$$$$     ###    ###                         ", end="\n")print("            $$$$$$$$       ###     ###                        ", end="\n")print("                                                              ", end="\n")print("##############################################################", end="\n")print("______________________________________________________________", end="\n")if __name__ == '__main__':# 入口地址urls = ['https://www.tsinghua.edu.cn/news/rcpy.htm']# 用来存储已经爬取过的地址，防止重复爬取urleds = []  # 队列 存放的是爬取过的url地址# 调用爬取函数getData(urls, urleds)

博客爬虫

# -*- coding:utf-8 -*-from bs4 import BeautifulSoup
import urllib.request, urllib.response, urllib.error, urllib.parse
from docx import Document
from docx.shared import Inches
import re# 爬取网页函数
def request(url):html = ""  # 爬取到的网页源代码URL = urllib.parse.quote(url, safe=':/.')  # 爬取的网页urlheaders = {"User-Agent": "Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / ""89.0.4389.114Safari / 537.36 "}  # 请求头，防止爬虫拦截request = urllib.request.Request(URL, headers=headers, method="GET")  # 封装请求对象try:response = urllib.request.urlopen(request)html = response.read().decode()except urllib.error.HTTPError as e:print("超时")return html# 获取数据函数
def getData(urllist, urleds):# 需要爬取的是urllist和urleds的差集newurl = list(set(urllist) - set(urleds))# 退出条件，urllist中没有新的urlif len(newurl) == 0:printok()returndata = []  # 存储每个页面的html数据urls = []  # 存储文章真是链接地址for url in newurl:data.append(request(url))  # 讲爬取到数据存入列表中，这样先爬取到的页面下标越小# 爬取之后讲地址放入列表urleds.append(url)# 处理data列表for dataone in data:# 使用正则分别拿到想要的数据bs = BeautifulSoup(dataone, "html.parser")# data_year = bs.select("div[class='collection-title'] > h1[class=archive-year]")data_title_url = bs.select("article > header > h2 > a")  # 用来获取文章真实地址# 1:获取文章的urlfor url in data_title_url:urls.append("https://www.kingname.info" + url['href'])data_title_name = bs.select("article > header > h2 > a > span")  # 用来获取文章标题data_title_detailtime = bs.select("article > header > div >time")  # 用来获取文章发布时间for i in range(0, len(data_title_name)):#调用方法，向word写标题# print(data_title_detailtime[i].get_text(), end="\t")# print(data_title_name[i].get_text(), end="\n")# 访问文章的连接，爬取文章的内容title_html = request(urls[i])s = BeautifulSoup(title_html, "html.parser")imgs = s.findAll("img")# 解析文章内容nav = s.select("div[class='post-body'] > p")save_to_doc(data_title_name[i].get_text(), data_title_detailtime[i].get_text(), nav,imgs)# for n in nav:#     print(n.get_text())# 1:获取其他页面的urlnexturl = []tem = bs.select("a[class='page-number']")for url in tem:nexturl.append("https://www.kingname.info" + url['href'])# 最后递归调用getData(nexturl, urleds)def save_to_doc(title,time,plist,imgs):doc = Document()save_title_name(doc,title,time)print(title)imgs.remove(imgs[len(imgs) - 1])src = []for img in imgs:src.append(img['src'])print(len(src))flag = 0#保存内容for n in plist:if str(n).find("img") ==-1:#说明是段落doc.add_paragraph(n.get_text())print(n.get_text())else:#说明是图片if flag < len(src):requestimg(doc, src[flag])flag = flag+1doc.save(title[1:-3]+".docx")def save_title_name(doc,title,time):doc.add_heading(title)print(title)doc.add_heading(time, level=1)print(time)def requestimg(doc,url):print(url)URL = urllib.parse.quote(url, safe=':/.')print(URL)path = ''if URL[-5:].find("jpg") ==-1 and URL[-5:].find("png") ==-1 and URL[-5:].find("png") and URL[-5:].find("JPEG"):name = URL[-13:]else:name = URL[-23:]if name.find("/") != -1:sub = name.index("/")name = name[sub:]print(name)try:urllib.request.urlretrieve(URL, name)except Exception as e:print("图片下载失败")try:doc.add_picture(name, width=Inches(5))except FileNotFoundError as e:print("图片路径未找到")print("ok")def test():html = request("https://www.kingname.info/2021/02/18/entry-file/")bs = BeautifulSoup(html, "html.parser")print("<p><img src='xxx'>".find("img"))imgs = bs.findAll("img")imgs.remove(imgs[len(imgs)-1])for img in imgs:print(img['src'])def printok():print("--------------------------------------------------------------", end="\n")print("##############################################################", end="\n")print("                                                              ", end="\n")print("           $$$$$$$$$$      ###     ###                        ", end="\n")print("          $$$$$$$$$$$$     ###    ###                         ", end="\n")print("         $############$    ###  ####                          ", end="\n")print("         $$$$$$$$$$$$$$    ########                           ", end="\n")print("         $$$$$$$$$$$$$$    ###  ###                           ", end="\n")print("          $$$$$$$$$$$$     ###    ###                         ", end="\n")print("            $$$$$$$$       ###     ###                        ", end="\n")print("                                                              ", end="\n")print("##############################################################", end="\n")print("______________________________________________________________", end="\n")
if __name__ == '__main__':# test()# 入口地址urls = ['https://www.kingname.info/archives/']# 用来存储已经爬取过的地址，防止重复爬取urleds = []# 调用爬取函数getData(urls, urleds)

python课程设计----简单爬虫相关推荐

Python课程设计大作业:获取比赛数据并进行机器学习智能预测NBA的比赛结果
前言该篇是之前遗漏的大三上的Python课程设计.刚好今天有空就补发了一篇文章.全部的代码在最后附录中.爬虫类的代码直接全部放到一起了,读者可以自行研究.百度网盘可以私聊我进行获取. 一.课程设计项 ...
北京交通大学Python课程设计大作业（四）——典籍词频统计
北京交通大学Python课程设计大作业(四)--典籍词频统计文章目录北京交通大学Python课程设计大作业(四)--典籍词频统计一.词频统计任务介绍二.典籍词频统计python源代码如下三. ...
夏敏捷第29本著作《Python课程设计》（微课视频版）
Python课程设计-微课视频版本书以Python 3.7为编程环境,逐步展开Python语言教学,是一本面向广大编程学习者的程序设计类图书.本书以案例为驱动介绍知识点,将Python知识点分解到不 ...
python课程设计文字游戏魔塔1
[python课程设计]魔塔文字游戏分享1 这是我的python课程设计魔塔游戏的DOS版,我觉得我做的还不错( 我太自恋了 .先给大家放个截图看一下效果. 因为代码很多,所以我打算分成几部分来分享给 ...
python课程设计博客网_python 课程设计
原博文 2019-12-09 14:56 − 泉州信息工程学院软件学院课程设计报告书课程名: python课程设计课程设计项目名称: &nb... 相关推荐 2019-09-28 ...
Python课程设计之俄罗斯方块
Python课程设计之俄罗斯方块演示效果下载地址运行效果基础页面界面动起来生成.移动.固定.变形消除与得分演示效果点击查看下载地址点击下载 Python课程设计之俄罗斯方块软件 ...
Python课程设计：点餐系统
开篇来了来了,又到了准备烦人的期末python课程设计的时候了,我相信你一定是摆烂了好久了吧,哈哈哈哈! 被我猜到了吧!不过不要担心!本初学者为大家奉上一个完全应付大作业的小代码啦! 话不多说!正文 ...
用python画简单图案图片_使用 Python Turtle 设计简单而又美丽的图形
使用 Python Turtle 设计简单而又美丽的图形看看本文中这些美丽的图画.正是这样的事情让我爱上了 turtle.优秀的程序员会认为turtle是"落后原始的",但我不敢 ...
用python画简单的图案-使用 Python Turtle 设计简单而又美丽的图形
原标题:使用 Python Turtle 设计简单而又美丽的图形正文看看本文中这些美丽的图画.正是这样的事情让我爱上了 turtle.优秀的程序员会认为turtle是"落后原始的&quo ...

python课程设计----简单爬虫

下面是清华大学新闻爬虫保存.docx文件

博客爬虫

python课程设计----简单爬虫相关推荐

最新文章

热门文章