#coding:utf8#author:Jery#datetime:2019/4/12 19:22#software:PyCharm#function:爬取百度百科关键词python1000个相关网页——标题和简介

from urllib.request importurlopenimportrefrom bs4 importBeautifulSoupclassSpiderMain(object):def __init__(self):

self.urls=UrlManager()

self.downloader=HtmlDownloader()

self.parser=HtmlParser()

self.outputer=DataOutputer()#主爬虫，调度四个类的方法执行爬虫

defcrawl(self, root_url):

count= 1self.urls.add_new_url(root_url)whileself.urls.has_new_url():try:

new_url=self.urls.get_new_url()print("crawl 第{} ：{}".format(count, new_url))

html_content=self.downloader.download(new_url)

new_urls, new_data=self.parser.parse(new_url, html_content)#新网页的url及数据

self.urls.add_new_urls(new_urls)

self.outputer.collect_data(new_data)if count == 1000:breakcount+= 1

except:print("crawl failed!")

self.outputer.output_html()#URL管理器，实现URL的增加与删除

classUrlManager:def __init__(self):

self.new_urls=set()

self.old_urls=set()defhas_new_url(self):return len(self.new_urls) !=0defget_new_url(self):

new_url=self.new_urls.pop()

self.old_urls.add(new_url)returnnew_urldefadd_new_url(self, url):if url isNone:return

if url not in self.new_urls and url not inself.old_urls:

self.new_urls.add(url)defadd_new_urls(self, urls):if urls is None or len(urls) ==0:return

for url inurls:

self.new_urls.add(url)#下载网页源代码

classHtmlDownloader:defdownload(self, url):if url inNone:returnresponse=urlopen(url)if response.getcode() != 200:return

returnresponse.read()#下载网页所需内容

classHtmlParser:defparse(self, page_url, html_content):if page_url is None or html_content isNone:returnsoup= BeautifulSoup(html_content, 'lxml', from_encoding='utf-8')

new_urls=self._get_new_urls(page_url, soup)

new_data=self._get_new_data(page_url, soup)returnnew_urls, new_datadef_get_new_urls(self, page_url, soup):

new_urls=set()

links= soup.find_all('a', href=re.compile(r'/view/.*'))for link inlinks:

new_url= "https://baike.baidu.com" + link['href']

new_urls.add(new_url)returnnew_urlsdef_get_new_data(self, page_url, soup):

res_data={}#

#

#

Python

title_node = soup.find("dl", {"class": "lemmaWgt-lemmaTitle lemmaWgt-lemmaTitle-"}).dd.h1

res_data['title'] =title_node.get_text()

summary_node= soup.find('div', {"class": "lemma-summary"})

res_data['summary'] =summary_node.get_text()returnres_data#将所搜集数据输出至html的表格中

classDataOutputer:def __init__(self):

self.datas=[]defcollect_data(self, data):if data isNone:returnself.datas.append(data)defoutput_html(self):

output= open('output.html', 'w')

output.write("")

output.write("

")

output.write("

output.write("

")

output.write("

{}".format(data['url']))

output.write("

{}".format(data['title'].encode('utf-8')))

output.write("

{}".format(data['summary'].encode('utf-8')))

output.write("

")

output.write("

")

output.write("

")

output.write("")

output.close()if __name__ == '__main__':

root_url= "https://baike.baidu.com/item/Python/407313"obj_spider=SpiderMain()

obj_spider.crawl(root_url)