1 #——游戏王卡片信息爬虫——
  2 #——最后修改时间：2017.11.02——
  3
  4 import requests
  5 import re
  6 import jieba
  7 import pandas
  8 import matplotlib.pyplot as plt
  9 from bs4 import BeautifulSoup
 10 from wordcloud import WordCloud
 11
 12 #函数功能：获取卡片的总页数
 13 def getpage():
 14     flag1 = '搜索结果     / 约 ' #该变量内容为总页数所在字段文本的前部分文字
 15     flag2 = '条'
 16     card_page = soup.select('.result')[0].text.lstrip(flag1) #获取到总页数所在的字段文本
 17     page_num = card_page.find(flag2) #查找“条”字符所在的位置
 18     allpage = int(card_page[0:page_num]) #输出“条”字前面的数字，即总卡片数
 19     page = allpage//18  #输出总页数，每页有18条卡片记录
 20     return page
 21
 22 #函数功能：获取魔法陷阱卡的信息
 23 def getdetail_MagicTrap(url_MT):
 24     resd = requests.get(url_MT)
 25     resd.encoding = 'utf-8'
 26     soup_MT = BeautifulSoup(resd.text,'html.parser')
 27     card_news_MagicTrap = {}
 28     for news in soup_MT.select('.card'):
 29         card_news_MagicTrap['名称'] = news.select('div')[3].text
 30         card_news_MagicTrap['链接'] = url_MT
 31         card_type = news.select('div')[12].text
 32         card_news_MagicTrap['类型'] = card_type[0:4]
 33         card_news_MagicTrap['属性'] = ' '
 34         card_news_MagicTrap['种族'] = ' '
 35         card_effect_text = news.select(".effect")[0].text
 36         card_news_MagicTrap['效果'] = card_effect_text[2:len(card_effect_text)-1]
 37     return card_news_MagicTrap
 38
 39 #函数功能：获取怪兽卡的信息
 40 def getdetail_Monster(url_M):
 41     resd = requests.get(url_M)
 42     resd.encoding = 'utf-8'
 43     soup_M = BeautifulSoup(resd.text,'html.parser')
 44     card_news_Monster = {}
 45     for news in soup_M.select('.card'):
 46         card_news_Monster['名称'] = news.select('div')[3].text
 47         card_news_Monster['链接'] = url_M
 48         card_type = news.select('div')[12].text
 49         card_news_Monster['类型'] = card_type[0:4]
 50         card_news_Monster['属性'] = news.select('div')[24].text
 51         card_news_Monster['种族'] = news.select('div')[22].text
 52         card_effect_text = news.select(".effect")[0].text
 53         card_news_Monster['效果'] = card_effect_text[0:len(card_effect_text)-1]
 54     return card_news_Monster
 55
 56 #函数功能：获取一个页面的卡片信息
 57 def onepage(url_page):
 58     res = requests.get(url_page)
 59     res.encoding = 'utf-8'
 60     soup_page = BeautifulSoup(res.text,'html.parser')
 61     card_newsls = [] #创建一个列表
 62     for allnews in soup_page.select('.card'):
 63         if len(allnews.select('h2'))>0:
 64             card_maintype = allnews.select('span')[0].text
 65             url = allnews.select('a')[0]['href']
 66             card_url = ('http://www.ourocg.cn{}'.format(url))
 67             if card_maintype in card_judge:   #这里用来判断卡片是怪兽卡还是魔法陷阱卡
 68                 card_newsls.append(getdetail_MagicTrap(card_url))
 69             else:
 70                 card_newsls.append(getdetail_Monster(card_url))
 71             #append（）：接受一个参数，这个参数可以是任何数据类型，并且简单地追加到list的尾部
 72     return card_newsls
 73
 74 card_judge = '魔法陷阱' #该字符串变量用于判断卡片是何类卡片
 75
 76 news_cardtotal = []  #存放所有卡片信息
 77 url_cards = 'http://www.ourocg.cn/Cards/Lists-5-1'  #网站首页
 78 news_cardtotal.extend(onepage(url_cards))  #把新list中的每个元素添加到原list中
 79
 80 res_card = requests.get(url_cards)
 81 res_card.encoding = 'utf-8'
 82 soup = BeautifulSoup(res_card.text,"html.parser")
 83
 84 for i in range(2,getpage()+1):   #通过循环获取首页之后页数的卡片信息列表
 85     url_list = 'http://www.ourocg.cn/Cards/Lists-5-{}'.format(i)
 86     news_cardtotal.extend(onepage(url_list))
 87
 88 df = pandas.DataFrame(news_cardtotal)  #转换成表格型数据（DataFrame型）
 89 df.to_excel('card_info.xlsx')   #以excel表的形式保存结果
 90
 91 for j in range(len(news_cardtotal)):
 92     if j>0:
 93         race = race+news_cardtotal[j].get('种族')
 94         race = race+' '
 95     else:
 96         race = news_cardtotal[j].get('种族')
 97         race = race+' '
 98 words = jieba.lcut(race)   #进行文本分词
 99
100 ls = []      #存放词组文本
101 counts = {}  #存放词组的计数
102 for word in words:
103     ls.append(word)
104     if len(word) == 0:
105         continue
106     else:
107         counts[word] = counts.get(word,0)+1  #计数字典
108 items = list(counts.items())  #计数列表
109 items.sort(key = lambda x:x[1], reverse = True)  #排序
110 for j in range(10):
111     word , count = items[j]
112     print ("{:<10}{:>5}".format(word,count))
113
114 cy = WordCloud().generate(race)  #词云的制作与显示
115 plt.imshow(cy)
116 plt.axis("off")  #不显示标尺
117 plt.show()