import sqlite3
with sqlite3.connect('gzccnewsdb.sqlite') as db:newsdf.to_sql('gzccnewsdb',con = db)
with sqlite3.connect('gzccnewsdb.sqlite') as db:df2 = pd.read_sql_query('SELECT * FROM gzccnewsdb',con=db)
df2

import pymysql
from sqlalchemy import create_engine
coninfo='mysql+pymysql://root:@localhost:3306/gzccnews?charset=utf8'
engine=create_engine(coninfo,encoding='utf-8')newsdf.to_sql(name='news',con = coninfo,if_exists='append',index=False)
conn=pymysql.connect(host='localhost',port=3306,user='root',passwd='',db='gzccnews',charset='utf8')xiaoguo　

# 构造函数def __init__(self, movie_id, start, type=''):''':type: 评论:movie_id: 影片的ID号:start: 开始的记录数，0-480'''self.movie_id = movie_idself.start = startself.type = typeself.url = 'https://movie.douban.com/subject/{id}/comments?start={start}&limit=20&sort=new_score\&status=P&percent_type={type}&comments_only=1'.format(id=str(self.movie_id),start=str(self.start),type=self.type)# 创建数据库连接self.session = create_session() f _get(self):self._random_UA()  # 调用随机产生的User-Agentres = ''  # res定义字符串# 获取网页信息并爬取try:res = requests.get(self.url, cookies=self.cookies, headers=self.headers,proxies=self.proxies)time.sleep(random.random() * 3)res = res.json()['html']# 返回的数据为json数据，需要提取里面的HTMLexcept Exception as e:print('IP被封，请使用代理IP')print('正在获取{} 开始的记录'.format(self.start))return res

def _parse(self):res = self._get()dom = etree.HTML(res)# id号self.id = dom.xpath(self.base_node + '/@data-cid')# 用户名 所有div节点下class名叫avatar的节点下面的a标签self.username = dom.xpath(self.base_node + '/div[@class="avatar"]/a/@title')# 用户连接self.user_center = dom.xpath(self.base_node + '/div[@class="avatar"]/a/@href')# 点赞数self.vote = dom.xpath(self.base_node + '//span[@class="votes"]/text()')# 评分self.star = dom.xpath(self.base_node + '//span[contains(@class,"rating")]/@title')# 发表时间self.time = dom.xpath(self.base_node + '//span[@class="comment-time "]/@title')# 评论内容 所有span标签class名为short的节点文本self.content = dom.xpath(self.base_node + '//span[@class="short"]/text()')if __name__ == '__main__':# [综合评论、好评、中评、差评]for i in ['', 'h', 'm', 'l']:# 最多爬取24页for j in range(1,30):fetcher = CommentFetcher(movie_id=26363254, start=j * 20, type=i)

IP =[" http://10.10.1.10:3128"," http://10.10.1.10:1080","http://110.88.126.70"]#随机调用代理IPdef _random_IP(self):self.proxies["http:"] = random.choice(IP)

from sqlalchemy import Column, String, create_engine, Integer, Text, DateTime
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker# 初始化数据库连接
engine = create_engine('mysql+pymysql://root:123456@127.0.0.1:3306/douban?charset=utf8')
Base = declarative_base()class Comments(Base):__tablename__ = 'test'id = Column(Integer, primary_key=True)   # IDusername = Column(String(64), nullable=False, index=True)   # 用户名user_center = Column(String(64), nullable=True)   # 用户主页链接vote = Column(Integer, nullable=True)   # 赞同这条评论的人数star = Column(String(10), nullable=True)  # 开始爬取的条数time = Column(DateTime, nullable=True)   # 时间content = Column(Text(), nullable=False, index=True)  # 评论def create_session():# 创建DBSession类型:Session = sessionmaker(bind=engine)session = Session()return sessionif __name__ == '__main__':#创建数据表Base.metadata.create_all(engine)

import pandas as pd
from wordcloud import WordCloud
import jieba
import matplotlib.pyplot as plt
import PIL.Image as image
import numpy as nptxt = open(r'G:\aa\zhanlang2.txt', 'r', encoding='utf8').read()  # 打开评论数据
jieba.load_userdict(r'G:\aa\ZL2.txt')  # 读取战狼2词库Filess= open(r'G:\aa\stops_chinese.txt', 'r', encoding='utf8')  # 打开中文停用词表
stops = Filess.read().split('\n')  # 以回车键作为标识符把停用词表放到stops列表中wordsls = jieba.lcut(txt)  # 使用jieba中文分词组件
wcdict = {}tokens=[token for token in wordsls if token not in stops]
print("过滤后中文内容对比:",len(tokens), len(wordsls))# 统计词频次数
for word in tokens:if len(word) == 1:continueelse:wcdict[word] = wcdict.get(word, 0) + 1# 词频排序
wcls = list(wcdict.items())
wcls.sort(key=lambda x: x[1], reverse=True)# 打印前30词频最高的中文
for i in range(20):print(wcls[i])# 存储过滤后的文本
pd.DataFrame(wcls).to_csv('2.csv', encoding='utf-8')# 读取csv词云
txt = open('2.csv', 'r', encoding='utf-8').read()# 用空格键隔开文本并把它弄进列表中
cut_text = "".join(jieba.lcut(txt))mywc = WordCloud().generate(cut_text)plt.imshow(mywc)
plt.axis("off")
plt.show()

import matplotlib
import matplotlib.pyplot as plt#显示中文
matplotlib.rcParams['font.sans-serif'] = ['SimHei']
matplotlib.rcParams['font.family']='sans-serif'
matplotlib.rcParams['axes.unicode_minus'] = False
#631137
alt=[124965,215218,198808,56171,35975]
index='力荐', '推荐', '还行', '较差','很差'
plt.axes(aspect=1)
plt.pie(x=alt,labels=index,autopct='%.0f%%')
plt.show()