毕设一:python 爬取苏宁的商品评论

毕设需要大量的商品评论,网上找的数据比较旧了,自己动手

代理池用的proxypool,github:https://github.com/jhao104/proxy_pool

ua:fake_useragent

 1 # 评价较多的店铺(苏宁推荐)
 2 https://tuijian.suning.com/recommend-portal/recommendv2/biz.jsonp?parameter=%E5%8D%8E%E4%B8%BA&sceneIds=2-1&count=10
 3
 4 # 评价
 5 https://review.suning.com/ajax/cluster_review_lists/general-30259269-000000010748901691-0000000000-total-1-default-10-----reviewList.htm?callback=reviewList
 6
 7 # clusterid
 8 # 在查看全部评论的href中,实际测试发现是执行js加上的,两种方案
 9 # 1.去匹配js中的clusterId
10 # 2.或者用selenium/phantomjs去请求执行js之后的页面然后解析html获得href

代码:

  1 # -*- coding: utf-8 -*-
  2 # @author: Tele
  3 # @Time  : 2019/04/15 下午 8:20
  4 import time
  5 import requests
  6 import os
  7 import json
  8 import re
  9 from fake_useragent import UserAgent
 10
 11
 12 class SNSplider:
 13     flag = True
 14     regex_cluser_id = re.compile("\"clusterId\":\"(.{8})\"")
 15     regex_comment = re.compile("reviewList\((.*)\)")
 16
 17     @staticmethod
 18     def get_proxy():
 19         return requests.get("http://127.0.0.1:5010/get/").content.decode()
 20
 21     @staticmethod
 22     def get_ua():
 23         ua = UserAgent()
 24         return ua.random
 25
 26     def __init__(self, kw_list):
 27         self.kw_list = kw_list
 28         # 评论url 参数顺序:cluser_id,sugGoodsCode,页码
 29         self.url_temp = "https://review.suning.com/ajax/cluster_review_lists/general-{}-{}-0000000000-total-{}-default-10-----reviewList.htm"
 30         self.headers = {
 31             "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
 32         }
 33         self.proxies = {
 34             "http": None
 35         }
 36         self.parent_dir = None
 37         self.file_dir = None
 38
 39     # ua,proxy
 40     def check(self):
 41         self.headers["User-Agent"] = SNSplider.get_ua()
 42         proxy = "http://" + SNSplider.get_proxy()
 43         self.proxies["http"] = proxy
 44         print("ua:", self.headers["User-Agent"])
 45         print("proxy:", self.proxies["http"])
 46
 47     # 评论
 48     def parse_url(self, cluster_id, sugGoodsCode, page):
 49         url = self.url_temp.format(cluster_id, sugGoodsCode, page)
 50         response = requests.get(url, headers=self.headers, proxies=self.proxies, verify=False)
 51         if response.status_code == 200:
 52             print(url)
 53             if len(response.content) < 0:
 54                 return
 55             data = json.loads(SNSplider.regex_comment.findall(response.content.decode())[0])
 56             if "commodityReviews" in data:
 57                 # 评论
 58                 comment_list = data["commodityReviews"]
 59                 if len(comment_list) > 0:
 60                     item_list = list()
 61                     for comment in comment_list:
 62                         item = dict()
 63                         try:
 64                             # 商品名
 65                             item["referenceName"] = comment["commodityInfo"]["commodityName"]
 66                         except:
 67                             item["referenceName"] = None
 68                         # 评论时间
 69                         item["creationTime"] = comment["publishTime"]
 70                         # 内容
 71                         item["content"] = comment["content"]
 72                         # label
 73                         item["label"] = comment["labelNames"]
 74                         item_list.append(item)
 75
 76                     # 保存
 77                     with open(self.file_dir, "a", encoding="utf-8") as file:
 78                         file.write(json.dumps(item_list, ensure_ascii=False, indent=2))
 79                         file.write("\n")
 80                     time.sleep(5)
 81             else:
 82                 SNSplider.flag = False
 83         else:
 84             print("评论页出错")
 85
 86     # 提取商品信息
 87     def get_product_info(self):
 88         url_temp = "https://tuijian.suning.com/recommend-portal/recommendv2/biz.jsonp?parameter={}&sceneIds=2-1&count=10"
 89         result_list = list()
 90         for kw in self.kw_list:
 91             url = url_temp.format(kw)
 92             response = requests.get(url, headers=self.headers, proxies=self.proxies, verify=False)
 93             if response.status_code == 200:
 94                 kw_dict = dict()
 95                 id_list = list()
 96                 data = json.loads(response.content.decode())
 97                 skus_list = data["sugGoods"][0]["skus"]
 98                 if len(skus_list) > 0:
 99                     for skus in skus_list:
100                         item = dict()
101                         sugGoodsCode = skus["sugGoodsCode"]
102                         # 请求cluserId
103                         item["sugGoodsCode"] = sugGoodsCode
104                         item["cluster_id"] = self.get_cluster_id(sugGoodsCode)
105                         id_list.append(item)
106                 kw_dict["title"] = kw
107                 kw_dict["id_list"] = id_list
108                 result_list.append(kw_dict)
109             else:
110                 pass
111         return result_list
112
113     # cluserid
114     def get_cluster_id(self, sugGoodsCode):
115         self.check()
116         url = "https://product.suning.com/0000000000/{}.html".format(sugGoodsCode[6::])
117         response = requests.get(url, headers=self.headers, proxies=self.proxies, verify=False)
118         if response.status_code == 200:
119             cluser_id = None
120             try:
121                 cluser_id = SNSplider.regex_cluser_id.findall(response.content.decode())[0]
122             except:
123                 pass
124             return cluser_id
125         else:
126             print("请求cluster id出错")
127
128     def get_comment(self, item_list):
129         if len(item_list) > 0:
130             for item in item_list:
131                 id_list = item["id_list"]
132                 item_title = item["title"]
133                 if len(id_list) > 0:
134                     self.parent_dir = "f:/sn_comment/" + item_title + time.strftime("-%Y-%m-%d-%H-%M-%S",
135                                                                                     time.localtime(time.time()))
136                     if not os.path.exists(self.parent_dir):
137                         os.makedirs(self.parent_dir)
138                     for product_code in id_list:
139                         # 检查proxy,ua
140                         sugGoodsCode = product_code["sugGoodsCode"]
141                         cluster_id = product_code["cluster_id"]
142                         if not cluster_id:
143                             continue
144                         page = 1
145                         # 检查目录
146                         self.file_dir = self.parent_dir + "/" + sugGoodsCode[6::] + "_ratecontent.txt"
147                         self.check()
148                         while SNSplider.flag:
149                             self.parse_url(cluster_id, sugGoodsCode, page)
150                             page += 1
151                         SNSplider.flag = True
152                 else:
153                     print("---error,empty id list---")
154         else:
155             print("---error,empty item list---")
156
157     def run(self):
158         self.check()
159         item_list = self.get_product_info()
160         print(item_list)
161         self.get_comment(item_list)
162
163
164 def main():
165     # , "华为mate20pro", "vivoz3", "oppok1", "荣耀8x", "小米9", "小米mix3", "三星s9", "iphonexr", "iphonexs"
166     # "华为p30pro", "华为mate20pro", "vivoz3""oppok1""荣耀8x", "小米9"
167     kw_list = ["小米mix3", "三星s9", "iphonexr", "iphonexs"]
168     splider = SNSplider(kw_list)
169     splider.run()
170
171
172 if __name__ == '__main__':
173     main()

转载于:https://www.cnblogs.com/tele-share/p/10719925.html

毕设一:python 爬取苏宁的商品评论相关推荐

python爬取苏宁商品评论
python爬取苏宁商品评论爬取其他电商物品评论的案例如下: https://blog.csdn.net/coffeetogether/article/details/114296159 https ...
利用python爬取东方财富网股吧评论并进行情感分析（一）
利用python爬取东方财富网股吧评论(一) python-东方财富网贴吧文本数据爬取分享一下写论文时爬数据用到的代码,有什么问题或者改善的建议的话小伙伴们一起评论区讨论.涉及内容在前人的研究基础之 ...
python 爬取携程旅游景点评论
python爬取携程旅游景点评论爬取网址:https://you.ctrip.com/ 爬取评论全部代码 import requests import json import re import t ...
python 爬取亚马逊评论_用Python爬取了三大相亲软件评论区，结果...
小三:怎么了小二?一副愁眉苦脸的样子. 小二:唉!这不是快过年了吗,家里又催相亲了 ... 小三:现在不是流行网恋吗,你可以试试相亲软件呀. 小二:这玩意靠谱吗? 小三:我也没用过,你自己看看软件评论 ...
python爬取国家男女比例_用python爬取3万多条评论，看韩国人如何评价韩国电影《寄生虫》？...
用python爬取3万多条评论,看韩国人如何评价韩国电影<寄生虫>? 朱小五凹凸数据大家好,我是朱小五今天给大家带来一个关于电影的数据分析文章. 别走啊,这次不是豆瓣,也不是猫眼真 ...
python爬取携程景区用户评论
python爬取携程景区用户评论(爬虫时遇到翻页但url不变问题时的解决办法) python爬取携程景区用户评论 Ajax页面加载的页面和普通页面的不同解决办法效果 python爬取携程景区用户评 ...
用Python爬取淘宝网商品信息
用Python爬取淘宝网商品信息转载请注明出处网购时经常会用到淘宝网点我去淘宝但淘宝网上的商品琳琅满目,于是我参照中国大学 MOOC的代码写了一个爬取淘宝网商品信息的程序代码如下: impor ...
python爬取网抑云音乐评论
提示:文章写完后,目录可以自动生成,如何生成可参考右边的帮助文档文章目录前言一.python爬取网抑云音乐评论前言提示:这里可以添加本文要记录的大概内容: python爬取网抑云音乐评论提 ...
python 爬取苏宁易购商品信息和评论的详细流程
总体说明爬取苏宁易购的最大难点就在于他这个页面是很多js动态加载的内容,寻找和构造链接比较麻烦,如果不用JS逆向工程.采用selenium等爬取的效率相对会比较差一点,下面我会先放上我找的js动态加 ...
python爬取B站动态的评论总数(不含用户评论内容详情)
目录前言需求方案分析方案一方案二接口分析请求流程抓包演示请求接口接口说明接口测试代码前言想看接口分析和代码的,可跳过前言. 更新,最核心的代码已删除,思路和其他代码保留. ...

毕设一:python 爬取苏宁的商品评论

毕设一:python 爬取苏宁的商品评论相关推荐

最新文章

热门文章