【Java爬虫】爬取淘宝买家秀

爬取目标

https://h5.m.taobao.com/ocean/privatenode/shop.html?&sellerId=50852803

需要sellerId=50852803的50852803

获取数据地址
https://acs.m.taobao.com/h5/mtop.taobao.social.feed.aggregate/1.0/?appKey=12574478&t=1582778795899&sign=367a770e5a56cfaafc350da1da6b7d76&api=mtop.taobao.social.feed.aggregate&v=1.0&timeout=300000&timer=300000&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22params%22%3A%22%7B%5C%22nodeId%5C%22%3A%5C%22%5C%22%2C%5C%22sellerId%5C%22%3A%5C%2250852803%5C%22%7D%22%2C%22cursor%22%3A%221%22%2C%22pageNum%22%3A%221%22%2C%22pageId%22%3A5703%2C%22env%22%3A%221%22%7D
其中
t为当前时间戳
sign 为 (token + "&" + t + "&" + appKey + "&" + data) 这几个参数拼接后转成MD5我们需要获取的就只有token,而token是服务器传过来的
所以伪造一次访问获取返回的token,然后再访问数据

加端端老师免费领取更多编程资料

pom.xml

    <dependency><groupId>org.apache.httpcomponents</groupId><artifactId>httpclient</artifactId><version>4.5.11</version></dependency><dependency><groupId>com.alibaba</groupId><artifactId>fastjson</artifactId><version>1.2.62</version></dependency><dependency><groupId>org.projectlombok</groupId><artifactId>lombok</artifactId><optional>true</optional></dependency>

TbBuyerShow.java

@Data
@NoArgsConstructor
public class TbBuyerShow {private String sellerId; //店铺类别IDprivate String title; //店铺名称private String userName; //用户名称private String userUrl; //用户链接private String userTitle; //用户评论private String imgId; //图片IDprivate String imgUrl; //图片衔接private String targetUrl; //图片来源private Integer pageNum;
}

BuyerShowReptile.Java

public class BuyerShowReptile {public static void main(String[] args) {List<TbBuyerShow> reptile = reptile("50852803", 1, 20);reptile.forEach(tbBuyerShow -> System.out.println(tbBuyerShow.getImgUrl()));}//ID,第几页,固定参数public static List<TbBuyerShow> reptile(String sellerId, int index, int num) {String url = "https://acs.m.taobao.com/h5/mtop.taobao.social.feed.aggregate/1.0/?";String appKey = "12574478";String t = String.valueOf(new Date().getTime());String sign = "af1fde903d6e32e57aaf3377e6a68f3a";String data = "{\"params\":" +"\"{\\\"nodeId\\\":" +"\\\"\\\",\\\"sellerId\\\":" +"\\\"" + sellerId + "\\\",\\\"pagination\\\":" +"{\\\"direction\\\":" +"\\\"1\\\",\\\"hasMore\\\":" +"\\\"true\\\",\\\"pageNum\\\":" +"\\\"" + index + "\\\",\\\"pageSize\\\":" +"\\\"" + num + "\\\"}}\",\"cursor\":" +"\"" + index + "\",\"pageNum\":" +"\"" + index + "\",\"pageId\":" +"5703,\"env\":" +"\"1\"}";Params params = newParams(appKey, t, sign, data);String str = htmlUrl(url, params);String mh5tk = "";String mh5tkenc = "";String token = "";String u;CookieStore cookieStore = new BasicCookieStore();CloseableHttpClient httpClient = HttpClientBuilder.create().setDefaultCookieStore(cookieStore).build();HttpGet httpGet = new HttpGet(str);CloseableHttpResponse response = null;try {response = httpClient.execute(httpGet);List<Cookie> cookies = cookieStore.getCookies();for (Cookie cookie : cookies) {if ("_m_h5_tk".equals(cookie.getName())) {mh5tk = cookie.getValue();token = mh5tk.split("_")[0];}if ("_m_h5_tk_enc".equals(cookie.getName())) {mh5tkenc = cookie.getValue();}}u = token + "&" + params.getT() + "&" + appKey + "&" + data;sign = DigestUtils.md5DigestAsHex(u.getBytes());params = newParams(appKey, t, sign, data);str = htmlUrl(url, params);Cookie cookie = new BasicClientCookie("_m_h5_tk", mh5tk);((BasicClientCookie) cookie).setAttribute("_m_h5_tk_enc", mh5tkenc);cookieStore.addCookie(cookie);httpClient = HttpClientBuilder.create().setDefaultCookieStore(cookieStore).build();httpGet = new HttpGet(str);response = httpClient.execute(httpGet);HttpEntity entity = response.getEntity();String conResult = EntityUtils.toString(entity, "UTF-8");return newTbBuyerShow(conResult, sellerId, index);} catch (IOException e) {e.printStackTrace();} finally {try {if (httpClient != null) {httpClient.close();}if (response != null) {response.close();}} catch (IOException e) {e.printStackTrace();}}return null;}static List<TbBuyerShow> newTbBuyerShow(String conResult, String sellerId, Integer index) {List<TbBuyerShow> tbBuyerShows = new ArrayList<>();String title = ""; //店铺名称String userName = ""; //用户名称String userUrl = ""; //用户链接String userTitle = ""; //用户评论String imgId; //图片IDString imgUrl; //图片衔接String targetUrl = ""; //图片来源Integer pageNum = index; //页码if (!StringUtils.isEmpty(conResult)) {conResult = conResult.replace("mtopjsonp(", "");conResult = conResult.replace(")", "");JSONObject jsonObject = JSON.parseObject(conResult);jsonObject = jsonObject.getJSONObject("data");if (!StringUtils.isEmpty(jsonObject)) {JSONObject header = jsonObject.getJSONObject("header");if (!StringUtils.isEmpty(header)) {title = (String) header.get("title");}JSONArray userList = jsonObject.getJSONArray("list");if (!StringUtils.isEmpty(userList)) {for (int i = 0; i < userList.size(); i++) {JSONObject list = userList.getJSONObject(i);JSONObject user = list.getJSONObject("user");if (!StringUtils.isEmpty(user)) {userName = (String) user.get("userNick");userUrl = (String) user.get("userUrl");}if (!StringUtils.isEmpty(list.get("title"))) {userTitle = (String) list.get("title");}if (!StringUtils.isEmpty(list.get("targetUrl"))) {targetUrl = (String) list.get("targetUrl");}JSONArray picsList = list.getJSONArray("pics");if (!StringUtils.isEmpty(picsList)) {for (int j = 0; j < picsList.size(); j++) {TbBuyerShow tbBuyerShow = new TbBuyerShow();JSONObject pics = picsList.getJSONObject(j);imgId = (String) pics.get("id");imgUrl = (String) pics.get("path");tbBuyerShow.setSellerId(sellerId);tbBuyerShow.setTitle(title);tbBuyerShow.setUserName(userName);tbBuyerShow.setUserUrl(userUrl);tbBuyerShow.setUserTitle(userTitle);tbBuyerShow.setImgId(imgId);tbBuyerShow.setImgUrl(imgUrl);tbBuyerShow.setTargetUrl(targetUrl);tbBuyerShow.setPageNum(pageNum);tbBuyerShows.add(tbBuyerShow);}}}}}}return tbBuyerShows;}static Params newParams(String appkey, String t, String sign, String data) {Params params = new Params();params.setAppKey(appkey);params.setT(t);params.setSign(sign);params.setApi("mtop.taobao.social.feed.aggregate");params.setV("1.0");params.setTimeout("300000");params.setTimer("300000");params.setType("jsonp");params.setDataType("jsonp");params.setCallback("mtopjsonp");params.setData(data);return params;}/*** * https://acs.m.taobao.com/h5/mtop.taobao.social.feed.aggregate/1.0/* * ?appKey=12574478* * &t=1581927984172* * &sign=e83a3add7b5fc1b70b0601a2ccd133e9* * &api=mtop.taobao.social.feed.aggregate* * &v=1.0* * &timeout=300000* * &timer=300000* * &type=jsonp* * &dataType=jsonp* * &callback=mtopjsonp1* * &data=%7B%22params%22%3A%22%7B%5C%22nodeId%5C%22%3A%5C%22%5C%22%2C%5C%22sellerId%5C%22%3A%5C%22109043255%5C%22%7D%22%2C%22cursor%22%3A%221%22%2C%22pageNum%22%3A%221%22%2C%22pageId%22%3A5703%2C%22env%22%3A%221%22%7D* *** @param url* @return*/static String htmlUrl(String url, Params params) {StringBuffer buffer = new StringBuffer();try {buffer.append(url).append("appkey=" + URLEncoder.encode(params.getAppKey(), "utf-8")).append("&t=" + URLEncoder.encode(params.getT(), "utf-8")).append("&sign=" + URLEncoder.encode(params.getSign(), "utf-8")).append("&api=" + URLEncoder.encode(params.getApi(), "utf-8")).append("&v=" + URLEncoder.encode(params.getV(), "utf-8")).append("&timeout=" + URLEncoder.encode(params.getTimeout(), "utf-8")).append("&timer=" + URLEncoder.encode(params.getTimer(), "utf-8")).append("&type=" + URLEncoder.encode(params.getType(), "utf-8")).append("&dataType=" + URLEncoder.encode(params.getDataType(), "utf-8")).append("&callback=" + URLEncoder.encode(params.getCallback(), "utf-8")).append("&data=" + URLEncoder.encode(params.getData(), "utf-8"));} catch (UnsupportedEncodingException e) {e.printStackTrace();}return buffer.toString();}
}

【Java爬虫】爬取淘宝买家秀相关推荐

网络爬虫爬取淘宝页面商品信息
网络爬虫爬取淘宝页面商品信息最近在MOOC上看嵩老师的网络爬虫课程,按照老师的写法并不能进行爬取,遇到了一个问题,就是关于如何"绕开"淘宝登录界面,正确的爬取相关信息.通过百度找 ...
利用Python爬虫爬取淘宝商品做数据挖掘分析实战篇，超详细教程
项目内容本案例选择>> 商品类目:沙发: 数量:共100页 4400个商品: 筛选条件:天猫.销量从高到低.价格500元以上. 项目目的 1. 对商品标题进行文本分析词云可视化 2. ...
python淘宝爬虫_python爬虫爬取淘宝商品信息
本文实例为大家分享了python爬取淘宝商品的具体代码,供大家参考,具体内容如下 import requests as req import re def getHTMLText(url): try: ...
python网络爬虫--爬取淘宝联盟
互联网是由一个个站点和网络设备组成的大网,我们通过浏览器访问站点,站点把HTML.JS.CSS代码返回给浏览器,这些代码经过浏览器解析.渲染,将丰富多彩的网页呈现我们眼前. 网络爬虫,也叫网络蜘蛛(W ...
简单使用Python爬虫爬取淘宝网页商品信息
最近在学习爬虫,本人还是入门级的小白,自己跟着老师写了一些代码,算是自己的总结,还有一些心得,跟大家分享一下,如果不当,还请各位前辈斧正. 这是代码: # 导入库 import requests im ...
python爬虫爬取淘宝，罗兰电钢琴和雅马哈电钢琴（参考崔大）
淘宝网上有很多商品,这些商品的信息就是一个很不错的数据来源,于是我参考资料后依葫芦画瓢弄了一个爬虫程序来爬一爬梦寐以求的电钢琴. 声明一下:电钢琴和电子琴是两种不同的琴,我在正则表达式里面设置了只要含 ...
使用java实现爬取淘宝商品页面数据
1.首先要排查淘宝获取商品数据的请求方法输入淘宝网后打开开发者工具(F12),点击网络.我用的是火狐浏览器 2.爬取数据首先需要获取如下请求头参数 cookie Referer 3.获取到基本信息后 ...
python爬虫 — 爬取淘宝商品信息
(一)确定需要爬取的信息在爬取前首先确定需要获取的信息,打开taobao,在搜索框中输入,需要获取的商品的信息,比如ipad,点击搜索就可以看到许多的ipad,选择其中的一款商品,比如第一个可以 ...
python爬虫爬取淘宝商品并保存至mongodb数据库
使用工具介绍 python3.8 selenium(请确保你已经成功安装了谷歌驱动chromedriver) mongodb数据库 mongo-compass 谷歌浏览器分析请求链接打开淘宝首页的 ...
python爬虫爬取淘宝搜索页面商品信息数据
主要使用的库: requests:爬虫请求并获取源码 re:使用正则表达式提取数据 json:使用JSON提取数据 pandas:使用pandans存储数据以下是源代码: #!coding=utf- ...

【Java爬虫】爬取淘宝买家秀

【Java爬虫】爬取淘宝买家秀相关推荐

最新文章

热门文章