java爬虫爬取网站使用多线程(虎嗅网站)

java爬虫爬取网站使用多线程(虎嗅网站)

图解虎嗅爬虫优化方案

pom 如下:

org.apache.httpcomponents

httpclient

4.5.3

org.jsoup

jsoup

1.10.3

org.springframework

spring-jdbc

4.2.6.RELEASE

mysql

mysql-connector-java

5.1.41

c3p0

c3p0

0.9.1.2

com.alibaba

fastjson

1.2.31

com.google.code.gson

gson

2.8.1

redis.clients

jedis

2.9.0

代码演示如下:

package cn.itcast.huxiu.query;

import java.util.ArrayList;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import org.apache.http.HttpEntity;

import org.apache.http.client.entity.UrlEncodedFormEntity;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import org.apache.http.message.BasicNameValuePair;

import org.apache.http.util.EntityUtils;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import com.google.gson.Gson;

import cn.itcast.huxiu.Article;

import cn.itcast.huxiu.ArticleDao;

import cn.itcast.huxiu.ResponseJson;

public class HuXiuTest {

public static final ArticleDao articleDao = new ArticleDao();

public static final ArrayBlockingQueue blockingQueue = new ArrayBlockingQueue(1000);

public static final ExecutorService threadPool = Executors.newFixedThreadPool(10);

public static final String prefix = "https://www.huxiu.com/article/";

public static final String end = ".html";

public static void main(String[] args) throws Exception {

//创建线程

for(int i=0;i<30;i++){

threadPool.execute(new ProcessPagingThreadQueue());

}

// 爬取首页的信息

String indexHtml = getIndex();

// 解析首页 得到首页里面的所有的id(根据id来查询每一个页面的信息) 存储到集合里面

parseIndexHtml(indexHtml);

/**

* 在首页的信息爬取了之后 就要准备爬取分页的信息 点击加载更多只时 就相当与是点击了下一页 点击之后 就会发送一个请求

* 这个请求就可以加载下一页的数据了 得到的下一页所有数据之后 就要解析每一页的数据

*

*/

// 根据首页的信息来得到加载下一页数据按钮的数据值

String last_dateline = getValueAndIndexHtml(indexHtml);// 得到没加载一页数据的数值

// 点击 加载下一页的数据

for (int page = 2; page < 10; page++) {

// 获得请求的路径

String url = "https://www.huxiu.com/v2_action/article_list";

HttpPost httpPost = new HttpPost(url);

// 请求参数

ArrayList list = new ArrayList();

list.add(new BasicNameValuePair("huxiu_hash_code", "647893ceb60219effa36193702fd89a3"));

list.add(new BasicNameValuePair("page", page + ""));

list.add(new BasicNameValuePair("last_dateline", last_dateline));

// 参数设置

httpPost.setEntity(new UrlEncodedFormEntity(list));

// User-Agent

httpPost.setHeader("User-Agent",

"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0");

// 发起请求

CloseableHttpClient httpClient = HttpClients.createDefault();

CloseableHttpResponse execute = httpClient.execute(httpPost);

// 在发送请求之后 页面没有跳转 因为是在和首页的同一个页面下 所以不用判断 只有页面跳转才有状态码的判定

// 请求发送之后 就有返回值了 主要注意的是返回值是json数据的形式来进行返回的

String jsonDate = EntityUtils.toString(execute.getEntity());

// 得到json数据值 就要水对json的数据进行解析 解析json的数据使用到的是gson

Gson gson = new Gson();

// 将数据进行解析并且映射到实体类中 实体类中是根据返回的参数来进行设置的

// 得到分页数据的所有的信息 也就是分页数据的url

ResponseJson fromJson = gson.fromJson(jsonDate, ResponseJson.class);

// 得到的分页的每一个数据 每一个URL信息

String data = fromJson.getData();// 得到分页的信息

// 对分页的数据信息进行解析 也就要取得每一个详情信息页面的id值

getDate(data);// 得到所有id值的集合

System.out.println(page+"&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&");

}

}

private static void getDate(String data) {

if (data != null) {

Document document = Jsoup.parse(data);

Elements elements = document.select("div[data-aid]");

for (Element element : elements) {

try {

blockingQueue.put(element.attr("data-aid"));

System.out.println(element.attr("data-aid"));

} catch (Exception e) {

// TODO Auto-generated catch block

e.printStackTrace();

}

}

}

}

// 得到加载下一页信息的数据值

private static String getValueAndIndexHtml(String indexHtml) {

if (indexHtml != null) {

Document document = Jsoup.parse(indexHtml);

Elements select = document.select("div[data-last_dateline]");

return select.get(0).attr("data-last_dateline");

}

return null;

}

public static Article parseXianQingYeMian(String html) {

if (html != null) {

Article article = new Article();

// 将详细页面的信息 转换为文档对象

Document document = Jsoup.parse(html);

// 获取文章的标题信息

String ownText = document.select(".t-h1").get(0).ownText();

article.setTitle(ownText);

// 获取作者

String author = document.select(".author-name").get(0).text();

article.setAuthor(author);

// 获取时间 根据页面上的信息可知时间有两种表示

Elements elements = document.select("span[class=article-time pull-left]");

if (elements.size() == 0) {

String createTime = document.select(".article-time").get(0).ownText();

article.setCreateTime(createTime);

} else {

String createTime = elements.get(0).ownText();

article.setCreateTime(createTime);

}

// 获取文章内容

String content = document.select(".article-content-wrap").get(0).text();

article.setContent(content);

// 获取点赞

article.setZan(document.select(".num").get(0).ownText());

// 获取评论

article.setPl(document.select(".article-pl").get(0).ownText());

System.out.println(article);

return article;

}

return null;

}

// 解析数据 得到url

private static void parseIndexHtml(String indexHtml) {

// TODO Auto-generated method stub

if (indexHtml != null) {

// 解析得到的页面的信息 将其变成文档对象

Document document = Jsoup.parse(indexHtml);

// 得到document对象后 就可以通过document对象来得到需要的东西

Elements elements = document.select(".mod-info-flow div[data-aid]");

for (Element element : elements) {

String aid = element.attr("data-aid");

try {

/**

* 在单线层转变多线程中   主要的几个是 队列 线程的创建 线程池的创建

* 这个主要是创建队列 并且把解析数据的详情的id传给了这个队列

* 队列的主要的作用是 防止单个线程被同时访问造成拥堵引起并发的问题

* 队列就把这几个问题给解决了

*/

blockingQueue.put(aid);

} catch (InterruptedException e) {

// TODO Auto-generated catch block

e.printStackTrace();

}

}

}

}

// 首页的获取

private static String getIndex() throws Exception {

String url = "https://www.huxiu.com";

// 发起一个get请求

HttpGet httpGet = new HttpGet(url);

// 设置请求头

httpGet.addHeader("User-Agent",

"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");

// 返回页面的信息

return getHtml(httpGet);

}

// 执行发送请求的方法

public static String getHtml(HttpGet httpGet) throws Exception {

// TODO Auto-generated method stub

String html = null;

CloseableHttpClient httpClient = HttpClients.createDefault();

CloseableHttpResponse execute = httpClient.execute(httpGet);

// 判断响应码是否为200

if (execute.getStatusLine().getStatusCode() == 200) {

HttpEntity entity = execute.getEntity();

html = EntityUtils.toString(entity);

System.out.println(html);// 返回的的页面的所有信息

}

return html;

}

}

**********************************************************************************************************************

代码创建线程

package cn.itcast.huxiu.query;

import org.apache.http.client.methods.HttpGet;

import cn.itcast.huxiu.Article;

public class ProcessPagingThreadQueue extends Thread {

public void run() {

// TODO Auto-generated method stub

while (true) {

//得到每一给详情页面的id 这里面用到了while循环  因为存在队列里面的id值不知道是多少个 而且出来也是一个一个的出来的 所以就使用到了循环

try {

String parseInt = HuXiuTest.blockingQueue.take();//得到详情页的id

int id = Integer.parseInt(parseInt);

//创建发送请求

HttpGet httpGet = new HttpGet(HuXiuTest.prefix + id + HuXiuTest.end);

// 消息头

httpGet.addHeader("user-agent",

"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");

String html = HuXiuTest.getHtml(httpGet);//得到页面的详情信息

Article article = HuXiuTest.parseXianQingYeMian(html);

if(article != null){

article.setId(id);

//article.setUrl(HuXiuTest.prefix + id + HuXiuTest.end);

HuXiuTest.articleDao.save(article);

}

} catch (Exception e) {

// TODO Auto-generated catch blockArticle

e.printStackTrace();

}

}

}

}

**********************************************************************************************************************

实体类

package cn.itcast.huxiu;

public class Article {

private int id;

private String title;

private String author;

private String createTime;

private String sc;

private String zan;

private String pl;

private String content;

private String url;

public int getId() {

return id;

}

public void setId(int id) {

this.id = id;

}

public String getTitle() {

return title;

}

public void setTitle(String title) {

this.title = title;

}

public String getAuthor() {

return author;

}

public void setAuthor(String author) {

this.author = author;

}

public String getCreateTime() {

return createTime;

}

public void setCreateTime(String createTime) {

this.createTime = createTime;

}

public String getSc() {

return sc;

}

public void setSc(String sc) {

this.sc = sc;

}

public String getZan() {

return zan;

}

public void setZan(String zan) {

this.zan = zan;

}

public String getPl() {

return pl;

}

public void setPl(String pl) {

this.pl = pl;

}

public String getContent() {

return content;

}

public void setContent(String content) {

this.content = content;

}

public String getUrl() {

return url;

}

public void setUrl(String url) {

this.url = url;

}

@Override

public String toString() {

return "Article [id=" + id + ", title=" + title + ", author=" + author + ", createTime=" + createTime + ", sc="

+ sc + ", zan=" + zan + ", pl=" + pl + ", content=" + content + ", url=" + url + "]";

}

}

数据库连接

package cn.itcast.huxiu;

import org.springframework.jdbc.core.JdbcTemplate;

import com.mchange.v2.c3p0.ComboPooledDataSource;

public class ArticleDao extends JdbcTemplate{

public ArticleDao() {

// 创建C3P0的datasource 1.配置 2.代码

ComboPooledDataSource dataSource = new ComboPooledDataSource();

// 1.url

// 2.driver

// 3.username&password

dataSource.setUser("root");

dataSource.setPassword("123");

dataSource.setJdbcUrl("jdbc:mysql://localhost:3306/spider?characterEncoding=utf-8");

setDataSource(dataSource);

}

public void save(Article article) {

String sql = "INSERT INTO huxiu_article (id, title, author, createTime, zan, pl, sc, content, url ) VALUES( ?,?,?,?,?,?,?,?,?)";

update(sql, article.getId(),article.getTitle(),article.getAuthor(),article.getCreateTime(),article.getZan(),article.getPl(),article.getSc(),article.getContent(),article.getUrl());

}

}

实体类

package cn.itcast.huxiu;

public class ResponseJson {

private int result;

private String msg;

private String data;

private double total_page;

private double last_dateline;

public int getResult() {

return result;

}

public void setResult(int result) {

this.result = result;

}

public String getMsg() {

return msg;

}

public void setMsg(String msg) {

this.msg = msg;

}

public String getData() {

return data;

}

public void setData(String data) {

this.data = data;

}

public double getTotal_page() {

return total_page;

}

public void setTotal_page(double total_page) {

this.total_page = total_page;

}

public double getLast_dateline() {

return last_dateline;

}

public void setLast_dateline(double last_dateline) {

this.last_dateline = last_dateline;

}

@Override

public String toString() {

return "ResponseJson [result=" + result + ", msg=" + msg + ", data=" + data + ", total_page=" + total_page

+ ", last_dateline=" + last_dateline + "]";

}

}

java爬虫爬取网站使用多线程(虎嗅网站)相关教程