事先说明不要用这技术，做一些违法的事情，网上玩爬虫的已经有还能多被抓了，此文就是为了学习webmagic

源码地址：https://github.com/smltq/spring-boot-demo/tree/master/webmagic

DetailPageProcesser类

package com.qf.webmagic.controller;

import us.codecraft.webmagic.Page;

import us.codecraft.webmagic.Site;

import us.codecraft.webmagic.processor.PageProcessor;

public class DetailPageProcesserimplements PageProcessor {

private Sitesite = Site.me().setDomain("127.0.0.1");

@Override

public void process(Page page) {

page.putField("download", page.getHtml().xpath("//*[@id=\"Zoom\"]/span/table/tbody/tr/td/a").toString());

}

@Override

public Site getSite() {

return site;

}

}

ListPageProcesser类

package com.qf.webmagic.controller;

import us.codecraft.webmagic.Page;

import us.codecraft.webmagic.Site;

import us.codecraft.webmagic.processor.PageProcessor;

public class ListPageProcesserimplements PageProcessor {

private Sitesite = Site.me().setDomain("127.0.0.1");

@Override

public void process(Page page) {

page.putField("title", page.getHtml().xpath("//a[@class='ulink']").all().toString());

}

@Override

public Site getSite() {

return site;

}

}

主启动类

package com.qf.webmagic.controller;

import us.codecraft.webmagic.Spider;

public class Main {

public static void main(String[] args) {

//获取影片标题和页面链接

Spider.create(new ListPageProcesser()).addUrl("https://www.dytt8.net/html/gndy/dyzz/list_23_1.html")

.addPipeline(new MyPipeline()).thread(1).run();

//获取指定详情页面的影片下载地址

Spider.create(new DetailPageProcesser()).addUrl("https://www.dytt8.net/html/gndy/dyzz/20191204/59453.html")

.addPipeline(new MyPipeline()).thread(1).run();

}

}

MyPipeline类

package com.qf.webmagic.controller;

import lombok.extern.slf4j.Slf4j;

import us.codecraft.webmagic.ResultItems;

import us.codecraft.webmagic.Task;

import us.codecraft.webmagic.pipeline.Pipeline;

import java.util.Map;

@Slf4j

public class MyPipelineimplements Pipeline {

@Override

public void process(ResultItems resultItems, Task task) {

log.info("get page: " + resultItems.getRequest().getUrl());

for (Map.Entry entry : resultItems.getAll().entrySet()) {

log.info(entry.getKey() +":\t" + entry.getValue());

}

}

}

WebMagicController类

package com.qf.webmagic.controller;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RestController;

@RestController

public class WebMagicController {

@RequestMapping("/hello")

public String hello() {

return "Hello World!";

}

}

WebMagicApplication项目启动

package com.qf.webmagic;

import org.springframework.boot.SpringApplication;

import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication

public class WebMagicApplication {

public static void main(String[] args) {

SpringApplication.run(WebMagicApplication.class, args);

}

}

依赖

xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

4.0.0

org.springframework.boot

spring-boot-starter-parent

2.1.8.RELEASE

com.easy

webmagic

0.0.1

webmagic

Demo project for Spring Boot

1.8

UTF-8

UTF-8

UTF-8

us.codecraft

webmagic-core

0.7.3

us.codecraft

webmagic-extension

0.7.3

org.springframework.boot

spring-boot-starter-web

org.springframework.boot

spring-boot-starter-test

test

org.apache.httpcomponents

httpclient

junit

junit

org.apache.commons

commons-lang3

us.codecraft

xsoup

com.github.dreamhead

moco-core

org.slf4j

slf4j-api

org.mockito

mockito-all

org.slf4j

slf4j-log4j12

commons-collections

commons-collections

org.assertj

assertj-core

org.jsoup

jsoup

org.apache.commons

commons-io

com.jayway.jsonpath

json-path

com.alibaba

fastjson

org.projectlombok

lombok

compile

org.quartz-scheduler

quartz

2.2.1

org.quartz-scheduler

quartz-jobs

2.2.1

org.apache.httpcomponents

httpcore

4.4.11

commons-io

commons-io

2.4

org.jsoup

jsoup

1.8.3

us.codecraft

xsoup

0.3.1

commons-collections

commons-collections

3.2.2

org.apache.maven.plugins

maven-compiler-plugin

1.8

1.8