【ELK】ElasticSearch插件demo

环境：

操作系统：win7

elasticsearch版本：5.4.4

java：1.8

参考文章：

1、Elasticsearch权威指南（中文版）

2、Elasticsearch笔记五之java操作es

3、java操作ElasticSearch（es）进行增删查改操作

4、ElasticSearch入门-增删改查(CRUD)

5、elasticsearch——Java API的使用

6、elasticsearch源码分析之plugin的开发

7、Elasticsearch 5.X下JAVA API使用指南

——————————————————————————————————————

8、elasticsearch5.2.2 插件开发（一）

9、elasticsearch5.2.2 插件开发（二）第一个有实际功能的插件 - CSDN博客

10、elasticsearch5.2.2 插件开发（三）ScriptPlugin 的实现 - CSDN博客

——————————————————————————————————————

11、Maven通俗讲解

12、如何添加本地JAR文件到Maven项目中 - CSDN博客

13、Maven Repository: weka

14、maven系列－－maven添加第三方、本地依赖 - CSDN博客

15、Maven配置项目依赖使用本地仓库的方法汇总 - EasonJim - 博客园

有关打包部分的视频在我的百度网盘

链接：https://pan.baidu.com/s/1DuIWN2uDtcA8guP_Ny91Vw 密码：tj65

第一部分：代码

一、创建maven项目

首先创建一个maven项目，填写Group Id为com.plugin和Artifact Id为esPlugin。其中Artifact Id是项目名，Group Id是项目中包的名字。

二、修改pom.xml配置文件

修改pom.xml文件，然后右键项目名->maven->update maven。会发现在Maven Dependencies文件夹下多了很多包（见下图），这些都是pom.xml文件添加进去的依赖，省去了手动导包的麻烦。（包全是在maven的中央仓库下载到本地的，pom.xml代码见页面底部。）

三、核心代码实现

代码目录如下（用的是之前的项目的截图，忽略顶部的包名），在文后的链接中下载项目，会有这么多包，真正用到的和需要修改的类是TasteEventRestAction.java，这里面包括注册URL，以及执行对应的操作。

package com.taste.elasticsearch_taste.rest;import static com.taste.elasticsearch_taste.action.LoggerUtils.emitErrorResponse;
import static org.elasticsearch.rest.RestRequest.Method.POST;import java.io.File;
import java.io.IOException;import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.rest.action.search.RestSearchAction;
import org.elasticsearch.search.SearchHits;import com.pligin.esPlugin.action.TasteEventAction;
import com.pligin.esPlugin.common.TasteEventRequestBuilder;
import com.pligin.esPlugin.common.TasteEventResponse;//我项目中的类
//import ding.util.ESUtil.ActionParameter;
//import ding.util.WEKAUtil.MeachineLearningBuilder;public class TasteEventRestAction extends BaseRestHandler{public ActionParameter parameter;@Injectpublic TasteEventRestAction(final Settings settings,final RestController restController) {super(settings);restController.registerHandler(RestRequest.Method.GET, "/_taste/{action}", this); // 注册URL1，该URL中最后的字符串赋给变量actionrestController.registerHandler(RestRequest.Method.GET, "/_taste", this); // 注册URL2}@Overrideprotected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {if (request.method() == POST && !request.hasContent()) {return channel -> emitErrorResponse(channel, logger, new IllegalArgumentException("Request body was expected for a POST request."));}String action = request.param("action");// 遇到URL1，执行createDoSomethingResponseif (action != null) { logger.info("do something action -- ESData & MeachineLearning");return createDoSomethingResponse(request, client);// 遇到URL2，执行createMessageResponse} else {logger.info("do Message action -- do anything without parameter");return createMessageResponse(request);  }}// 一、对应URL为 /_taste/{action}private RestChannelConsumer createDoSomethingResponse(RestRequest request, NodeClient client){/*这里request存放的是URL，使用request.param("action")可以获得URL中的action变量值（可作为参数使用）我这里对action做了一个解析，我的参数规格是http://127.0.0.1:9200/_taste/index=logstash-20180323&from=0&size=100&clfName=J48&classifyAttributeName=tes_tims可以通过 parameter.Index, parameter.From, parameter.Size获得action中的参数然后通过前几个包里面的TasteEventRequestBuilder等类，在es里面进行查询，得到输出myresponse最后使用异步编程，即return channel，可以对es传出来的数据进行处理，处理结果作为String，加入builder里面，由channel返回*/String action = request.param("action");this.parameter= new ActionParameter(action);// 1、通过parameter进行数据查询，构造自定义的request请求final TasteEventRequestBuilder actionBuilder=new TasteEventRequestBuilder(client);SearchRequestBuilder requestBuilder = client.prepareSearch(parameter.Index).setQuery(QueryBuilders.matchAllQuery()).setFrom(parameter.From).setSize(parameter.Size).setExplain(true);SearchRequest searchRequest=new SearchRequest();try {RestSearchAction.parseSearchRequest(searchRequest, request, null);} catch (IOException e1) {logger.debug("Failed to emit response.", e1);e1.printStackTrace();}actionBuilder.setSearchRequest(requestBuilder);return channel -> client.execute(TasteEventAction.INSTANSE, actionBuilder.request(),new ActionListener<TasteEventResponse>() {@Overridepublic void onResponse(TasteEventResponse response) {try{// 2、获得查询数据SearchResponse myresponse=response.getSearchResponse();SearchHits ESHits = myresponse.getHits();// 3、调用机器学习功能String res = new MeachineLearningBuilder().getOutput(ESHits, parameter);//将response转换成json类型，不过这个未转化成功XContentBuilder builder = channel.newBuilder();builder.startObject();builder.field("res", res);//response.toXContent(builder, request); // 此处可以写个覆盖函数builder.endObject();channel.sendResponse( new BytesRestResponse(RestStatus.OK, builder));}catch(Exception e){logger.debug("Failed to emit response.", e);onFailure(e);}}@Overridepublic void onFailure(Exception e) {emitErrorResponse(channel, logger, e);}});}// 2、对应URL为 /_tasteprivate RestChannelConsumer createMessageResponse(RestRequest request) {  // 也可以像这里面一样，不使用es里面的通信，直接对builder进行处理，然后在builder里面写入自定义的一些东西，最后可以输出在网页上return channel -> {  Message message = new Message();XContentBuilder builder = channel.newBuilder();  builder.startObject();message.toXContent(builder, request);builder.endObject();  channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));  };  }class Message implements ToXContent {  @Override  public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {  return builder.field("message", "This is my first plugin      Run without error");  }  }

四、插件打包与安装

在./src/main下创建 resources 文件夹，在文件夹中创建 plugin-descriptor.properties 文件，

description=elasticsearch-esPlugin-5.4.0.1 for Elasticsearch #随便写
version=5.4.1 #es的版本
name=esPlugin #写项目名
site=${elasticsearch.plugin.site}
jvm=true
classname=com.plugin.esPlugin.plugin.TastePlugin #根据自己的TastePlugin的目录修改
java.version=1.8
elasticsearch.version=5.4.1
isolated=${elasticsearch.plugin.isolated}

打包，在pom.xml所在目录，进入cmd，输入目录 mvn clean install，会在打包后在target目录下将看到新生成的jar包。然后新建一个文件夹，文件夹名字为elasticsearch,将这个jar包放入这个文件夹，并将之前写的plugin-descriptor.properties文件复制一份放入这个文件夹（注意文件夹的名字一定为elasticsearch，不能为其他名字！！）

最后将这个文件夹压缩为zip，压缩后文件的名字可以随便起，这里起为elasticseach_taste_5.4.1.zip，然后在那个最开始下载的软件版的elasticsearch5.4.1的bin目录下执行elasticsearch-plugin install file:F:/java_programming/elasticsearch_taste1/target/elasticseach_taste_5.4.1.zip操作安装插件，安装好可以在plugins文件夹下看到自定义的插件，将这个插件复制一份到源码的core中的plugins文件夹下也可以。

五、demo的实验效果

第二部分、超简洁的命令介绍

—————————————————————————————————

1、使用该命令对插件（maven项目）进行打包

mvn clean install

—————————————————————————————————

2、删除之前安装的插件（不删除的话会重复占用URL报错）

elasticsearch-plugin remove esmlplugin2

elasticsearch-plugin remove taste

3、安装新的插件（已经压缩为F盘下的taste-5.4.0.1.zip）
elasticsearch-plugin install file:F:/esmlplugin2-5.4.0.1.zip
elasticsearch-plugin install file:F:/taste-5.4.0.1.zip

这个命令效果如下图所示，完成之后，会在 elasticsearch5.4.4/plugin 目录下生成名为taste的文件夹（文件夹名字根据.property中的name确定），可以直接将此文件夹复制到源码中 /core/plugin 目录下，在源码中运行。

—————————————————————————————————

4、启动
elasticsearch.bat

第三部分：测试类介绍

测试类在com.taste.elasticsearch_taste包下，核心代码是TasteEventRestTest类。该类功能是发出http的get请求，获取数据。代码如下：

package com.taste.elasticsearch_taste;import java.net.InetSocketAddress;import org.apache.http.HttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.common.network.NetworkAddress;//import org.elasticsearch.common.xcontent.XContentType;public class TasteEventRestTest extends TastePluginTest {public void test_recommended_items_from_user() throws Exception {
//      final String index = "blog";
//      XContentType type = randomFrom(XContentType.values());try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) {InetSocketAddress[] endpoint = cluster().httpAddresses();this.restBaseUrl = "http://" + NetworkAddress.format(endpoint[0]);String s1 = "/_taste/parameter?index=blog&from=0&size=10&clfName=RandomForest&classifyAttributeName=classify";
//          String s2 = "/_taste";HttpGet get=new HttpGet(restBaseUrl + s1);System.out.println("post请求已发送11111111111");HttpResponse response = httpClient.execute(get);System.out.println("post请求已发送");System.out.println("得到返回值：");if(response.getStatusLine().getStatusCode()==200){//如果状态码为200,就是正常返回String result=EntityUtils.toString(response.getEntity());//得到返回的字符串System.out.println(result);}}}
}

在测试类中开启的es里是没有任何数据的，需要在HttpGet get=new HttpGet(restBaseUrl + s1);这里加上断点，debug到这句话，然后使用git想es里面输入数据，下面给出一个数据样例。然后继续debug，会执行项目第一部分中的核心代码。

curl -XPUT http://127.0.0.1:9200/blog
curl -XPUT http://127.0.0.1:9200/blog/es/1 -d '{"a":"1","b":"3","c":"4","d":"6","e":"8","path":"xxx","classify":"a"}'
curl -XPUT http://127.0.0.1:9200/blog/es/2 -d '{"a":"4","b":"1","c":"7","d":"7","e":"7","path":"yyy","classify":"a"}'
curl -XPUT http://127.0.0.1:9200/blog/es/3 -d '{"a":"0","b":"0","c":"1","d":"3","e":"8","path":"zzz","classify":"b"}'
curl -XPUT http://127.0.0.1:9200/blog/es/4 -d '{"a":"4","b":"3","c":"6","d":"1","e":"4","path":"eee","classify":"a"}'
curl -XPUT http://127.0.0.1:9200/blog/es/5 -d '{"a":"1","b":"7","c":"5","d":"3","e":"1","path":"fff","classify":"b"}'
curl -XPUT http://127.0.0.1:9200/blog/es/6 -d '{"a":"4","b":"0","c":"4","d":"9","e":"6","path":"ggg","classify":"a"}'
curl -XPUT http://127.0.0.1:9200/blog/es/7 -d '{"a":"1","b":"3","c":"0","d":"1","e":"3","path":"hhh","classify":"b"}'
curl -XPUT http://127.0.0.1:9200/blog/es/8 -d '{"a":"4","b":"2","c":"1","d":"0","e":"3","path":"iii","classify":"b"}'
curl -XPUT http://127.0.0.1:9200/blog/es/9 -d '{"a":"1","b":"1","c":"3","d":"0","e":"9","path":"jjj","classify":"b"}'
curl -XPUT http://127.0.0.1:9200/blog/es/10 -d '{"a":"1","b":"0","c":"9","d":"9","e":"5","path":"kkk","classify":"a"}'

这是正确运行的结果：

项目链接：

链接：https://pan.baidu.com/s/1Wi3ywvjCFVOOxUNdP_ND7w 密码：uqn7

下面给出pom.xml文件

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>com.taste</groupId><artifactId>elasticsearch-taste</artifactId><version>0.0.1-SNAPSHOT</version><packaging>jar</packaging><name>elasticsearch-taste</name><url>http://maven.apache.org</url><properties><project.build.sourceEncoding>UTF-8</project.build.sourceEncoding><maven.compiler.source>1.8</maven.compiler.source><maven.compiler.target>1.8</maven.compiler.target></properties><dependencies><!-- https://mvnrepository.com/artifact/nz.ac.waikato.cms.moa/moa -->
<dependency><groupId>nz.ac.waikato.cms.moa</groupId><artifactId>moa</artifactId><version>2017.06</version>
</dependency><dependency><groupId>org.carrot2</groupId><artifactId>carrot2-mini</artifactId><version>3.15.1</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.simpleframework</groupId><artifactId>simple-xml</artifactId><version>2.7.1</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>com.carrotsearch</groupId><artifactId>hppc</artifactId><version>0.7.1</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.carrot2.attributes</groupId><artifactId>attributes-binder</artifactId><version>1.3.1</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.carrot2.shaded</groupId><artifactId>carrot2-guava</artifactId><version>18.0</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>commons-lang</groupId><artifactId>commons-lang</artifactId><version>2.6</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>com.fasterxml.jackson.core</groupId><artifactId>jackson-core</artifactId><version>2.8.6</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>com.fasterxml.jackson.core</groupId><artifactId>jackson-annotations</artifactId><version>2.8.6</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>com.fasterxml.jackson.core</groupId><artifactId>jackson-databind</artifactId><version>2.8.6</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.slf4j</groupId><artifactId>slf4j-api</artifactId><version>1.7.13</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.slf4j</groupId><artifactId>slf4j-log4j12</artifactId><version>1.7.13</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.carrot2</groupId><artifactId>morfologik-stemming</artifactId><version>2.1.1</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.carrot2</groupId><artifactId>morfologik-fsa</artifactId><version>2.1.1</version><scope>compile</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.elasticsearch</groupId><artifactId>elasticsearch</artifactId><version>5.4.1</version><scope>provided</scope></dependency><dependency><groupId>org.locationtech.spatial4j</groupId><artifactId>spatial4j</artifactId><version>0.6</version><scope>provided</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>com.vividsolutions</groupId><artifactId>jts</artifactId><version>1.13</version><scope>provided</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.apache.logging.log4j</groupId><artifactId>log4j-api</artifactId><version>2.8.2</version><scope>provided</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.apache.logging.log4j</groupId><artifactId>log4j-core</artifactId><version>2.8.2</version><scope>provided</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.elasticsearch</groupId><artifactId>jna</artifactId><version>4.4.0</version><scope>provided</scope></dependency><dependency><groupId>org.elasticsearch.test</groupId><artifactId>framework</artifactId><version>5.4.1</version><scope>test</scope></dependency><dependency><groupId>org.assertj</groupId><artifactId>assertj-core</artifactId><version>2.1.0</version><scope>test</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.json</groupId><artifactId>json</artifactId><version>20140107</version><scope>test</scope><exclusions><exclusion><artifactId>*</artifactId><groupId>*</groupId></exclusion></exclusions></dependency><dependency><groupId>org.elasticsearch.client</groupId><artifactId>transport</artifactId><version>5.4.1</version><scope>test</scope></dependency></dependencies>
</project>