ElasticSearch最新版(8.4.3) IK分词器基于mysql实现热更新词库
2024-05-23 14:50:04
1 源码阅读思路
阅读Dictionary类下的initial方法，基于该方法进行改造
2 下载源码
https://github.com/medcl/elasticsearch-analysis-ik/releases/tag/v8.4.3
3 导入idea
4 创建jdbc配置文件
jdbc.url=jdbc:mysql://192.168.38.80:3306/word_db?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai
jdbc.user=root
jdbc.password=123456
jdbc.reload.word.extend.sql=select word from t_extend_word
jdbc.reload.word.stop.sql=select word from t_stop_word
jdbc.driver.class.name=com.mysql.jdbc.Driver
5 防止初始化数据库驱动报错ExceptionInInitializerError 修改文件resource/plugin-security.policy
添加权限 permission java.lang.RuntimePermission “setContextClassLoader”;
grant {// needed because of the hot reload functionalitypermission java.net.SocketPermission "*", "connect,resolve";permission java.lang.RuntimePermission "setContextClassLoader";
};
6 Dictionary 新增方法
void reloadMysqlStopWord(DictSegment _StopWords) {logger.info("start to reload mysql stopWord.");this._StopWords = _StopWords;}void reloadMysqlExtendWord(DictSegment _MainDict) {logger.info("start to reload mysql extendWord.");this._MainDict = _MainDict;}
7 创建WordMysqlRunnable
package org.wltea.analyzer.dic;import org.apache.logging.log4j.Logger;
import org.elasticsearch.SpecialPermission;
import org.wltea.analyzer.help.ESPluginLoggerFactory;import java.security.AccessController;
import java.security.PrivilegedAction;
import java.sql.*;
import java.util.List;
import java.util.Properties;public class WordMysqlRunnable implements Runnable {private static final Logger logger = ESPluginLoggerFactory.getLogger(WordMysqlRunnable.class.getName());/*** mysql配置文件*/private final Properties properties;/*** type: extend、stop*/private final String type;public static final String EXTEND = "extend";public static final String STOP = "stop";public WordMysqlRunnable(Properties properties, String type) {AccessController.doPrivileged((PrivilegedAction<Void>) () -> {try {Class.forName(properties.getProperty("jdbc.driver.class.name"));} catch (ClassNotFoundException e) {throw new RuntimeException(e);}return null;});this.properties = properties;this.type = type;}@Overridepublic void run() {SpecialPermission.check();AccessController.doPrivileged((PrivilegedAction<Void>) () -> {this.loadWord();return null;});}private void loadWord() {logger.info("WordMysqlRunnable invoke run");Connection conn = null;Statement stmt = null;try {conn = DriverManager.getConnection(properties.getProperty("jdbc.url"), properties.getProperty("jdbc.user"), properties.getProperty("jdbc.password"));stmt = conn.createStatement();if (type.equals(EXTEND)) {loadExtendWord(stmt);} else if (type.equals(STOP)) {loadStopWord(stmt);}} catch (Exception e) {logger.error(e);} finally {if (stmt != null) {try {stmt.close();} catch (SQLException e) {logger.error(e);}}if (conn != null) {try {conn.close();} catch (SQLException e) {logger.error(e);}}}}private void loadStopWord(Statement stmt) throws SQLException {logger.info("WordMysqlRunnable invoke loadStopWord");// 建立主词典实例DictSegment _StopWords = new DictSegment((char) 0);//执行sqlString sql = properties.getProperty("jdbc.reload.word.stop.sql");ResultSet resultSet = stmt.executeQuery(sql);while (resultSet.next()) {String word = resultSet.getString("word");_StopWords.fillSegment(word.trim().toLowerCase().toCharArray());}//加载Dictionary.getSingleton().reloadMysqlStopWord(_StopWords);}private void loadExtendWord(Statement stmt) throws SQLException {logger.info("WordMysqlRunnable invoke loadExtendWord");// 建立一个主词典实例DictSegment _MainDict = new DictSegment((char) 0);//执行sqlString sql = properties.getProperty("jdbc.reload.word.extend.sql");ResultSet resultSet = stmt.executeQuery(sql);while (resultSet.next()) {String word = resultSet.getString("word");_MainDict.fillSegment(word.trim().toLowerCase().toCharArray());}//加载Dictionary.getSingleton().reloadMysqlExtendWord(_MainDict);}
}
8 修改方法Dictionary initial 方法
public static synchronized void initial(Configuration cfg) {if (singleton == null) {synchronized (Dictionary.class) {if (singleton == null) {singleton = new Dictionary(cfg);singleton.loadMainDict();singleton.loadSurnameDict();singleton.loadQuantifierDict();singleton.loadSuffixDict();singleton.loadPrepDict();singleton.loadStopWordDict();//自定义mysql词汇singleton.loadMysqlWord();if (cfg.isEnableRemoteDict()) {// 建立监控线程for (String location : singleton.getRemoteExtDictionarys()) {// 10 秒是初始延迟可以修改的 60是间隔时间 单位秒pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);}for (String location : singleton.getRemoteExtStopWordDictionarys()) {pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);}}}}}}private void loadMysqlWord() {try {Properties properties = new Properties();Path path = PathUtils.get(getDictRoot(), "jdbc-reload.properties");properties.load(Files.newInputStream(path.toFile().toPath()));pool.scheduleAtFixedRate(new WordMysqlRunnable(properties, WordMysqlRunnable.EXTEND), 10, 60, TimeUnit.SECONDS);pool.scheduleAtFixedRate(new WordMysqlRunnable(properties, WordMysqlRunnable.STOP), 10, 60, TimeUnit.SECONDS);} catch (IOException e) {logger.error("e", e);}}
9 pom版本需要跟ik分词版本一致
10 pom中新增mysql依赖
<!-- MySql --><dependency><groupId>mysql</groupId><artifactId>mysql-connector-java</artifactId><version>5.1.47</version></dependency>
11 修改plugin.xml 添加mysql配置
<dependencySet><outputDirectory>/</outputDirectory><useProjectArtifact>true</useProjectArtifact><useTransitiveFiltering>true</useTransitiveFiltering><includes><include>mysql:mysql-connector-java</include></includes></dependencySet>
12 打包
13 替换、重启
unzip elasticsearch-analysis-ik-8.4.3.zip -d ik-analyzer
rm -rf elasticsearch-analysis-ik-8.4.3.zip
chmod +777 ik-analyzer/
docker restart elasticsearch
14 测试
ElasticSearch最新版(8.4.3) IK分词器基于mysql实现热更新词库相关推荐
- 六Elasticsearch之中文分词器插件es-ik的热更新词库
参考:http://www.mamicode.com/info-detail-1705113.html 先声明,热更新词库,需要用到,web项目和Tomcat.不会的,请移步 Eclipse下Mave ...
- Elasticsearch 分词器详解（热更新词库）
1 分词器 text类型数据存入ES经过的步骤: 2 规范化(normalization) #采用默认分词器分词 GET _analyze {"analyzer": "s ...
- ElasticSearch ik分词器从MySQL里面获取分词信息
出自 中华石杉 老师的一个课程, 缘由 每次都是在es的扩展词典中,手动添加新词语,很坑 (1)每次添加完,都要重启es才能生效,非常麻烦 (2)ElasticSearch是分布式的,可能有数百个节点 ...
- 拾忆Elasticsearch01：Elasticsearch概述及相关ES、IK分词器等下载安装
为了防止老年痴呆记不住学过的Elasticsearch,在这里写点blog回顾一下 写在前面,建议下载以下出现的Elasticsearch版本,我之前下的7.6.1在这里找不到对应的springboo ...
- es ik分词热更新MySQL,ElasticSearch（25）- 改IK分词器源码来基于mysql热更新词库
代码地址 已经修改过的支持定期从数据库中提取新词库,来实现热更新.代码: https://github.com/csy512889371/learndemo/tree/master/elasticse ...
- Elasticsearch之中文分词器插件es-ik的自定义热更新词库
1: 部署 http 服务 在这使用 tomcat7 作为 web 容器, 先下载一个 tomcat7, 然后上传到某一台服务器上(192.168.80.10). 再执行以下命令 tar -zxvf ...
- 31_ElasticSearch 修改IK分词器源码来基于mysql热更新词库
31_ElasticSearch 修改IK分词器源码来基于mysql热更新词库 更多干货 分布式实战(干货) spring cloud 实战(干货) mybatis 实战(干货) spring boo ...
- ik mysql热加载分词_Elasticsearch 之（25）重写IK分词器源码来基于mysql热更新词库...
热更新在上一节< IK分词器配置文件讲解以及自定义词库>自定义词库,每次都是在es的扩展词典中,手动添加新词语,很坑 (1)每次添加完,都要重启es才能生效,非常麻烦 (2)es是分布式的 ...
- ElasticSearch 中文分词器ik的安装、测试、使用、自定义词库、热更新词库
文章目录 # 实验环境 # ik分词器的下载.安装.测试 ## 安装方法一:使用elasticsearch-plugin 安装 ## 安装方法二:下载编译好的包进行安装 1.下载 2.安装 3.重启` ...
最新文章
- 直播 | Python Web开发者的破局之道
- Google更新最大的带注释图像数据集，添加本地化叙述
- 测试mysql安装成功_MySQL安装之“测试”
- 算法工程与计算机科学,合并的分水岭算法-计算机工程与科学.PDF
- python 消息机制_Python并发编程之线程消息通信机制任务协调（四）
- 圆桌的项目Alpha冲刺（团队）
- 术语-服务：PaaS
- .Net Core 之 Ubuntu 14.04 部署过程
- 我要重构“软件行业”......
- Xilinx Zynq-7000 嵌入式系统设计与实现
- 最短路径算法——迪杰斯特拉（Dijkstra）
- 使用For XML与XSL（XSLT）配套快速输出查询结果到Web页面
- linux eclipse中文设置字体,完美调整Ubuntu下的Eclipse字体及界面显示
- PotPlayer获取获取播放文件的实时码率信息
- 国家区域代码关系整理
- 织梦dedecms 模板代码标签学习
- 基于wine葡萄酒数据集的分类方法研究
- leetcode 1276. 不浪费原料的汉堡制作方案（C++）
- python中len的用法_Python len函数用法
- 第三方软件MOOS-IvP扩展