java 集成 protobuf及二次压缩
2024-06-30 04:49:01
一、创建springboot项目
1、创建proto文件目录 和 proto文件生成的java对象包
2、pom 中添加依赖包
## 版本
<grpc-version>1.6.1</grpc-version>
<protobuf-version>3.12.4</protobuf-version>
## 依赖
<dependency><groupId>com.google.protobuf</groupId><artifactId>protobuf-java</artifactId><version>${protobuf-version}</version>
</dependency>
<dependency><groupId>com.google.protobuf</groupId><artifactId>protobuf-java-util</artifactId><version>${protobuf-version}</version>
</dependency>
<dependency><groupId>com.googlecode.protobuf-java-format</groupId><artifactId>protobuf-java-format</artifactId><version>1.2</version>
</dependency>
<dependency><groupId>io.grpc</groupId><artifactId>grpc-netty</artifactId><version>${grpc-version}</version>
</dependency>
<dependency><groupId>io.grpc</groupId><artifactId>grpc-protobuf</artifactId><version>${grpc-version}</version>
</dependency>
<dependency><groupId>io.grpc</groupId><artifactId>grpc-stub</artifactId><version>${grpc-version}</version>
</dependency>
<dependency><groupId>io.protostuff</groupId><artifactId>protostuff-core</artifactId><version>1.7.4</version>
</dependency>
<dependency><groupId>io.protostuff</groupId><artifactId>protostuff-runtime</artifactId><version>1.7.4</version>
</dependency>
3、编译proto文件，会自动生成class文件放到 proto包下
二、protobuf文件定义
1、proto3 对应各语言类型
| .proto Type | Notes | C++ Type | Java Type | Python Type[2] | Go Type |
|---|---|---|---|---|---|
| double | double | double | float | *float64 | |
| float | float | float | float | *float32 | |
| int32 | 使用可变长度编码。编码负数的效率低 - 如果你的字段可能有负值，请改用 sint32 | int32 | int | int | *int32 |
| int64 | 使用可变长度编码。编码负数的效率低 - 如果你的字段可能有负值，请改用 sint64 | int64 | long | int/long[3] | *int64 |
| uint32 | 使用可变长度编码 | uint32 | int[1] | int/long[3] | *uint32 |
| uint64 | 使用可变长度编码 | uint64 | long[1] | int/long[3] | *uint64 |
| sint32 | 使用可变长度编码。有符号的 int 值。这些比常规 int32 对负数能更有效地编码 | int32 | int | int | *int32 |
| sint64 | 使用可变长度编码。有符号的 int 值。这些比常规 int64 对负数能更有效地编码 | int64 | long | int/long[3] | *int64 |
| fixed32 | 总是四个字节。如果值通常大于 228，则比 uint32 更有效。 | uint32 | int[1] | int/long[3] | *uint32 |
| fixed64 | 总是八个字节。如果值通常大于 256，则比 uint64 更有效。 | uint64 | long[1] | int/long[3] | *uint64 |
| sfixed32 | 总是四个字节 | int32 | int | int | *int32 |
| sfixed64 | 总是八个字节 | int64 | long | int/long[3] | *int64 |
| bool | bool | boolean | bool | *bool | |
| string | 字符串必须始终包含 UTF-8 编码或 7 位 ASCII 文本 | string | String | str/unicode[4] | *string |
| bytes | 可以包含任意字节序列 | string | ByteString | str | []byte |
2、定义proto文件
# 统一返回报文对象定义
syntax = "proto3";
option java_outer_classname = "ResultVo";
option java_package = "com.demo.proto";
import "google/protobuf/any.proto";message ResultResponse {int32 code = 1;string msg = 2;google.protobuf.Any data = 3;# Any类型可代表proto对象
}
#返回业务Vo对象定义
syntax = "proto3";
option java_outer_classname = "AccountVo"; # java_outer_classname 的名字 要与 message 名字不同不然会编译不过
option java_package = "com.demo.proto";
message Account {//账号uint64 ID = 1;//名字string name = 2;//密码string password = 3;//临时string test = 4;
}syntax = "proto3";
option java_outer_classname = "UserVo";
option java_package = "com.demo.proto";
import "Account.proto"; # 引入依赖对象
message User {Account user = 1;
}
三、自定义消息转换器HttpMessageConverter
转换器
package com.demo.configs.converter;import com.demo.common.util.ZlibUtil;
import com.google.common.base.Stopwatch;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
import net.jpountz.lz4.LZ4BlockOutputStream;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpInputMessage;
import org.springframework.http.HttpOutputMessage;
import org.springframework.http.MediaType;
import org.springframework.http.converter.AbstractHttpMessageConverter;
import org.springframework.http.converter.HttpMessageConversionException;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.http.converter.HttpMessageNotWritableException;
import org.springframework.util.ConcurrentReferenceHashMap;
import org.xerial.snappy.SnappyOutputStream;import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Method;
import java.nio.charset.Charset;
import java.util.Map;public class ProtobufHttpMessageConverter extends AbstractHttpMessageConverter<Message> {private Logger logger = LoggerFactory.getLogger(getClass());public static final Charset DEFAULT_CHARSET = Charset.forName("UTF-8");public static final MediaType MEDIA_TYPE = new MediaType("application", "x-protobuf", DEFAULT_CHARSET);public static final MediaType MEDIA_TYPE_GZIP = new MediaType("application", "x-protobuf-zip", DEFAULT_CHARSET);private static final Map<Class<?>, Method> methodCache = new ConcurrentReferenceHashMap();/*** Construct a new instance.*/public ProtobufHttpMessageConverter() {super(MEDIA_TYPE, MEDIA_TYPE_LZ4, MEDIA_TYPE_GZIP, MEDIA_TYPE_SNAPPY);}/*** {@inheritDoc}*/@Overridepublic boolean canRead(final Class<?> clazz, final MediaType mediaType) {if(MEDIA_TYPE.isCompatibleWith(mediaType)||MEDIA_TYPE_GZIP.isCompatibleWith(mediaType)){return true;}return false;}/*** {@inheritDoc}* 根据 mediaType判断需不需要走protobuf的转换*/@Overridepublic boolean canWrite(final Class<?> clazz, final MediaType mediaType) {if(MEDIA_TYPE.isCompatibleWith(mediaType)||MEDIA_TYPE_GZIP.isCompatibleWith(mediaType)){return true;}return false;}/*** {@inheritDoc}*/@Overrideprotected Message readInternal(final Class<? extends Message> clazz, final HttpInputMessage inputMessage)throws IOException, HttpMessageNotReadableException {Message.Builder builder = this.getMessageBuilder(clazz);// protobuf 序列化方式if (MEDIA_TYPE.isCompatibleWith(inputMessage.getHeaders().getContentType())) {builder.mergeFrom(inputMessage.getBody());//protobuf 序列化 + zlib 压缩 }else if(MEDIA_TYPE_GZIP.isCompatibleWith(inputMessage.getHeaders().getContentType())){ByteArrayOutputStream outputStream = null;try {long time = System.currentTimeMillis();InputStream inputStream = inputMessage.getBody();HttpHeaders httpHeaders = inputMessage.getHeaders();Descriptors.Descriptor descriptor = builder.getDescriptorForType();Descriptors.FieldDescriptor fieldDescriptorLength = descriptor.findFieldByNumber(25);Descriptors.FieldDescriptor fieldDescriptorTime = descriptor.findFieldByNumber(26);Long length = httpHeaders.getContentLength();byte[] bytes = new byte[1024];outputStream = new ByteArrayOutputStream(length.intValue());int i =0;while ( (i = inputStream.read(bytes)) > 0){outputStream.write(bytes,0,i);}byte[] data = ZlibUtil.uncompress(outputStream.toByteArray(),false);logger.info("压缩前的数据长度={}",data.length);//httpHeaders.add("uncompressLength",data.length+"");builder.setField(fieldDescriptorLength,data.length+"");builder.mergeFrom(data);long time2 = System.currentTimeMillis();logger.info("解压数据耗时---{}--长度={}",time2-time,length);//httpHeaders.add("uncompressTime",(time2-time)+"");builder.setField(fieldDescriptorTime,((time2-time)==0?1:(time2-time))+"");} catch (Exception e) {e.printStackTrace();}finally {if(outputStream != null){outputStream.close();}}}return builder.build();}private Message.Builder getMessageBuilder(Class<? extends Message> clazz) {try {Method method = (Method)methodCache.get(clazz);if (method == null) {method = clazz.getMethod("newBuilder");methodCache.put(clazz, method);}return (Message.Builder)method.invoke(clazz);} catch (Exception var3) {throw new HttpMessageConversionException("Invalid Protobuf Message type: no invocable newBuilder() method on " + clazz, var3);}}/*** {@inheritDoc}*/@Overrideprotected boolean supports(final Class<?> clazz) {// Should not be called, since we override canRead/canWrite.throw new UnsupportedOperationException();}/*** {@inheritDoc}*/@Overrideprotected void writeInternal(Message message, HttpOutputMessage outputMessage)throws IOException, HttpMessageNotWritableException {if (logger.isDebugEnabled()) {logger.info("Current type: {}", outputMessage.getHeaders().getContentType());}Stopwatch stopwatch = Stopwatch.createStarted();OutputStream stream = null;try {stream = outputMessage.getBody();if (MEDIA_TYPE.isCompatibleWith(outputMessage.getHeaders().getContentType())) {this.setProtoHeader(outputMessage, message);CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(outputMessage.getBody());message.writeTo(codedOutputStream);codedOutputStream.flush();return;} else if (MEDIA_TYPE_GZIP.isCompatibleWith(outputMessage.getHeaders().getContentType())) {byte[] bytes = message.toByteArray();stream.write(ZlibUtil.compress(bytes,false));stream.flush();} else if (MEDIA_TYPE_LZ4.isCompatibleWith(outputMessage.getHeaders().getContentType())) {stream = new LZ4BlockOutputStream(stream);} else if (MEDIA_TYPE_SNAPPY.isCompatibleWith(outputMessage.getHeaders().getContentType())) {stream = new SnappyOutputStream(stream);} else {return;}message.writeTo(stream);} finally {IOUtils.closeQuietly(stream);}if (logger.isDebugEnabled()) {logger.info("Output spend {}", stopwatch.toString());}}private void setProtoHeader(HttpOutputMessage response, Message message) {response.getHeaders().set("X-Protobuf-Schema", message.getDescriptorForType().getFile().getName());response.getHeaders().set("X-Protobuf-Message", message.getDescriptorForType().getFullName());}
}
zlib解压缩工具
public class ZlibUtil {public static byte[] uncompress(byte[] input, boolean nowrap) throws IOException {Inflater inflater = new Inflater(nowrap);inflater.setInput(input);ByteArrayOutputStream baos = new ByteArrayOutputStream(input.length);try {byte[] buff = new byte[1024];while (!inflater.finished()) {int count = inflater.inflate(buff);/*if(count == 0){break;}*/baos.write(buff, 0, count);}} catch (Exception e) {e.printStackTrace();} finally {baos.close();}inflater.end();byte[] output = baos.toByteArray();return output;}public static byte[] compress(byte[] data, boolean nowrap) throws IOException {byte[] output;Deflater compress = new Deflater(Deflater.BEST_COMPRESSION, nowrap);compress.reset();compress.setInput(data);compress.finish();ByteArrayOutputStream bos = new ByteArrayOutputStream(data.length);try {byte[] buf = new byte[1024];while (!compress.finished()) {int i = compress.deflate(buf);bos.write(buf, 0, i);}output = bos.toByteArray();} catch (Exception e) {output = data;e.printStackTrace();} finally {bos.close();}compress.end();return output;}
}四、Controller 层定义
/*** 查询数据接口 ,请求接口时 ，content-type = 'application/x-protobuf-zip'* produces = "application/x-protobuf-zip" 自定义消息转换器会根据 这个mediaType类型判断，是否将返回信息封装成protobuf系列化 +zlib压缩* @param request* @param calResultQuery* @return*/@PostMapping(value = "queryResult", produces = "application/x-protobuf-zip" )@ResponseBodypublic ResultVo.ResultResponse queryResult(HttpServletRequest request,@RequestBody CalResultQueryVO calResultQuery){ResultVo.ResultResponse.Builder resultResponse = ResultVo.ResultResponse.newBuilder();try {String globalId = (String) request.getAttribute("globalId");CalResultVO.CalResult calResult = parseProtoDataManager.parseResultData(calResultQuery,globalId);resultResponse.setData(Any.pack(calResult));resultResponse.setCode(Integer.valueOf(ErrorCode.SYS_CODE_200));resultResponse.setMsg("下载数据成功");}catch (Exception e){e.printStackTrace();resultResponse.setCode(Integer.valueOf(ErrorCode.SYS_CODE_500));resultResponse.setMsg("下载数据失败"+e.getMessage());}return resultResponse.build();}
java 集成 protobuf及二次压缩相关推荐
- 解压并安装protobuf库_通过protobuf自带gzip压缩对报文进行二次压缩
全部在windows平台进行测试 一.编译zlib 从github上下载最新zlib代码,根据教程进行编译,是cmake工程很简单.编译安装好,找到安装目录的include.lib.bin.默认是路径 ...
- protobuf 3.5 java使用介绍（二）
protobuf 3.5 java使用介绍(二) 上一篇遗留了两个问题: 1,数据模型中有可能会出现数组格式,而数组里面是一个其他的模型,这个怎么来做? 2,构建数据消息的时候,通常会有一个头,一个体 ...
- Netty中集成Protobuf实现Java对象数据传递
场景 Netty的Socket编程详解-搭建服务端与客户端并进行数据传输: https://blog.csdn.net/BADAO_LIUMANG_QIZHI/article/details/1086 ...
- Java学习笔记（二）--Java开发环境
Java开发环境 Java语言程序设计的开发环境可以是基本JDK,也可以使用Java集成开发环境. 利用JDK是一个基本的技能,完成每一项任务都要在shell窗口中键入命令.有些许麻烦,但是可以通过执 ...
- Java：Java编程实现导出二维码
Java:Java编程实现导出二维码 目录 输出结果 代码设计 输出结果 更新-- 代码设计 public class QRCodeUtil {private static final String ...
- java集成jpush实现客户端推送
代码地址如下: http://www.demodashi.com/demo/13700.html 前言 java 集成jpush 实现客户端推送 ###一.准备工作 开发环境: jdk1.6 Ecli ...
- Java kryo/protobuf/protostuff序列化 or Json 性能对比
Java kryo/protobuf/protostuff序列化 or Json 性能对比 - 腾飞的鹰 - 博客园 对于一个java object的序列化,想测一下使用json和使用一般序列化工具, ...
- java利用zxing生成二维码
什么是ZXing? ZXing是一个开源的,用Java实现的多种格式的1D/2D条码图像处理库. 利用ZXing这个工具库来生成二维码,具体操作如下: 引入相关资源包 <dependency&g ...
- Java基础学习笔记(二)_Java核心技术(进阶)
本篇文章的学习资源来自Java学习视频教程:Java核心技术(进阶)_华东师范大学_中国大学MOOC(慕课) 本篇文章的学习笔记即是对Java核心技术课程的总结,也是对自己学习的总结 文章目录 Jav ...
最新文章
- RedHat Enterprise Linux 5.0之LVM逻辑卷管理
- 数值分析 pdf_统计分析前，要做哪些数据准备工作？
- 图形界面不卡的linux,图形化界面linux（linaro）的安装小结
- gdb调试时查看内存
- Polygon Cruncher减边用法
- usr/bin/expect方式免密码登录和发送文件脚本
- 什么是JVM？JVM概述——初识JVM（类加载器，垃圾回收器，执行引擎）
- LeetCode 动态规划《简单》部分 Python实现
- 你可能不知道的10条SQL技巧，涨知识了！
- 网管学习笔记-hybrid口配置
- Win10 家庭版找不到 gpedit.msc；win10怎样关闭windows defender
- linux ubuntu系统远程连接linux ubuntu
- 鲁大师2014绿色版 v3.75.14.1058 免费版
- 消息队列MQ技术的介绍和原理
- 单片机及开发板介绍（学习笔记）
- [转]网银支付接口编程资料汇总
- web端实现视频播放，视频地址切换，清晰度切换，断点续播
- 数据库系统的核心：数据模型
- spring mybatis 多数据源配置 jeesite 多数据源配置
- 如何使用Grafana轻松实现OVL数据可视化