MTCNN移植java_MTCNN移植安卓并检测视频中人脸
继续此前的文章,使用vlc播放了rtsp流媒体视频后,想检测视频中的人脸,之前采用了opencv但是遇到低头、抬头和侧脸时候,效果就不太好。所以本篇介绍如何使用mtcnn来检测视频中的人脸。
大致流程:
一、Tensorflow 模型固化
将PNet、ONet、RNet 网络参数.npy固化成.pb格式,方便java载入, 固化后的文件在assets中,文件名mtcnn_freezed_model.pb。
二、引入android tensorflow lite 库
只需在build.gradle(module)最后添加以下几行语句即可。dependencies {
implementation fileTree(include: ['*.jar'], dir: 'libs')
implementation 'com.android.support:appcompat-v7:27.1.1'
implementation 'com.android.support.constraint:constraint-layout:1.1.3'
testImplementation 'junit:junit:4.12'
androidTestImplementation 'com.android.support.test:runner:1.0.2'
androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2'
compile(name: 'libvlc-3.0.0', ext: 'aar')
implementation files('libs/androidutils.jar')
compile 'org.tensorflow:tensorflow-android:+'
implementation files('libs/libutils.jar')
}
三、新建MTCNN类
该类包含加载模型文件,并检测bitmap中的人脸package com.cayden.face.facenet;import android.content.res.AssetManager;import android.graphics.Bitmap;import android.graphics.Matrix;import android.graphics.Point;import android.util.Log;import com.cayden.face.FaceApplication;import org.tensorflow.contrib.android.TensorFlowInferenceInterface;import java.util.Vector;import static java.lang.Math.max;import static java.lang.Math.min;/**
* Created by caydencui on 2018/9/6.
*/public class MTCNN { //参数
private float factor=0.709f; private float PNetThreshold=0.6f; private float RNetThreshold=0.7f; private float ONetThreshold=0.7f; //MODEL PATH
private static final String MODEL_FILE = "file:///android_asset/mtcnn_freezed_model.pb"; //tensor name
private static final String PNetInName ="pnet/input:0"; private static final String[] PNetOutName =new String[]{"pnet/prob1:0","pnet/conv4-2/BiasAdd:0"}; private static final String RNetInName ="rnet/input:0"; private static final String[] RNetOutName =new String[]{ "rnet/prob1:0","rnet/conv5-2/conv5-2:0",}; private static final String ONetInName ="onet/input:0"; private static final String[] ONetOutName =new String[]{ "onet/prob1:0","onet/conv6-2/conv6-2:0","onet/conv6-3/conv6-3:0"}; private static class SingletonInstance { private static final MTCNN INSTANCE = new MTCNN();
} public static MTCNN getInstance() { return SingletonInstance.INSTANCE;
} //安卓相关
public long lastProcessTime; //最后一张图片处理的时间ms
private static final String TAG="MTCNN"; private AssetManager assetManager; private TensorFlowInferenceInterface inferenceInterface; private MTCNN() {
assetManager= FaceApplication.getMyApplication().getAssets();
loadModel();
} private boolean loadModel() { //AssetManager
try {
inferenceInterface = new TensorFlowInferenceInterface(assetManager, MODEL_FILE);
Log.d("MTCNN","[*]load model success");
}catch(Exception e){
Log.e("MTCNN","[*]load model failed"+e); return false;
} return true;
} //读取Bitmap像素值,预处理(-127.5 /128),转化为一维数组返回
private float[] normalizeImage(Bitmap bitmap){ int w=bitmap.getWidth(); int h=bitmap.getHeight(); float[] floatValues=new float[w*h*3]; int[] intValues=new int[w*h];
bitmap.getPixels(intValues,0,bitmap.getWidth(),0,0,bitmap.getWidth(),bitmap.getHeight()); float imageMean=127.5f; float imageStd=128; for (int i=0;i
floatValues[i * 3 + 0] = (((val >> 16) & 0xFF) - imageMean) / imageStd;
floatValues[i * 3 + 1] = (((val >> 8) & 0xFF) - imageMean) / imageStd;
floatValues[i * 3 + 2] = ((val & 0xFF) - imageMean) / imageStd;
} return floatValues;
} /*
检测人脸,minSize是最小的人脸像素值
*/
private Bitmap bitmapResize(Bitmap bm, float scale) { int width = bm.getWidth(); int height = bm.getHeight(); // CREATE A MATRIX FOR THE MANIPULATION。matrix指定图片仿射变换参数
Matrix matrix = new Matrix(); // RESIZE THE BIT MAP
matrix.postScale(scale, scale);
Bitmap resizedBitmap = Bitmap.createBitmap(
bm, 0, 0, width, height, matrix, true); return resizedBitmap;
} //输入前要翻转,输出也要翻转
private int PNetForward(Bitmap bitmap, float [][]PNetOutProb, float[][][]PNetOutBias){ int w=bitmap.getWidth(); int h=bitmap.getHeight(); float[] PNetIn=normalizeImage(bitmap);
Utils.flip_diag(PNetIn,h,w,3); //沿着对角线翻转
inferenceInterface.feed(PNetInName,PNetIn,1,w,h,3);
inferenceInterface.run(PNetOutName,false); int PNetOutSizeW=(int) Math.ceil(w*0.5-5); int PNetOutSizeH=(int) Math.ceil(h*0.5-5); float[] PNetOutP=new float[PNetOutSizeW*PNetOutSizeH*2]; float[] PNetOutB=new float[PNetOutSizeW*PNetOutSizeH*4];
inferenceInterface.fetch(PNetOutName[0],PNetOutP);
inferenceInterface.fetch(PNetOutName[1],PNetOutB); //【写法一】先翻转,后转为2/3维数组
Utils.flip_diag(PNetOutP,PNetOutSizeW,PNetOutSizeH,2);
Utils.flip_diag(PNetOutB,PNetOutSizeW,PNetOutSizeH,4);
Utils.expand(PNetOutB,PNetOutBias);
Utils.expandProb(PNetOutP,PNetOutProb); /*
*【写法二】这个比较快,快了3ms。意义不大,用上面的方法比较直观
for (int y=0;y
for (int x=0;x
int idx=PNetOutSizeH*x+y;
PNetOutProb[y][x]=PNetOutP[idx*2+1];
for(int i=0;i<4;i++)
PNetOutBias[y][x][i]=PNetOutB[idx*4+i];
}
*/
return 0;
} //Non-Maximum Suppression
//nms,不符合条件的deleted设置为true
private void nms(Vector boxes, float threshold, String method){ //NMS.两两比对
//int delete_cnt=0;
for(int i=0;i
Box box = boxes.get(i); if (!box.deleted) { //score<0表示当前矩形框被删除
for (int j = i + 1; j
Box box2=boxes.get(j); if (!box2.deleted) { int x1 = max(box.box[0], box2.box[0]); int y1 = max(box.box[1], box2.box[1]); int x2 = min(box.box[2], box2.box[2]); int y2 = min(box.box[3], box2.box[3]); if (x2
iou = 1.0f*areaIoU / (box.area() + box2.area() - areaIoU); else if (method.equals("Min"))
iou= 1.0f*areaIoU / (min(box.area(),box2.area())); if (iou >= threshold) { //删除prob小的那个框
if (box.score>box2.score)
box2.deleted=true; else
box.deleted=true; //delete_cnt++;
}
}
}
}
} //Log.i(TAG,"[*]sum:"+boxes.size()+" delete:"+delete_cnt);
} private int generateBoxes(float[][] prob,float[][][]bias,float scale,float threshold,Vector boxes){ int h=prob.length; int w=prob[0].length; //Log.i(TAG,"[*]height:"+prob.length+" width:"+prob[0].length);
for (int y=0;ythreadshold(0.6 here)
if (score>PNetThreshold){
Box box=new Box(); //score
box.score=score; //box
box.box[0]= Math.round(x*2/scale);
box.box[1]= Math.round(y*2/scale);
box.box[2]= Math.round((x*2+11)/scale);
box.box[3]= Math.round((y*2+11)/scale); //bbr
for(int i=0;i<4;i++)
box.bbr[i]=bias[y][x][i]; //add
boxes.addElement(box);
}
} return 0;
} private void BoundingBoxReggression(Vector boxes){ for (int i=0;i
boxes.get(i).calibrate();
} //Pnet + Bounding Box Regression + Non-Maximum Regression
/* NMS执行完后,才执行Regression
* (1) For each scale , use NMS with threshold=0.5
* (2) For all candidates , use NMS with threshold=0.7
* (3) Calibrate Bounding Box
* 注意:CNN输入图片最上面一行,坐标为[0..width,0]。所以Bitmap需要对折后再跑网络;网络输出同理.
*/
private Vector PNet(Bitmap bitmap, int minSize){ int whMin=min(bitmap.getWidth(),bitmap.getHeight()); float currentFaceSize=minSize; //currentFaceSize=minSize/(factor^k) k=0,1,2... until excced whMin
Vector totalBoxes=new Vector(); //【1】Image Paramid and Feed to Pnet
while (currentFaceSize<=whMin){ float scale=12.0f/currentFaceSize; //(1)Image Resize
Bitmap bm=bitmapResize(bitmap,scale); int w=bm.getWidth(); int h=bm.getHeight(); //(2)RUN CNN
int PNetOutSizeW=(int)(Math.ceil(w*0.5-5)+0.5); int PNetOutSizeH=(int)(Math.ceil(h*0.5-5)+0.5); float[][] PNetOutProb=new float[PNetOutSizeH][PNetOutSizeW];; float[][][] PNetOutBias=new float[PNetOutSizeH][PNetOutSizeW][4];
PNetForward(bm,PNetOutProb,PNetOutBias); //(3)数据解析
Vector curBoxes=new Vector();
generateBoxes(PNetOutProb,PNetOutBias,scale,PNetThreshold,curBoxes); //Log.i(TAG,"[*]CNN Output Box number:"+curBoxes.size()+" Scale:"+scale);
//(4)nms 0.5
nms(curBoxes,0.5f,"Union"); //(5)add to totalBoxes
for (int i=0;i
totalBoxes.addElement(curBoxes.get(i)); //Face Size等比递增
currentFaceSize/=factor;
} //NMS 0.7
nms(totalBoxes,0.7f,"Union"); //BBR
BoundingBoxReggression(totalBoxes); return Utils.updateBoxes(totalBoxes);
} //截取box中指定的矩形框(越界要处理),并resize到size*size大小,返回数据存放到data中。
public Bitmap tmp_bm; private void crop_and_resize(Bitmap bitmap, Box box, int size, float[] data){ //(2)crop and resize
Matrix matrix = new Matrix(); float scale=1.0f*size/box.width();
matrix.postScale(scale, scale);
Bitmap croped= Bitmap.createBitmap(bitmap, box.left(),box.top(),box.width(), box.height(),matrix,true); //(3)save
int[] pixels_buf=new int[size*size];
croped.getPixels(pixels_buf,0,croped.getWidth(),0,0,croped.getWidth(),croped.getHeight()); float imageMean=127.5f; float imageStd=128; for (int i=0;i
data[i * 3 + 0] = (((val >> 16) & 0xFF) - imageMean) / imageStd;
data[i * 3 + 1] = (((val >> 8) & 0xFF) - imageMean) / imageStd;
data[i * 3 + 2] = ((val & 0xFF) - imageMean) / imageStd;
}
} /*
* RNET跑神经网络,将score和bias写入boxes
*/
private void RNetForward(float[] RNetIn,Vector boxes){ int num=RNetIn.length/24/24/3; //feed & run
inferenceInterface.feed(RNetInName,RNetIn,num,24,24,3);
inferenceInterface.run(RNetOutName,false); //fetch
float[] RNetP=new float[num*2]; float[] RNetB=new float[num*4];
inferenceInterface.fetch(RNetOutName[0],RNetP);
inferenceInterface.fetch(RNetOutName[1],RNetB); //转换
for (int i=0;i
boxes.get(i).score = RNetP[i * 2 + 1]; for (int j=0;j<4;j++)
boxes.get(i).bbr[j]=RNetB[i*4+j];
}
} //Refine Net
private Vector RNet(Bitmap bitmap, Vector boxes){ //RNet Input Init
int num=boxes.size(); float[] RNetIn=new float[num*24*24*3]; float[] curCrop=new float[24*24*3]; int RNetInIdx=0; for (int i=0;i
crop_and_resize(bitmap,boxes.get(i),24,curCrop);
Utils.flip_diag(curCrop,24,24,3); //Log.i(TAG,"[*]Pixels values:"+curCrop[0]+" "+curCrop[1]);
for (int j=0;j
} //Run RNet
RNetForward(RNetIn,boxes); //RNetThreshold
for (int i=0;i
boxes.get(i).deleted=true; //Nms
nms(boxes,0.7f,"Union");
BoundingBoxReggression(boxes); return Utils.updateBoxes(boxes);
} /*
* ONet跑神经网络,将score和bias写入boxes
*/
private void ONetForward(float[] ONetIn,Vector boxes){ int num=ONetIn.length/48/48/3; //feed & run
inferenceInterface.feed(ONetInName,ONetIn,num,48,48,3);
inferenceInterface.run(ONetOutName,false); //fetch
float[] ONetP=new float[num*2]; //prob
float[] ONetB=new float[num*4]; //bias
float[] ONetL=new float[num*10]; //landmark
inferenceInterface.fetch(ONetOutName[0],ONetP);
inferenceInterface.fetch(ONetOutName[1],ONetB);
inferenceInterface.fetch(ONetOutName[2],ONetL); //转换
for (int i=0;i
boxes.get(i).score = ONetP[i * 2 + 1]; //bias
for (int j=0;j<4;j++)
boxes.get(i).bbr[j]=ONetB[i*4+j]; //landmark
for (int j=0;j<5;j++) { int x=boxes.get(i).left()+(int) (ONetL[i * 10 + j]*boxes.get(i).width()); int y= boxes.get(i).top()+(int) (ONetL[i * 10 + j +5]*boxes.get(i).height());
boxes.get(i).landmark[j] = new Point(x,y); //Log.i(TAG,"[*] landmarkd "+x+ " "+y);
}
}
} //ONet
private Vector ONet(Bitmap bitmap, Vector boxes){ //ONet Input Init
int num=boxes.size(); float[] ONetIn=new float[num*48*48*3]; float[] curCrop=new float[48*48*3]; int ONetInIdx=0; for (int i=0;i
crop_and_resize(bitmap,boxes.get(i),48,curCrop);
Utils.flip_diag(curCrop,48,48,3); for (int j=0;j
} //Run ONet
ONetForward(ONetIn,boxes); //ONetThreshold
for (int i=0;i
boxes.get(i).deleted=true;
BoundingBoxReggression(boxes); //Nms
nms(boxes,0.7f,"Min"); return Utils.updateBoxes(boxes);
} private void square_limit(Vector boxes, int w, int h){ //square
for (int i=0;i
boxes.get(i).toSquareShape();
boxes.get(i).limit_square(w,h);
}
} /*
* 参数:
* bitmap:要处理的图片
* minFaceSize:最小的人脸像素值.(此值越大,检测越快)
* 返回:
* 人脸框
*/
public Vector detectFaces(Bitmap bitmap, int minFaceSize) { long t_start = System.currentTimeMillis(); //【1】PNet generate candidate boxes
Vector boxes=PNet(bitmap,minFaceSize);
square_limit(boxes,bitmap.getWidth(),bitmap.getHeight()); //【2】RNet
boxes=RNet(bitmap,boxes);
square_limit(boxes,bitmap.getWidth(),bitmap.getHeight()); //【3】ONet
boxes=ONet(bitmap,boxes); //return
Log.i(TAG,"[*]Mtcnn Detection Time:"+(System.currentTimeMillis()-t_start));
lastProcessTime=(System.currentTimeMillis()-t_start); return boxes;
}
}
四、视频流数据处理
由于vlc播放的流媒体视频格式是nv12,需要将其转为nv21,并保存为bitmap1、首先给出nv12转为nv21方法private void NV12ToNV21(byte[] nv12, byte[] nv21, int width, int height) { if (nv21 == null || nv12 == null) return; int framesize = width * height; int i = 0, j = 0; //System.arraycopy(nv21, test, nv12, test, framesize);
for (i = 0; i
nv21[i] = nv12[i];
} for (j = 0; j
nv21[framesize + j] = nv12[j + framesize + 1];
} for (j = 0; j
nv21[framesize + j + 1] = nv12[j + framesize];
}
}2、然后给出nv21高效率转换为bitmap的方法
可以查看此前的一篇文章 nv21高效率转换为bitmap
3、调用mtcnn的人脸检测方法Vector boxes = mtcnn.detectFaces(bitmap, 20);4、根据返回的数据 ,标记人脸protected void drawAnim(Vector faces, SurfaceView outputView, float scale_bit, int cameraId, String fps) {
Paint paint = new Paint();
Canvas canvas = ((SurfaceView) outputView).getHolder().lockCanvas(); if (canvas != null) { try { int viewH = outputView.getHeight(); int viewW = outputView.getWidth();// DLog.d("viewW:"+viewW+",viewH:"+viewH);
canvas.drawColor(0, PorterDuff.Mode.CLEAR); if (faces == null || faces.size() == 0) return; for (int i = 0; i
paint.setColor(Color.BLUE); int size = DisplayUtil.dip2px(this, 3);
paint.setStrokeWidth(size);
paint.setStyle(Paint.Style.STROKE);
Box box = faces.get(i); float[] rect = box.transform2float(); float x1 = rect[0] * scale_bit; float y1 = rect[1] * scale_bit; float rect_width = rect[2] * 0.5F;
RectF rectf = new RectF(x1, y1, x1 + rect_width, y1 + rect_width);
canvas.drawRect(rectf, paint);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
((SurfaceView) outputView).getHolder().unlockCanvasAndPost(canvas);
}
}
}
作者:一杯茶一本书
链接:https://www.jianshu.com/p/1787ccde11d5
MTCNN移植java_MTCNN移植安卓并检测视频中人脸相关推荐
- 用python画微笑脸表情_一种检测视频中人脸微笑表情的方法与流程
本发明涉及视频检测技术领域,特别涉及一种视频中人脸微笑表情的检测方法. 背景技术: 近几年来,表情识别技术在计算机视觉和模式识别领域逐步成为一个重要的研究热点,已经有越来越多的科研成果是基于图像或视频 ...
- 检测视频中的人脸,并画出矩形框
检测视频中的人脸,并画出矩形框,这是一个测试程序,由于很多人经常会用到,写下以备不时之需. #include"stdafx.h" #include <opencv2/core ...
- opencv 视频中人脸检测
opencv 视频中人脸检测 先看一下运行结果: 源代码: //头文件 #include<opencv2/objdetect/objdetect.hpp> #include<ope ...
- 【行人检测】检测视频中的行人
[行人检测]检测视频中的行人 在上一节检测图片中的行人的基础上,实现检测视频中的行人. 检测行人的视频可戳:https://download.csdn.net/download/u012679707/ ...
- 边缘设备上的实时AI人员检测:以实时模式检测视频中的人员
下载数据-19.3 MB 下载模型-43.5 MB 下载结果-36.66 MB 这是七篇系列文章中的最后一篇.到目前为止,我们已经有了用于人员检测的DNN模型和用于在Raspberry Pi设备上启动 ...
- 边缘设备上的实时AI人员检测:检测视频中的人员
下载数据-19.3 MB 下载模型-43.5 MB 下载结果-36.66 MB 从本系列的前几篇文章中,我们获得了使用SSD DNN模型检测图像中人物的Python代码.而且我们已经展示了该代码可以在 ...
- Python3 调用 FaceAPI 读取并检测视频中的人脸
本文地址:https://blog.csdn.net/shanglianlm/article/details/80727006 faceAPI.py 封装Face ++ 的Face Detection ...
- 如何检测视频中的绿屏、绿帧问题
今天给项目拷机,发现视频会偶现绿屏,非常偶现,很难复现出来. 由于问题暂时没有定位,只能先表面解决一下,就是过滤掉出现绿屏的帧. 当然,首先要把绿帧检测出来,才能做后续的补救措施. 绿屏.绿帧出现的时 ...
- 给视频中人脸加上墨镜(基于python+opencv)
先声明,本人菜鸟一只,欢迎指导,这个代码跑起来还会有点卡顿,问题在于加模型的方法中存在多个循环,菜鸟本人暂时没有优化的方案,希望各位大哥能提供一下意见. import cv2# 覆盖图像 def ov ...
最新文章
- MyBatis常规CURD详解及拓展~
- 通用AI咋发展?向大脑学习是条路子
- ubuntu 配置网络
- 源码时代php中级项目,PHP学科项目评比圆满结束
- html怎么让表格连接数据库,【前端】如何将html的table空白单元格合并?数据是循环从数据库里面读取的。...
- angular ng-href
- 格子箱被评选为12家最值得注意的亚洲初创科技公司之一
- aws ec2 选择可用区_AWS Messaging Services:选择合适的服务
- python---(2)Python库资源大全
- 高效管理http连接
- 谷歌停止中国版搜索引擎;李楠宣布离职魅族;微软用 Rust 替代 C/C++ | 极客头条...
- 使用Shell工具连接虚拟机
- python 保存视频为图片
- 企业资源计划(ERP)原理与实践 第三章 需求计划
- winform TreeView节点中的CheckBox 禁用
- 计算机科学中的哲学思想,冯_诺依曼的计算机科学哲学思想.doc
- gradient设置上下渐变_CSS3 线性渐变(linear-gradient)
- 随机手机号码_微信绑定了手机号码怎么解绑
- 视频截帧:javacv实现视频截帧功能
- BLOCK PVSE 230/24-5电源acim-jouanin AJ7003.J.2000温度传感器
热门文章
- 计算机网络知识点总结之物理层(二)
- SQL server 2005安装问题汇总(转)
- thinkphp3.2.3模糊查询搜索分页,完整实例。
- 解决:FLASK中Warning: (1366, Incorrect string value: '\\xD6\\xD0\\xB9\\xFA\\xB1\\xEA...'
- R语言 ggplot2 多图排列 Part(1)
- pythonnumpy生成二进制流_Python 读写二进制文件 以及Numpy读写二进制文件
- MSN pk QQ - 看软件重点用户体验
- 【工具】Chrome浏览器书签误删恢复
- Qt之创建桌面和开始菜单快捷方式
- Unity中的坐标与绘制准心