ZF/SmartJavaAI
1
0
Fork 0
mirror of https://github.com/geekwenjie/SmartJavaAI.git synced 2026-09-11 04:08:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

1、【核心升级】升级DJL版本到0.34.0

Browse Source 
2、【平台支持】新增对 Linux ARM64 架构的全面支持
3、【通用视觉】集成零样本目标检测模型
4、【活体检测】优化视频检测流程，实现 Predictor 视频会话级复用
5、【人脸识别】SQLite人脸查询改进线程池
6、【人脸识别】修复 Milvus 向量库下 listFaces 接口的调用异常
This commit is contained in:
dengwenjie
2025-11-26 18:51:59 +08:00
parent 06cb54bbf7
commit efce16834b
49 changed files with 1517 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
face/pom.xml
View File

@@ -6,11 +6,11 @@
<parent>
<groupId>cn.smartjavaai</groupId>
<artifactId>smartjavaai-parent</artifactId>
<version>1.0.27</version>
<version>1.1.0</version>
</parent>
<artifactId>face</artifactId>
<version>1.0.27</version>
<version>1.1.0</version>
<name>face</name>
<description>SmartJavaAI</description>
<url>https://github.com/geekwenjie/SmartJavaAI</url>
@@ -26,7 +26,7 @@
<!-- <maven.compiler.target>11</maven.compiler.target>-->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.test.skip>true</maven.test.skip>
<javacv.version>1.5.8</javacv.version>
<javacv.version>1.5.10</javacv.version>
<javacv.ffmpeg.version>5.1.2-1.5.8</javacv.ffmpeg.version>
</properties>
@@ -57,6 +57,8 @@
</dependency>
</dependencies>
<build>

8
face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java
View File

@@ -176,7 +176,7 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel {
imageData.data = BufferedImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("无人脸数据");
}
for(SeetaRect seetaRect : seetaResult){
@@ -456,7 +456,7 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel {
imageData.data = BufferedImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("无人脸数据");
}
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
@@ -510,7 +510,7 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel {
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("无人脸数据");
}
for(SeetaRect seetaRect : seetaResult){
@@ -642,7 +642,7 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel {
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("无人脸数据");
}
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];

25
face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java
View File

@@ -9,6 +9,7 @@ import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.translate.TranslateException;
import cn.smartjavaai.common.cv.SmartImageFactory;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
@@ -50,6 +51,30 @@ public class CommonFaceDetModel implements FaceDetModel{
private FaceDetConfig config;
@Override
public Predictor<Image, DetectedObjects> borrowPredictor() throws Exception {
if(predictorPool == null){
throw new FaceException("请先加载模型");
}
return predictorPool.borrowObject();
}
@Override
public void returnPredictor(Predictor<Image, DetectedObjects> predictor){
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
/**
* 加载模型

16
face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java
View File

@@ -131,4 +131,20 @@ public interface FaceDetModel extends AutoCloseable{
default void setFromFactory(boolean fromFactory){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 获取Predictor
* @return
*/
default Predictor<Image, DetectedObjects> borrowPredictor() throws Exception{
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 归还Predictor
* @param predictor
*/
default void returnPredictor(Predictor<Image, DetectedObjects> predictor){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

99
face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetectManager.java Normal file
View File

@@ -0,0 +1,99 @@
package cn.smartjavaai.face.model.facedect;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import cn.smartjavaai.common.entity.DetectionInfo;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.facedect.mtcnn.MtcnnPredictors;
import cn.smartjavaai.face.seetaface.SeetaFace6FaceDetPredictors;
import cn.smartjavaai.face.utils.FaceUtils;
import java.util.Objects;
/**
* @author dwj
* @date 2025/11/24
*/
public class FaceDetectManager implements AutoCloseable{
private FaceDetModel faceDetModel;
public FaceDetectManager(FaceDetModel faceDetModel) {
this.faceDetModel = faceDetModel;
}
private MtcnnPredictors mtcnnPredictors;
private SeetaFace6FaceDetPredictors seetaFace6FaceDetPredictors;
private Predictor<Image, DetectedObjects> commonPredictor;
public void borrowPredictors(){
try {
//mtcnn
if(faceDetModel instanceof MtcnnFaceDetModel){
MtcnnFaceDetModel mtcnnFaceDetModel = (MtcnnFaceDetModel) faceDetModel;
mtcnnPredictors = mtcnnFaceDetModel.borrowPredictors();
}else if(faceDetModel instanceof SeetaFace6FaceDetModel){
//SeetaFace6
SeetaFace6FaceDetModel seetaFace6FaceDetModel = (SeetaFace6FaceDetModel) faceDetModel;
seetaFace6FaceDetPredictors = seetaFace6FaceDetModel.borrowPredictors();
}else{
//其他通用模型
commonPredictor = faceDetModel.borrowPredictor();
}
} catch (Exception e) {
throw new FaceException("获取predictors异常", e);
}
}
public R<DetectionInfo> detectTopFace(Image image){
DetectionResponse detectionResponse = null;
try {
//mtcnn
if(faceDetModel instanceof MtcnnFaceDetModel){
MtcnnFaceDetModel mtcnnFaceDetModel = (MtcnnFaceDetModel) faceDetModel;
DetectedObjects detections = mtcnnFaceDetModel.detectCoreByPredictors(image, mtcnnPredictors);
detectionResponse = FaceUtils.convertToDetectionResponse(detections, image);
}else if(faceDetModel instanceof SeetaFace6FaceDetModel){
//SeetaFace6
SeetaFace6FaceDetModel seetaFace6FaceDetModel = (SeetaFace6FaceDetModel) faceDetModel;
detectionResponse = seetaFace6FaceDetModel.detectByPredictors(image, seetaFace6FaceDetPredictors);
}else{
DetectedObjects detections = commonPredictor.predict(image);
detectionResponse = FaceUtils.convertToDetectionResponse(detections, image);
}
} catch (Exception e) {
throw new FaceException("获取predictors异常", e);
}
if(Objects.isNull(detectionResponse) || Objects.isNull(detectionResponse.getDetectionInfoList()) || detectionResponse.getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
DetectionInfo detectionInfo = detectionResponse.getDetectionInfoList().get(0);
return R.ok(detectionInfo);
}
@Override
public void close(){
try {
//mtcnn
if(faceDetModel instanceof MtcnnFaceDetModel){
mtcnnPredictors.close();
}else if(faceDetModel instanceof SeetaFace6FaceDetModel){
//SeetaFace6
seetaFace6FaceDetPredictors.close();
}else{
faceDetModel.getPool().returnObject(commonPredictor);
}
} catch (Exception e) {
throw new FaceException("归还predictors异常", e);
}
}
}

140
face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java
View File

@@ -221,52 +221,51 @@ public class MtcnnFaceDetModel extends CommonFaceDetModel{
}
/**
* 使用MtcnnPredictors进行人脸检测
* @param image
* @param predictors
* @return
*/
public DetectedObjects detectCoreByPredictors(Image image, MtcnnPredictors predictors){
Predictor<NDList, NDList> pNetPredictor = predictors.pNetPredictor;
Predictor<NDList, NDList> rNetPredictor = predictors.rNetPredictor;
Predictor<NDList, NDList> oNetPredictor = predictors.oNetPredictor;
try (NDManager manager = pNetModel.getNDManager().newSubManager();){
int h = image.getHeight();
int w = image.getWidth();
//第一阶段
NDList outputPnet = PNetModel.firstStage(manager, pNetPredictor, image);
// /**
// * 转换为FaceDetectedResult
// * @param mtcnnBatchResult
// * @return
// */
// public static DetectionResponse convertToDetectionResponse(MtcnnBatchResult mtcnnBatchResult){
// if(Objects.isNull(mtcnnBatchResult) || CollectionUtils.isEmpty(mtcnnBatchResult.boxes)
// || CollectionUtils.isEmpty(mtcnnBatchResult.points)
// || CollectionUtils.isEmpty(mtcnnBatchResult.probs)){
// return null;
// }
// DetectionResponse detectionResponse = new DetectionResponse();
// List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
//
// NDArray boxes = mtcnnBatchResult.boxes.get(0);
// NDArray probs = mtcnnBatchResult.probs.get(0);
// NDArray points = mtcnnBatchResult.points.get(0);
//
// if (DJLCommonUtils.isNDArrayEmpty(boxes) || DJLCommonUtils.isNDArrayEmpty(probs) || DJLCommonUtils.isNDArrayEmpty(points)){
// return null;
// }
// long numBoxes = boxes.getShape().get(0);
// for (int i = 0; i < numBoxes; i++) {
// float[] boxCoords = boxes.get(i).toFloatArray(); // [x1, y1, x2, y2]
// float score = probs.getFloat(i);
// NDArray pointND = points.get(i); // shape [5,2]
// float[] flatPoints = pointND.toFloatArray(); // 一维长度 10
// List<Point> keyPoints = new ArrayList<Point>();
// for (int p = 0; p < 5; p++) {
// keyPoints.add(new Point(flatPoints[p * 2], flatPoints[p * 2 + 1]));
// }
// int x = Math.round(boxCoords[0]);
// int y = Math.round(boxCoords[1]);
// int w = Math.round(boxCoords[2] - boxCoords[0]);
// int h = Math.round(boxCoords[3] - boxCoords[1]);
//
// DetectionRectangle rectangle = new DetectionRectangle(x, y, w, h);
// FaceInfo faceInfo = new FaceInfo(keyPoints);
// DetectionInfo detectionInfo = new DetectionInfo(rectangle, score, faceInfo);
// detectionInfoList.add(detectionInfo);
// }
// detectionResponse.setDetectionInfoList(detectionInfoList);
// return detectionResponse;
// }
if(CollectionUtils.isEmpty(outputPnet)){
return DJLCommonUtils.buildEmptyDetectedObjects();
}
NDArray boxes = outputPnet.get(0);
NDArray image_inds = outputPnet.get(1);
NDArray imgs = outputPnet.get(2);
if(DJLCommonUtils.isNDArrayEmpty(boxes) || DJLCommonUtils.isNDArrayEmpty(image_inds) || DJLCommonUtils.isNDArrayEmpty(imgs)){
return DJLCommonUtils.buildEmptyDetectedObjects();
}
NDList pad = MtcnnUtils.pad(boxes, w, h);
//第二阶段
NDList outputRnet = RNetModel.secondStage(manager, rNetPredictor, imgs, boxes, pad, image_inds);
if(CollectionUtils.isEmpty(outputRnet)){
return DJLCommonUtils.buildEmptyDetectedObjects();
}
NDArray image_indsFiltered = outputRnet.get(0);
NDArray scoresFiltered = outputRnet.get(1);
boxes = outputRnet.get(2);
if(DJLCommonUtils.isNDArrayEmpty(boxes) || DJLCommonUtils.isNDArrayEmpty(image_indsFiltered) || DJLCommonUtils.isNDArrayEmpty(scoresFiltered)){
return DJLCommonUtils.buildEmptyDetectedObjects();
}
//第三阶段
MtcnnBatchResult oNetResult = ONetModel.thirdStage(manager, oNetPredictor, imgs, boxes, w, h, scoresFiltered, image_indsFiltered);
return FaceUtils.toDetectedObjects(oNetResult, w, h);
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
@@ -293,6 +292,57 @@ public class MtcnnFaceDetModel extends CommonFaceDetModel{
return fromFactory;
}
public MtcnnPredictors borrowPredictors() throws Exception {
if(pnetPredictorPool == null || rnetPredictorPool == null || onetPredictorPool == null){
return null;
}
Predictor<NDList, NDList> p = pnetPredictorPool.borrowObject();
Predictor<NDList, NDList> r = rnetPredictorPool.borrowObject();
Predictor<NDList, NDList> o = onetPredictorPool.borrowObject();
return new MtcnnPredictors(p, r, o, this);
}
public void returnPredictor(Predictor<NDList, NDList> pNetPredictor, Predictor<NDList, NDList> rNetPredictor, Predictor<NDList, NDList> oNetPredictor) {
if (pNetPredictor != null) {
try {
pnetPredictorPool.returnObject(pNetPredictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
pNetPredictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
if (rNetPredictor != null) {
try {
rnetPredictorPool.returnObject(rNetPredictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
rNetPredictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
if (oNetPredictor != null) {
try {
onetPredictorPool.returnObject(oNetPredictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
oNetPredictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
@Override
public void close() {
if (fromFactory) {

52
face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java
View File

@@ -1,7 +1,9 @@
package cn.smartjavaai.face.model.facedect;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDList;
import cn.smartjavaai.common.cv.SmartImageFactory;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
@@ -13,7 +15,9 @@ import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.factory.FaceDetModelFactory;
import cn.smartjavaai.face.model.facedect.mtcnn.MtcnnPredictors;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.seetaface.SeetaFace6FaceDetPredictors;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.pool.*;
import com.seeta.sdk.*;
@@ -124,6 +128,27 @@ public class SeetaFace6FaceDetModel implements FaceDetModel{
}
}
public DetectionResponse detectByPredictors(Image image, SeetaFace6FaceDetPredictors predictors) {
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector predictor = predictors.faceDetector;
FaceLandmarker faceLandmarker = predictors.faceLandmarker;
try {
SeetaRect[] seetaResult = predictor.Detect(imageData);
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
seetaPointFSList.add(pointFS);
}
return FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList);
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}
}
@Override
public R<DetectionResponse> detectAndDraw(Image image) {
R<DetectionResponse> result = detect(image);
@@ -276,6 +301,33 @@ public class SeetaFace6FaceDetModel implements FaceDetModel{
return R.ok(drawnImage);
}
public SeetaFace6FaceDetPredictors borrowPredictors() throws Exception {
if(faceDetectorPool == null || faceLandmarkerPool == null){
return null;
}
FaceDetector predictor = faceDetectorPool.borrowObject();
predictor.set(FaceDetector.Property.PROPERTY_THRESHOLD, config.getConfidenceThreshold() > 0 ? config.getConfidenceThreshold() : THRESHOLD);
FaceLandmarker faceLandmarker = faceLandmarkerPool.borrowObject();
return new SeetaFace6FaceDetPredictors(predictor, faceLandmarker, this);
}
public void returnPredictor(FaceDetector predictor, FaceLandmarker faceLandmarker) {
if (predictor != null) {
try {
faceDetectorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}

31
face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/MtcnnPredictors.java Normal file
View File

@@ -0,0 +1,31 @@
package cn.smartjavaai.face.model.facedect.mtcnn;
import ai.djl.inference.Predictor;
import ai.djl.ndarray.NDList;
import cn.smartjavaai.face.model.facedect.MtcnnFaceDetModel;
/**
* @author dwj
* @date 2025/11/24
*/
public class MtcnnPredictors implements AutoCloseable{
public Predictor<NDList, NDList> pNetPredictor;
public Predictor<NDList, NDList> rNetPredictor;
public Predictor<NDList, NDList> oNetPredictor;
// 标记是否由外部借用，用于控制 close 行为
private MtcnnFaceDetModel model;
public MtcnnPredictors(Predictor<NDList, NDList> p, Predictor<NDList, NDList> r, Predictor<NDList, NDList> o, MtcnnFaceDetModel m) {
this.pNetPredictor = p;
this.rNetPredictor = r;
this.oNetPredictor = o;
this.model = m;
}
@Override
public void close() throws Exception {
model.returnPredictor(pNetPredictor, rNetPredictor, oNetPredictor);
}
}

70
face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java
View File

@@ -6,6 +6,7 @@ import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
@@ -28,8 +29,14 @@ import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.factory.FaceDetModelFactory;
import cn.smartjavaai.face.factory.LivenessModelFactory;
import cn.smartjavaai.face.model.facedect.FaceDetectManager;
import cn.smartjavaai.face.model.facedect.MtcnnFaceDetModel;
import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel;
import cn.smartjavaai.face.model.facedect.mtcnn.MtcnnPredictors;
import cn.smartjavaai.face.model.liveness.criterial.LivenessCriteriaFactory;
import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator;
import cn.smartjavaai.face.seetaface.SeetaFace6FaceDetPredictors;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.sdk.FaceAntiSpoofing;
import lombok.extern.slf4j.Slf4j;
import nu.pattern.OpenCV;
@@ -124,8 +131,12 @@ public class CommonLivenessModel implements LivenessDetModel{
return detectVideo(new FFmpegFrameGrabber(videoPath));
}
private R<LivenessResult> detectVideo(FFmpegFrameGrabber grabber) {
try {
protected R<LivenessResult> detectVideo(FFmpegFrameGrabber grabber) {
Predictor<Image, Float> predictor = null;
try (FaceDetectManager faceDetectManager = new FaceDetectManager(config.getDetectModel())){
//初始化predictors
faceDetectManager.borrowPredictors();
predictor = predictorPool.borrowObject();
//滑动窗口
Deque<Float> scoreWindow = new ArrayDeque<>();
grabber.start();
@@ -147,7 +158,8 @@ public class CommonLivenessModel implements LivenessDetModel{
converterToMat = new OpenCVFrameConverter.ToOrgOpenCvCoreMat();
}
Mat mat = converterToMat.convert(frame);
R<LivenessResult> livenessScore = detectTopFace(SmartImageFactory.getInstance().fromMat(mat));
Image image = SmartImageFactory.getInstance().fromMat(mat);
R<LivenessResult> livenessScore = detectVideoFrame(faceDetectManager, image, predictor);
mat.release();
if(!livenessScore.isSuccess()){
log.debug("第" + frameIndex + "帧处理失败：" + livenessScore.getMessage());
@@ -175,6 +187,24 @@ public class CommonLivenessModel implements LivenessDetModel{
}
} catch (Exception e) {
throw new FaceException(e);
} finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
try {
grabber.release();
} catch (FFmpegFrameGrabber.Exception e) {
throw new RuntimeException(e);
}
}
return R.fail(R.Status.Unknown);
}
@@ -262,6 +292,40 @@ public class CommonLivenessModel implements LivenessDetModel{
}
}
private R<LivenessResult> detectVideoFrame(FaceDetectManager faceDetectManager, Image image, Predictor<Image, Float> predictor) {
//预处理图片
Image processedImage = null;
try {
//检测人脸
R<DetectionInfo> detectResult = faceDetectManager.detectTopFace(image);
if(!detectResult.isSuccess()){
return R.fail(detectResult.getCode(), detectResult.getMessage());
}
DetectionInfo detectionInfo = detectResult.getData();
if(config.getModelEnum() == LivenessModelEnum.IIC_FL_MODEL){
processedImage = new DJLImagePreprocessor(image, detectionInfo.getDetectionRectangle())
.setExtendRatio(96f / 112f)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(128)
.enableCenterCrop(true)
.setCenterCropSize(112)
.process();
}
Float result = null;
if(processedImage != null){
result = predictor.predict(processedImage);
ImageUtils.releaseOpenCVMat(processedImage);
}else{
result = predictor.predict(image);
}
LivenessStatus status = result >= config.getRealityThreshold() ? LivenessStatus.LIVE : LivenessStatus.NON_LIVE;
return R.ok(new LivenessResult(status, result));
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
}
}
@Override
public R<LivenessResult> detectTopFace(Image image) {
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);

149
face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java
View File

@@ -10,6 +10,7 @@ import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.cv.SmartImageFactory;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.LivenessResult;
@@ -21,14 +22,19 @@ import cn.smartjavaai.common.preprocess.DJLImagePreprocessor;
import cn.smartjavaai.common.utils.*;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.constant.MiniVisionConstant;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.factory.LivenessModelFactory;
import cn.smartjavaai.face.model.facedect.FaceDetectManager;
import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.opencv.core.Mat;
import javax.imageio.ImageIO;
@@ -59,6 +65,8 @@ public class MiniVisionLivenessModel extends CommonLivenessModel{
private GenericObjectPool<Predictor<Image, float[]>> sePredictorPool;
private OpenCVFrameConverter.ToOrgOpenCvCoreMat converterToMat = null;
/**
* 模型策略
@@ -224,6 +232,147 @@ public class MiniVisionLivenessModel extends CommonLivenessModel{
}
}
protected R<LivenessResult> detectVideo(FFmpegFrameGrabber grabber) {
Predictor<Image, float[]> predictor = null;
Predictor<Image, float[]> sePredictor = null;
try (FaceDetectManager faceDetectManager = new FaceDetectManager(config.getDetectModel())){
//初始化predictors
faceDetectManager.borrowPredictors();
predictor = predictorPool.borrowObject();
sePredictor = sePredictorPool.borrowObject();
//滑动窗口
Deque<Float> scoreWindow = new ArrayDeque<>();
grabber.start();
// 获取视频总帧数
int totalFrames = grabber.getLengthInFrames();
log.debug("视频总帧数：{}，检测帧数：{}", totalFrames, config.getFrameCount());
if(totalFrames < config.getFrameCount()){
return R.fail(10001, "视频帧数低于检测帧数");
}
// 逐帧处理视频
for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) {
if(frameIndex >= config.getMaxVideoDetectFrames()){
return R.fail(10002, "超出最大检测帧数：" + config.getMaxVideoDetectFrames());
}
// 获取当前帧
Frame frame = grabber.grabImage();
if (frame != null) {
if(converterToMat == null){
converterToMat = new OpenCVFrameConverter.ToOrgOpenCvCoreMat();
}
Mat mat = converterToMat.convert(frame);
Image image = SmartImageFactory.getInstance().fromMat(mat);
R<LivenessResult> livenessScore = detectVideoFrame(faceDetectManager, image, predictor, sePredictor);
mat.release();
if(!livenessScore.isSuccess()){
log.debug("第" + frameIndex + "帧处理失败：" + livenessScore.getMessage());
continue;
}else{
log.debug("第" + frameIndex + "帧活体检测结果：" + livenessScore);
scoreWindow.add(livenessScore.getData().getScore());
}
// 如果累计检测帧数 >= 配置值，开始判断
if (scoreWindow.size() >= config.getFrameCount()) {
float avgScore = (float) scoreWindow.stream()
.mapToDouble(Float::doubleValue)
.average()
.orElse(0.0);
log.debug("滑动窗口平均得分: {}", avgScore);
grabber.stop();
LivenessStatus livenessStatus = avgScore > config.getRealityThreshold() ? LivenessStatus.LIVE : LivenessStatus.NON_LIVE;
return R.ok(new LivenessResult(livenessStatus, avgScore));
}
}
}
grabber.stop();
if(scoreWindow.size() < config.getFrameCount()){
return R.fail(1000, "有效帧数量不足，无法完成活体检测");
}
} catch (Exception e) {
throw new FaceException(e);
} finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
if (sePredictor != null) {
try {
sePredictorPool.returnObject(sePredictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
sePredictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
try {
grabber.release();
} catch (FFmpegFrameGrabber.Exception e) {
throw new RuntimeException(e);
}
}
return R.fail(R.Status.Unknown);
}
private R<LivenessResult> detectVideoFrame(FaceDetectManager faceDetectManager, Image image, Predictor<Image, float[]> predictor, Predictor<Image, float[]> sePredictor) {
try {
//检测人脸
R<DetectionInfo> detectResult = faceDetectManager.detectTopFace(image);
if(!detectResult.isSuccess()){
return R.fail(detectResult.getCode(), detectResult.getMessage());
}
DetectionInfo detectionInfo = detectResult.getData();
float[] result = null;
float[] seResult = null;
//预处理图片
Image processedImage = new DJLImagePreprocessor(image, detectionInfo.getDetectionRectangle())
.setExtendRatio(2.7f)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(80)
.process();
result = predictor.predict(processedImage);
ImageUtils.releaseOpenCVMat(processedImage);
//预处理图片
Image seProcessedImage = new DJLImagePreprocessor(image, detectionInfo.getDetectionRectangle())
.setExtendRatio(4)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(80)
.process();
seResult = sePredictor.predict(seProcessedImage);
ImageUtils.releaseOpenCVMat(seProcessedImage);
if(Objects.isNull(result) && Objects.isNull(seResult)){
throw new FaceException("活体检测错误");
}
//计算结果
int maxIndex = ArrayUtils.sumAndFindMaxIndex(result, seResult, 3);
BigDecimal score = Objects.isNull(result) ? BigDecimal.ZERO : BigDecimal.valueOf(result[maxIndex]);
BigDecimal seScore = Objects.isNull(seResult) ? BigDecimal.ZERO : BigDecimal.valueOf(seResult[maxIndex]);
BigDecimal avgSocre = score.add(seScore).divide(BigDecimal.valueOf(2), 2, RoundingMode.HALF_UP);
//活体
if(maxIndex == 1){
LivenessStatus livenessStatus = avgSocre.floatValue() > config.getRealityThreshold() ? LivenessStatus.LIVE : LivenessStatus.NON_LIVE;
return R.ok(new LivenessResult(livenessStatus, avgSocre.floatValue()));
}else{//非活体
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, BigDecimal.ONE.subtract(avgSocre).floatValue()));
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
}
}
public GenericObjectPool<Predictor<Image, float[]>> getPredictorPool() {
return predictorPool;
}

56
face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java
View File

@@ -1,7 +1,9 @@
package cn.smartjavaai.face.model.liveness;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import cn.smartjavaai.common.cv.SmartImageFactory;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceInfo;
@@ -15,6 +17,8 @@ import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.face.constant.LivenessConstant;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.factory.LivenessModelFactory;
import cn.smartjavaai.face.model.facedect.FaceDetectManager;
import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.utils.Seetaface6Utils;
@@ -23,6 +27,7 @@ import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import nu.pattern.OpenCV;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameUtils;
@@ -60,6 +65,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
}
if(Objects.isNull(config.getDetectModel())){
throw new FaceException("未指定人脸检测模型");
}
this.config = config;
//加载依赖库
NativeLoader.loadNativeLibraries(config.getDevice());
@@ -176,10 +184,41 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
}
private R<LivenessResult> detectVideoFrame(Image image, FaceDetectManager faceDetectManager, FaceAntiSpoofing faceAntiSpoofing) {
//检测人脸
R<DetectionInfo> detectResult = faceDetectManager.detectTopFace(image);
if(!detectResult.isSuccess()){
return R.fail(detectResult.getCode(), detectResult.getMessage());
}
DetectionInfo detectionInfo = detectResult.getData();
if(Objects.isNull(detectionInfo)){
return R.fail(R.Status.NO_FACE_DETECTED);
}
if(detectionInfo.getFaceInfo().getKeyPoints() == null || detectionInfo.getFaceInfo().getKeyPoints().isEmpty()){
return R.fail(1002,"人脸关键点keyPoints为空");
}
FaceAntiSpoofing.Status status = null;
try {
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = Seetaface6Utils.convertToSeetaRect(detectionInfo.getDetectionRectangle());
SeetaPointF[] landmarks = Seetaface6Utils.convertToSeetaPointF(detectionInfo.getFaceInfo().getKeyPoints());
//检测视频
status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks);
return R.ok(new LivenessResult(Seetaface6Utils.convertToLivenessStatus(status)));
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
}
}
private R<LivenessResult> detectVideo(FFmpegFrameGrabber grabber) {
FaceAntiSpoofing faceAntiSpoofing = null;
try {
try (FaceDetectManager faceDetectManager = new FaceDetectManager(config.getDetectModel())){
//初始化predictors
faceDetectManager.borrowPredictors();
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
//重置视频
faceAntiSpoofing.ResetVideo();
@@ -194,14 +233,14 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
// 逐帧处理视频
for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) {
if(frameIndex >= config.getMaxVideoDetectFrames()){
return R.fail(10002, "超出最大检测帧数：" + config.getMaxVideoDetectFrames());
return R.fail(10002, "视频中未检测到人脸，超出最大检测帧数：" + config.getMaxVideoDetectFrames());
}
// 获取当前帧
Frame frame = grabber.grabImage();
if (frame != null) {
BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame);
Image image = SmartImageFactory.getInstance().fromBufferedImage(bufferedImage);
R<LivenessResult> livenessStatus = detectTopFace(image, false);
R<LivenessResult> livenessStatus = detectVideoFrame(image, faceDetectManager, faceAntiSpoofing);
if(!livenessStatus.isSuccess()){
log.debug("第" + frameIndex + "帧处理失败：" + livenessStatus.getMessage());
continue;
@@ -225,10 +264,17 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
log.warn("归还Predictor失败", e);
}
}
try {
grabber.release();
} catch (FFmpegFrameGrabber.Exception e) {
throw new RuntimeException(e);
}
}
return R.fail(1000, "有效帧数量不足，无法完成活体检测");
}
@Override
public R<DetectionResponse> detect(Image image) {
FaceAntiSpoofing faceAntiSpoofing = null;
@@ -246,7 +292,7 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(SeetaRect seetaRect : seetaResult){
@@ -346,7 +392,7 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];

27
face/src/main/java/cn/smartjavaai/face/seetaface/SeetaFace6FaceDetPredictors.java Normal file
View File

@@ -0,0 +1,27 @@
package cn.smartjavaai.face.seetaface;
import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel;
import com.seeta.sdk.FaceDetector;
import com.seeta.sdk.FaceLandmarker;
/**
* SeetaFace6 人脸检测Detector
* @author dwj
*/
public class SeetaFace6FaceDetPredictors implements AutoCloseable{
public FaceDetector faceDetector;
public FaceLandmarker faceLandmarker;
public SeetaFace6FaceDetModel model;
public SeetaFace6FaceDetPredictors(FaceDetector faceDetector, FaceLandmarker faceLandmarker, SeetaFace6FaceDetModel model) {
this.faceDetector = faceDetector;
this.faceLandmarker = faceLandmarker;
this.model = model;
}
@Override
public void close(){
model.returnPredictor(faceDetector, faceLandmarker);
}
}

3
face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java
View File

@@ -637,7 +637,8 @@ public class MilvusClient implements VectorDBClient {
List<FaceVector> result = new ArrayList<>();
for (QueryResultsWrapper.RowRecord row : records) {
String id = (String) row.get(VectorDBConstants.FieldNames.ID_FIELD);
Object idObj = row.get(VectorDBConstants.FieldNames.ID_FIELD);
String id = idObj != null ? idObj.toString() : null;
Object vectorObj = row.get(VectorDBConstants.FieldNames.VECTOR_FIELD);
float[] vector = null;
if (vectorObj instanceof List<?>) {

16
face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java
View File

@@ -2,6 +2,7 @@ package cn.smartjavaai.face.vector.core;
import cn.hutool.core.util.IdUtil;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.common.executor.GlobalExecutor;
import cn.smartjavaai.common.utils.SimilarityUtil;
import cn.smartjavaai.face.dao.FaceDao;
import cn.smartjavaai.face.entity.FaceSearchParams;
@@ -23,12 +24,9 @@ import java.util.stream.Collectors;
public class SQLiteClient implements VectorDBClient {
private final FaceDao faceDao;
//private final List<FaceVector> memoryIndex = new CopyOnWriteArrayList<>();
private final ConcurrentHashMap<String, FaceVector> memoryIndex = new ConcurrentHashMap<>();
private int featureDimension; // 维度
private final ExecutorService executor = Executors.newFixedThreadPool(4);
private SQLiteConfig config;
/**
@@ -162,7 +160,7 @@ public class SQLiteClient implements VectorDBClient {
return similarity >= faceSearchParams.getThreshold() ?
new FaceSearchResult(vector.getId(), similarity, vector.getMetadata()) :
null;
}, executor))
}, GlobalExecutor.getExecutor()))
.collect(Collectors.toList());
// 收集结果并过滤null
@@ -185,15 +183,7 @@ public class SQLiteClient implements VectorDBClient {
@Override
public void close() {
executor.shutdown();
try {
if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
executor.shutdownNow();
Thread.currentThread().interrupt();
}
}
@Override