diff --git a/README.md b/README.md index c718f5f..8694a68 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,19 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 + + +
+

零样本目标检测
(ZeroShot Object Detection)

+ - YOLO-World 模型
+
+ + +
+ +
+ +
@@ -407,6 +420,8 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 - 支持KINETICS400数据集中400个人类动作识别 - **姿态估计** - 集成YOLOv8-pose、YOLOv11-pose等模型 +- **零样本目标检测** + - 集成YOLOv8s_worldv2、owlv2_base_patch16模型 - **CLIP** - 支持提取图片及文本特征 - 支持文搜图、图搜文、图搜图 @@ -483,7 +498,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 cn.smartjavaai all - 1.0.27 + 1.1.0 ``` @@ -682,6 +697,14 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 | YOLOV11-OBB | OnnxRuntime | Ultralytics在DOTAv1 数据集 上训练的模型、通过引入一个额外的角度来更准确地定位图像中的对象 | [Github](https://docs.ultralytics.com/zh/tasks/segment/) | --- +#### 零样本目标检测模型 + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|---------|--------------------------------|----------------------------------------------------------| +| YOLOv8s-worldv2 | PyTorch | 可根据描述性文本检测图像中的任何物体 | [官网](https://docs.ultralytics.com/zh/models/yolo-world/) | +| owlv2-base-patch16 | PyTorch | OWLv2是一种多模态模型，通过结合CLIP的骨干和ViT样的Transformer，实现零样本文本对象检测| [官网](https://huggingface.co/google/owlv2-base-patch16) | +--- + #### 行人检测模型 | 模型名称 | 引擎 | 模型开源网站 | diff --git a/all/pom.xml b/all/pom.xml index be65c94..63c7d14 100644 --- a/all/pom.xml +++ b/all/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.27 + 1.1.0 all - 1.0.27 + 1.1.0 ${project.artifactId} SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/bom/pom.xml b/bom/pom.xml index c50ce45..815bba0 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -6,10 +6,10 @@ cn.smartjavaai smartjavaai-parent - 1.0.27 + 1.1.0 - 1.0.27 + 1.1.0 bom bom 统一版本管理的 BOM 包，同时支持 import 和全量依赖 diff --git a/common/pom.xml b/common/pom.xml index 23ad847..e8d4e2c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -6,7 +6,7 @@ cn.smartjavaai smartjavaai-parent - 1.0.27 + 1.1.0 common diff --git a/common/src/main/java/cn/smartjavaai/common/executor/GlobalExecutor.java b/common/src/main/java/cn/smartjavaai/common/executor/GlobalExecutor.java new file mode 100644 index 0000000..035f8b5 --- /dev/null +++ b/common/src/main/java/cn/smartjavaai/common/executor/GlobalExecutor.java @@ -0,0 +1,53 @@ +package cn.smartjavaai.common.executor; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author dwj + * @date 2025/11/26 + */ +public class GlobalExecutor { + + private static volatile ExecutorService executor; + + public static ExecutorService getExecutor() { + if (executor == null) { + synchronized (GlobalExecutor.class) { + if (executor == null) { + int cores = Runtime.getRuntime().availableProcessors(); + executor = new ThreadPoolExecutor( + cores, + cores * 2, + 60L, TimeUnit.SECONDS, + new SynchronousQueue<>(), + runnable -> { + Thread t = new Thread(runnable); + t.setDaemon(true); // 守护线程 + return t; + }, + new ThreadPoolExecutor.DiscardOldestPolicy() + ); + } + } + } + return executor; + } + + public static void shutdown() { + if (executor != null) { + executor.shutdown(); + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + executor.shutdownNow(); + } + } catch (InterruptedException e) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + } + +} diff --git a/examples/face-example/pom.xml b/examples/face-example/pom.xml index ba03477..78c8918 100644 --- a/examples/face-example/pom.xml +++ b/examples/face-example/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.27 + 1.1.0 smartai.examples.face.facedet.FaceDetDemo @@ -88,11 +88,10 @@ face - ai.djl.pytorch pytorch-jni - 2.5.1-0.32.0 + 2.7.1-0.34.0 runtime @@ -129,7 +128,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.windows-x86_64} - 2.5.1 + 2.7.1 runtime @@ -167,19 +166,45 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.linux-x86_64} - 2.5.1 + 2.7.1 runtime + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.linux-arm64} + - ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-x86_64} - 2.5.1 - runtime + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.linux-arm64} + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.linux-arm64} + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.linux-arm64} + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.linux-aarch64} + 2.7.1 + runtime + @@ -213,7 +238,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.osx-aarch64} - 2.5.1 + 2.7.1 runtime diff --git a/examples/ocr-examples/pom.xml b/examples/ocr-examples/pom.xml index ce4ec26..b444855 100644 --- a/examples/ocr-examples/pom.xml +++ b/examples/ocr-examples/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.27 + 1.1.0 smartai.examples.ocr.common.OcrRecognizeDemo @@ -95,7 +95,7 @@ ai.djl.pytorch pytorch-jni - 2.5.1-0.32.0 + 2.7.1-0.34.0 runtime @@ -132,7 +132,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.windows-x86_64} - 2.5.1 + 2.7.1 runtime @@ -170,14 +170,43 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.linux-x86_64} - 2.5.1 + 2.7.1 runtime + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.linux-arm64} + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.linux-arm64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.linux-arm64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.linux-arm64} + + ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-x86_64} - 2.5.1 + pytorch-native-cpu + ${djl.platform.linux-aarch64} + 2.7.1 runtime @@ -214,7 +243,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.osx-aarch64} - 2.5.1 + 2.7.1 runtime diff --git a/examples/speech-examples/pom.xml b/examples/speech-examples/pom.xml index e077903..cc0c81c 100644 --- a/examples/speech-examples/pom.xml +++ b/examples/speech-examples/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.27 + 1.1.0 smartai.examples.speech.asr.common.OcrRecognizeDemo diff --git a/examples/translation-example/pom.xml b/examples/translation-example/pom.xml index d0012b1..b4f85e4 100644 --- a/examples/translation-example/pom.xml +++ b/examples/translation-example/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.27 + 1.1.0 smartai.examples.nlp.translation.TranslationDemo @@ -93,7 +93,7 @@ ai.djl.pytorch pytorch-jni - 2.5.1-0.32.0 + 2.7.1-0.34.0 runtime @@ -104,7 +104,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.windows-x86_64} - 2.5.1 + 2.7.1 runtime @@ -116,25 +116,25 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.linux-x86_64} - 2.5.1 + 2.7.1 runtime + ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-x86_64} - 2.5.1 + pytorch-native-cpu + ${djl.platform.linux-aarch64} + 2.7.1 runtime - ai.djl.pytorch pytorch-native-cpu ${djl.platform.osx-aarch64} - 2.5.1 + 2.7.1 runtime diff --git a/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java b/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java index b667123..e12b62f 100644 --- a/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java +++ b/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java @@ -44,7 +44,7 @@ public class TranslationDemo { //指定翻译模型：NLLB,切换模型需同时修改modelEnum及modelPath config.setModelEnum(TranslationModeEnum.NLLB_MODEL); //指定模型路径，需将模型路径修改为本地的模型路径 - config.setModelPath("/Users/xxx/Documents/develop/model/trans/traced_translation_cpu.pt"); + config.setModelPath("/Users/wenjie/Documents/develop/model/translate/nllb/traced_translation_cpu.pt"); config.setDevice(DeviceEnum.CPU); return TranslationModelFactory.getInstance().getModel(config); } diff --git a/examples/vision-example/pom.xml b/examples/vision-example/pom.xml index 54d2a27..db39394 100644 --- a/examples/vision-example/pom.xml +++ b/examples/vision-example/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.27 + 1.1.0 smartai.examples.vision.ObjectDetectionDemo @@ -92,7 +92,7 @@ ai.djl.pytorch pytorch-jni - 2.5.1-0.32.0 + 2.7.1-0.34.0 runtime @@ -129,7 +129,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.windows-x86_64} - 2.5.1 + 2.7.1 runtime @@ -179,12 +179,11 @@ ${javacv.platform.linux-x86_64} - ai.djl.pytorch pytorch-native-cpu ${djl.platform.linux-x86_64} - 2.5.1 + 2.7.1 runtime @@ -203,14 +202,57 @@ 1.9.1 + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.linux-arm64} + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.linux-arm64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.linux-arm64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.linux-arm64} + + ai.djl.pytorch - pytorch-native-cpu-precxx11 - ${djl.platform.linux-x86_64} - 2.5.1 + pytorch-native-cpu + ${djl.platform.linux-aarch64} + 2.7.1 runtime + + ai.djl.tensorflow + tensorflow-native-cpu + ${javacv.platform.linux-arm64} + runtime + 2.16.1 + + + ai.djl.mxnet + mxnet-native-mkl + ${javacv.platform.linux-arm64} + runtime + 1.9.1 + + @@ -244,7 +286,7 @@ ai.djl.pytorch pytorch-native-cpu ${djl.platform.osx-aarch64} - 2.5.1 + 2.7.1 runtime diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/ClsDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/ClsDemo.java index 955f7ee..c6a93b0 100644 --- a/examples/vision-example/src/main/java/smartai/examples/vision/ClsDemo.java +++ b/examples/vision-example/src/main/java/smartai/examples/vision/ClsDemo.java @@ -46,7 +46,7 @@ public class ClsDemo { public ClsModel getModel(){ ClsModelConfig config = new ClsModelConfig(); - //实例分割模型，切换模型需要同时修改modelEnum及modelPath + //切换模型需要同时修改modelEnum及modelPath config.setModelEnum(ClsModelEnum.YOLOV8); //模型所在路径，synset.txt也需要放在同目录下 config.setModelPath("/Users/wenjie/Documents/develop/model/vision/cls/yolo11m-cls.onnx"); diff --git a/examples/vision-example/src/main/java/smartai/examples/vision/ZeroShotObjectDetectionDemo.java b/examples/vision-example/src/main/java/smartai/examples/vision/ZeroShotObjectDetectionDemo.java new file mode 100644 index 0000000..529cf8b --- /dev/null +++ b/examples/vision-example/src/main/java/smartai/examples/vision/ZeroShotObjectDetectionDemo.java @@ -0,0 +1,137 @@ +package smartai.examples.vision; + +import ai.djl.modality.cv.Image; + +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.zeroshot.config.ZeroDetConfig; +import cn.smartjavaai.zeroshot.enums.ZeroDetModelEnum; +import cn.smartjavaai.zeroshot.model.ZeroDetModel; +import cn.smartjavaai.zeroshot.model.ZeroDetModelFactory; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * 零样本目标检测 + * @author dwj + */ +@Slf4j +public class ZeroShotObjectDetectionDemo { + + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + @BeforeClass + public static void beforeAll() throws IOException { + //修改缓存路径 +// Config.setCachePath("/Users/xxx/smartjavaai_cache"); + } + + /** + * 获取零样本目标检测模型 + */ + public ZeroDetModel getModel(){ + ZeroDetConfig config = new ZeroDetConfig(); + //零样本目标检测模型，切换模型需要同时修改modelEnum及modelPath + config.setModelEnum(ZeroDetModelEnum.OWLV2_BASE_PATCH16); + //模型所在路径 + config.setModelPath("/Users/wenjie/Documents/develop/model/vision/zero/owlv2-base-patch16"); + config.setDevice(device); + //置信度阈值 + config.setThreshold(0.5f); + return ZeroDetModelFactory.getInstance().getModel(config); + } + + + + /** + * 零样本目标检测 + * 特性： + * 1、零样本检测能力：无需针对特定类别进行训练，可直接通过文本查询检测新类别物体 + * 2、开放词汇识别：能够识别训练时未见过的类别名称，突破传统检测模型的类别限制 + * 3、多查询支持：支持同时使用多个文本查询进行目标检测，提高检测效率 + */ + @Test + public void zeroDetection(){ + try { + ZeroDetModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/zero/000000039769.jpg")); + //输入图片以及条件 + R result = detectorModel.detect(image, new String[]{"cat","remote control"}); + if(result.isSuccess()){ + log.info("零样本目标检测结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("零样本目标检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 零样本目标检测并绘制检测结果 + * 特性： + * 1、零样本检测能力：无需针对特定类别进行训练，可直接通过文本查询检测新类别物体 + * 2、开放词汇识别：能够识别训练时未见过的类别名称，突破传统检测模型的类别限制 + * 3、多查询支持：支持同时使用多个文本查询进行目标检测，提高检测效率 + */ + @Test + public void zeroDetectionAndDraw() { + try { + ZeroDetModel detectorModel = getModel(); + String[] candidates = new String[]{"cat","remote control"}; + //保存绘制后图片以及返回检测结果 + R result = detectorModel.detectAndDraw(candidates, "src/main/resources/zero/000000039769.jpg","output/cat_detected.png"); + if(result.isSuccess()){ + log.info("零样本目标检测结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("零样本目标检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + /** + * 零样本目标检测并绘制检测结果 + * 特性： + * 1、零样本检测能力：无需针对特定类别进行训练，可直接通过文本查询检测新类别物体 + * 2、开放词汇识别：能够识别训练时未见过的类别名称，突破传统检测模型的类别限制 + * 3、多查询支持：支持同时使用多个文本查询进行目标检测，提高检测效率 + */ + @Test + public void zeroDetectionAndDraw2(){ + try { + ZeroDetModel detectorModel = getModel(); + //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档 + Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/zero/000000039769.jpg")); + String[] candidates = new String[]{"cat","remote control"}; + R result = detectorModel.detectAndDraw(image, candidates); + if(result.isSuccess()){ + log.info("零样本目标检测结果：{}", JSONObject.toJSONString(result.getData())); + //保存图片 + ImageUtils.save(result.getData().getDrawnImage(), "output/cat_detected.png"); + }else{ + log.info("零样本目标检测失败：{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + +} diff --git a/face/pom.xml b/face/pom.xml index 189c135..e5ea7ce 100644 --- a/face/pom.xml +++ b/face/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.27 + 1.1.0 face - 1.0.27 + 1.1.0 face SmartJavaAI https://github.com/geekwenjie/SmartJavaAI @@ -26,7 +26,7 @@ UTF-8 true - 1.5.8 + 1.5.10 5.1.2-1.5.8 @@ -57,6 +57,8 @@ + + diff --git a/face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java b/face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java index 29f6bfa..c5e43f3 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java @@ -176,7 +176,7 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel { imageData.data = BufferedImageUtils.getMatrixBGR(image); //检测人脸 SeetaRect[] seetaResult = detectPredictor.Detect(imageData); - if(Objects.isNull(seetaResult)){ + if(Objects.isNull(seetaResult) || seetaResult.length == 0){ throw new FaceException("无人脸数据"); } for(SeetaRect seetaRect : seetaResult){ @@ -456,7 +456,7 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel { imageData.data = BufferedImageUtils.getMatrixBGR(image); //检测人脸 SeetaRect[] seetaResult = detectPredictor.Detect(imageData); - if(Objects.isNull(seetaResult)){ + if(Objects.isNull(seetaResult) || seetaResult.length == 0){ throw new FaceException("无人脸数据"); } SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()]; @@ -510,7 +510,7 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel { imageData.data = ImageUtils.getMatrixBGR(image); //检测人脸 SeetaRect[] seetaResult = detectPredictor.Detect(imageData); - if(Objects.isNull(seetaResult)){ + if(Objects.isNull(seetaResult) || seetaResult.length == 0){ throw new FaceException("无人脸数据"); } for(SeetaRect seetaRect : seetaResult){ @@ -642,7 +642,7 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel { imageData.data = ImageUtils.getMatrixBGR(image); //检测人脸 SeetaRect[] seetaResult = detectPredictor.Detect(imageData); - if(Objects.isNull(seetaResult)){ + if(Objects.isNull(seetaResult) || seetaResult.length == 0){ throw new FaceException("无人脸数据"); } SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()]; diff --git a/face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java b/face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java index a1dc181..fcb1dd9 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java @@ -9,6 +9,7 @@ import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; +import ai.djl.translate.TranslateException; import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; @@ -50,6 +51,30 @@ public class CommonFaceDetModel implements FaceDetModel{ private FaceDetConfig config; + @Override + public Predictor borrowPredictor() throws Exception { + if(predictorPool == null){ + throw new FaceException("请先加载模型"); + } + return predictorPool.borrowObject(); + } + + @Override + public void returnPredictor(Predictor predictor){ + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + } + /** * 加载模型 diff --git a/face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java b/face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java index 228c4b3..5d3768f 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java @@ -131,4 +131,20 @@ public interface FaceDetModel extends AutoCloseable{ default void setFromFactory(boolean fromFactory){ throw new UnsupportedOperationException("默认不支持该功能"); } + + /** + * 获取Predictor + * @return + */ + default Predictor borrowPredictor() throws Exception{ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 归还Predictor + * @param predictor + */ + default void returnPredictor(Predictor predictor){ + throw new UnsupportedOperationException("默认不支持该功能"); + } } diff --git a/face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetectManager.java b/face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetectManager.java new file mode 100644 index 0000000..588d237 --- /dev/null +++ b/face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetectManager.java @@ -0,0 +1,99 @@ +package cn.smartjavaai.face.model.facedect; + +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.DetectedObjects; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.model.facedect.mtcnn.MtcnnPredictors; +import cn.smartjavaai.face.seetaface.SeetaFace6FaceDetPredictors; +import cn.smartjavaai.face.utils.FaceUtils; + +import java.util.Objects; + +/** + * @author dwj + * @date 2025/11/24 + */ +public class FaceDetectManager implements AutoCloseable{ + + + private FaceDetModel faceDetModel; + + public FaceDetectManager(FaceDetModel faceDetModel) { + this.faceDetModel = faceDetModel; + } + + private MtcnnPredictors mtcnnPredictors; + + private SeetaFace6FaceDetPredictors seetaFace6FaceDetPredictors; + + private Predictor commonPredictor; + + + + public void borrowPredictors(){ + try { + //mtcnn + if(faceDetModel instanceof MtcnnFaceDetModel){ + MtcnnFaceDetModel mtcnnFaceDetModel = (MtcnnFaceDetModel) faceDetModel; + mtcnnPredictors = mtcnnFaceDetModel.borrowPredictors(); + }else if(faceDetModel instanceof SeetaFace6FaceDetModel){ + //SeetaFace6 + SeetaFace6FaceDetModel seetaFace6FaceDetModel = (SeetaFace6FaceDetModel) faceDetModel; + seetaFace6FaceDetPredictors = seetaFace6FaceDetModel.borrowPredictors(); + }else{ + //其他通用模型 + commonPredictor = faceDetModel.borrowPredictor(); + } + } catch (Exception e) { + throw new FaceException("获取predictors异常", e); + } + } + + public R detectTopFace(Image image){ + DetectionResponse detectionResponse = null; + try { + //mtcnn + if(faceDetModel instanceof MtcnnFaceDetModel){ + MtcnnFaceDetModel mtcnnFaceDetModel = (MtcnnFaceDetModel) faceDetModel; + DetectedObjects detections = mtcnnFaceDetModel.detectCoreByPredictors(image, mtcnnPredictors); + detectionResponse = FaceUtils.convertToDetectionResponse(detections, image); + }else if(faceDetModel instanceof SeetaFace6FaceDetModel){ + //SeetaFace6 + SeetaFace6FaceDetModel seetaFace6FaceDetModel = (SeetaFace6FaceDetModel) faceDetModel; + detectionResponse = seetaFace6FaceDetModel.detectByPredictors(image, seetaFace6FaceDetPredictors); + }else{ + DetectedObjects detections = commonPredictor.predict(image); + detectionResponse = FaceUtils.convertToDetectionResponse(detections, image); + } + } catch (Exception e) { + throw new FaceException("获取predictors异常", e); + } + if(Objects.isNull(detectionResponse) || Objects.isNull(detectionResponse.getDetectionInfoList()) || detectionResponse.getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + DetectionInfo detectionInfo = detectionResponse.getDetectionInfoList().get(0); + return R.ok(detectionInfo); + } + + + @Override + public void close(){ + try { + //mtcnn + if(faceDetModel instanceof MtcnnFaceDetModel){ + mtcnnPredictors.close(); + }else if(faceDetModel instanceof SeetaFace6FaceDetModel){ + //SeetaFace6 + seetaFace6FaceDetPredictors.close(); + }else{ + faceDetModel.getPool().returnObject(commonPredictor); + } + } catch (Exception e) { + throw new FaceException("归还predictors异常", e); + } + } +} diff --git a/face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java b/face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java index e878b12..ecf9ed5 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java @@ -221,52 +221,51 @@ public class MtcnnFaceDetModel extends CommonFaceDetModel{ } + /** + * 使用MtcnnPredictors进行人脸检测 + * @param image + * @param predictors + * @return + */ + public DetectedObjects detectCoreByPredictors(Image image, MtcnnPredictors predictors){ + Predictor pNetPredictor = predictors.pNetPredictor; + Predictor rNetPredictor = predictors.rNetPredictor; + Predictor oNetPredictor = predictors.oNetPredictor; + try (NDManager manager = pNetModel.getNDManager().newSubManager();){ + int h = image.getHeight(); + int w = image.getWidth(); + //第一阶段 + NDList outputPnet = PNetModel.firstStage(manager, pNetPredictor, image); - -// /** -// * 转换为FaceDetectedResult -// * @param mtcnnBatchResult -// * @return -// */ -// public static DetectionResponse convertToDetectionResponse(MtcnnBatchResult mtcnnBatchResult){ -// if(Objects.isNull(mtcnnBatchResult) || CollectionUtils.isEmpty(mtcnnBatchResult.boxes) -// || CollectionUtils.isEmpty(mtcnnBatchResult.points) -// || CollectionUtils.isEmpty(mtcnnBatchResult.probs)){ -// return null; -// } -// DetectionResponse detectionResponse = new DetectionResponse(); -// List detectionInfoList = new ArrayList(); -// -// NDArray boxes = mtcnnBatchResult.boxes.get(0); -// NDArray probs = mtcnnBatchResult.probs.get(0); -// NDArray points = mtcnnBatchResult.points.get(0); -// -// if (DJLCommonUtils.isNDArrayEmpty(boxes) || DJLCommonUtils.isNDArrayEmpty(probs) || DJLCommonUtils.isNDArrayEmpty(points)){ -// return null; -// } -// long numBoxes = boxes.getShape().get(0); -// for (int i = 0; i < numBoxes; i++) { -// float[] boxCoords = boxes.get(i).toFloatArray(); // [x1, y1, x2, y2] -// float score = probs.getFloat(i); -// NDArray pointND = points.get(i); // shape [5,2] -// float[] flatPoints = pointND.toFloatArray(); // 一维长度 10 -// List keyPoints = new ArrayList(); -// for (int p = 0; p < 5; p++) { -// keyPoints.add(new Point(flatPoints[p * 2], flatPoints[p * 2 + 1])); -// } -// int x = Math.round(boxCoords[0]); -// int y = Math.round(boxCoords[1]); -// int w = Math.round(boxCoords[2] - boxCoords[0]); -// int h = Math.round(boxCoords[3] - boxCoords[1]); -// -// DetectionRectangle rectangle = new DetectionRectangle(x, y, w, h); -// FaceInfo faceInfo = new FaceInfo(keyPoints); -// DetectionInfo detectionInfo = new DetectionInfo(rectangle, score, faceInfo); -// detectionInfoList.add(detectionInfo); -// } -// detectionResponse.setDetectionInfoList(detectionInfoList); -// return detectionResponse; -// } + if(CollectionUtils.isEmpty(outputPnet)){ + return DJLCommonUtils.buildEmptyDetectedObjects(); + } + NDArray boxes = outputPnet.get(0); + NDArray image_inds = outputPnet.get(1); + NDArray imgs = outputPnet.get(2); + if(DJLCommonUtils.isNDArrayEmpty(boxes) || DJLCommonUtils.isNDArrayEmpty(image_inds) || DJLCommonUtils.isNDArrayEmpty(imgs)){ + return DJLCommonUtils.buildEmptyDetectedObjects(); + } + NDList pad = MtcnnUtils.pad(boxes, w, h); + //第二阶段 + NDList outputRnet = RNetModel.secondStage(manager, rNetPredictor, imgs, boxes, pad, image_inds); + if(CollectionUtils.isEmpty(outputRnet)){ + return DJLCommonUtils.buildEmptyDetectedObjects(); + } + NDArray image_indsFiltered = outputRnet.get(0); + NDArray scoresFiltered = outputRnet.get(1); + boxes = outputRnet.get(2); + if(DJLCommonUtils.isNDArrayEmpty(boxes) || DJLCommonUtils.isNDArrayEmpty(image_indsFiltered) || DJLCommonUtils.isNDArrayEmpty(scoresFiltered)){ + return DJLCommonUtils.buildEmptyDetectedObjects(); + } + //第三阶段 + MtcnnBatchResult oNetResult = ONetModel.thirdStage(manager, oNetPredictor, imgs, boxes, w, h, scoresFiltered, image_indsFiltered); + return FaceUtils.toDetectedObjects(oNetResult, w, h); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } @@ -293,6 +292,57 @@ public class MtcnnFaceDetModel extends CommonFaceDetModel{ return fromFactory; } + + public MtcnnPredictors borrowPredictors() throws Exception { + if(pnetPredictorPool == null || rnetPredictorPool == null || onetPredictorPool == null){ + return null; + } + Predictor p = pnetPredictorPool.borrowObject(); + Predictor r = rnetPredictorPool.borrowObject(); + Predictor o = onetPredictorPool.borrowObject(); + return new MtcnnPredictors(p, r, o, this); + } + + public void returnPredictor(Predictor pNetPredictor, Predictor rNetPredictor, Predictor oNetPredictor) { + if (pNetPredictor != null) { + try { + pnetPredictorPool.returnObject(pNetPredictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + pNetPredictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + if (rNetPredictor != null) { + try { + rnetPredictorPool.returnObject(rNetPredictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + rNetPredictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + if (oNetPredictor != null) { + try { + onetPredictorPool.returnObject(oNetPredictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + oNetPredictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + } + + @Override public void close() { if (fromFactory) { diff --git a/face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java b/face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java index ef275c4..32b3bce 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java @@ -1,7 +1,9 @@ package cn.smartjavaai.face.model.facedect; import ai.djl.engine.Engine; +import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; +import ai.djl.ndarray.NDList; import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; @@ -13,7 +15,9 @@ import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.face.config.FaceDetConfig; import cn.smartjavaai.face.exception.FaceException; import cn.smartjavaai.face.factory.FaceDetModelFactory; +import cn.smartjavaai.face.model.facedect.mtcnn.MtcnnPredictors; import cn.smartjavaai.face.seetaface.NativeLoader; +import cn.smartjavaai.face.seetaface.SeetaFace6FaceDetPredictors; import cn.smartjavaai.face.utils.FaceUtils; import com.seeta.pool.*; import com.seeta.sdk.*; @@ -124,6 +128,27 @@ public class SeetaFace6FaceDetModel implements FaceDetModel{ } } + public DetectionResponse detectByPredictors(Image image, SeetaFace6FaceDetPredictors predictors) { + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + FaceDetector predictor = predictors.faceDetector; + FaceLandmarker faceLandmarker = predictors.faceLandmarker; + try { + SeetaRect[] seetaResult = predictor.Detect(imageData); + List seetaPointFSList = new ArrayList(); + for(SeetaRect seetaRect : seetaResult){ + //提取人脸的5点人脸标识 + SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()]; + faceLandmarker.mark(imageData, seetaRect, pointFS); + seetaPointFSList.add(pointFS); + } + return FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList); + } catch (Exception e) { + throw new FaceException("目标检测错误", e); + } + } + + @Override public R detectAndDraw(Image image) { R result = detect(image); @@ -276,6 +301,33 @@ public class SeetaFace6FaceDetModel implements FaceDetModel{ return R.ok(drawnImage); } + public SeetaFace6FaceDetPredictors borrowPredictors() throws Exception { + if(faceDetectorPool == null || faceLandmarkerPool == null){ + return null; + } + FaceDetector predictor = faceDetectorPool.borrowObject(); + predictor.set(FaceDetector.Property.PROPERTY_THRESHOLD, config.getConfidenceThreshold() > 0 ? config.getConfidenceThreshold() : THRESHOLD); + FaceLandmarker faceLandmarker = faceLandmarkerPool.borrowObject(); + return new SeetaFace6FaceDetPredictors(predictor, faceLandmarker, this); + } + + public void returnPredictor(FaceDetector predictor, FaceLandmarker faceLandmarker) { + if (predictor != null) { + try { + faceDetectorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + if (faceLandmarker != null) { + try { + faceLandmarkerPool.returnObject(faceLandmarker); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + diff --git a/face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/MtcnnPredictors.java b/face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/MtcnnPredictors.java new file mode 100644 index 0000000..f051c48 --- /dev/null +++ b/face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/MtcnnPredictors.java @@ -0,0 +1,31 @@ +package cn.smartjavaai.face.model.facedect.mtcnn; + +import ai.djl.inference.Predictor; +import ai.djl.ndarray.NDList; +import cn.smartjavaai.face.model.facedect.MtcnnFaceDetModel; + +/** + * @author dwj + * @date 2025/11/24 + */ +public class MtcnnPredictors implements AutoCloseable{ + + public Predictor pNetPredictor; + public Predictor rNetPredictor; + public Predictor oNetPredictor; + + // 标记是否由外部借用，用于控制 close 行为 + private MtcnnFaceDetModel model; + + public MtcnnPredictors(Predictor p, Predictor r, Predictor o, MtcnnFaceDetModel m) { + this.pNetPredictor = p; + this.rNetPredictor = r; + this.oNetPredictor = o; + this.model = m; + } + + @Override + public void close() throws Exception { + model.returnPredictor(pNetPredictor, rNetPredictor, oNetPredictor); + } +} diff --git a/face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java b/face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java index 9a73be3..854cefb 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java @@ -6,6 +6,7 @@ import ai.djl.engine.Engine; import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; +import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; @@ -28,8 +29,14 @@ import cn.smartjavaai.face.enums.LivenessModelEnum; import cn.smartjavaai.face.exception.FaceException; import cn.smartjavaai.face.factory.FaceDetModelFactory; import cn.smartjavaai.face.factory.LivenessModelFactory; +import cn.smartjavaai.face.model.facedect.FaceDetectManager; +import cn.smartjavaai.face.model.facedect.MtcnnFaceDetModel; +import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel; +import cn.smartjavaai.face.model.facedect.mtcnn.MtcnnPredictors; import cn.smartjavaai.face.model.liveness.criterial.LivenessCriteriaFactory; import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator; +import cn.smartjavaai.face.seetaface.SeetaFace6FaceDetPredictors; +import cn.smartjavaai.face.utils.FaceUtils; import com.seeta.sdk.FaceAntiSpoofing; import lombok.extern.slf4j.Slf4j; import nu.pattern.OpenCV; @@ -124,8 +131,12 @@ public class CommonLivenessModel implements LivenessDetModel{ return detectVideo(new FFmpegFrameGrabber(videoPath)); } - private R detectVideo(FFmpegFrameGrabber grabber) { - try { + protected R detectVideo(FFmpegFrameGrabber grabber) { + Predictor predictor = null; + try (FaceDetectManager faceDetectManager = new FaceDetectManager(config.getDetectModel())){ + //初始化predictors + faceDetectManager.borrowPredictors(); + predictor = predictorPool.borrowObject(); //滑动窗口 Deque scoreWindow = new ArrayDeque<>(); grabber.start(); @@ -147,7 +158,8 @@ public class CommonLivenessModel implements LivenessDetModel{ converterToMat = new OpenCVFrameConverter.ToOrgOpenCvCoreMat(); } Mat mat = converterToMat.convert(frame); - R livenessScore = detectTopFace(SmartImageFactory.getInstance().fromMat(mat)); + Image image = SmartImageFactory.getInstance().fromMat(mat); + R livenessScore = detectVideoFrame(faceDetectManager, image, predictor); mat.release(); if(!livenessScore.isSuccess()){ log.debug("第" + frameIndex + "帧处理失败：" + livenessScore.getMessage()); @@ -175,6 +187,24 @@ public class CommonLivenessModel implements LivenessDetModel{ } } catch (Exception e) { throw new FaceException(e); + } finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + try { + grabber.release(); + } catch (FFmpegFrameGrabber.Exception e) { + throw new RuntimeException(e); + } } return R.fail(R.Status.Unknown); } @@ -262,6 +292,40 @@ public class CommonLivenessModel implements LivenessDetModel{ } } + private R detectVideoFrame(FaceDetectManager faceDetectManager, Image image, Predictor predictor) { + //预处理图片 + Image processedImage = null; + try { + //检测人脸 + R detectResult = faceDetectManager.detectTopFace(image); + if(!detectResult.isSuccess()){ + return R.fail(detectResult.getCode(), detectResult.getMessage()); + } + DetectionInfo detectionInfo = detectResult.getData(); + if(config.getModelEnum() == LivenessModelEnum.IIC_FL_MODEL){ + processedImage = new DJLImagePreprocessor(image, detectionInfo.getDetectionRectangle()) + .setExtendRatio(96f / 112f) + .enableSquarePadding(true) + .enableScaling(true) + .setTargetSize(128) + .enableCenterCrop(true) + .setCenterCropSize(112) + .process(); + } + Float result = null; + if(processedImage != null){ + result = predictor.predict(processedImage); + ImageUtils.releaseOpenCVMat(processedImage); + }else{ + result = predictor.predict(image); + } + LivenessStatus status = result >= config.getRealityThreshold() ? LivenessStatus.LIVE : LivenessStatus.NON_LIVE; + return R.ok(new LivenessResult(status, result)); + } catch (Exception e) { + throw new FaceException("活体检测错误", e); + } + } + @Override public R detectTopFace(Image image) { R faceDetectionResponse = config.getDetectModel().detect(image); diff --git a/face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java b/face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java index 247a6c9..5013961 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java @@ -10,6 +10,7 @@ import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.*; import cn.smartjavaai.common.entity.face.FaceInfo; import cn.smartjavaai.common.entity.face.LivenessResult; @@ -21,14 +22,19 @@ import cn.smartjavaai.common.preprocess.DJLImagePreprocessor; import cn.smartjavaai.common.utils.*; import cn.smartjavaai.face.config.LivenessConfig; import cn.smartjavaai.face.constant.MiniVisionConstant; +import cn.smartjavaai.face.enums.LivenessModelEnum; import cn.smartjavaai.face.exception.FaceException; import cn.smartjavaai.face.factory.LivenessModelFactory; +import cn.smartjavaai.face.model.facedect.FaceDetectManager; import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator; import com.seeta.sdk.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.impl.GenericObjectPool; +import org.bytedeco.javacv.FFmpegFrameGrabber; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.OpenCVFrameConverter; import org.opencv.core.Mat; import javax.imageio.ImageIO; @@ -59,6 +65,8 @@ public class MiniVisionLivenessModel extends CommonLivenessModel{ private GenericObjectPool> sePredictorPool; + private OpenCVFrameConverter.ToOrgOpenCvCoreMat converterToMat = null; + /** * 模型策略 @@ -224,6 +232,147 @@ public class MiniVisionLivenessModel extends CommonLivenessModel{ } } + + protected R detectVideo(FFmpegFrameGrabber grabber) { + Predictor predictor = null; + Predictor sePredictor = null; + try (FaceDetectManager faceDetectManager = new FaceDetectManager(config.getDetectModel())){ + //初始化predictors + faceDetectManager.borrowPredictors(); + predictor = predictorPool.borrowObject(); + sePredictor = sePredictorPool.borrowObject(); + //滑动窗口 + Deque scoreWindow = new ArrayDeque<>(); + grabber.start(); + // 获取视频总帧数 + int totalFrames = grabber.getLengthInFrames(); + log.debug("视频总帧数：{}，检测帧数：{}", totalFrames, config.getFrameCount()); + if(totalFrames < config.getFrameCount()){ + return R.fail(10001, "视频帧数低于检测帧数"); + } + // 逐帧处理视频 + for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) { + if(frameIndex >= config.getMaxVideoDetectFrames()){ + return R.fail(10002, "超出最大检测帧数：" + config.getMaxVideoDetectFrames()); + } + // 获取当前帧 + Frame frame = grabber.grabImage(); + if (frame != null) { + if(converterToMat == null){ + converterToMat = new OpenCVFrameConverter.ToOrgOpenCvCoreMat(); + } + Mat mat = converterToMat.convert(frame); + Image image = SmartImageFactory.getInstance().fromMat(mat); + R livenessScore = detectVideoFrame(faceDetectManager, image, predictor, sePredictor); + mat.release(); + if(!livenessScore.isSuccess()){ + log.debug("第" + frameIndex + "帧处理失败：" + livenessScore.getMessage()); + continue; + }else{ + log.debug("第" + frameIndex + "帧活体检测结果：" + livenessScore); + scoreWindow.add(livenessScore.getData().getScore()); + } + // 如果累计检测帧数 >= 配置值，开始判断 + if (scoreWindow.size() >= config.getFrameCount()) { + float avgScore = (float) scoreWindow.stream() + .mapToDouble(Float::doubleValue) + .average() + .orElse(0.0); + log.debug("滑动窗口平均得分: {}", avgScore); + grabber.stop(); + LivenessStatus livenessStatus = avgScore > config.getRealityThreshold() ? LivenessStatus.LIVE : LivenessStatus.NON_LIVE; + return R.ok(new LivenessResult(livenessStatus, avgScore)); + } + } + } + grabber.stop(); + if(scoreWindow.size() < config.getFrameCount()){ + return R.fail(1000, "有效帧数量不足，无法完成活体检测"); + } + } catch (Exception e) { + throw new FaceException(e); + } finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + if (sePredictor != null) { + try { + sePredictorPool.returnObject(sePredictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + sePredictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + try { + grabber.release(); + } catch (FFmpegFrameGrabber.Exception e) { + throw new RuntimeException(e); + } + } + return R.fail(R.Status.Unknown); + } + + private R detectVideoFrame(FaceDetectManager faceDetectManager, Image image, Predictor predictor, Predictor sePredictor) { + try { + //检测人脸 + R detectResult = faceDetectManager.detectTopFace(image); + if(!detectResult.isSuccess()){ + return R.fail(detectResult.getCode(), detectResult.getMessage()); + } + DetectionInfo detectionInfo = detectResult.getData(); + float[] result = null; + float[] seResult = null; + //预处理图片 + Image processedImage = new DJLImagePreprocessor(image, detectionInfo.getDetectionRectangle()) + .setExtendRatio(2.7f) + .enableSquarePadding(true) + .enableScaling(true) + .setTargetSize(80) + .process(); + result = predictor.predict(processedImage); + ImageUtils.releaseOpenCVMat(processedImage); + //预处理图片 + Image seProcessedImage = new DJLImagePreprocessor(image, detectionInfo.getDetectionRectangle()) + .setExtendRatio(4) + .enableSquarePadding(true) + .enableScaling(true) + .setTargetSize(80) + .process(); + seResult = sePredictor.predict(seProcessedImage); + ImageUtils.releaseOpenCVMat(seProcessedImage); + if(Objects.isNull(result) && Objects.isNull(seResult)){ + throw new FaceException("活体检测错误"); + } + //计算结果 + int maxIndex = ArrayUtils.sumAndFindMaxIndex(result, seResult, 3); + BigDecimal score = Objects.isNull(result) ? BigDecimal.ZERO : BigDecimal.valueOf(result[maxIndex]); + BigDecimal seScore = Objects.isNull(seResult) ? BigDecimal.ZERO : BigDecimal.valueOf(seResult[maxIndex]); + BigDecimal avgSocre = score.add(seScore).divide(BigDecimal.valueOf(2), 2, RoundingMode.HALF_UP); + //活体 + if(maxIndex == 1){ + LivenessStatus livenessStatus = avgSocre.floatValue() > config.getRealityThreshold() ? LivenessStatus.LIVE : LivenessStatus.NON_LIVE; + return R.ok(new LivenessResult(livenessStatus, avgSocre.floatValue())); + }else{//非活体 + return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, BigDecimal.ONE.subtract(avgSocre).floatValue())); + } + } catch (Exception e) { + throw new FaceException("活体检测错误", e); + } + } + public GenericObjectPool> getPredictorPool() { return predictorPool; } diff --git a/face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java b/face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java index ec1ae5e..87fa5b7 100644 --- a/face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java +++ b/face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java @@ -1,7 +1,9 @@ package cn.smartjavaai.face.model.liveness; import ai.djl.engine.Engine; +import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.DetectedObjects; import cn.smartjavaai.common.cv.SmartImageFactory; import cn.smartjavaai.common.entity.*; import cn.smartjavaai.common.entity.face.FaceInfo; @@ -15,6 +17,8 @@ import cn.smartjavaai.common.enums.face.LivenessStatus; import cn.smartjavaai.face.constant.LivenessConstant; import cn.smartjavaai.face.exception.FaceException; import cn.smartjavaai.face.factory.LivenessModelFactory; +import cn.smartjavaai.face.model.facedect.FaceDetectManager; +import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel; import cn.smartjavaai.face.seetaface.NativeLoader; import cn.smartjavaai.face.utils.FaceUtils; import cn.smartjavaai.face.utils.Seetaface6Utils; @@ -23,6 +27,7 @@ import com.seeta.sdk.*; import lombok.extern.slf4j.Slf4j; import nu.pattern.OpenCV; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.impl.GenericObjectPool; import org.bytedeco.javacv.FFmpegFrameGrabber; import org.bytedeco.javacv.Frame; import org.bytedeco.javacv.Java2DFrameUtils; @@ -60,6 +65,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ if(StringUtils.isBlank(config.getModelPath())){ throw new FaceException("modelPath is null"); } + if(Objects.isNull(config.getDetectModel())){ + throw new FaceException("未指定人脸检测模型"); + } this.config = config; //加载依赖库 NativeLoader.loadNativeLibraries(config.getDevice()); @@ -176,10 +184,41 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } } + private R detectVideoFrame(Image image, FaceDetectManager faceDetectManager, FaceAntiSpoofing faceAntiSpoofing) { + //检测人脸 + R detectResult = faceDetectManager.detectTopFace(image); + if(!detectResult.isSuccess()){ + return R.fail(detectResult.getCode(), detectResult.getMessage()); + } + DetectionInfo detectionInfo = detectResult.getData(); + if(Objects.isNull(detectionInfo)){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + if(detectionInfo.getFaceInfo().getKeyPoints() == null || detectionInfo.getFaceInfo().getKeyPoints().isEmpty()){ + return R.fail(1002,"人脸关键点keyPoints为空"); + } + FaceAntiSpoofing.Status status = null; + try { + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaRect seetaRect = Seetaface6Utils.convertToSeetaRect(detectionInfo.getDetectionRectangle()); + SeetaPointF[] landmarks = Seetaface6Utils.convertToSeetaPointF(detectionInfo.getFaceInfo().getKeyPoints()); + //检测视频 + status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks); + return R.ok(new LivenessResult(Seetaface6Utils.convertToLivenessStatus(status))); + } catch (Exception e) { + throw new FaceException("活体检测错误", e); + } + } + + + private R detectVideo(FFmpegFrameGrabber grabber) { FaceAntiSpoofing faceAntiSpoofing = null; - try { + try (FaceDetectManager faceDetectManager = new FaceDetectManager(config.getDetectModel())){ + //初始化predictors + faceDetectManager.borrowPredictors(); faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); //重置视频 faceAntiSpoofing.ResetVideo(); @@ -194,14 +233,14 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ // 逐帧处理视频 for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) { if(frameIndex >= config.getMaxVideoDetectFrames()){ - return R.fail(10002, "超出最大检测帧数：" + config.getMaxVideoDetectFrames()); + return R.fail(10002, "视频中未检测到人脸，超出最大检测帧数：" + config.getMaxVideoDetectFrames()); } // 获取当前帧 Frame frame = grabber.grabImage(); if (frame != null) { BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame); Image image = SmartImageFactory.getInstance().fromBufferedImage(bufferedImage); - R livenessStatus = detectTopFace(image, false); + R livenessStatus = detectVideoFrame(image, faceDetectManager, faceAntiSpoofing); if(!livenessStatus.isSuccess()){ log.debug("第" + frameIndex + "帧处理失败：" + livenessStatus.getMessage()); continue; @@ -225,10 +264,17 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ log.warn("归还Predictor失败", e); } } + try { + grabber.release(); + } catch (FFmpegFrameGrabber.Exception e) { + throw new RuntimeException(e); + } } return R.fail(1000, "有效帧数量不足，无法完成活体检测"); } + + @Override public R detect(Image image) { FaceAntiSpoofing faceAntiSpoofing = null; @@ -246,7 +292,7 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ imageData.data = ImageUtils.getMatrixBGR(image); //检测人脸 SeetaRect[] seetaResult = detectPredictor.Detect(imageData); - if(Objects.isNull(seetaResult)){ + if(Objects.isNull(seetaResult) || seetaResult.length == 0){ return R.fail(R.Status.NO_FACE_DETECTED); } for(SeetaRect seetaRect : seetaResult){ @@ -346,7 +392,7 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ imageData.data = ImageUtils.getMatrixBGR(image); //检测人脸 SeetaRect[] seetaResult = detectPredictor.Detect(imageData); - if(Objects.isNull(seetaResult)){ + if(Objects.isNull(seetaResult) || seetaResult.length == 0){ return R.fail(R.Status.NO_FACE_DETECTED); } SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()]; diff --git a/face/src/main/java/cn/smartjavaai/face/seetaface/SeetaFace6FaceDetPredictors.java b/face/src/main/java/cn/smartjavaai/face/seetaface/SeetaFace6FaceDetPredictors.java new file mode 100644 index 0000000..5e5bda3 --- /dev/null +++ b/face/src/main/java/cn/smartjavaai/face/seetaface/SeetaFace6FaceDetPredictors.java @@ -0,0 +1,27 @@ +package cn.smartjavaai.face.seetaface; + +import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel; +import com.seeta.sdk.FaceDetector; +import com.seeta.sdk.FaceLandmarker; + +/** + * SeetaFace6 人脸检测Detector + * @author dwj + */ +public class SeetaFace6FaceDetPredictors implements AutoCloseable{ + + public FaceDetector faceDetector; + public FaceLandmarker faceLandmarker; + public SeetaFace6FaceDetModel model; + + public SeetaFace6FaceDetPredictors(FaceDetector faceDetector, FaceLandmarker faceLandmarker, SeetaFace6FaceDetModel model) { + this.faceDetector = faceDetector; + this.faceLandmarker = faceLandmarker; + this.model = model; + } + + @Override + public void close(){ + model.returnPredictor(faceDetector, faceLandmarker); + } +} diff --git a/face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java b/face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java index 39595ff..ea915f1 100644 --- a/face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java +++ b/face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java @@ -637,7 +637,8 @@ public class MilvusClient implements VectorDBClient { List result = new ArrayList<>(); for (QueryResultsWrapper.RowRecord row : records) { - String id = (String) row.get(VectorDBConstants.FieldNames.ID_FIELD); + Object idObj = row.get(VectorDBConstants.FieldNames.ID_FIELD); + String id = idObj != null ? idObj.toString() : null; Object vectorObj = row.get(VectorDBConstants.FieldNames.VECTOR_FIELD); float[] vector = null; if (vectorObj instanceof List) { diff --git a/face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java b/face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java index f1f28ae..95ba8bf 100644 --- a/face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java +++ b/face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java @@ -2,6 +2,7 @@ package cn.smartjavaai.face.vector.core; import cn.hutool.core.util.IdUtil; import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.common.executor.GlobalExecutor; import cn.smartjavaai.common.utils.SimilarityUtil; import cn.smartjavaai.face.dao.FaceDao; import cn.smartjavaai.face.entity.FaceSearchParams; @@ -23,12 +24,9 @@ import java.util.stream.Collectors; public class SQLiteClient implements VectorDBClient { private final FaceDao faceDao; - //private final List memoryIndex = new CopyOnWriteArrayList<>(); private final ConcurrentHashMap memoryIndex = new ConcurrentHashMap<>(); private int featureDimension; // 维度 - private final ExecutorService executor = Executors.newFixedThreadPool(4); - private SQLiteConfig config; /** @@ -162,7 +160,7 @@ public class SQLiteClient implements VectorDBClient { return similarity >= faceSearchParams.getThreshold() ? new FaceSearchResult(vector.getId(), similarity, vector.getMetadata()) : null; - }, executor)) + }, GlobalExecutor.getExecutor())) .collect(Collectors.toList()); // 收集结果并过滤null @@ -185,15 +183,7 @@ public class SQLiteClient implements VectorDBClient { @Override public void close() { - executor.shutdown(); - try { - if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { - executor.shutdownNow(); - } - } catch (InterruptedException e) { - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } + } @Override diff --git a/ocr/pom.xml b/ocr/pom.xml index d5d1cef..67ad098 100644 --- a/ocr/pom.xml +++ b/ocr/pom.xml @@ -6,7 +6,7 @@ cn.smartjavaai smartjavaai-parent - 1.0.27 + 1.1.0 ocr @@ -42,7 +42,7 @@ - 1.0.27 + 1.1.0 ocr SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/pom.xml b/pom.xml index 0ca7dfd..c07ea22 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ SmartJavaAI cn.smartjavaai smartjavaai-parent - 1.0.27 + 1.1.0 pom SmartJavaAI @@ -26,7 +26,7 @@ 8 8 UTF-8 - 0.32.0 + 0.34.0 diff --git a/speech/pom.xml b/speech/pom.xml index ee84d79..bfaf7c7 100644 --- a/speech/pom.xml +++ b/speech/pom.xml @@ -6,7 +6,7 @@ cn.smartjavaai smartjavaai-parent - 1.0.27 + 1.1.0 speech @@ -57,7 +57,7 @@ - 1.0.27 + 1.1.0 speech SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/translate/pom.xml b/translate/pom.xml index 9d8746d..2469f02 100644 --- a/translate/pom.xml +++ b/translate/pom.xml @@ -25,7 +25,7 @@ - 1.0.27 + 1.1.0 translate SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/vision/pom.xml b/vision/pom.xml index 7dfc497..ed6baf0 100644 --- a/vision/pom.xml +++ b/vision/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.27 + 1.1.0 vision - 1.0.27 + 1.1.0 vision SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModelFactory.java b/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModelFactory.java index 27d43cf..5312f14 100644 --- a/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/action/model/ActionRecModelFactory.java @@ -55,7 +55,7 @@ public class ActionRecModelFactory { throw new DetectionException("未配置模型"); } return modelMap.computeIfAbsent(config.getModelEnum(), k -> { - return createFaceDetModel(config); + return createModel(config); }); } @@ -64,7 +64,7 @@ public class ActionRecModelFactory { * @param config * @return */ - private ActionRecModel createFaceDetModel(ActionRecModelConfig config) { + private ActionRecModel createModel(ActionRecModelConfig config) { Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new DetectionException("Unsupported model"); diff --git a/vision/src/main/java/cn/smartjavaai/clip/model/ClipModel.java b/vision/src/main/java/cn/smartjavaai/clip/model/ClipModel.java index c5c8718..7b8f169 100644 --- a/vision/src/main/java/cn/smartjavaai/clip/model/ClipModel.java +++ b/vision/src/main/java/cn/smartjavaai/clip/model/ClipModel.java @@ -66,8 +66,9 @@ public interface ClipModel extends AutoCloseable{ /** * 图片特征比较 - * @param image1 图1 - * @param image2 图2 + * @param image1 + * @param image2 + * @param scale * @return */ default R compareImage(Image image1, Image image2, float scale){ @@ -115,10 +116,12 @@ public interface ClipModel extends AutoCloseable{ throw new UnsupportedOperationException("默认不支持该功能"); } + /** - * 文本特征比较 - * @param feature1 文本1 - * @param feature2 文本2 + * 特征比较 + * @param feature1 + * @param feature2 + * @param scale * @return */ default R compareFeatures(float[] feature1, float[] feature2, float scale){ diff --git a/vision/src/main/java/cn/smartjavaai/clip/model/ClipModelFactory.java b/vision/src/main/java/cn/smartjavaai/clip/model/ClipModelFactory.java index e87b796..b9cab11 100644 --- a/vision/src/main/java/cn/smartjavaai/clip/model/ClipModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/clip/model/ClipModelFactory.java @@ -56,7 +56,7 @@ public class ClipModelFactory { throw new DetectionException("未配置模型"); } return modelMap.computeIfAbsent(config.getModelEnum(), k -> { - return createFaceDetModel(config); + return createModel(config); }); } @@ -65,7 +65,7 @@ public class ClipModelFactory { * @param config * @return */ - private ClipModel createFaceDetModel(ClipModelConfig config) { + private ClipModel createModel(ClipModelConfig config) { Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new DetectionException("Unsupported model"); diff --git a/vision/src/main/java/cn/smartjavaai/cls/model/ClsModelFactory.java b/vision/src/main/java/cn/smartjavaai/cls/model/ClsModelFactory.java index 03953e8..0863a3b 100644 --- a/vision/src/main/java/cn/smartjavaai/cls/model/ClsModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/cls/model/ClsModelFactory.java @@ -55,7 +55,7 @@ public class ClsModelFactory { throw new DetectionException("未配置模型"); } return modelMap.computeIfAbsent(config.getModelEnum(), k -> { - return createFaceDetModel(config); + return createModel(config); }); } @@ -64,7 +64,7 @@ public class ClsModelFactory { * @param config * @return */ - private ClsModel createFaceDetModel(ClsModelConfig config) { + private ClsModel createModel(ClsModelConfig config) { Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new DetectionException("Unsupported model"); diff --git a/vision/src/main/java/cn/smartjavaai/instanceseg/model/InstanceSegModelFactory.java b/vision/src/main/java/cn/smartjavaai/instanceseg/model/InstanceSegModelFactory.java index ca52395..e5549e6 100644 --- a/vision/src/main/java/cn/smartjavaai/instanceseg/model/InstanceSegModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/instanceseg/model/InstanceSegModelFactory.java @@ -54,7 +54,7 @@ public class InstanceSegModelFactory { throw new DetectionException("未配置模型"); } return modelMap.computeIfAbsent(config.getModelEnum(), k -> { - return createFaceDetModel(config); + return createModel(config); }); } @@ -63,7 +63,7 @@ public class InstanceSegModelFactory { * @param config * @return */ - private InstanceSegModel createFaceDetModel(InstanceSegModelConfig config) { + private InstanceSegModel createModel(InstanceSegModelConfig config) { Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new DetectionException("Unsupported model"); diff --git a/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModelFactory.java b/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModelFactory.java index 2cb4f79..16d01a4 100644 --- a/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModelFactory.java @@ -54,7 +54,7 @@ public class ObbDetModelFactory { throw new DetectionException("未配置模型"); } return modelMap.computeIfAbsent(config.getModelEnum(), k -> { - return createFaceDetModel(config); + return createModel(config); }); } @@ -63,7 +63,7 @@ public class ObbDetModelFactory { * @param config * @return */ - private ObbDetModel createFaceDetModel(ObbDetModelConfig config) { + private ObbDetModel createModel(ObbDetModelConfig config) { Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new DetectionException("Unsupported model"); diff --git a/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/PersonDetModelFactory.java b/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/PersonDetModelFactory.java index d480cba..af0eab0 100644 --- a/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/PersonDetModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/objectdetection/model/person/PersonDetModelFactory.java @@ -56,7 +56,7 @@ public class PersonDetModelFactory { throw new DetectionException("未配置模型"); } return modelMap.computeIfAbsent(config.getModelEnum(), k -> { - return createFaceDetModel(config); + return createModel(config); }); } @@ -65,7 +65,7 @@ public class PersonDetModelFactory { * @param config * @return */ - private PersonDetModel createFaceDetModel(PersonDetModelConfig config) { + private PersonDetModel createModel(PersonDetModelConfig config) { Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new DetectionException("Unsupported model"); diff --git a/vision/src/main/java/cn/smartjavaai/pose/model/PoseDetModelFactory.java b/vision/src/main/java/cn/smartjavaai/pose/model/PoseDetModelFactory.java index 8b5cc18..13172cd 100644 --- a/vision/src/main/java/cn/smartjavaai/pose/model/PoseDetModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/pose/model/PoseDetModelFactory.java @@ -55,7 +55,7 @@ public class PoseDetModelFactory { throw new DetectionException("未配置模型"); } return modelMap.computeIfAbsent(config.getModelEnum(), k -> { - return createFaceDetModel(config); + return createModel(config); }); } @@ -64,7 +64,7 @@ public class PoseDetModelFactory { * @param config * @return */ - private PoseModel createFaceDetModel(PoseModelConfig config) { + private PoseModel createModel(PoseModelConfig config) { Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new DetectionException("Unsupported model"); diff --git a/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModelFactory.java b/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModelFactory.java index 285ad1c..7da0849 100644 --- a/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModelFactory.java +++ b/vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModelFactory.java @@ -54,7 +54,7 @@ public class SemSegModelFactory { throw new DetectionException("未配置模型"); } return modelMap.computeIfAbsent(config.getModelEnum(), k -> { - return createFaceDetModel(config); + return createModel(config); }); } @@ -63,7 +63,7 @@ public class SemSegModelFactory { * @param config * @return */ - private SemSegModel createFaceDetModel(SemSegModelConfig config) { + private SemSegModel createModel(SemSegModelConfig config) { Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new DetectionException("Unsupported model"); diff --git a/vision/src/main/java/cn/smartjavaai/zeroshot/config/ZeroDetConfig.java b/vision/src/main/java/cn/smartjavaai/zeroshot/config/ZeroDetConfig.java new file mode 100644 index 0000000..30ae953 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/zeroshot/config/ZeroDetConfig.java @@ -0,0 +1,48 @@ +package cn.smartjavaai.zeroshot.config; + +import cn.smartjavaai.common.config.ModelConfig; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.zeroshot.enums.ZeroDetModelEnum; +import lombok.Data; + +import java.util.List; + +/** + * 零样本目标检测模型参数配置 + * + * @author dwj + */ +@Data +public class ZeroDetConfig extends ModelConfig { + + /** + * 模型 + */ + private ZeroDetModelEnum modelEnum; + + + /** + * 模型路径 + */ + private String modelPath; + + /** + * 置信度阈值 + */ + private float threshold = 0.3f; + + + public ZeroDetConfig() { + } + + public ZeroDetConfig(ZeroDetModelEnum modelEnum, DeviceEnum device) { + this.modelEnum = modelEnum; + setDevice(device); + } + + public ZeroDetConfig(ZeroDetModelEnum modelEnum) { + this.modelEnum = modelEnum; + } + + +} diff --git a/vision/src/main/java/cn/smartjavaai/zeroshot/criteria/ZeroDetCriteriaFactory.java b/vision/src/main/java/cn/smartjavaai/zeroshot/criteria/ZeroDetCriteriaFactory.java new file mode 100644 index 0000000..d953eb6 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/zeroshot/criteria/ZeroDetCriteriaFactory.java @@ -0,0 +1,53 @@ +package cn.smartjavaai.zeroshot.criteria; + +import ai.djl.Device; +import ai.djl.huggingface.translator.ZeroShotObjectDetectionTranslatorFactory; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.VisionLanguageInput; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.translator.YoloWorldTranslatorFactory; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.TranslatorFactory; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.zeroshot.config.ZeroDetConfig; +import cn.smartjavaai.zeroshot.enums.ZeroDetModelEnum; +import org.apache.commons.lang3.StringUtils; + +import java.nio.file.Paths; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 零样本目标检测Criteria工厂 + * @author dwj + */ +public class ZeroDetCriteriaFactory { + + + public static Criteria createCriteria(ZeroDetConfig config) { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId()); + } + TranslatorFactory translatorFactory = null; + if(config.getModelEnum() == ZeroDetModelEnum.OWLV2_BASE_PATCH16){ + translatorFactory = new ZeroShotObjectDetectionTranslatorFactory(); + }else if(config.getModelEnum() == ZeroDetModelEnum.YOLOV8S_WORLDV2){ + translatorFactory = new YoloWorldTranslatorFactory(); + } + Criteria criteria = + Criteria.builder() + .setTypes(VisionLanguageInput.class, DetectedObjects.class) + .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : + config.getModelEnum().getModelUri()) + .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) + .optDevice(device) + .optEngine(config.getModelEnum().getEngine()) + .optTranslatorFactory(translatorFactory) + .optProgress(new ProgressBar()) + .build(); + + return criteria; + } +} diff --git a/vision/src/main/java/cn/smartjavaai/zeroshot/entity/DetectParams.java b/vision/src/main/java/cn/smartjavaai/zeroshot/entity/DetectParams.java new file mode 100644 index 0000000..a297396 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/zeroshot/entity/DetectParams.java @@ -0,0 +1,17 @@ +package cn.smartjavaai.zeroshot.entity; + +import lombok.Data; + +/** + * 检测参数 + * @author dwj + */ +@Data +public class DetectParams { + + /** + * 置信度阈值 + */ + private float threshold = 0.3f; + +} diff --git a/vision/src/main/java/cn/smartjavaai/zeroshot/enums/ZeroDetModelEnum.java b/vision/src/main/java/cn/smartjavaai/zeroshot/enums/ZeroDetModelEnum.java new file mode 100644 index 0000000..140397e --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/zeroshot/enums/ZeroDetModelEnum.java @@ -0,0 +1,45 @@ +package cn.smartjavaai.zeroshot.enums; + +/** + * 零样本目标检测模型枚举 + * @author dwj + */ +public enum ZeroDetModelEnum { + + YOLOV8S_WORLDV2("PyTorch", "djl://ai.djl.pytorch/yolov8s-worldv2"), + OWLV2_BASE_PATCH16("PyTorch", "djl://ai.djl.huggingface.pytorch/google/owlv2-base-patch16"); + + + /** + * 根据名称获取枚举 (忽略大小写和下划线变体) + */ + public static ZeroDetModelEnum fromName(String name) { + String formatted = name.trim().toUpperCase().replaceAll("[-_]", ""); + for (ZeroDetModelEnum model : values()) { + if (model.name().replaceAll("_", "").equals(formatted)) { + return model; + } + } + throw new IllegalArgumentException("未知模型名称: " + name); + } + + private final String modelUri; + + /** + * 模型引擎 + */ + private final String engine; + + ZeroDetModelEnum(String engine, String modelUri) { + this.modelUri = modelUri; + this.engine = engine; + } + + public String getModelUri() { + return modelUri; + } + + public String getEngine() { + return engine; + } +} diff --git a/vision/src/main/java/cn/smartjavaai/zeroshot/exception/ZeroDetException.java b/vision/src/main/java/cn/smartjavaai/zeroshot/exception/ZeroDetException.java new file mode 100644 index 0000000..e13918e --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/zeroshot/exception/ZeroDetException.java @@ -0,0 +1,29 @@ +package cn.smartjavaai.zeroshot.exception; + +/** + * 零样本目标检测异常 + * @author dwj + */ +public class ZeroDetException extends RuntimeException{ + + public ZeroDetException() { + super(); + } + + public ZeroDetException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public ZeroDetException(String message, Throwable cause) { + super(message, cause); + } + + public ZeroDetException(String message) { + super(message); + } + + public ZeroDetException(Throwable cause) { + super(cause); + } + +} diff --git a/vision/src/main/java/cn/smartjavaai/zeroshot/model/CommonZeroDetModel.java b/vision/src/main/java/cn/smartjavaai/zeroshot/model/CommonZeroDetModel.java new file mode 100644 index 0000000..0d2c1f9 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/zeroshot/model/CommonZeroDetModel.java @@ -0,0 +1,162 @@ +package cn.smartjavaai.zeroshot.model; + +import ai.djl.MalformedModelException; +import ai.djl.engine.Engine; +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.VisionLanguageInput; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ZooModel; +import cn.smartjavaai.common.cv.SmartImageFactory; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.pool.PredictorFactory; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import cn.smartjavaai.vision.utils.DetectedObjectsFilter; +import cn.smartjavaai.vision.utils.DetectorUtils; +import cn.smartjavaai.zeroshot.config.ZeroDetConfig; +import cn.smartjavaai.zeroshot.criteria.ZeroDetCriteriaFactory; +import cn.smartjavaai.zeroshot.exception.ZeroDetException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.pool2.impl.GenericObjectPool; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Objects; + +/** + * 零样本目标检测模型 + * @author dwj + */ +@Slf4j +public class CommonZeroDetModel implements ZeroDetModel { + + + private ZeroDetConfig config; + + private ZooModel model; + + private GenericObjectPool> predictorPool; + + @Override + public void loadModel(ZeroDetConfig config) { + if(Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型枚举"); + } + Criteria criteria = ZeroDetCriteriaFactory.createCriteria(config); + this.config = config; + try { + model = criteria.loadModel(); + // 创建池子：每个线程独享 Predictor + this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); + int predictorPoolSize = config.getPredictorPoolSize(); + if(config.getPredictorPoolSize() <= 0){ + predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数 + } + predictorPool.setMaxTotal(predictorPoolSize); + log.debug("当前设备: " + model.getNDManager().getDevice()); + log.debug("当前引擎: " + Engine.getInstance().getEngineName()); + log.debug("模型推理器线程池最大数量: " + predictorPoolSize); + } catch (IOException | ModelNotFoundException | MalformedModelException e) { + throw new DetectionException("模型加载失败", e); + } + } + + @Override + public R detect(Image image, String[] candidates) { + DetectedObjects detectedObjects = detectCore(new VisionLanguageInput(image, candidates)); + DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, image); + return R.ok(detectionResponse); + } + + + /** + * 模型核心推理方法 + * @param input + * @return + */ + @Override + public DetectedObjects detectCore(VisionLanguageInput input) { + Predictor predictor = null; + try { + predictor = predictorPool.borrowObject(); + DetectedObjects detectedObjects = predictor.predict(input); + //过滤 + if(Objects.nonNull(detectedObjects) && detectedObjects.getNumberOfObjects() > 0){ + DetectedObjectsFilter detectedObjectsFilter = new DetectedObjectsFilter(null, config.getThreshold()); + detectedObjects = detectedObjectsFilter.filter(detectedObjects); + } + return detectedObjects; + } catch (Exception e) { + throw new DetectionException("零样本目标检测错误", e); + }finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + log.debug("释放资源"); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + } + } + + @Override + public R detectAndDraw(Image image, String[] candidates) { + DetectedObjects detectedObjects = detectCore(new VisionLanguageInput(image, candidates)); + image.drawBoundingBoxes(detectedObjects); + DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, image); + detectionResponse.setDrawnImage(image); + return R.ok(detectionResponse); + } + + @Override + public R detectAndDraw(String[] candidates, String imagePath, String outputPath) { + try { + Image img = SmartImageFactory.getInstance().fromFile(Paths.get(imagePath)); + DetectedObjects detectedObjects = detectCore(new VisionLanguageInput(img, candidates)); + img.drawBoundingBoxes(detectedObjects); + img.save(Files.newOutputStream(Paths.get(outputPath)), "png"); + DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, img); + return R.ok(detectionResponse); + } catch (IOException e) { + throw new ZeroDetException(e); + } + } + + private boolean fromFactory = false; + + @Override + public void setFromFactory(boolean fromFactory) { + this.fromFactory = fromFactory; + } + public boolean isFromFactory() { + return fromFactory; + } + + @Override + public void close() throws Exception { + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + } +} diff --git a/vision/src/main/java/cn/smartjavaai/zeroshot/model/ZeroDetModel.java b/vision/src/main/java/cn/smartjavaai/zeroshot/model/ZeroDetModel.java new file mode 100644 index 0000000..9cd83fe --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/zeroshot/model/ZeroDetModel.java @@ -0,0 +1,50 @@ +package cn.smartjavaai.zeroshot.model; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.VisionLanguageInput; +import ai.djl.modality.cv.output.DetectedObjects; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.zeroshot.config.ZeroDetConfig; + +/** + * 零样本目标检测模型 + * @author dwj + */ + +public interface ZeroDetModel extends AutoCloseable{ + + + /** + * 加载模型 + * @param config + */ + void loadModel(ZeroDetConfig config); + + /** + * 零样本目标检测 + * @param image + * @return + */ + default R detect(Image image, String[] candidates){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default DetectedObjects detectCore(VisionLanguageInput input){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default R detectAndDraw(Image image, String[] candidates){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default R detectAndDraw(String[] candidates, String imagePath, String outputPath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default void setFromFactory(boolean fromFactory){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + +} diff --git a/vision/src/main/java/cn/smartjavaai/zeroshot/model/ZeroDetModelFactory.java b/vision/src/main/java/cn/smartjavaai/zeroshot/model/ZeroDetModelFactory.java new file mode 100644 index 0000000..ead9603 --- /dev/null +++ b/vision/src/main/java/cn/smartjavaai/zeroshot/model/ZeroDetModelFactory.java @@ -0,0 +1,109 @@ +package cn.smartjavaai.zeroshot.model; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import cn.smartjavaai.zeroshot.config.ZeroDetConfig; +import cn.smartjavaai.zeroshot.enums.ZeroDetModelEnum; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 零样本目标检测 模型工厂 + * @author dwj + */ +@Slf4j +public class ZeroDetModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile ZeroDetModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + // 私有构造函数，防止外部创建实例 + private ZeroDetModelFactory() {} + + // 双重检查锁定的单例方法 + public static ZeroDetModelFactory getInstance() { + if (instance == null) { + synchronized (ZeroDetModelFactory.class) { + if (instance == null) { + instance = new ZeroDetModelFactory(); + } + } + } + return instance; + } + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public ZeroDetModel getModel(ZeroDetConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new DetectionException("未配置模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private ZeroDetModel createModel(ZeroDetConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new DetectionException("Unsupported model"); + } + ZeroDetModel model = null; + try { + model = (ZeroDetModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new DetectionException(e); + } + model.loadModel(config); + model.setFromFactory(true); + return model; + } + + + /** + * 注册模型 + * @param modelEnum + * @param clazz + */ + private static void registerAlgorithm(ZeroDetModelEnum modelEnum, Class clazz) { + registry.put(modelEnum, clazz); + } + + + /** + * 移除缓存的模型 + * @param modelEnum + */ + public static void removeFromCache(ZeroDetModelEnum modelEnum) { + modelMap.remove(modelEnum); + } + + + // 初始化默认算法 + static { + registerAlgorithm(ZeroDetModelEnum.YOLOV8S_WORLDV2, CommonZeroDetModel.class); + registerAlgorithm(ZeroDetModelEnum.OWLV2_BASE_PATCH16, CommonZeroDetModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } +} +