diff --git a/README.md b/README.md index 7f299ed..75a1c0a 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ### ✅ 已实现功能 - **人脸识别** + - 支持模型： - 人脸检测、人脸识别、人脸比对1:1、人脸比对1:N(支持向量数据库milvus/sqlite)、人脸库注册、人脸库删除 - 5点人脸关键点定位 - 人脸属性检测（性别、年龄、口罩、眼睛状态、脸部姿态） @@ -279,7 +280,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 cn.smartjavaai smartjavaai-all - 1.0.17 + 1.0.19 ``` ### 3、完整示例代码 @@ -318,6 +319,19 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ## 近期更新日志 +## [v1.0.19] - 2025-07-06 +- 人脸模块：新增小视科技（MiniVision）活体检测模型 +- 人脸模块：新增阿里通义工作室活体检测模型 +- 人脸模块：新增 2 个表情识别模型 +- 人脸模块：新增 InsightFace 和 ElasticFace 人脸识别模型 +- 人脸模块：新增 Seetaface6 质量评估模型 +- 目标检测模块：支持更多自定义模型参数配置 +- 人脸模块：支持 Base64 编码图片输入 +- 通用功能：实现 AutoCloseable 接口，支持资源自动释放 +- OCR 模块：修复加方向矫正后无法连续识别的问题 +- 人脸模块：修复人脸更新后的缓存异常问题 +- 其他：优化部分功能与细节体验 + ## [v1.0.17] - 2025-06-18 - 新增机器翻译模块：支持 200+ 种语言之间的相互翻译 - 人脸识别模块：修复批量删除人脸数据时的异常问题 diff --git a/examples/face-example/.gitignore b/examples/face-example/.gitignore new file mode 100644 index 0000000..93dbf83 --- /dev/null +++ b/examples/face-example/.gitignore @@ -0,0 +1,7 @@ +.idea +.idea/ +target +log +*.iml +/.settings/ +/logging.file_IS_UNDEFINED/ diff --git a/examples/face-example/README.md b/examples/face-example/README.md new file mode 100644 index 0000000..cbdc26b --- /dev/null +++ b/examples/face-example/README.md @@ -0,0 +1,103 @@ +# 人脸识别示例 + +本项目提供了一系列关于人脸识别相关功能的 Java 示例代码，适用于图像处理、人脸检测、活体检测等场景。所有示例基于 SmartJavaAI 的 SDK 实现。 + +## 📁 项目结构 + +``` +src/main/java/smartai/examples/face/ +├── attribute/ # 人脸属性检测模块 +│ └── FaceAttributeDetDemo.java # 检测性别、年龄等人脸属性 +├── expression/ # 表情识别模块 +│ └── ExpressionRecDemo.java # 识别中性、高兴、悲伤等7种表情 +├── facedet/ # 人脸检测模块 +│ └── FaceDetDemo.java # 检测图片或视频中的人脸并绘制人脸框 +├── facerec/ # 人脸识别模块（1:1, 1:N） +│ └── FaceRecDemo.java # 提取人脸特征、比对、注册与搜索人脸库 +├── liveness/ # 活体检测模块 +│ └── LivenessDetDemo.java # 判断是否为真人（静态图或摄像头视频流） +├── quality/ # 人脸质量评估模块 +│ └── FaceQualityDetDemo.java # 评估亮度、清晰度、完整性、姿态、分辨率 +└── ViewerFrame.java # 图像显示窗口工具类（用于在 GUI 中展示图像） +``` + + +--- + +## 🧩 功能模块说明 + +### 1. 人脸属性检测 ([FaceAttributeDetDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java)) +- **功能**：识别性别、年龄、眼镜佩戴状态、种族等属性。 +- **使用模型**：SeetaFace6 等。 + +--- + +### 2. 表情识别 ([ExpressionRecDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java)) +- **功能**：识别 7 种面部表情：中性、高兴、悲伤、惊讶、恐惧、厌恶、愤怒。 +- **支持模式**：单人、多人、摄像头实时检测。 + +--- + +### 3. 人脸检测 ([FaceDetDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java)) +- **功能**：识别图像或视频中的人脸区域，并返回人脸边界框。 +- **支持模型**：RetinaFace、SeetaFace6。 +--- + +### 4. 人脸识别 ([FaceRecDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java)) +- **功能**：提取人脸特征、进行人脸比对（1:1）、人脸搜索（1:N）、人脸注册管理。 +- **支持数据库**：SQLite、Milvus 向量数据库。 + +--- + +### 5. 活体检测 ([LivenessDetDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java)) +- **功能**：判断输入图像中人脸是否为真实人脸（非照片、视频伪造）。 +- **支持模型**：IIC-FL、MiniVision（双模型融合）。 + +--- + +### 6. 人脸质量评估 ([FaceQualityDetDemo.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java)) +- **功能**：评估人脸图像的质量指标，包括： + - 亮度 (Brightness) + - 完整度 (Completeness) + - 清晰度 (Clarity) + - 姿态 (Pose) + - 分辨率 (Resolution) + +--- + +### 7. 工具类 ([ViewerFrame.java](file:///Users/xxx/Documents/idea_workplace/SmartJavaAI/examples/face-example/src/main/java/smartai/examples/face/ViewerFrame.java)) +- **功能**：GUI 显示组件，用于展示图像处理结果（如人脸框、表情、活体状态等）。 +- **用途**：支持摄像头实时检测时的结果可视化。 + +--- + +## ⚙️ 配置要求 + +- **运行环境**： + - JDK 1.8 或更高版本 + - IntelliJ IDEA 推荐作为开发 IDE +- **依赖库**： + - OpenCV、DJL、SmartJavaAI SDK +- **模型路径**： + - 所有模型需下载并配置正确的路径（参考各 demo 注释中的链接） + +--- + +## 🚀 快速开始 + +1. 克隆项目到本地： + +2. 导入项目至 IntelliJ IDEA。 + +3. 根据需要修改模型路径（见各 demo 中注释）。 + +4. 运行对应的 JUnit 测试类方法即可体验各项功能。 + +--- + +## 📄 文档 + +有关完整使用说明，请查阅 SmartJavaAI 官方文档： +[http://doc.smartjavaai.cn](http://doc.smartjavaai.cn) + +--- diff --git a/examples/face-example/pom.xml b/examples/face-example/pom.xml new file mode 100644 index 0000000..3c21eef --- /dev/null +++ b/examples/face-example/pom.xml @@ -0,0 +1,308 @@ + + + 4.0.0 + + cn.smartjavaai + face-example + 1.0.0-SNAPSHOT + + + 11 + 11 + UTF-8 + 1.0.19 + + smartai.examples.face.facedet.FaceDetDemo + + 1.5.10 + + macosx-arm64 + linux-x86_64 + linux-arm64 + windows-x86_64 + + + win-x86_64 + linux-x86_64 + linux-aarch64 + osx-aarch64 + + + + + + cn.smartjavaai + smartjavaai-bom + ${smartjavaai.version} + pom + + import + + + + + + + + commons-cli + commons-cli + 1.9.0 + + + commons-io + commons-io + 2.17.0 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.1 + + + org.testng + testng + 7.10.2 + test + + + + + ch.qos.logback + logback-classic + 1.2.3 + + + org.slf4j + slf4j-api + 1.7.30 + + + + com.alibaba + fastjson + 1.2.83 + + + + junit + junit + 4.13.2 + + + + + cn.smartjavaai + smartjavaai-face + + + + + + ai.djl.pytorch + pytorch-jni + 2.5.1-0.32.0 + runtime + + + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.windows-x86_64} + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.windows-x86_64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.windows-x86_64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.windows-x86_64} + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.windows-x86_64} + 2.5.1 + runtime + + + + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.linux-x86_64} + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.linux-x86_64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.linux-x86_64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.linux-x86_64} + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.linux-x86_64} + 2.5.1 + runtime + + + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.macosx-arm64} + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.macosx-arm64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.macosx-arm64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.macosx-arm64} + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.osx-aarch64} + 2.5.1 + runtime + + + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.linux-arm64} + + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.linux-arm64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.linux-arm64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.linux-arm64} + + + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-aarch64} + 2.5.1 + runtime + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.0 + + + package + shade + + false + + + + ${exec.mainClass} + + + + + + + + + + + + aliyunmaven + 阿里云公共仓库 + https://maven.aliyun.com/repository/public + + true + + + false + + + + + + + + + + + diff --git a/examples/face-example/src/main/java/smartai/examples/face/ViewerFrame.java b/examples/face-example/src/main/java/smartai/examples/face/ViewerFrame.java new file mode 100644 index 0000000..5c1ad32 --- /dev/null +++ b/examples/face-example/src/main/java/smartai/examples/face/ViewerFrame.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package smartai.examples.face; + +import javax.swing.*; +import java.awt.*; +import java.awt.image.BufferedImage; + +public class ViewerFrame { + + private JFrame frame; + private ImagePanel imagePanel; + + public ViewerFrame(int width, int height) { + frame = new JFrame("Demo"); + imagePanel = new ImagePanel(); + frame.setLayout(new BorderLayout()); + frame.add(BorderLayout.CENTER, imagePanel); + + JOptionPane.setRootFrame(frame); + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); + if (width > screenSize.width) { + width = screenSize.width; + } + Dimension frameSize = new Dimension(width, height); + frame.setSize(frameSize); + frame.setLocation((screenSize.width - width) / 2, (screenSize.height - height) / 2); + frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE); + frame.setVisible(true); + } + + public void showImage(BufferedImage image) { + imagePanel.setImage(image); + SwingUtilities.invokeLater( + () -> { + frame.repaint(); + frame.pack(); + }); + } + + private static final class ImagePanel extends JPanel { + + private BufferedImage image; + + void setImage(BufferedImage image) { + this.image = image; + } + + @Override + public void paintComponent(Graphics g) { + super.paintComponent(g); + if (image == null) { + return; + } + + g.drawImage(image, 0, 0, null); + setPreferredSize(new Dimension(image.getWidth(), image.getHeight())); + } + } +} diff --git a/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java new file mode 100644 index 0000000..bb664aa --- /dev/null +++ b/examples/face-example/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java @@ -0,0 +1,130 @@ +package smartai.examples.face.attribute; + +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.entity.face.FaceAttribute; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.face.config.FaceAttributeConfig; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.enums.FaceAttributeModelEnum; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import cn.smartjavaai.face.factory.FaceAttributeModelFactory; +import cn.smartjavaai.face.factory.FaceDetModelFactory; +import cn.smartjavaai.face.model.attribute.FaceAttributeModel; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.utils.FaceUtils; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; + +/** + * 人脸属性检测demo + * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * @author dwj + */ +@Slf4j +public class FaceAttributeDetDemo { + + + public FaceAttributeModel getFaceAttributeModel() { + FaceAttributeConfig config = new FaceAttributeConfig(); + config.setModelEnum(FaceAttributeModelEnum.SEETA_FACE6_MODEL); + //需替换为实际模型存储路径 + config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); + return FaceAttributeModelFactory.getInstance().getModel(config); + } + + public FaceDetModel getFaceDetModel() { + //需替换为实际模型存储路径 + String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; + FaceDetConfig faceDetectModelConfig = new FaceDetConfig(); + faceDetectModelConfig.setModelEnum(FaceDetModelEnum.SEETA_FACE6_MODEL); + faceDetectModelConfig.setModelPath(modelPath); + return FaceDetModelFactory.getInstance().getModel(faceDetectModelConfig); + } + + + /** + * 人脸属性检测（多人脸） + */ + @Test + public void testFaceAttributeDetect(){ + try (FaceAttributeModel faceAttributeModel = getFaceAttributeModel()){ + DetectionResponse detectionResponse = faceAttributeModel.detect("src/main/resources/iu_1.jpg"); + //绘制并导出人脸属性图片，小人脸仅有人脸框 + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + FaceUtils.drawBoxesWithFaceAttribute(image, detectionResponse,"C:/Users/Administrator/Downloads/double_person_.png"); + log.info("人脸属性检测结果：{}", JSONObject.toJSONString(detectionResponse)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 图片人脸属性检测（分数最高人脸） + */ + @Test + public void testFaceAttributeDetect2(){ + try (FaceAttributeModel faceAttributeModel = getFaceAttributeModel()){ + FaceAttribute faceAttribute = faceAttributeModel.detectTopFace("src/main/resources/iu_1.jpg"); + log.info("人脸属性检测结果：{}", JSONObject.toJSONString(faceAttribute)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 图片多人脸属性检测（基于已检测出的人脸区域和关键点） + */ + @Test + public void testFaceAttributeDetect3(){ + try (FaceAttributeModel faceAttributeModel = getFaceAttributeModel()){ + FaceAttribute faceAttribute = faceAttributeModel.detectTopFace("src/main/resources/iu_1.jpg"); + log.info("人脸属性检测结果：{}", JSONObject.toJSONString(faceAttribute)); + } catch (Exception e) { + e.printStackTrace(); + } + //人脸检测 + + + + } + + /** + * 图片单人脸人脸属性检测（基于已检测出的人脸区域和关键点） + */ + @Test + public void testFaceAttributeDetect4(){ + try (FaceDetModel faceDetModel = getFaceDetModel(); + FaceAttributeModel faceAttributeModel = getFaceAttributeModel()){ + //人脸检测 + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + R detectionResponse = faceDetModel.detect(image); + if(detectionResponse.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); + //检测到人脸 + if(detectionResponse.getData() != null && detectionResponse.getData().getDetectionInfoList() != null && detectionResponse.getData().getDetectionInfoList().size() > 0){ + for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + FaceAttribute faceAttribute = faceAttributeModel.detect(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + log.info("人脸属性检测结果：{}", JSONObject.toJSONString(faceAttribute)); + } + } + }else{ + log.info("人脸检测失败：{}", detectionResponse.getMessage()); + } + } catch (Exception e){ + e.printStackTrace(); + } + } + + +} diff --git a/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java b/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java new file mode 100644 index 0000000..2c74f78 --- /dev/null +++ b/examples/face-example/src/main/java/smartai/examples/face/expression/ExpressionRecDemo.java @@ -0,0 +1,253 @@ +package smartai.examples.face.expression; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.entity.face.ExpressionResult; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.enums.face.FacialExpression; +import cn.smartjavaai.common.enums.face.LivenessStatus; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.config.FaceExpressionConfig; +import cn.smartjavaai.face.constant.FaceDetectConstant; +import cn.smartjavaai.face.enums.ExpressionModelEnum; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.factory.ExpressionModelFactory; +import cn.smartjavaai.face.factory.FaceDetModelFactory; +import cn.smartjavaai.face.model.expression.ExpressionModel; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.liveness.LivenessDetModel; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import nu.pattern.OpenCV; +import org.junit.Test; +import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; +import org.opencv.videoio.VideoCapture; +import org.opencv.videoio.Videoio; +import smartai.examples.face.ViewerFrame; + +import javax.imageio.ImageIO; +import javax.swing.*; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; + +/** + * 表情识别demo + * 支持识别7种表情：neutral（中性）、happy（高兴）、sad（悲伤）、surprise（惊讶）、fear（恐惧）、disgust（厌恶）、anger（愤怒） + * @author dwj + */ +@Slf4j +public class ExpressionRecDemo { + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + /** + * 获取人脸检测模型 + * @return + */ + public FaceDetModel getFaceDetModel(){ + FaceDetConfig config = new FaceDetConfig(); + config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型 + config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setDevice(device); + return FaceDetModelFactory.getInstance().getModel(config); + } + + + /** + * 获取表情识别模型 + * @return + */ + public ExpressionModel getExpressionModel(){ + FaceExpressionConfig config = new FaceExpressionConfig(); + config.setModelEnum(ExpressionModelEnum.FrEmotion); + config.setModelPath("/Users/xxx/Documents/develop/model/emotion/fr_expression.onnx"); + config.setDevice(device); + config.setAlign(true); + config.setDetectModel(getFaceDetModel()); + return ExpressionModelFactory.getInstance().getModel(config); + } + + /** + * 表情识别（单人脸） + * 支持识别7种表情：neutral（中性）、happy（高兴）、sad（悲伤）、surprise（惊讶）、fear（恐惧）、disgust（厌恶）、anger（愤怒） + */ + @Test + public void testExpressionDetect() { + ExpressionModel model = getExpressionModel(); + R result = model.detectTopFace("src/main/resources/emotion/happy.png"); + if(result.isSuccess()){ + log.info("识别结果：{}", JSONObject.toJSONString(result.getData().getExpression().getDescription())); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } + + /** + * 表情识别（多人脸） + * 支持识别7种表情：neutral（中性）、happy（高兴）、sad（悲伤）、surprise（惊讶）、fear（恐惧）、disgust（厌恶）、anger（愤怒） + */ + @Test + public void testExpressionDetect2() { + ExpressionModel model = getExpressionModel(); + R result = model.detect("src/main/resources/emotion/happy.png"); + if(result.isSuccess()){ + //log.info("识别结果：{}", JSONObject.toJSONString(result.getData())); + for (DetectionInfo detectionInfo : result.getData().getDetectionInfoList()) { + log.info("识别结果：{}", JSONObject.toJSONString(detectionInfo.getFaceInfo().getExpressionResult().getExpression().getDescription())); + } + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } + + /** + * 表情识别(基于人脸检测检测框-多人) + * 流程：人脸检测 -》表情识别 + * 支持识别7种表情：neutral（中性）、happy（高兴）、sad（悲伤）、surprise（惊讶）、fear（恐惧）、disgust（厌恶）、anger（愤怒） + */ + @Test + public void testExpressionDetect3() { + FaceDetModel faceDetModel = getFaceDetModel(); + ExpressionModel model = getExpressionModel(); + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get("src/main/resources/emotion/happy.png").toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + R detResult = faceDetModel.detect(image); + if(detResult.isSuccess()){ + R> result = model.detect(image, detResult.getData()); + if(result.isSuccess()){ + result.getData().forEach(expressionResult -> { + log.info("识别结果：{}", JSONObject.toJSONString(expressionResult.getExpression().getDescription())); + }); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + }else{ + log.info("人脸检测失败：{}", detResult.getMessage()); + } + } + + /** + * 表情识别(基于人脸检测检测框-单人) + * 流程：人脸检测 -》表情识别 + * 支持识别7种表情：neutral（中性）、happy（高兴）、sad（悲伤）、surprise（惊讶）、fear（恐惧）、disgust（厌恶）、anger（愤怒） + */ + @Test + public void testExpressionDetect4() { + FaceDetModel faceDetModel = getFaceDetModel(); + ExpressionModel model = getExpressionModel(); + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get("src/main/resources/emotion/happy.png").toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + R detResult = faceDetModel.detect(image); + if(detResult.isSuccess()){ + for (DetectionInfo detectionInfo : detResult.getData().getDetectionInfoList()) { + R result = model.detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints()); + if(result.isSuccess()){ + log.info("识别结果：{}", JSONObject.toJSONString(result.getData().getExpression().getDescription())); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } + }else{ + log.info("人脸检测失败：{}", detResult.getMessage()); + } + } + + /** + * 摄像头表情识别 + * 注意事项：如果视频比较卡，可以使用轻量的人脸检测模型 + */ + @Test + public void testLivenessDetectCamera(){ + try (ExpressionModel expressionModel = getExpressionModel()){ + OpenCV.loadShared(); + VideoCapture capture = new VideoCapture(0); + if (!capture.isOpened()) { + System.out.println("No camera detected"); + return; + } + + double ratio = + capture.get(Videoio.CAP_PROP_FRAME_WIDTH) + / capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); + int height = (int) (screenSize.height * 0.65f); + int width = (int) (height * ratio); + if (width > screenSize.width) { + width = screenSize.width; + } + + Mat image = new Mat(); + boolean captured = false; + for (int i = 0; i < 10; ++i) { + captured = capture.read(image); + if (captured) { + break; + } + + try { + Thread.sleep(50); + } catch (InterruptedException ignore) { + // ignore + } + } + if (!captured) { + JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); + } + ViewerFrame frame = new ViewerFrame(width, height); + ImageFactory factory = ImageFactory.getInstance(); + Size size = new Size(width, height); + + while (capture.isOpened()) { + if (!capture.read(image)) { + break; + } + Mat resizeImage = new Mat(); + Imgproc.resize(image, resizeImage, size); + Image img = factory.fromImage(resizeImage); + BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); + R detectedResult = expressionModel.detect(bufferedImage); + if(!detectedResult.isSuccess()){ + log.debug("识别失败：{}", detectedResult.getMessage()); + continue; + } + for(DetectionInfo detectionInfo : detectedResult.getData().getDetectionInfoList()){ + DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); + String text = detectionInfo.getFaceInfo().getExpressionResult().getExpression().getDescription() + ":" + detectionInfo.getFaceInfo().getExpressionResult().getScore(); + ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, Color.red); + } + frame.showImage(bufferedImage); + } + + capture.release(); + System.exit(0); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java new file mode 100644 index 0000000..d03286e --- /dev/null +++ b/examples/face-example/src/main/java/smartai/examples/face/facedet/FaceDetDemo.java @@ -0,0 +1,289 @@ +package smartai.examples.face.facedet; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.enums.face.LivenessStatus; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.constant.FaceDetectConstant; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import cn.smartjavaai.face.factory.FaceDetModelFactory; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.liveness.LivenessDetModel; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import nu.pattern.OpenCV; +import org.junit.Test; +import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; +import org.opencv.videoio.VideoCapture; +import org.opencv.videoio.Videoio; +import smartai.examples.face.ViewerFrame; + +import javax.imageio.ImageIO; +import javax.swing.*; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.nio.file.Paths; + +/** + * 人脸检测模型demo + * 支持系统：windows 64位，linux 64位, macos M系列 + * 支持功能：人脸检测 + * 模型下载地址：https://pan.baidu.com/s/1d2YlJ2YOdGn3Y-AegyAhmQ?pwd=1234 提取码: 1234 + * @author dwj + */ +@Slf4j +public class FaceDetDemo { + + + public static String imgPath = "src/main/resources/iu_1.jpg"; + + + /** + * 获取人脸检测模型 + * 注意事项：高精度模型，速度较慢 + * @return + */ + public FaceDetModel getFaceDetModel(){ + FaceDetConfig config = new FaceDetConfig(); + config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型 + config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + return FaceDetModelFactory.getInstance().getModel(config); + } + + /** + * 获取Seetaface6 人脸检测模型 + * 注意：不支持macos + * @return + */ + public FaceDetModel getSeetaface6DetModel(){ + FaceDetConfig config = new FaceDetConfig(); + //指定模型 + config.setModelEnum(FaceDetModelEnum.SEETA_FACE6_MODEL); + //指定模型路径：请根据实际情况替换为本地模型文件的绝对路径（模型下载地址请查看文档） + config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); + return FaceDetModelFactory.getInstance().getModel(config); + } + + /** + * 人脸检测(默认配置) + * 使用默认模型参数检测，默认模型：retinaface，需联网，会自动下载模型 + * 图片参数：图片路径 + */ + @Test + public void testFaceDetect(){ + try (FaceDetModel faceModel = FaceDetModelFactory.getInstance().getModel()) { + R detectedResult = faceModel.detect(imgPath); + if(detectedResult.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); + }else{ + log.info("人脸检测失败：{}", detectedResult.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 人脸检测(自定义模型参数) + * 图片参数：图片路径 + */ + @Test + public void testFaceDetectCustomConfig(){ + try (FaceDetModel faceModel = getFaceDetModel()){ + R detectedResult = faceModel.detect(imgPath); + if(detectedResult.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); + }else{ + log.info("人脸检测失败：{}", detectedResult.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + /** + * 人脸检测并绘制人脸框 + */ + @Test + public void testFaceDetectAndDraw(){ + try (FaceDetModel faceModel = getFaceDetModel()){ + faceModel.detectAndDraw("src/main/resources/largest_selfie.jpg","output/largest_selfie_detected.png"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 人脸检测并绘制人脸框,返回BufferedImage + * + */ + @Test + public void testFaceDetectAndDraw2(){ + try (FaceDetModel faceModel = getFaceDetModel()){ + BufferedImage image = null; + String imagePath = "src/main/resources/largest_selfie.jpg"; + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + //可以根据后续业务场景使用detectedImage + R detectedImage = faceModel.detectAndDraw(image); + if(detectedImage.isSuccess()){ + log.info("人脸检测成功"); + }else{ + log.info("人脸检测失败：{}", detectedImage.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + + } + + /** + * 人脸检测（离线模型） + */ + @Test + public void testDetectFaceOffine(){ + FaceDetConfig config = new FaceDetConfig(); + config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸模型 + //模型路径,不同模型下载路径请参看文档 + config.setModelPath("/Users/xxx/Documents/develop/face_model/retinaface.pt"); + try (FaceDetModel faceModel = FaceDetModelFactory.getInstance().getModel(config)) { + R detectedResult = faceModel.detect(imgPath); + if(detectedResult.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); + }else{ + log.info("人脸检测失败：{}", detectedResult.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 人脸检测（GPU模式） + */ + @Test + public void testDetectFaceGPU(){ + FaceDetConfig config = new FaceDetConfig(); + config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸模型 + config.setDevice(DeviceEnum.GPU); + try (FaceDetModel faceModel = FaceDetModelFactory.getInstance().getModel(config)) { + R detectedResult = faceModel.detect(imgPath); + if(detectedResult.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); + }else{ + log.info("人脸检测失败：{}", detectedResult.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 人脸检测(Seetaface6) + * 图片参数：图片路径 + */ + @Test + public void testFaceDetectSeetaface6(){ + try (FaceDetModel faceModel = getSeetaface6DetModel()){ + R detectedResult = faceModel.detect(imgPath); + if(detectedResult.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult.getData())); + }else{ + log.info("人脸检测失败：{}", detectedResult.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + /** + * 摄像头人脸检测 + * 注意事项：如果视频比较卡，可以使用轻量的人脸检测模型 + */ + @Test + public void testDetectCamera(){ + try (FaceDetModel faceModel = getFaceDetModel()){ + OpenCV.loadShared(); + VideoCapture capture = new VideoCapture(0); + if (!capture.isOpened()) { + System.out.println("No camera detected"); + return; + } + + double ratio = + capture.get(Videoio.CAP_PROP_FRAME_WIDTH) + / capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); + int height = (int) (screenSize.height * 0.65f); + int width = (int) (height * ratio); + if (width > screenSize.width) { + width = screenSize.width; + } + + Mat image = new Mat(); + boolean captured = false; + for (int i = 0; i < 10; ++i) { + captured = capture.read(image); + if (captured) { + break; + } + + try { + Thread.sleep(50); + } catch (InterruptedException ignore) { + // ignore + } + } + if (!captured) { + JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); + } + ViewerFrame frame = new ViewerFrame(width, height); + ImageFactory factory = ImageFactory.getInstance(); + Size size = new Size(width, height); + + while (capture.isOpened()) { + if (!capture.read(image)) { + break; + } + Mat resizeImage = new Mat(); + Imgproc.resize(image, resizeImage, size); + Image img = factory.fromImage(resizeImage); + BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); + R detectedResult = faceModel.detect(bufferedImage); + if(!detectedResult.isSuccess()){ + log.debug("识别失败：{}", detectedResult.getMessage()); + continue; + } + for(DetectionInfo detectionInfo : detectedResult.getData().getDetectionInfoList()){ + DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); + String text = null; + if(detectionInfo.getScore() > 0){ + text = detectionInfo.getScore() + ""; + } + ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, Color.red); + } + frame.showImage(bufferedImage); + } + + capture.release(); + System.exit(0); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + +} diff --git a/examples/src/main/java/smartai/examples/face/facerec/FaceNetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java similarity index 58% rename from examples/src/main/java/smartai/examples/face/facerec/FaceNetDemo.java rename to examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java index 8ff03b8..b6670d4 100644 --- a/examples/src/main/java/smartai/examples/face/facerec/FaceNetDemo.java +++ b/examples/face-example/src/main/java/smartai/examples/face/facerec/FaceRecDemo.java @@ -1,33 +1,30 @@ package smartai.examples.face.facerec; import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.entity.FaceSearchResult; import cn.smartjavaai.common.entity.R; -import cn.smartjavaai.face.config.FaceExtractConfig; -import cn.smartjavaai.face.config.FaceModelConfig; +import cn.smartjavaai.common.entity.face.FaceSearchResult; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.config.FaceRecConfig; +import cn.smartjavaai.face.constant.FaceDetectConstant; import cn.smartjavaai.face.entity.FaceRegisterInfo; -import cn.smartjavaai.face.entity.FaceResult; import cn.smartjavaai.face.entity.FaceSearchParams; -import cn.smartjavaai.face.enums.FaceModelEnum; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import cn.smartjavaai.face.enums.FaceRecModelEnum; import cn.smartjavaai.face.enums.IdStrategy; import cn.smartjavaai.face.enums.SimilarityType; -import cn.smartjavaai.face.factory.FaceModelFactory; -import cn.smartjavaai.face.model.facerec.FaceModel; +import cn.smartjavaai.face.factory.FaceDetModelFactory; +import cn.smartjavaai.face.factory.FaceRecModelFactory; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.facerec.FaceRecModel; +import cn.smartjavaai.face.utils.SimilarityUtil; import cn.smartjavaai.face.vector.config.MilvusConfig; import cn.smartjavaai.face.vector.config.SQLiteConfig; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import io.milvus.param.MetricType; import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; import org.junit.Test; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.List; /** @@ -39,11 +36,97 @@ import java.util.List; * @date 2025/4/11 */ @Slf4j -public class FaceNetDemo { +public class FaceRecDemo { + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + + /** + * 获取人脸检测模型 + * @return + */ + public FaceDetModel getFaceDetModel(){ + FaceDetConfig config = new FaceDetConfig(); + config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型 + config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setDevice(device); + return FaceDetModelFactory.getInstance().getModel(config); + } + + /** + * 获取人脸识别模型 + * @return + */ + public FaceRecModel getFaceRecModel(){ + FaceRecConfig config = new FaceRecConfig(); + config.setModelEnum(FaceRecModelEnum.FACENET_MODEL); +// config.setModelPath("/Users/xxx/Documents/develop/model/elasticface.pt"); +// config.setModelPath("/Users/xxx/Documents/develop/model/InsightFace/model_mobilefacenet.pt"); + //裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false + config.setCropFace(true); + //开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能 + config.setAlign(true); + config.setDevice(device); + //指定人脸检测模型 + config.setDetectModel(getFaceDetModel()); + return FaceRecModelFactory.getInstance().getModel(config); + } + + /** + * 获取人脸识别模型(带向量数据库配置) + * @return + */ + public FaceRecModel getFaceRecModelWithDbConfig(){ + FaceRecConfig config = new FaceRecConfig(); + config.setModelEnum(FaceRecModelEnum.ELASTIC_FACE_MODEL);//人脸检测模型 + config.setModelPath("/Users/xxx/Documents/develop/model/elasticface.pt"); + //裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false + config.setCropFace(true); + //开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能 + config.setAlign(true); + //指定人脸检测模型 + config.setDetectModel(getFaceDetModel()); + config.setDevice(device); + + //初始化向量数据库：Milvus数据库配置 + MilvusConfig vectorDBConfig = new MilvusConfig(); + vectorDBConfig.setHost("127.0.0.1"); + vectorDBConfig.setPort(19530); + //vectorDBConfig.setCollectionName("face5"); + //ID策略：自动生成 + vectorDBConfig.setIdStrategy(IdStrategy.AUTO); + //索引类型:内积 (Inner Product) 不建议修改 + //vectorDBConfig.setMetricType(MetricType.IP); + config.setVectorDBConfig(vectorDBConfig); + return FaceRecModelFactory.getInstance().getModel(config); + } + + /** + * 获取人脸识别模型(带SQLite数据库配置) + * @return + */ + public FaceRecModel getFaceRecModelWithSQLiteConfig(){ + FaceRecConfig config = new FaceRecConfig(); + config.setModelEnum(FaceRecModelEnum.FACENET_MODEL);//人脸检测模型 + //裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false + config.setCropFace(true); + //开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能 + config.setAlign(true); + //指定人脸检测模型 + config.setDetectModel(getFaceDetModel()); + config.setDevice(device); + + //初始化SQLite数据库 + SQLiteConfig vectorDBConfig = new SQLiteConfig(); + vectorDBConfig.setSimilarityType(SimilarityType.IP); + config.setVectorDBConfig(vectorDBConfig); + return FaceRecModelFactory.getInstance().getModel(config); + } /** * 提取人脸特征(多人脸场景) - * 默认使用检测模型：ULTRA_LIGHT_FAST_GENERIC_FACE * 自动裁剪人脸（处理耗时略有增加） * 注意事项： * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 @@ -51,14 +134,9 @@ public class FaceNetDemo { */ @Test public void testExtractFeatures(){ - try { - //人脸特征提取模型 - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.FACENET_MODEL); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - log.info("人脸特征提取模型加载成功"); + try (FaceRecModel faceRecModel = getFaceRecModel()){ //提取图片中所有人脸特征 - R faceResult = faceModel.extractFeatures("src/main/resources/face/iu_1.jpg"); + R faceResult = faceRecModel.extractFeatures("src/main/resources/iu_1.jpg"); if(faceResult.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(faceResult.getData())); }else{ @@ -69,46 +147,6 @@ public class FaceNetDemo { } } - /** - * 提取人脸特征(自定义配置) - * 注意事项： - * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 - * 2、若人脸朝向不正，可开启人脸对齐以提升特征提取准确度。（方法参考自定义配置人脸特征提取） - */ - @Test - public void testExtractFeaturesWithCustomConfig(){ - try { - //人脸模型参数 - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.FACENET_MODEL); - //人脸特征提取参数 - FaceExtractConfig extractConfig = new FaceExtractConfig(); - //当关闭人脸裁剪时，程序将跳过人脸检测与裁剪流程，直接进行特征提取，适用于输入已为标准人脸区域的图像，有助于提升处理效率。 - extractConfig.setCropFace(true); - //开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能 - extractConfig.setAlign(true); - //人脸检测模型配置，指定人脸检测模型：ULTRA_LIGHT_FAST_GENERIC_FACE - FaceModelConfig detectModelConfig = new FaceModelConfig(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE); - //设置人脸检测置信度阈值 - detectModelConfig.setConfidenceThreshold(0.98); - extractConfig.setDetectModel(FaceModelFactory.getInstance().getModel(detectModelConfig)); - config.setExtractConfig(extractConfig); - //获取人脸模型 - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult = faceModel.extractTopFaceFeature("src/main/resources/face/iu_1.jpg"); - if(featureResult.isSuccess()){ - log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult.getData())); - }else{ - log.info("人脸特征提取失败：{}", featureResult.getMessage()); - } - }catch (Exception e){ - e.printStackTrace(); - } - } - - - /** * 人脸比对1：1（基于图像直接比对） * 流程：从输入图像中裁剪分数最高的人脸 → 提取其人脸特征 → 比对两张图片中提取的人脸特征。（接口内自动完成） @@ -119,14 +157,15 @@ public class FaceNetDemo { */ @Test public void featureComparison(){ - try { - FaceModelConfig config = new FaceModelConfig(); - //人脸模型 - config.setModelEnum(FaceModelEnum.FACENET_MODEL); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); + try (FaceRecModel faceRecModel = getFaceRecModel()){ //基于图像直接比对人脸特征 - float similar = faceModel.featureComparison("src/main/resources/face/iu_1.jpg","src/main/resources/face/iu_2.jpg"); - log.info("相似度：{}", similar); + R similarResult = faceRecModel.featureComparison("src/main/resources/iu_1.jpg","src/main/resources/iu_2.jpg"); + if(similarResult.isSuccess()){ + //相似度阈值不同模型不同，具体参看文档 + log.info("人脸比对相似度：{}", JSONObject.toJSONString(similarResult.getData())); + }else{ + log.info("人脸比对失败：{}", similarResult.getMessage()); + } } catch (Exception e){ e.printStackTrace(); @@ -143,13 +182,9 @@ public class FaceNetDemo { */ @Test public void featureComparison2(){ - try { - FaceModelConfig config = new FaceModelConfig(); - //人脸模型 - config.setModelEnum(FaceModelEnum.FACENET_MODEL); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); + try (FaceRecModel faceRecModel = getFaceRecModel()){ //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult1 = faceModel.extractTopFaceFeature("src/main/resources/face/iu_1.jpg"); + R featureResult1 = faceRecModel.extractTopFaceFeature("src/main/resources/iu_1.jpg"); if(featureResult1.isSuccess()){ log.info("图片1人脸特征提取成功：{}", JSONObject.toJSONString(featureResult1.getData())); }else{ @@ -157,7 +192,7 @@ public class FaceNetDemo { return; } //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceModel.extractTopFaceFeature("src/main/resources/face/iu_2.jpg"); + R featureResult2 = faceRecModel.extractTopFaceFeature("src/main/resources/iu_2.jpg"); if(featureResult2.isSuccess()){ log.info("图片2人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); }else{ @@ -165,7 +200,7 @@ public class FaceNetDemo { return; } //计算相似度 - float similar = faceModel.calculSimilar(featureResult1.getData(), featureResult2.getData()); + float similar = faceRecModel.calculSimilar(featureResult1.getData(), featureResult2.getData()); log.info("相似度：{}", similar); } catch (Exception e){ @@ -184,28 +219,14 @@ public class FaceNetDemo { */ @Test public void searchFace(){ - try { - FaceModelConfig config = new FaceModelConfig(); - //人脸模型 - config.setModelEnum(FaceModelEnum.FACENET_MODEL); - //初始化向量数据库：Milvus数据库配置 - MilvusConfig vectorDBConfig = new MilvusConfig(); - vectorDBConfig.setHost("127.0.0.1"); - vectorDBConfig.setPort(19530); - //vectorDBConfig.setCollectionName("face5"); - //ID策略：自动生成 - vectorDBConfig.setIdStrategy(IdStrategy.AUTO); - //索引类型:内积 (Inner Product) 不建议修改 - //vectorDBConfig.setMetricType(MetricType.IP); - config.setVectorDBConfig(vectorDBConfig); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); + try (FaceRecModel faceRecModel = getFaceRecModelWithDbConfig()){ //等待加载人脸库结束 - while (!faceModel.isLoadFaceCompleted()){ + while (!faceRecModel.isLoadFaceCompleted()){ Thread.sleep(100); } log.info("====================人脸注册=========================="); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult = faceModel.extractTopFaceFeature("src/main/resources/face/iu_1.jpg"); + R featureResult = faceRecModel.extractTopFaceFeature("src/main/resources/iu_1.jpg"); if(featureResult.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult.getData())); }else{ @@ -218,9 +239,10 @@ public class FaceNetDemo { JSONObject metadataJson = new JSONObject(); metadataJson.put("name", "iu"); metadataJson.put("age", "25"); + //faceRegisterInfo.setId("001"); faceRegisterInfo.setMetadata(metadataJson.toJSONString()); //人脸注册，返回人脸库ID - R registerResult = faceModel.register(faceRegisterInfo, featureResult.getData()); + R registerResult = faceRecModel.register(faceRegisterInfo, featureResult.getData()); if(registerResult.isSuccess()){ log.info("注册成功：ID-{}", registerResult.getData()); }else{ @@ -236,11 +258,11 @@ public class FaceNetDemo { updateInfo.setMetadata(metadataJsonUpdate.toJSONString()); //更新必须设置ID,只有 updateInfo.setId(registerResult.getData()); - faceModel.upsertFace(updateInfo, "src/main/resources/face/iu_2.jpg"); + faceRecModel.upsertFace(updateInfo, "src/main/resources/iu_2.jpg"); log.info("更新人脸成功");*/ log.info("====================人脸查询=========================="); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceModel.extractTopFaceFeature("src/main/resources/face/iu_3.jpg"); + R featureResult2 = faceRecModel.extractTopFaceFeature("src/main/resources/iu_3.jpg"); if(featureResult2.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); }else{ @@ -251,11 +273,11 @@ public class FaceNetDemo { faceSearchParams.setTopK(1); faceSearchParams.setThreshold(0.8f); - List faceSearchResults = faceModel.search(featureResult2.getData(), faceSearchParams); + List faceSearchResults = faceRecModel.search(featureResult2.getData(), faceSearchParams); // R faceSearchResults = faceModel.search("src/main/resources/face/iu_3.jpg", faceSearchParams); log.info("人脸查询结果：{}", JSONArray.toJSONString(faceSearchResults)); log.info("====================人脸删除=========================="); - faceModel.removeRegister(registerResult.getData()); + faceRecModel.removeRegister(registerResult.getData()); log.info("人脸删除成功"); } catch (Exception e){ @@ -273,23 +295,14 @@ public class FaceNetDemo { */ @Test public void searchFace2(){ - try { - FaceModelConfig config = new FaceModelConfig(); - //人脸模型 - config.setModelEnum(FaceModelEnum.FACENET_MODEL); - //初始化向量数据库：Milvus数据库配置 - SQLiteConfig vectorDBConfig = new SQLiteConfig(); - vectorDBConfig.setDbPath("/Users/wenjie/Downloads/face.db"); - vectorDBConfig.setSimilarityType(SimilarityType.IP); - config.setVectorDBConfig(vectorDBConfig); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); + try (FaceRecModel faceRecModel = getFaceRecModelWithSQLiteConfig()){ //等待加载人脸库结束 - while (!faceModel.isLoadFaceCompleted()){ + while (!faceRecModel.isLoadFaceCompleted()){ Thread.sleep(100); } log.info("====================人脸注册=========================="); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult = faceModel.extractTopFaceFeature("src/main/resources/face/iu_1.jpg"); + R featureResult = faceRecModel.extractTopFaceFeature("src/main/resources/iu_1.jpg"); if(featureResult.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult.getData())); }else{ @@ -306,7 +319,7 @@ public class FaceNetDemo { //可自定义 ID，若未设置则自动生成。 //faceRegisterInfo.setId("00001"); //人脸注册，返回人脸库ID - R registerResult = faceModel.register(faceRegisterInfo, featureResult.getData()); + R registerResult = faceRecModel.register(faceRegisterInfo, featureResult.getData()); if(registerResult.isSuccess()){ log.info("注册成功：ID-{}", registerResult.getData()); }else{ @@ -321,11 +334,11 @@ public class FaceNetDemo { updateInfo.setMetadata(metadataJsonUpdate.toJSONString()); //更新必须设置ID,只有 updateInfo.setId(registerResult.getData()); - faceModel.upsertFace(updateInfo, "src/main/resources/face/iu_2.jpg"); + faceRecModel.upsertFace(updateInfo, "src/main/resources/iu_2.jpg"); log.info("更新人脸成功"); log.info("====================人脸查询=========================="); //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceModel.extractTopFaceFeature("src/main/resources/face/iu_3.jpg"); + R featureResult2 = faceRecModel.extractTopFaceFeature("src/main/resources/iu_3.jpg"); if(featureResult2.isSuccess()){ log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); }else{ @@ -335,10 +348,10 @@ public class FaceNetDemo { FaceSearchParams faceSearchParams = new FaceSearchParams(); faceSearchParams.setTopK(1); faceSearchParams.setThreshold(0.8f); - List faceSearchResults = faceModel.search(featureResult2.getData(), faceSearchParams); + List faceSearchResults = faceRecModel.search(featureResult2.getData(), faceSearchParams); log.info("人脸查询结果：{}", JSONArray.toJSONString(faceSearchResults)); log.info("====================人脸删除=========================="); - faceModel.removeRegister(registerResult.getData()); + faceRecModel.removeRegister(registerResult.getData()); log.info("人脸删除成功"); } catch (Exception e){ @@ -347,31 +360,6 @@ public class FaceNetDemo { } - /** - * 使用离线模型 - * @throws Exception - */ - @Test - public void featureComparisonOffline(){ - try { - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.FACENET_MODEL);//人脸模型 - //设置人脸识别模型文件路径，请根据实际情况替换为本地模型文件的绝对路径 - config.setModelPath("/Users/xxx/Documents/develop/face_model/face_feature.pt"); - //人脸特征提取参数 - FaceExtractConfig extractConfig = new FaceExtractConfig(); - FaceModelConfig detectModelConfig = new FaceModelConfig(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE); - //设置人脸检测模型文件路径，请根据实际情况替换为本地模型文件的绝对路径 - detectModelConfig.setModelPath("/Users/xxx/Documents/develop/face_model/ultranet.pt"); - //人脸检测模型配置 - extractConfig.setDetectModel(FaceModelFactory.getInstance().getModel(detectModelConfig)); - config.setExtractConfig(extractConfig); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - } - catch (Exception e){ - e.printStackTrace(); - } - } } diff --git a/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java new file mode 100644 index 0000000..14c33d3 --- /dev/null +++ b/examples/face-example/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java @@ -0,0 +1,322 @@ +package smartai.examples.face.liveness; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import cn.hutool.core.lang.UUID; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.entity.face.ExpressionResult; +import cn.smartjavaai.common.entity.face.LivenessResult; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.enums.face.LivenessStatus; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.config.LivenessConfig; +import cn.smartjavaai.face.constant.FaceDetectConstant; +import cn.smartjavaai.face.constant.LivenessConstant; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import cn.smartjavaai.face.enums.LivenessModelEnum; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.factory.FaceDetModelFactory; +import cn.smartjavaai.face.factory.LivenessModelFactory; +import cn.smartjavaai.face.model.expression.ExpressionModel; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.liveness.LivenessDetModel; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import nu.pattern.OpenCV; +import org.bytedeco.javacv.FFmpegFrameGrabber; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.Java2DFrameUtils; +import org.junit.Test; +import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; +import org.opencv.videoio.VideoCapture; +import org.opencv.videoio.Videoio; +import smartai.examples.face.ViewerFrame; + +import javax.imageio.ImageIO; +import javax.swing.*; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; + +/** + * 静态活体检测demo + * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * @author dwj + * @date 2025/5/1 + */ +@Slf4j +public class LivenessDetDemo { + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + + + /** + * 获取活体检测模型 + * @return + */ + public LivenessDetModel getLivenessDetModel(){ + LivenessConfig config = new LivenessConfig(); + config.setModelEnum(LivenessModelEnum.IIC_FL_MODEL); + config.setDevice(device); + //需替换为实际模型存储路径 + config.setModelPath("/Users/xxx/Documents/develop/model/anti/model.onnx"); + //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 + config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); + /*视频检测帧数，可选，默认10，输出帧数超过这个number之后，就可以输出识别结果。 + 这个数量相当于多帧识别结果融合的融合的帧数。当输入的帧数超过设定帧数的时候，会采用滑动窗口的方式，返回融合的最近输入的帧融合的识别结果。 + 一般来说，在10以内，帧数越多，结果越稳定，相对性能越好，但是得到结果的延时越高。*/ + config.setFrameCount(LivenessConstant.DEFAULT_FRAME_COUNT); + //指定人脸检测模型 + config.setDetectModel(getFaceDetModel()); + return LivenessModelFactory.getInstance().getModel(config); + } + + /** + * 获取活体检测模型(小视科技模型) + * 备注：小视科技活体检测是两个模型融合结果 + * @return + */ + public LivenessDetModel getMiniVisionLivenessDetModel(){ + LivenessConfig config = new LivenessConfig(); + config.setModelEnum(LivenessModelEnum.MINI_VISION_MODEL); + config.setDevice(device); + //模型1路径：需替换为实际模型存储路径 + config.setModelPath("/Users/xxx/Documents/develop/model/live/2.7_80x80_MiniFASNetV2.onnx"); + //SE模型路径：需替换为实际模型存储路径 + config.putCustomParam("seModelPath", "/Users/xxx/Documents/develop/model/live/4_0_0_80x80_MiniFASNetV1SE.onnx"); + //人脸活体阈值,可选,超过阈值则认为是真人，低于阈值是非活体 + config.setRealityThreshold(0.5f); + /*视频检测帧数，可选，默认10，输出帧数超过这个number之后，就可以输出识别结果。 + 这个数量相当于多帧识别结果融合的融合的帧数。当输入的帧数超过设定帧数的时候，会采用滑动窗口的方式，返回融合的最近输入的帧融合的识别结果。 + 一般来说，在10以内，帧数越多，结果越稳定，相对性能越好，但是得到结果的延时越高。*/ + config.setFrameCount(LivenessConstant.DEFAULT_FRAME_COUNT); + //指定人脸检测模型 + config.setDetectModel(getFaceDetModel()); + return LivenessModelFactory.getInstance().getModel(config); + } + + + /** + * 获取人脸检测模型 + * @return + */ + public FaceDetModel getFaceDetModel(){ + FaceDetConfig config = new FaceDetConfig(); + config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型 + config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 + config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 + config.setDevice(device); + return FaceDetModelFactory.getInstance().getModel(config); + } + + + + /** + * 图片活体检测（多人脸） + */ + @Test + public void testLivenessDetect(){ + try (LivenessDetModel livenessDetModel = getLivenessDetModel()){ + R response = livenessDetModel.detect("src/main/resources/liveness/1.jpg"); + if(response.isSuccess()){ + for (DetectionInfo detectionInfo : response.getData().getDetectionInfoList()){ + log.info("活体检测结果：{}", JSONObject.toJSONString(detectionInfo.getFaceInfo().getLivenessStatus().getStatus().getDescription())); + } + }else{ + log.info("活体检测失败：{}", response.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 图片活体检测（分数最高人脸） + */ + @Test + public void testLivenessDetect2(){ + try (LivenessDetModel livenessDetModel = getLivenessDetModel()){ + //指定文件夹路径 + File dir = new File("face-example/src/main/resources/liveness"); + File[] files = dir.listFiles(); + for (File file : files) { + R response = livenessDetModel.detectTopFace(ImageIO.read(file)); + if(response.isSuccess()){ + log.info("{}活体检测结果：{},分数：{}", file.getName(), response.getData().getStatus().getDescription(), response.getData().getScore()); + }else{ + log.info("{}活体检测失败：{}", file.getName(), response.getMessage()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + /** + * 图片多人脸活体检测（基于已检测出的人脸区域和关键点） + */ + @Test + public void testLivenessDetect3(){ + try (FaceDetModel faceDetectModel = getFaceDetModel(); + LivenessDetModel livenessDetModel = getLivenessDetModel()){ + // 将图片路径转换为 BufferedImage + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/liveness/1.jpg").toAbsolutePath().toString())); + //人脸检测 + R detectionResponse = faceDetectModel.detect(image); + if(detectionResponse.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); + //检测到人脸 + if(detectionResponse.getData() != null && detectionResponse.getData().getDetectionInfoList() != null && detectionResponse.getData().getDetectionInfoList().size() > 0){ + R> livenessResult = livenessDetModel.detect(image, detectionResponse.getData()); + if(livenessResult.isSuccess()){ + log.info("活体检测结果：{}", JSONObject.toJSONString(livenessResult.getData())); + }else{ + log.error("活体检测失败：{}", livenessResult.getMessage()); + } + }else{ + log.info("未检测到人脸"); + } + }else{ + log.error("人脸检测失败：{}", detectionResponse.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 图片单人脸活体检测（基于已检测出的人脸区域和关键点） + */ + @Test + public void testLivenessDetect4(){ + try (FaceDetModel faceDetModel = getFaceDetModel(); + LivenessDetModel livenessDetModel = getMiniVisionLivenessDetModel()){ + // 将图片路径转换为 BufferedImage + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/liveness/1.jpg").toAbsolutePath().toString())); + R detResult = faceDetModel.detect(image); + if(detResult.isSuccess()){ + for (DetectionInfo detectionInfo : detResult.getData().getDetectionInfoList()) { + //seetaface6 需要有5点人脸关键点 + //R result = livenessDetModel.detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints()); + R result = livenessDetModel.detect(image, detectionInfo.getDetectionRectangle()); + if(result.isSuccess()){ + log.info("识别结果：{}", JSONObject.toJSONString(result.getData())); + }else{ + log.info("识别失败：{}", result.getMessage()); + } + } + }else{ + log.info("人脸检测失败：{}", detResult.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 视频活体检测 + */ + @Test + public void testLivenessDetectVideo(){ + try (LivenessDetModel livenessDetModel = getLivenessDetModel()){ + //视频路径 + R livenessStatus = livenessDetModel.detectVideo("video.mp4"); + if (livenessStatus.isSuccess()){ + log.info("识别结果：{}", JSONObject.toJSONString(livenessStatus.getData())); + }else{ + log.info("识别失败：{}", livenessStatus.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 摄像头活体检测 + * 注意事项：如果视频比较卡，可以使用轻量的人脸检测模型 + */ + @Test + public void testLivenessDetectCamera(){ + try (LivenessDetModel livenessDetModel = getLivenessDetModel()){ + OpenCV.loadShared(); + VideoCapture capture = new VideoCapture(0); + if (!capture.isOpened()) { + System.out.println("No camera detected"); + return; + } + + double ratio = + capture.get(Videoio.CAP_PROP_FRAME_WIDTH) + / capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); + int height = (int) (screenSize.height * 0.65f); + int width = (int) (height * ratio); + if (width > screenSize.width) { + width = screenSize.width; + } + + Mat image = new Mat(); + boolean captured = false; + for (int i = 0; i < 10; ++i) { + captured = capture.read(image); + if (captured) { + break; + } + + try { + Thread.sleep(50); + } catch (InterruptedException ignore) { + // ignore + } + } + if (!captured) { + JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); + } + ViewerFrame frame = new ViewerFrame(width, height); + ImageFactory factory = ImageFactory.getInstance(); + Size size = new Size(width, height); + + while (capture.isOpened()) { + if (!capture.read(image)) { + break; + } + Mat resizeImage = new Mat(); + Imgproc.resize(image, resizeImage, size); + Image img = factory.fromImage(resizeImage); + BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); + R detectedResult = livenessDetModel.detect(bufferedImage); + if(!detectedResult.isSuccess()){ + log.debug("识别失败：{}", detectedResult.getMessage()); + continue; + } + for(DetectionInfo detectionInfo : detectedResult.getData().getDetectionInfoList()){ + DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); + Color color = detectionInfo.getFaceInfo().getLivenessStatus().getStatus() == LivenessStatus.LIVE ? Color.GREEN : Color.RED; + String text = detectionInfo.getFaceInfo().getLivenessStatus().getStatus().getDescription() + ":" + detectionInfo.getFaceInfo().getLivenessStatus().getScore(); + ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, color); + } + frame.showImage(bufferedImage); + } + + capture.release(); + System.exit(0); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + +} diff --git a/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java b/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java new file mode 100644 index 0000000..01a7351 --- /dev/null +++ b/examples/face-example/src/main/java/smartai/examples/face/quality/FaceQualityDetDemo.java @@ -0,0 +1,271 @@ +package smartai.examples.face.quality; + +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.entity.face.FaceAttribute; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.config.FaceAttributeConfig; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.config.QualityConfig; +import cn.smartjavaai.face.entity.FaceQualityResult; +import cn.smartjavaai.face.entity.FaceQualitySummary; +import cn.smartjavaai.face.enums.FaceAttributeModelEnum; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import cn.smartjavaai.face.enums.QualityModelEnum; +import cn.smartjavaai.face.factory.FaceAttributeModelFactory; +import cn.smartjavaai.face.factory.FaceDetModelFactory; +import cn.smartjavaai.face.factory.FaceQualityModelFactory; +import cn.smartjavaai.face.model.attribute.FaceAttributeModel; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.quality.FaceQualityModel; +import cn.smartjavaai.face.utils.FaceUtils; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.nio.file.Paths; + +/** + * 人脸质量评估 demo + * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 + * @author dwj + */ +@Slf4j +public class FaceQualityDetDemo { + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + + + + /** + * 获取质量评估模型 + * @return + */ + public FaceQualityModel getFaceQualityModel() { + QualityConfig config = new QualityConfig(); + config.setModelEnum(QualityModelEnum.SEETA_FACE6_MODEL); + //需替换为实际模型存储路径 + config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); + config.setDevice(device); + return FaceQualityModelFactory.getInstance().getModel(config); + } + + + /** + * 获取人脸检测模型 + * @return + */ + public FaceDetModel getFaceDetModel() { + //需替换为实际模型存储路径 + String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; + FaceDetConfig faceDetectModelConfig = new FaceDetConfig(); + faceDetectModelConfig.setModelEnum(FaceDetModelEnum.SEETA_FACE6_MODEL); + faceDetectModelConfig.setModelPath(modelPath); + faceDetectModelConfig.setDevice(device); + return FaceDetModelFactory.getInstance().getModel(faceDetectModelConfig); + } + + + /** + * 人脸亮度评估 + */ + @Test + public void evaluateBrightness(){ + try (FaceQualityModel faceQualityModel = getFaceQualityModel(); + FaceDetModel faceDetModel = getFaceDetModel()){ + //人脸检测 + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + R detectionResponse = faceDetModel.detect(image); + if(detectionResponse.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); + //检测到人脸 + if(detectionResponse.getData() != null && detectionResponse.getData().getDetectionInfoList() != null && detectionResponse.getData().getDetectionInfoList().size() > 0){ + for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + R faceQualityResultR = faceQualityModel.evaluateBrightness(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + if(faceQualityResultR.isSuccess()){ + log.info("人脸亮度评估结果：{}", JSONObject.toJSONString(faceQualityResultR.getData())); + }else{ + log.info("人脸亮度评估失败：{}", faceQualityResultR.getMessage()); + } + } + } + }else{ + log.info("人脸检测失败：{}", detectionResponse.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 人脸完整度评估 + */ + @Test + public void evaluateCompleteness(){ + try (FaceQualityModel faceQualityModel = getFaceQualityModel(); + FaceDetModel faceDetModel = getFaceDetModel()){ + //人脸检测 + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + R detectionResponse = faceDetModel.detect(image); + if(detectionResponse.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); + //检测到人脸 + if(detectionResponse.getData() != null && detectionResponse.getData().getDetectionInfoList() != null && detectionResponse.getData().getDetectionInfoList().size() > 0){ + for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + R faceQualityResultR = faceQualityModel.evaluateCompleteness(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + if(faceQualityResultR.isSuccess()){ + log.info("人脸完整度评估结果：{}", JSONObject.toJSONString(faceQualityResultR.getData())); + }else{ + log.info("人脸完整度评估失败：{}", faceQualityResultR.getMessage()); + } + } + } + }else{ + log.info("人脸检测失败：{}", detectionResponse.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 人脸清晰度评估 + */ + @Test + public void evaluateClarity(){ + try (FaceQualityModel faceQualityModel = getFaceQualityModel(); + FaceDetModel faceDetModel = getFaceDetModel()){ + //人脸检测 + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + R detectionResponse = faceDetModel.detect(image); + if(detectionResponse.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); + //检测到人脸 + if(detectionResponse.getData() != null && detectionResponse.getData().getDetectionInfoList() != null && detectionResponse.getData().getDetectionInfoList().size() > 0){ + for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + R faceQualityResultR = faceQualityModel.evaluateClarity(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + if(faceQualityResultR.isSuccess()){ + log.info("人脸清晰度评估结果：{}", JSONObject.toJSONString(faceQualityResultR.getData())); + }else{ + log.info("人脸清晰度评估失败：{}", faceQualityResultR.getMessage()); + } + } + } + }else{ + log.info("人脸检测失败：{}", detectionResponse.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 人脸姿态评估 + */ + @Test + public void evaluatePose(){ + try (FaceQualityModel faceQualityModel = getFaceQualityModel(); + FaceDetModel faceDetModel = getFaceDetModel()){ + //人脸检测 + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + R detectionResponse = faceDetModel.detect(image); + if(detectionResponse.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); + //检测到人脸 + if(detectionResponse.getData() != null && detectionResponse.getData().getDetectionInfoList() != null && detectionResponse.getData().getDetectionInfoList().size() > 0){ + for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + R faceQualityResultR = faceQualityModel.evaluatePose(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + if(faceQualityResultR.isSuccess()){ + log.info("人脸姿态评估结果：{}", JSONObject.toJSONString(faceQualityResultR.getData())); + }else{ + log.info("人脸姿态评估失败：{}", faceQualityResultR.getMessage()); + } + } + } + }else{ + log.info("人脸检测失败：{}", detectionResponse.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 人脸分辨率评估 + */ + @Test + public void evaluateResolution(){ + try (FaceQualityModel faceQualityModel = getFaceQualityModel(); + FaceDetModel faceDetModel = getFaceDetModel()){ + //人脸检测 + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + R detectionResponse = faceDetModel.detect(image); + if(detectionResponse.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); + //检测到人脸 + if(detectionResponse.getData() != null && detectionResponse.getData().getDetectionInfoList() != null && detectionResponse.getData().getDetectionInfoList().size() > 0){ + for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + R faceQualityResultR = faceQualityModel.evaluateResolution(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + if(faceQualityResultR.isSuccess()){ + log.info("人脸分辨率评估结果：{}", JSONObject.toJSONString(faceQualityResultR.getData())); + }else{ + log.info("人脸分辨率评估失败：{}", faceQualityResultR.getMessage()); + } + } + } + }else{ + log.info("人脸检测失败：{}", detectionResponse.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 评估所有 + */ + @Test + public void evaluateAll(){ + try (FaceQualityModel faceQualityModel = getFaceQualityModel(); + FaceDetModel faceDetModel = getFaceDetModel()){ + //人脸检测 + BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/iu_1.jpg").toAbsolutePath().toString())); + R detectionResponse = faceDetModel.detect(image); + if(detectionResponse.isSuccess()){ + log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse.getData())); + //检测到人脸 + if(detectionResponse.getData() != null && detectionResponse.getData().getDetectionInfoList() != null && detectionResponse.getData().getDetectionInfoList().size() > 0){ + for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + R faceQualityResultR = faceQualityModel.evaluateAll(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + if(faceQualityResultR.isSuccess()){ + log.info("人脸评估结果：{}", JSONObject.toJSONString(faceQualityResultR.getData())); + }else{ + log.info("人脸评估失败：{}", faceQualityResultR.getMessage()); + } + } + } + }else{ + log.info("人脸检测失败：{}", detectionResponse.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + +} diff --git a/examples/src/main/resources/META-INF/MANIFEST.MF b/examples/face-example/src/main/resources/META-INF/MANIFEST.MF similarity index 100% rename from examples/src/main/resources/META-INF/MANIFEST.MF rename to examples/face-example/src/main/resources/META-INF/MANIFEST.MF diff --git a/examples/face-example/src/main/resources/emotion/angry.png b/examples/face-example/src/main/resources/emotion/angry.png new file mode 100644 index 0000000..9f19edd Binary files /dev/null and b/examples/face-example/src/main/resources/emotion/angry.png differ diff --git a/examples/face-example/src/main/resources/emotion/disgust.png b/examples/face-example/src/main/resources/emotion/disgust.png new file mode 100644 index 0000000..e8beac7 Binary files /dev/null and b/examples/face-example/src/main/resources/emotion/disgust.png differ diff --git a/examples/face-example/src/main/resources/emotion/fear.png b/examples/face-example/src/main/resources/emotion/fear.png new file mode 100644 index 0000000..d69cddc Binary files /dev/null and b/examples/face-example/src/main/resources/emotion/fear.png differ diff --git a/examples/face-example/src/main/resources/emotion/happy.png b/examples/face-example/src/main/resources/emotion/happy.png new file mode 100644 index 0000000..103a23b Binary files /dev/null and b/examples/face-example/src/main/resources/emotion/happy.png differ diff --git a/examples/face-example/src/main/resources/emotion/neutral.png b/examples/face-example/src/main/resources/emotion/neutral.png new file mode 100644 index 0000000..08c993c Binary files /dev/null and b/examples/face-example/src/main/resources/emotion/neutral.png differ diff --git a/examples/face-example/src/main/resources/emotion/sad.png b/examples/face-example/src/main/resources/emotion/sad.png new file mode 100644 index 0000000..94752b1 Binary files /dev/null and b/examples/face-example/src/main/resources/emotion/sad.png differ diff --git a/examples/face-example/src/main/resources/emotion/surprise.png b/examples/face-example/src/main/resources/emotion/surprise.png new file mode 100644 index 0000000..0f0312f Binary files /dev/null and b/examples/face-example/src/main/resources/emotion/surprise.png differ diff --git a/examples/src/main/resources/face/iu_1.jpg b/examples/face-example/src/main/resources/iu_1.jpg similarity index 100% rename from examples/src/main/resources/face/iu_1.jpg rename to examples/face-example/src/main/resources/iu_1.jpg diff --git a/examples/src/main/resources/face/iu_2.jpg b/examples/face-example/src/main/resources/iu_2.jpg similarity index 100% rename from examples/src/main/resources/face/iu_2.jpg rename to examples/face-example/src/main/resources/iu_2.jpg diff --git a/examples/src/main/resources/face/iu_3.jpg b/examples/face-example/src/main/resources/iu_3.jpg similarity index 100% rename from examples/src/main/resources/face/iu_3.jpg rename to examples/face-example/src/main/resources/iu_3.jpg diff --git a/examples/src/main/resources/jsy.jpg b/examples/face-example/src/main/resources/jsy.jpg similarity index 100% rename from examples/src/main/resources/jsy.jpg rename to examples/face-example/src/main/resources/jsy.jpg diff --git a/examples/src/main/resources/largest_selfie.jpg b/examples/face-example/src/main/resources/largest_selfie.jpg similarity index 100% rename from examples/src/main/resources/largest_selfie.jpg rename to examples/face-example/src/main/resources/largest_selfie.jpg diff --git a/examples/face-example/src/main/resources/liveness/1.jpg b/examples/face-example/src/main/resources/liveness/1.jpg new file mode 100644 index 0000000..3a30e7a Binary files /dev/null and b/examples/face-example/src/main/resources/liveness/1.jpg differ diff --git a/examples/face-example/src/main/resources/liveness/2.png b/examples/face-example/src/main/resources/liveness/2.png new file mode 100644 index 0000000..bd912b9 Binary files /dev/null and b/examples/face-example/src/main/resources/liveness/2.png differ diff --git a/examples/face-example/src/main/resources/liveness/4.png b/examples/face-example/src/main/resources/liveness/4.png new file mode 100644 index 0000000..515eab7 Binary files /dev/null and b/examples/face-example/src/main/resources/liveness/4.png differ diff --git a/examples/face-example/src/main/resources/liveness/5.png b/examples/face-example/src/main/resources/liveness/5.png new file mode 100644 index 0000000..b72c061 Binary files /dev/null and b/examples/face-example/src/main/resources/liveness/5.png differ diff --git a/examples/face-example/src/main/resources/liveness/6.png b/examples/face-example/src/main/resources/liveness/6.png new file mode 100644 index 0000000..b44db27 Binary files /dev/null and b/examples/face-example/src/main/resources/liveness/6.png differ diff --git a/examples/src/main/resources/logback.xml b/examples/face-example/src/main/resources/logback.xml similarity index 100% rename from examples/src/main/resources/logback.xml rename to examples/face-example/src/main/resources/logback.xml diff --git a/examples/objectdetection-example/.gitignore b/examples/objectdetection-example/.gitignore new file mode 100644 index 0000000..93dbf83 --- /dev/null +++ b/examples/objectdetection-example/.gitignore @@ -0,0 +1,7 @@ +.idea +.idea/ +target +log +*.iml +/.settings/ +/logging.file_IS_UNDEFINED/ diff --git a/examples/objectdetection-example/README.md b/examples/objectdetection-example/README.md new file mode 100644 index 0000000..45b23eb --- /dev/null +++ b/examples/objectdetection-example/README.md @@ -0,0 +1,58 @@ +# 目标检测示例 + + +## 📁 项目结构 + +``` + +objectdetection-example/ +├── src/ +│ ├── main/ +│ │ ├── java/ +│ │ │ └── smartai/examples/objectdetection/ +│ │ │ ├── ObjectDetection.java +│ │ │ └── ViewerFrame.java + +``` + + +--- + +## 🧩 功能模块说明 + +### 1. 目标检测 [ObjectDetection.java] +- **功能**：核心目标检测类，包含多个测试方法，展示了如何使用不同的模型进行目标检测 + +--- + + +## ⚙️ 配置要求 + +- **运行环境**： + - JDK 1.8 或更高版本 + - IntelliJ IDEA 推荐作为开发 IDE +- **依赖库**： + - OpenCV、DJL、SmartJavaAI SDK +- **模型路径**： + - 所有模型需下载并配置正确的路径（参考各 demo 注释中的链接） + +--- + +## 🚀 快速开始 + +1. 克隆项目到本地： + +2. 导入项目至 IntelliJ IDEA。 + +3. 根据需要修改模型路径（见各 demo 中注释）。 + +4. 运行对应的 JUnit 测试类方法即可体验各项功能。 + +--- + +## 📄 文档 + +有关完整使用说明，请查阅 SmartJavaAI 官方文档： +[http://doc.smartjavaai.cn](http://doc.smartjavaai.cn) + +--- diff --git a/examples/pom.xml b/examples/objectdetection-example/pom.xml similarity index 94% rename from examples/pom.xml rename to examples/objectdetection-example/pom.xml index f87acde..7b5bdf9 100644 --- a/examples/pom.xml +++ b/examples/objectdetection-example/pom.xml @@ -12,9 +12,9 @@ 11 11 UTF-8 - 1.0.17 + 1.0.19 - smartai.examples.face.facerec.RetinaFaceDemo + smartai.examples.objectdetection.ObjectDetection 1.5.10 @@ -91,30 +91,12 @@ 4.13.2 - - - cn.smartjavaai - smartjavaai-face - - cn.smartjavaai smartjavaai-objectdetection - - - cn.smartjavaai - smartjavaai-ocr - - - - - cn.smartjavaai - smartjavaai-translate - - ai.djl.pytorch diff --git a/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java b/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java new file mode 100644 index 0000000..0a65e3a --- /dev/null +++ b/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ObjectDetection.java @@ -0,0 +1,249 @@ +package smartai.examples.objectdetection; + +import ai.djl.Application; +import ai.djl.MalformedModelException; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.modality.cv.output.*; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ModelZoo; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.enums.face.LivenessStatus; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.face.model.liveness.LivenessDetModel; +import cn.smartjavaai.objectdetection.config.DetectorModelConfig; +import cn.smartjavaai.objectdetection.enums.DetectorModelEnum; +import cn.smartjavaai.objectdetection.exception.DetectionException; +import cn.smartjavaai.objectdetection.model.DetectorModel; +import cn.smartjavaai.objectdetection.model.ObjectDetectionModelFactory; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import nu.pattern.OpenCV; +import org.junit.Assert; +import org.junit.Test; +import org.opencv.core.Mat; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; +import org.opencv.videoio.VideoCapture; +import org.opencv.videoio.Videoio; + +import javax.imageio.ImageIO; +import javax.swing.*; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * 目标检测模型demo + * 支持功能：目标检测 + * 模型下载地址：https://pan.baidu.com/s/10aTOLBlR6EG-sq6g0OkAWg?pwd=1234 提取码: 1234 + * @author dwj + */ +@Slf4j +public class ObjectDetection { + + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + + + /** + * 使用默认模型检测：YOLO11N + */ + @Test + public void objectDetection(){ + //默认cpu + try (DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel()){ + DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/object_detection.jpg"); + log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 指定模型检测(19种模型可选) + */ + @Test + public void objectDetection2(){ + DetectorModelConfig config = new DetectorModelConfig(); + config.setModelEnum(DetectorModelEnum.SSD_300_RESNET50);//检测模型，目前支持19种预置模型 + config.setDevice(device); + try (DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config)){ + DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); + log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 人脸检测并绘制检测结果 + */ + @Test + public void objectDetectionAndDraw(){ + try (DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel()){ + detectorModel.detectAndDraw("src/main/resources/object_detection.jpg","output/object_detection_detected.png"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 人脸检测并绘制检测结果,返回BufferedImage + */ + @Test + public void objectDetectionAndDraw2(){ + try (DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel()){ + String imagePath = "src/main/resources/object_detection.jpg"; + BufferedImage image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + //可以根据后续业务场景使用detectedImage + BufferedImage detectedImage = detectorModel.detectAndDraw(image); + Assert.assertNotNull("detectedImage null", detectedImage); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + + /** + * 使用yolo官方模型检测物品识别 + */ + @Test + public void objectDetectionWithOfficialModel(){ + DetectorModelConfig config = new DetectorModelConfig(); + config.setThreshold(0.3f); + //也支持YoloV8：YOLOV8_OFFICIAL 模型可以从文档中提供的地址下载 + config.setModelEnum(DetectorModelEnum.YOLOV12_OFFICIAL);//检测模型，目前支持19种模型 + // 指定模型路径，需要更改为自己的模型路径 + config.setModelPath("/Users/xxx/Documents/yolov12n.onnx"); + config.setDevice(device); + //一定要将yolo官方的类别文件：synset.txt（文档中下载）放在模型同目录下，否则报错 + try (DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config)){ + DetectionResponse detect = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); + log.info("目标检测结果：{}", JSONObject.toJSONString(detect)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 使用自己训练的模型检测 + */ + @Test + public void objectDetectionWithCustomModel(){ + DetectorModelConfig config = new DetectorModelConfig(); + //也支持YoloV8：YOLOV8_CUSTOM 模型需要自己训练，训练教程可以查看文档 + config.setModelEnum(DetectorModelEnum.YOLOV12_CUSTOM);//自定义YOLOV12模型 + // 指定模型路径，需要更改为自己的模型路径 + config.setModelPath("/Users/xxx/Documents/develop/fire_model/best.onnx"); + config.putCustomParam("width", 640);//resize 宽 + config.putCustomParam("height", 640);// resize 高 + config.putCustomParam("nmsThreshold", 0.5f); + config.setDevice(device); + try (DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config)){ + DetectionResponse detect = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); + log.info("目标检测结果：{}", JSONObject.toJSONString(detect)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 摄像头目标检测 + * 注意事项：如果视频比较卡，可以使用轻量的检测模型 + */ + @Test + public void testDetectCamera(){ + try (DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel()){ + OpenCV.loadShared(); + VideoCapture capture = new VideoCapture(0); + if (!capture.isOpened()) { + System.out.println("No camera detected"); + return; + } + + double ratio = + capture.get(Videoio.CAP_PROP_FRAME_WIDTH) + / capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); + int height = (int) (screenSize.height * 0.65f); + int width = (int) (height * ratio); + if (width > screenSize.width) { + width = screenSize.width; + } + + Mat image = new Mat(); + boolean captured = false; + for (int i = 0; i < 10; ++i) { + captured = capture.read(image); + if (captured) { + break; + } + + try { + Thread.sleep(50); + } catch (InterruptedException ignore) { + // ignore + } + } + if (!captured) { + JOptionPane.showConfirmDialog(null, "Failed to capture image from WebCam."); + } + ViewerFrame frame = new ViewerFrame(width, height); + ImageFactory factory = ImageFactory.getInstance(); + Size size = new Size(width, height); + + while (capture.isOpened()) { + if (!capture.read(image)) { + break; + } + Mat resizeImage = new Mat(); + Imgproc.resize(image, resizeImage, size); + Image img = factory.fromImage(resizeImage); + BufferedImage bufferedImage = OpenCVUtils.mat2Image(resizeImage); + DetectionResponse detectedResult = detectorModel.detect(bufferedImage); + if (Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getDetectionInfoList()) || detectedResult.getDetectionInfoList().size() == 0){ + log.debug("未检测到物体"); + continue; + } + for(DetectionInfo detectionInfo : detectedResult.getDetectionInfoList()){ + DetectionRectangle detectionRectangle = detectionInfo.getDetectionRectangle(); + String text = detectionInfo.getObjectDetInfo().getClassName(); + ImageUtils.drawImageRectWithText(bufferedImage, detectionRectangle, text, Color.RED); + } + frame.showImage(bufferedImage); + } + + capture.release(); + System.exit(0); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + +} diff --git a/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ViewerFrame.java b/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ViewerFrame.java new file mode 100644 index 0000000..eb83ce4 --- /dev/null +++ b/examples/objectdetection-example/src/main/java/smartai/examples/objectdetection/ViewerFrame.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package smartai.examples.objectdetection; + +import javax.swing.*; +import java.awt.*; +import java.awt.image.BufferedImage; + +public class ViewerFrame { + + private JFrame frame; + private ImagePanel imagePanel; + + public ViewerFrame(int width, int height) { + frame = new JFrame("Demo"); + imagePanel = new ImagePanel(); + frame.setLayout(new BorderLayout()); + frame.add(BorderLayout.CENTER, imagePanel); + + JOptionPane.setRootFrame(frame); + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); + if (width > screenSize.width) { + width = screenSize.width; + } + Dimension frameSize = new Dimension(width, height); + frame.setSize(frameSize); + frame.setLocation((screenSize.width - width) / 2, (screenSize.height - height) / 2); + frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE); + frame.setVisible(true); + } + + public void showImage(BufferedImage image) { + imagePanel.setImage(image); + SwingUtilities.invokeLater( + () -> { + frame.repaint(); + frame.pack(); + }); + } + + private static final class ImagePanel extends JPanel { + + private BufferedImage image; + + void setImage(BufferedImage image) { + this.image = image; + } + + @Override + public void paintComponent(Graphics g) { + super.paintComponent(g); + if (image == null) { + return; + } + + g.drawImage(image, 0, 0, null); + setPreferredSize(new Dimension(image.getWidth(), image.getHeight())); + } + } +} diff --git a/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF b/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000..91b424f --- /dev/null +++ b/examples/objectdetection-example/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Main-Class: smartai.examples.face.SeetaFace6LinuxDemo + diff --git a/examples/src/main/resources/dog_bike_car.jpg b/examples/objectdetection-example/src/main/resources/dog_bike_car.jpg similarity index 100% rename from examples/src/main/resources/dog_bike_car.jpg rename to examples/objectdetection-example/src/main/resources/dog_bike_car.jpg diff --git a/examples/objectdetection-example/src/main/resources/largest_selfie.jpg b/examples/objectdetection-example/src/main/resources/largest_selfie.jpg new file mode 100644 index 0000000..605ec97 Binary files /dev/null and b/examples/objectdetection-example/src/main/resources/largest_selfie.jpg differ diff --git a/examples/objectdetection-example/src/main/resources/logback.xml b/examples/objectdetection-example/src/main/resources/logback.xml new file mode 100644 index 0000000..809ebab --- /dev/null +++ b/examples/objectdetection-example/src/main/resources/logback.xml @@ -0,0 +1,14 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{36}) - %msg%n + + + + + + + diff --git a/examples/src/main/resources/object_detection.jpg b/examples/objectdetection-example/src/main/resources/object_detection.jpg similarity index 100% rename from examples/src/main/resources/object_detection.jpg rename to examples/objectdetection-example/src/main/resources/object_detection.jpg diff --git a/examples/ocr-examples/.gitignore b/examples/ocr-examples/.gitignore new file mode 100644 index 0000000..93dbf83 --- /dev/null +++ b/examples/ocr-examples/.gitignore @@ -0,0 +1,7 @@ +.idea +.idea/ +target +log +*.iml +/.settings/ +/logging.file_IS_UNDEFINED/ diff --git a/examples/ocr-examples/README.md b/examples/ocr-examples/README.md new file mode 100644 index 0000000..4b395a1 --- /dev/null +++ b/examples/ocr-examples/README.md @@ -0,0 +1,74 @@ +# OCR文字识别示例 + + +## 📁 项目结构 + +``` + +src +├── main +│ ├── java +│ │ └── smartai/examples/ocr +│ │ ├── OcrDetectionDemo.java # 文本检测示例 +│ │ ├── OcrDirectionDetDemo.java # 文本方向检测示例 +│ │ └── OcrRecognizeDemo.java # 文本识别示例 +│ └── resources +│ ├── logback.xml # 日志配置文件 +└── test + + +``` + + +--- + +## 🧩 功能说明 + +### 1. 文本检测 - [OcrDetectionDemo] + +- **功能**：检测图像中的文本区域，仅返回文本框位置，不识别文字内容。 + + +### 2. 文本方向检测 - [OcrDirectionDetDemo] + +- **功能**：在文本检测基础上，判断文本整体方向（0°, 90°, 180°, 270°）。 + +### 3. 文本识别 - [OcrRecognizeDemo] + +- **功能**：对检测到的文本区域进行文字识别，支持简体中文、繁体中文、英文、日文等。 +- **流程**： + - 文本检测 → 文本识别（或加上方向矫正） + +--- + + +## ⚙️ 配置要求 + +- **运行环境**： + - JDK 1.8 或更高版本 + - IntelliJ IDEA 推荐作为开发 IDE +- **依赖库**： + - OpenCV、DJL、SmartJavaAI SDK +- **模型路径**： + - 所有模型需下载并配置正确的路径（参考各 demo 注释中的链接） + +--- + +## 🚀 快速开始 + +1. 克隆项目到本地： + +2. 导入项目至 IntelliJ IDEA。 + +3. 根据需要修改模型路径（见各 demo 中注释）。 + +4. 运行对应的 JUnit 测试类方法即可体验各项功能。 + +--- + +## 📄 文档 + +有关完整使用说明，请查阅 SmartJavaAI 官方文档： +[http://doc.smartjavaai.cn](http://doc.smartjavaai.cn) + +--- diff --git a/examples/ocr-examples/pom.xml b/examples/ocr-examples/pom.xml new file mode 100644 index 0000000..5e8700b --- /dev/null +++ b/examples/ocr-examples/pom.xml @@ -0,0 +1,309 @@ + + + 4.0.0 + + cn.smartjavaai + examples + 1.0.0-SNAPSHOT + + + 11 + 11 + UTF-8 + 1.0.19 + + smartai.examples.ocr.OcrRecognizeDemo + + 1.5.10 + + macosx-arm64 + linux-x86_64 + linux-arm64 + windows-x86_64 + + + win-x86_64 + linux-x86_64 + linux-aarch64 + osx-aarch64 + + + + + + cn.smartjavaai + smartjavaai-bom + ${smartjavaai.version} + pom + + import + + + + + + + + commons-cli + commons-cli + 1.9.0 + + + commons-io + commons-io + 2.17.0 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.1 + + + org.testng + testng + 7.10.2 + test + + + + + ch.qos.logback + logback-classic + 1.2.3 + + + org.slf4j + slf4j-api + 1.7.30 + + + + com.alibaba + fastjson + 1.2.83 + + + + junit + junit + 4.13.2 + + + + + + + cn.smartjavaai + smartjavaai-ocr + + + + + ai.djl.pytorch + pytorch-jni + 2.5.1-0.32.0 + runtime + + + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.windows-x86_64} + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.windows-x86_64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.windows-x86_64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.windows-x86_64} + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.windows-x86_64} + 2.5.1 + runtime + + + + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.linux-x86_64} + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.linux-x86_64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.linux-x86_64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.linux-x86_64} + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.linux-x86_64} + 2.5.1 + runtime + + + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.macosx-arm64} + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.macosx-arm64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.macosx-arm64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.macosx-arm64} + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.osx-aarch64} + 2.5.1 + runtime + + + + + + org.bytedeco + javacpp + ${javacv.version} + ${javacv.platform.linux-arm64} + + + + org.bytedeco + ffmpeg + 6.1.1-1.5.10 + ${javacv.platform.linux-arm64} + + + + org.bytedeco + openblas + 0.3.26-1.5.10 + ${javacv.platform.linux-arm64} + + + + org.bytedeco + opencv + 4.9.0-1.5.10 + ${javacv.platform.linux-arm64} + + + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-aarch64} + 2.5.1 + runtime + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.0 + + + package + shade + + false + + + + ${exec.mainClass} + + + + + + + + + + + + aliyunmaven + 阿里云公共仓库 + https://maven.aliyun.com/repository/public + + true + + + false + + + + + + + + + + + diff --git a/examples/src/main/java/smartai/examples/ocr/OcrDetectionDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrDetectionDemo.java similarity index 68% rename from examples/src/main/java/smartai/examples/ocr/OcrDetectionDemo.java rename to examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrDetectionDemo.java index 1f0487f..ead2d71 100644 --- a/examples/src/main/java/smartai/examples/ocr/OcrDetectionDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrDetectionDemo.java @@ -1,8 +1,7 @@ package smartai.examples.ocr; import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.objectdetection.model.DetectorModel; -import cn.smartjavaai.objectdetection.model.ObjectDetectionModelFactory; +import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.ocr.config.OcrDetModelConfig; import cn.smartjavaai.ocr.entity.OcrBox; import cn.smartjavaai.ocr.enums.CommonDetModelEnum; @@ -20,14 +19,17 @@ import java.util.List; /** * OCR 文本检测 示例 - * 模型下载地址：https://pan.baidu.com/s/1MLfd73Vjdpnuls9-oqc9uw?pwd=1234 提取码: 1234 + * 模型下载地址：https://pan.baidu.com/s/15Noz2xHQzqMQSl1B19BobQ?pwd=1234 提取码: 1234 * @author dwj - * @date 2025/5/25 */ @Slf4j public class OcrDetectionDemo { + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + /** * 文本检测 * 检测图像中的文本区域，仅返回文本框位置，不识别文字内容 @@ -40,9 +42,13 @@ public class OcrDetectionDemo { config.setModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） config.setDetModelPath("/Users/xxx/Documents/develop/ocr模型/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); - OcrCommonDetModel model = OcrModelFactory.getInstance().getDetModel(config); - List boxes = model.detect("src/main/resources/ocr_1.jpg"); - log.info("OCR检测结果：{}", JSONObject.toJSONString(boxes)); + config.setDevice(device); + try (OcrCommonDetModel model = OcrModelFactory.getInstance().getDetModel(config)){ + List boxes = model.detect("src/main/resources/ocr_1.jpg"); + log.info("OCR检测结果：{}", JSONObject.toJSONString(boxes)); + } catch (Exception e) { + e.printStackTrace(); + } } /** @@ -57,8 +63,11 @@ public class OcrDetectionDemo { config.setModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); //指定模型位置，需要更改为自己的模型路径（下载地址请查看文档） config.setDetModelPath("/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); - OcrCommonDetModel model = OcrModelFactory.getInstance().getDetModel(config); - model.detectAndDraw("src/main/resources/ocr_1.jpg", "output/ocr_1_detected.jpg"); + try (OcrCommonDetModel model = OcrModelFactory.getInstance().getDetModel(config)){ + model.detectAndDraw("src/main/resources/ocr_1.jpg", "output/ocr_1_detected.jpg"); + } catch (Exception e) { + e.printStackTrace(); + } } diff --git a/examples/src/main/java/smartai/examples/ocr/OcrDirectionDetDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrDirectionDetDemo.java similarity index 64% rename from examples/src/main/java/smartai/examples/ocr/OcrDirectionDetDemo.java rename to examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrDirectionDetDemo.java index 9b04416..c2165fd 100644 --- a/examples/src/main/java/smartai/examples/ocr/OcrDirectionDetDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrDirectionDetDemo.java @@ -1,5 +1,6 @@ package smartai.examples.ocr; +import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.ocr.config.DirectionModelConfig; import cn.smartjavaai.ocr.config.OcrDetModelConfig; import cn.smartjavaai.ocr.entity.OcrBox; @@ -24,6 +25,27 @@ import java.util.List; @Slf4j public class OcrDirectionDetDemo { + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + /** + * 获取方向检测模型 + * @return + */ + public OcrDirectionModel getDirectionModel(){ + DirectionModelConfig directionModelConfig = new DirectionModelConfig(); + //指定检测模型 + directionModelConfig.setDetModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); + //指定检测模型位置，需要更改为自己的模型路径（下载地址请查看文档） + directionModelConfig.setDetModelPath("/Users/xxx/Documents/develop/ocr模型/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); + //指定文本方向检测模型 + directionModelConfig.setModelEnum(DirectionModelEnum.CH_PPOCR_MOBILE_V2_CLS); + //指定文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） + directionModelConfig.setModelPath("/Users/xxx/Documents/develop/ocr模型/ch_ppocr_mobile_v2.0_cls.onnx"); + directionModelConfig.setDevice(device); + return OcrModelFactory.getInstance().getDirectionModel(directionModelConfig); + } + /** * 文本方向检测 @@ -34,18 +56,13 @@ public class OcrDirectionDetDemo { */ @Test public void detect(){ - DirectionModelConfig directionModelConfig = new DirectionModelConfig(); - //指定检测模型 - directionModelConfig.setDetModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); - //指定检测模型位置，需要更改为自己的模型路径（下载地址请查看文档） - directionModelConfig.setDetModelPath("/Users/xxx/Documents/develop/ocr模型/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); - //指定文本方向检测模型 - directionModelConfig.setModelEnum(DirectionModelEnum.CH_PPOCR_MOBILE_V2_CLS); - //指定文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） - directionModelConfig.setModelPath("/Users/xxx/Documents/develop/ocr模型/ch_ppocr_mobile_v2.0_cls.onnx"); - OcrDirectionModel directionModel = OcrModelFactory.getInstance().getDirectionModel(directionModelConfig); - List itemList = directionModel.detect("src/main/resources/ocr_3.jpg"); - log.info("OCR方向检测结果：{}", JSONObject.toJSONString(itemList)); + try (OcrDirectionModel directionModel = getDirectionModel()){ + List itemList = directionModel.detect("src/main/resources/ocr_3.jpg"); + log.info("OCR方向检测结果：{}", JSONObject.toJSONString(itemList)); + } catch (Exception e) { + e.printStackTrace(); + } + } /** @@ -56,20 +73,13 @@ public class OcrDirectionDetDemo { */ @Test public void detectAndDraw(){ - DirectionModelConfig directionModelConfig = new DirectionModelConfig(); - //指定检测模型 - directionModelConfig.setDetModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); - //指定检测模型位置，需要更改为自己的模型路径（下载地址请查看文档） - directionModelConfig.setDetModelPath("/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); - //指定文本方向检测模型 - directionModelConfig.setModelEnum(DirectionModelEnum.CH_PPOCR_MOBILE_V2_CLS); - //指定文本方向检测模型路径，需要更改为自己的模型路径（下载地址请查看文档） - directionModelConfig.setModelPath("/cls/ch_ppocr_mobile_v2.0_cls.onnx"); - OcrDirectionModel directionModel = OcrModelFactory.getInstance().getDirectionModel(directionModelConfig); - directionModel.detectAndDraw("src/main/resources/ocr_3.jpg", "output/ocr_3_detected.png"); + try (OcrDirectionModel directionModel = getDirectionModel()){ + directionModel.detectAndDraw("src/main/resources/ocr_3.jpg", "output/ocr_3_detected.png"); + } catch (Exception e) { + e.printStackTrace(); + } } - } diff --git a/examples/src/main/java/smartai/examples/ocr/OcrRecognizeDemo.java b/examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrRecognizeDemo.java similarity index 59% rename from examples/src/main/java/smartai/examples/ocr/OcrRecognizeDemo.java rename to examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrRecognizeDemo.java index d0b7640..57b3310 100644 --- a/examples/src/main/java/smartai/examples/ocr/OcrRecognizeDemo.java +++ b/examples/ocr-examples/src/main/java/smartai/examples/ocr/OcrRecognizeDemo.java @@ -1,5 +1,6 @@ package smartai.examples.ocr; +import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.ocr.config.OcrDetModelConfig; import cn.smartjavaai.ocr.config.OcrRecModelConfig; import cn.smartjavaai.ocr.entity.OcrBox; @@ -14,6 +15,7 @@ import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import org.junit.Test; +import java.io.File; import java.util.List; /** @@ -25,15 +27,14 @@ import java.util.List; @Slf4j public class OcrRecognizeDemo { + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; /** - * 文本识别 - * 支持简体中文、繁体中文、英文、日文四种主要语言，以及手写、竖版、拼音、生僻字 - * 流程：文本检测 -> 文本识别 - * 模型需要放在单独文件夹 + * 获取通用识别模型（不带方向矫正） + * @return */ - @Test - public void recognize(){ + public OcrCommonRecModel getRecModel(){ OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); //指定检测模型 recModelConfig.setDetModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); @@ -43,9 +44,47 @@ public class OcrRecognizeDemo { recModelConfig.setRecModelEnum(CommonRecModelEnum.PADDLEOCR_V5_REC_MODEL); //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） recModelConfig.setRecModelPath("/Users/xxx/Documents/develop/ocr模型/PP-OCRv5_server_rec_infer/PP-OCRv5_server_rec.onnx"); - OcrCommonRecModel recModel = OcrModelFactory.getInstance().getRecModel(recModelConfig); - OcrInfo ocrInfo = recModel.recognize("src/main/resources/ocr_1.jpg"); - log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); + recModelConfig.setDevice(device); + return OcrModelFactory.getInstance().getRecModel(recModelConfig); + } + + /** + * 获取通用识别模型(带方向矫正) + * @return + */ + public OcrCommonRecModel getRecModelWithDirection() { + OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); + //指定检测模型 + recModelConfig.setDetModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); + //指定检测模型位置，需要更改为自己的模型路径（下载地址请查看文档） + recModelConfig.setDetModelPath("/Users/xxx/Documents/develop/ocr模型/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); + //指定识别模型 + recModelConfig.setRecModelEnum(CommonRecModelEnum.PADDLEOCR_V5_REC_MODEL); + //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） + recModelConfig.setRecModelPath("/Users/xxx/Documents/develop/ocr模型/PP-OCRv5_server_rec_infer/PP-OCRv5_server_rec.onnx"); + //指定方向检测模型 + recModelConfig.setDirectionModelEnum(DirectionModelEnum.CH_PPOCR_MOBILE_V2_CLS); + //指定方向模型位置，需要更改为自己的模型路径（下载地址请查看文档） + recModelConfig.setDirectionModelPath("/Users/xxx/Documents/develop/ocr模型/ch_ppocr_mobile_v2.0_cls.onnx"); + recModelConfig.setDevice(device); + return OcrModelFactory.getInstance().getRecModel(recModelConfig); + } + + + /** + * 文本识别 + * 支持简体中文、繁体中文、英文、日文四种主要语言，以及手写、竖版、拼音、生僻字 + * 流程：文本检测 -> 文本识别 + * 模型需要放在单独文件夹 + */ + @Test + public void recognize(){ + try (OcrCommonRecModel recModel = getRecModel()){ + OcrInfo ocrInfo = recModel.recognize("src/main/resources/ocr_2.jpg"); + log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); + } catch (Exception e) { + e.printStackTrace(); + } } @@ -57,18 +96,12 @@ public class OcrRecognizeDemo { */ @Test public void recognizeHandWriting(){ - OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定检测模型 - recModelConfig.setDetModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); - //指定检测模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setDetModelPath("/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); - //指定识别模型 - recModelConfig.setRecModelEnum(CommonRecModelEnum.PADDLEOCR_V5_REC_MODEL); - //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setRecModelPath("/PP-OCRv5_server_rec_infer/PP-OCRv5_server_rec.onnx"); - OcrCommonRecModel recModel = OcrModelFactory.getInstance().getRecModel(recModelConfig); - OcrInfo ocrInfo = recModel.recognize("src/main/resources/handwriting_1.jpg"); - log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); + try (OcrCommonRecModel recModel = getRecModel()){ + OcrInfo ocrInfo = recModel.recognize("src/main/resources/handwriting_1.jpg"); + log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); + } catch (Exception e) { + e.printStackTrace(); + } } /** @@ -80,22 +113,12 @@ public class OcrRecognizeDemo { */ @Test public void recognize2(){ - OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定检测模型 - recModelConfig.setDetModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); - //指定检测模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setDetModelPath("/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); - //指定识别模型 - recModelConfig.setRecModelEnum(CommonRecModelEnum.PADDLEOCR_V5_REC_MODEL); - //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setRecModelPath("/PP-OCRv5_server_rec_infer/PP-OCRv5_server_rec.onnx"); - //指定方向检测模型 - recModelConfig.setDirectionModelEnum(DirectionModelEnum.CH_PPOCR_MOBILE_V2_CLS); - //指定方向模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setDirectionModelPath("/cls/ch_ppocr_mobile_v2.0_cls.onnx"); - OcrCommonRecModel recModel = OcrModelFactory.getInstance().getRecModel(recModelConfig); - OcrInfo ocrInfo = recModel.recognize("src/main/resources/ocr_4.jpg"); - log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); + try (OcrCommonRecModel recModel = getRecModelWithDirection()){ + OcrInfo ocrInfo = recModel.recognize("src/main/resources/ocr_4.jpg"); + log.info("OCR识别结果：{}", JSONObject.toJSONString(ocrInfo)); + } catch (Exception e) { + e.printStackTrace(); + } } @@ -108,23 +131,12 @@ public class OcrRecognizeDemo { */ @Test public void recognizeAndDraw(){ - OcrRecModelConfig recModelConfig = new OcrRecModelConfig(); - //指定检测模型 - recModelConfig.setDetModelEnum(CommonDetModelEnum.PADDLEOCR_V5_DET_MODEL); - //指定检测模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setDetModelPath("/Users/xxx/Documents/develop/ocr模型/PP-OCRv5_server_det_infer/PP-OCRv5_server_det.onnx"); - //指定识别模型 - recModelConfig.setRecModelEnum(CommonRecModelEnum.PADDLEOCR_V5_REC_MODEL); - //directionModelConfig.setDirectionModelEnum(DirectionModelEnum.CH_PPOCR_MOBILE_V2_CLS); - //指定识别模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setRecModelPath("/Users/xxx/Documents/develop/ocr模型/PP-OCRv5_server_rec_infer/PP-OCRv5_server_rec.onnx"); - //指定方向检测模型 - recModelConfig.setDirectionModelEnum(DirectionModelEnum.CH_PPOCR_MOBILE_V2_CLS); - //指定方向模型位置，需要更改为自己的模型路径（下载地址请查看文档） - recModelConfig.setDirectionModelPath("/Users/xxx/Documents/develop/ocr模型/ch_ppocr_mobile_v2.0_cls.onnx"); - OcrCommonRecModel recModel = OcrModelFactory.getInstance().getRecModel(recModelConfig); - int fontSize = 25; - recModel.recognizeAndDraw("src/main/resources/ocr_4.jpg", "output/ocr_4_recognized.jpg", fontSize); + try (OcrCommonRecModel recModel = getRecModelWithDirection()){ + int fontSize = 25; + recModel.recognizeAndDraw("src/main/resources/ocr_4.jpg", "output/ocr_4_recognized.jpg", fontSize); + } catch (Exception e) { + e.printStackTrace(); + } } diff --git a/examples/ocr-examples/src/main/resources/META-INF/MANIFEST.MF b/examples/ocr-examples/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000..91b424f --- /dev/null +++ b/examples/ocr-examples/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Main-Class: smartai.examples.face.SeetaFace6LinuxDemo + diff --git a/examples/src/main/resources/general_ocr_002.png b/examples/ocr-examples/src/main/resources/general_ocr_002.png similarity index 100% rename from examples/src/main/resources/general_ocr_002.png rename to examples/ocr-examples/src/main/resources/general_ocr_002.png diff --git a/examples/src/main/resources/handwriting_1.jpg b/examples/ocr-examples/src/main/resources/handwriting_1.jpg similarity index 100% rename from examples/src/main/resources/handwriting_1.jpg rename to examples/ocr-examples/src/main/resources/handwriting_1.jpg diff --git a/examples/ocr-examples/src/main/resources/logback.xml b/examples/ocr-examples/src/main/resources/logback.xml new file mode 100644 index 0000000..809ebab --- /dev/null +++ b/examples/ocr-examples/src/main/resources/logback.xml @@ -0,0 +1,14 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{36}) - %msg%n + + + + + + + diff --git a/examples/src/main/resources/ocr_1.jpg b/examples/ocr-examples/src/main/resources/ocr_1.jpg similarity index 100% rename from examples/src/main/resources/ocr_1.jpg rename to examples/ocr-examples/src/main/resources/ocr_1.jpg diff --git a/examples/src/main/resources/ocr_2.jpg b/examples/ocr-examples/src/main/resources/ocr_2.jpg similarity index 100% rename from examples/src/main/resources/ocr_2.jpg rename to examples/ocr-examples/src/main/resources/ocr_2.jpg diff --git a/examples/src/main/resources/ocr_3.jpg b/examples/ocr-examples/src/main/resources/ocr_3.jpg similarity index 100% rename from examples/src/main/resources/ocr_3.jpg rename to examples/ocr-examples/src/main/resources/ocr_3.jpg diff --git a/examples/src/main/resources/ocr_4.jpg b/examples/ocr-examples/src/main/resources/ocr_4.jpg similarity index 100% rename from examples/src/main/resources/ocr_4.jpg rename to examples/ocr-examples/src/main/resources/ocr_4.jpg diff --git a/examples/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java b/examples/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java deleted file mode 100644 index a11f8d4..0000000 --- a/examples/src/main/java/smartai/examples/face/attribute/FaceAttributeDetDemo.java +++ /dev/null @@ -1,136 +0,0 @@ -package smartai.examples.face.attribute; - -import cn.smartjavaai.common.entity.*; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.face.config.FaceAttributeConfig; -import cn.smartjavaai.face.constant.LivenessConstant; -import cn.smartjavaai.face.enums.FaceModelEnum; -import cn.smartjavaai.face.enums.FaceAttributeModelEnum; -import cn.smartjavaai.face.exception.FaceException; -import cn.smartjavaai.face.factory.FaceModelFactory; -import cn.smartjavaai.face.factory.FaceAttributeModelFactory; -import cn.smartjavaai.face.model.attribute.FaceAttributeModel; -import cn.smartjavaai.face.model.facerec.FaceModel; -import cn.smartjavaai.face.utils.FaceUtils; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.bytedeco.javacv.FFmpegFrameGrabber; -import org.bytedeco.javacv.Frame; -import org.bytedeco.javacv.Java2DFrameUtils; -import org.junit.Test; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.List; - -/** - * 人脸属性检测demo - * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 - * @author dwj - * @date 2025/5/1 - */ -@Slf4j -public class FaceAttributeDetDemo { - - - /** - * 人脸属性检测（多人脸） - */ - @Test - public void testFaceAttributeDetect(){ - FaceAttributeConfig config = new FaceAttributeConfig(); - config.setModelEnum(FaceAttributeModelEnum.SEETA_FACE6_MODEL); - //需替换为实际模型存储路径 - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - FaceAttributeModel faceAttributeModel = FaceAttributeModelFactory.getInstance().getModel(config); - DetectionResponse detectionResponse = faceAttributeModel.detect("src/main/resources/double_person.png"); - try { - //绘制并导出人脸属性图片，小人脸仅有人脸框 - BufferedImage image = ImageIO.read(new File(Paths.get("src/main/resources/double_person.png").toAbsolutePath().toString())); - FaceUtils.drawBoxesWithFaceAttribute(image, detectionResponse,"C:/Users/Administrator/Downloads/double_person_.png"); - } catch (IOException e) { - e.printStackTrace(); - } - log.info("人脸属性检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } - - /** - * 图片人脸属性检测（分数最高人脸） - */ - @Test - public void testFaceAttributeDetect2(){ - FaceAttributeConfig config = new FaceAttributeConfig(); - config.setModelEnum(FaceAttributeModelEnum.SEETA_FACE6_MODEL); - //需替换为实际模型存储路径 - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - FaceAttributeModel faceAttributeModel = FaceAttributeModelFactory.getInstance().getModel(config); - FaceAttribute faceAttribute = faceAttributeModel.detectTopFace("src/main/resources/double_person.png"); - log.info("人脸属性检测结果：{}", JSONObject.toJSONString(faceAttribute)); - } - - /** - * 图片多人脸属性检测（基于已检测出的人脸区域和关键点） - */ - @Test - public void testFaceAttributeDetect3(){ - //人脸检测 - //需替换为实际模型存储路径 - String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; - FaceModelConfig faceDetectModelConfig = new FaceModelConfig(); - faceDetectModelConfig.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - faceDetectModelConfig.setModelPath(modelPath); - FaceModel faceDetectModel = FaceModelFactory.getInstance().getModel(faceDetectModelConfig); - DetectionResponse detectionResponse = faceDetectModel.detect("src/main/resources/double_person.png"); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse)); - //检测到人脸 - if(detectionResponse != null && detectionResponse.getDetectionInfoList() != null && detectionResponse.getDetectionInfoList().size() > 0){ - //人脸属性检测 - FaceAttributeConfig config = new FaceAttributeConfig(); - config.setModelEnum(FaceAttributeModelEnum.SEETA_FACE6_MODEL); - config.setModelPath(modelPath); - FaceAttributeModel faceAttributeModel = FaceAttributeModelFactory.getInstance().getModel(config); - List livenessStatusList = faceAttributeModel.detect("src/main/resources/double_person.png",detectionResponse); - log.info("人脸属性检测结果：{}", JSONObject.toJSONString(livenessStatusList)); - } - } - - /** - * 图片单人脸人脸属性检测（基于已检测出的人脸区域和关键点） - */ - @Test - public void testFaceAttributeDetect4(){ - try { - //人脸检测 - //需替换为实际模型存储路径 - String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; - String imagePath = "src/main/resources/double_person.png"; - FaceModelConfig faceDetectModelConfig = new FaceModelConfig(); - faceDetectModelConfig.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - faceDetectModelConfig.setModelPath(modelPath); - FaceModel faceDetectModel = FaceModelFactory.getInstance().getModel(faceDetectModelConfig); - DetectionResponse detectionResponse = faceDetectModel.detect(imagePath); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse)); - //检测到人脸 - if(detectionResponse != null && detectionResponse.getDetectionInfoList() != null && detectionResponse.getDetectionInfoList().size() > 0){ - //人脸属性检测 - FaceAttributeConfig config = new FaceAttributeConfig(); - config.setModelEnum(FaceAttributeModelEnum.SEETA_FACE6_MODEL); - config.setModelPath(modelPath); - FaceAttributeModel faceAttributeModel = FaceAttributeModelFactory.getInstance().getModel(config); - BufferedImage image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - for (DetectionInfo detectionInfo : detectionResponse.getDetectionInfoList()){ - FaceInfo faceInfo = detectionInfo.getFaceInfo(); - FaceAttribute faceAttribute = faceAttributeModel.detect(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); - log.info("人脸属性检测结果：{}", JSONObject.toJSONString(faceAttribute)); - } - } - } catch (Exception e){ - e.printStackTrace(); - } - } - - -} diff --git a/examples/src/main/java/smartai/examples/face/facerec/GpuFaceDemo.java b/examples/src/main/java/smartai/examples/face/facerec/GpuFaceDemo.java deleted file mode 100644 index fd997b7..0000000 --- a/examples/src/main/java/smartai/examples/face/facerec/GpuFaceDemo.java +++ /dev/null @@ -1,36 +0,0 @@ -package smartai.examples.face.facerec; - -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.face.enums.FaceModelEnum; -import cn.smartjavaai.face.factory.FaceModelFactory; -import cn.smartjavaai.face.model.facerec.FaceModel; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -/** - * GPU 人脸检测 - * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 - * @author dwj - * @date 2025/4/14 - */ -@Slf4j -public class GpuFaceDemo { - - /** - * 人脸检测(GPU) - * 图片参数：图片路径 - */ - @Test - public void testFaceGpu(){ - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.RETINA_FACE);//人脸模型 - config.setDevice(DeviceEnum.GPU); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - DetectionResponse detectedResult = faceModel.detect("src/main/resources/largest_selfie.jpg"); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult)); - } - -} diff --git a/examples/src/main/java/smartai/examples/face/facerec/LightFaceDemo.java b/examples/src/main/java/smartai/examples/face/facerec/LightFaceDemo.java deleted file mode 100644 index a8632c1..0000000 --- a/examples/src/main/java/smartai/examples/face/facerec/LightFaceDemo.java +++ /dev/null @@ -1,99 +0,0 @@ -package smartai.examples.face.facerec; - -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.face.enums.FaceModelEnum; -import cn.smartjavaai.face.factory.FaceModelFactory; -import cn.smartjavaai.face.model.facerec.FaceModel; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - -/** - * UltraLightFastGenericFaceModel 轻量人脸算法模型demo - * 支持功能：人脸检测（不支持人脸特征提取） - * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 - * @author dwj - * @date 2025/4/11 - */ -@Slf4j -public class LightFaceDemo { - - - /** - * 人脸检测-自定义参数 - * 图片参数：图片路径 - */ - @Test - public void testFaceDetectCustomConfig(){ - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);//人脸模型 - //config.setConfidenceThreshold(FaceConfig.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 - //config.setNmsThresh(FaceConfig.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - DetectionResponse detectedResult = faceModel.detect("src/main/resources/largest_selfie.jpg"); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult)); - } - - - /** - * 人脸检测并绘制人脸框 - */ - @Test - public void testFaceDetectAndDraw(){ - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);//人脸模型 - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - faceModel.detectAndDraw("src/main/resources/largest_selfie.jpg","output/largest_selfie_detected.png"); - } - - /** - * 人脸检测并绘制人脸框,返回BufferedImage - * - */ - @Test - public void testFaceDetectAndDraw2(){ - try { - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);//人脸模型 - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - BufferedImage image = null; - String imagePath = "src/main/resources/largest_selfie.jpg"; - image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - //可以根据后续业务场景使用detectedImage - BufferedImage detectedImage = faceModel.detectAndDraw(image); - Assert.assertNotNull("detectedImage null", detectedImage); - } catch (IOException e) { - e.printStackTrace(); - } - - } - - /** - * 人脸检测（离线模型） - */ - @Test - public void testDetectFaceOffine(){ - try { - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);//人脸模型 - //模型路径,不同模型下载路径请参看文档 - config.setModelPath("/Users/xxx/Documents/develop/face_model/ultranet.pt"); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - DetectionResponse detectedResult = faceModel.detect("src/main/resources/largest_selfie.jpg"); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - -} diff --git a/examples/src/main/java/smartai/examples/face/facerec/RetinaFaceDemo.java b/examples/src/main/java/smartai/examples/face/facerec/RetinaFaceDemo.java deleted file mode 100644 index cc6bd29..0000000 --- a/examples/src/main/java/smartai/examples/face/facerec/RetinaFaceDemo.java +++ /dev/null @@ -1,108 +0,0 @@ -package smartai.examples.face.facerec; - -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.face.constant.FaceDetectConstant; -import cn.smartjavaai.face.enums.FaceModelEnum; -import cn.smartjavaai.face.exception.FaceException; -import cn.smartjavaai.face.factory.FaceModelFactory; -import cn.smartjavaai.face.model.facerec.FaceModel; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - -/** - * RetinaFace人脸算法模型demo - * 支持功能：人脸检测（不支持人脸特征提取） - * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 - * @author dwj - * @date 2025/4/11 - */ -@Slf4j -public class RetinaFaceDemo { - - /** - * 人脸检测(默认配置) - * 使用默认模型参数检测，默认模型：retinaface，需联网，会自动下载模型 - * 图片参数：图片路径 - */ - @Test - public void testFaceDetect(){ - FaceModel faceModel = FaceModelFactory.getInstance().getModel(); - DetectionResponse detectedResult = faceModel.detect("src/main/resources/largest_selfie.jpg"); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult)); - } - - /** - * 人脸检测(自定义模型参数) - * 图片参数：图片路径 - */ - @Test - public void testFaceDetectCustomConfig(){ - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.RETINA_FACE);//人脸模型 - config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸 - config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个 - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - DetectionResponse detectedResult = faceModel.detect("src/main/resources/largest_selfie.jpg"); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult)); - } - - - /** - * 人脸检测并绘制人脸框 - */ - @Test - public void testFaceDetectAndDraw(){ - FaceModel faceModel = FaceModelFactory.getInstance().getModel(); - faceModel.detectAndDraw("src/main/resources/largest_selfie.jpg","output/largest_selfie_detected.png"); - } - - /** - * 人脸检测并绘制人脸框,返回BufferedImage - * - */ - @Test - public void testFaceDetectAndDraw2(){ - try { - FaceModel faceModel = FaceModelFactory.getInstance().getModel(); - BufferedImage image = null; - String imagePath = "src/main/resources/largest_selfie.jpg"; - image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - //可以根据后续业务场景使用detectedImage - BufferedImage detectedImage = faceModel.detectAndDraw(image); - Assert.assertNotNull("detectedImage null", detectedImage); - } catch (IOException e) { - e.printStackTrace(); - } - - } - - /** - * 人脸检测（离线模型） - */ - @Test - public void testDetectFaceOffine(){ - try { - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.RETINA_FACE);//人脸模型 - //模型路径,不同模型下载路径请参看文档 - config.setModelPath("/Users/xxx/Documents/develop/face_model/retinaface.pt"); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - DetectionResponse detectedResult = faceModel.detect("src/main/resources/largest_selfie.jpg"); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectedResult)); - } catch (Exception e) { - e.printStackTrace(); - } - } - - -} diff --git a/examples/src/main/java/smartai/examples/face/facerec/SeetaFace6Demo.java b/examples/src/main/java/smartai/examples/face/facerec/SeetaFace6Demo.java deleted file mode 100644 index 5a485b1..0000000 --- a/examples/src/main/java/smartai/examples/face/facerec/SeetaFace6Demo.java +++ /dev/null @@ -1,347 +0,0 @@ -package smartai.examples.face.facerec; - -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.entity.FaceSearchResult; -import cn.smartjavaai.common.entity.R; -import cn.smartjavaai.face.config.FaceExtractConfig; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.face.entity.FaceRegisterInfo; -import cn.smartjavaai.face.entity.FaceResult; -import cn.smartjavaai.face.entity.FaceSearchParams; -import cn.smartjavaai.face.enums.FaceModelEnum; -import cn.smartjavaai.face.enums.IdStrategy; -import cn.smartjavaai.face.enums.SimilarityType; -import cn.smartjavaai.face.factory.FaceModelFactory; -import cn.smartjavaai.face.model.facerec.FaceModel; -import cn.smartjavaai.face.utils.SimilarityUtil; -import cn.smartjavaai.face.vector.config.MilvusConfig; -import cn.smartjavaai.face.vector.config.SQLiteConfig; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; -import io.milvus.param.MetricType; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; - -/** - * SeetaFace6人脸算法模型demo - * 支持系统：windows 64位，linux 64位 - * 支持功能：人脸检测、人脸特征提取、人脸比对（1：1）、人脸比对（1：N）、人脸注册 - * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 - * @author dwj - * @date 2025/4/11 - */ -@Slf4j -public class SeetaFace6Demo { - - - /** - * 提取人脸特征(多人脸场景) - * 默认使用SEETA_FACE6_MODEL自己的检测模型 - * 注意事项： - * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 - */ - @Test - public void testExtractFeatures(){ - try { - //人脸特征提取模型 - FaceModelConfig config = new FaceModelConfig(); - //指定模型 - config.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - //指定模型路径：请根据实际情况替换为本地模型文件的绝对路径（模型下载地址请查看文档） - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - //提取图片中所有人脸特征 - R faceResult = faceModel.extractFeatures("src/main/resources/face/iu_1.jpg"); - if(faceResult.isSuccess()){ - log.info("人脸特征提取成功：{}", JSONObject.toJSONString(faceResult.getData())); - }else{ - log.info("人脸特征提取失败：{}", faceResult.getMessage()); - } - }catch (Exception e){ - e.printStackTrace(); - } - } - - /** - * 提取人脸特征(只提取图片中分数最高人脸特征) - * 默认使用SEETA_FACE6_MODEL自己的检测模型 - * 注意事项： - * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 - */ - @Test - public void testExtractFeatures2(){ - try { - //人脸特征提取模型 - FaceModelConfig config = new FaceModelConfig(); - //指定模型 - config.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - //指定模型路径：请根据实际情况替换为本地模型文件的绝对路径（模型下载地址请查看文档） - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - //提取图片中检测分数最高人脸特征 - R faceResult = faceModel.extractTopFaceFeature("src/main/resources/face/iu_1.jpg"); - if(faceResult.isSuccess()){ - log.info("人脸特征提取成功：{}", faceResult.getData()); - }else{ - log.info("人脸特征提取失败：{}", faceResult.getMessage()); - } - }catch (Exception e){ - e.printStackTrace(); - } - } - - - - - - /** - * 人脸比对1：1（基于图像直接比对） - * 流程：从输入图像中裁剪分数最高的人脸 → 提取其人脸特征 → 比对两张图片中提取的人脸特征。（接口内自动完成） - * 注意事项： - * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 - * @throws Exception - */ - @Test - public void featureComparison(){ - try { - FaceModelConfig config = new FaceModelConfig(); - //指定模型 - config.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - //指定模型路径：请根据实际情况替换为本地模型文件的绝对路径（模型下载地址请查看文档） - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - //基于图像直接比对人脸特征 - float similar = faceModel.featureComparison("src/main/resources/face/iu_1.jpg","src/main/resources/face/iu_2.jpg"); - log.info("相似度：{}", similar); - } - catch (Exception e){ - e.printStackTrace(); - } - } - - /** - * 人脸比对1：1（基于特征值比对） - * 流程：从输入图像中裁剪分数最高的人脸 → 提取其人脸特征 → 比对两张图片中提取的人脸特征。 - * 注意事项： - * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 - * @throws Exception - */ - @Test - public void featureComparison2(){ - try { - FaceModelConfig config = new FaceModelConfig(); - //指定模型 - config.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - //指定模型路径：请根据实际情况替换为本地模型文件的绝对路径（模型下载地址请查看文档） - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult1 = faceModel.extractTopFaceFeature("src/main/resources/face/iu_1.jpg"); - if(featureResult1.isSuccess()){ - log.info("图片1人脸特征提取成功：{}", JSONObject.toJSONString(featureResult1.getData())); - }else{ - log.info("图片1人脸特征提取失败：{}", featureResult1.getMessage()); - return; - } - //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceModel.extractTopFaceFeature("src/main/resources/face/iu_2.jpg"); - if(featureResult2.isSuccess()){ - log.info("图片2人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); - }else{ - log.info("图片2人脸特征提取失败：{}", featureResult2.getMessage()); - return; - } - //计算相似度 - float similar = faceModel.calculSimilar(featureResult1.getData(), featureResult2.getData()); - log.info("相似度：{}", similar); - } - catch (Exception e){ - e.printStackTrace(); - } - } - - - /** - * 人脸注册 + 人脸更新 + 人脸查询 + 人脸删除（使用向量数据库Milvus） - * 流程：从输入图像中裁剪分数最高的人脸 → 提取其人脸特征 → 注册人脸 - * 注意事项： - * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 - * 2、若人脸朝向较正，可关闭人脸对齐以提升性能。（方法参考自定义配置人脸特征提取） - * @throws Exception - */ - @Test - public void searchFace(){ - try { - FaceModelConfig config = new FaceModelConfig(); - //人脸模型 - config.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - //初始化向量数据库：Milvus数据库配置 - MilvusConfig vectorDBConfig = new MilvusConfig(); - vectorDBConfig.setHost("127.0.0.1"); - vectorDBConfig.setPort(19530); - //vectorDBConfig.setCollectionName("face10"); - //ID策略：自动生成 - vectorDBConfig.setIdStrategy(IdStrategy.AUTO); - //索引类型:内积 (Inner Product) 不建议修改 - vectorDBConfig.setMetricType(MetricType.COSINE); - config.setVectorDBConfig(vectorDBConfig); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - - //等待加载人脸库结束 - while (!faceModel.isLoadFaceCompleted()) { - Thread.sleep(50); // 避免 CPU 占用过高 - } - - log.info("====================人脸注册=========================="); - //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult = faceModel.extractTopFaceFeature("src/main/resources/face/iu_1.jpg"); - if(featureResult.isSuccess()){ - log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult.getData())); - }else{ - log.info("人脸特征提取失败：{}", featureResult.getMessage()); - return; - } - - //人脸注册信息 - FaceRegisterInfo faceRegisterInfo = new FaceRegisterInfo(); - //设置人脸注册的自定义元数据，本例中使用 JSON 格式存储用户信息 - JSONObject metadataJson = new JSONObject(); - metadataJson.put("name", "iu"); - metadataJson.put("age", "25"); - faceRegisterInfo.setMetadata(metadataJson.toJSONString()); - //人脸注册，返回人脸库ID - R registerResult = faceModel.register(faceRegisterInfo, featureResult.getData()); - if(registerResult.isSuccess()){ - log.info("注册成功：ID-{}", registerResult.getData()); - }else{ - log.info("注册失败：{}", registerResult.getMessage()); - } - /*log.info("====================人脸更新=========================="); - //更新人脸 只支持自定义ID：vectorDBConfig.setIdStrategy(IdStrategy.CUSTOM); - FaceRegisterInfo updateInfo = new FaceRegisterInfo(); - //设置人脸注册的自定义元数据，本例中使用 JSON 格式存储用户信息 - JSONObject metadataJsonUpdate = new JSONObject(); - metadataJsonUpdate.put("name", "iu_update"); - metadataJsonUpdate.put("age", "25"); - updateInfo.setMetadata(metadataJsonUpdate.toJSONString()); - //更新必须设置ID,只有 - updateInfo.setId(registerResult.getData()); - faceModel.upsertFace(updateInfo, "src/main/resources/face/iu_2.jpg"); - log.info("更新人脸成功");*/ - log.info("====================人脸查询=========================="); - //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceModel.extractTopFaceFeature("src/main/resources/face/iu_2.jpg"); - if(featureResult2.isSuccess()){ - log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); - }else{ - log.info("人脸特征提取失败：{}", featureResult2.getMessage()); - return; - } - FaceSearchParams faceSearchParams = new FaceSearchParams(); - faceSearchParams.setTopK(1); - faceSearchParams.setThreshold(0.8f); - List faceSearchResults = faceModel.search(featureResult2.getData(), faceSearchParams); - log.info("人脸查询结果：{}", JSONArray.toJSONString(faceSearchResults)); - log.info("====================人脸删除=========================="); - faceModel.removeRegister(registerResult.getData()); - log.info("人脸删除成功"); - } - catch (Exception e){ - e.printStackTrace(); - } - } - - /** - * 人脸注册 + 人脸更新 + 人脸查询 + 人脸删除（使用轻量数据库SQLite） - * 流程：从输入图像中裁剪分数最高的人脸 → 提取其人脸特征 → 注册人脸 - * 注意事项： - * 1、首次调用接口，可能会较慢。只要不关闭程序，后续调用会明显加快。若每次重启程序，则每次首次调用都将重新加载，仍会较慢。 - * 2、若人脸朝向较正，可关闭人脸对齐以提升性能。（方法参考自定义配置人脸特征提取） - * @throws Exception - */ - @Test - public void searchFace2(){ - try { - FaceModelConfig config = new FaceModelConfig(); - //人脸模型 - config.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - //使用轻量数据库SQLite - config.setVectorDBConfig(new SQLiteConfig()); - FaceModel faceModel = FaceModelFactory.getInstance().getModel(config); - log.info("====================人脸注册=========================="); - //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult = faceModel.extractTopFaceFeature("src/main/resources/face/iu_1.jpg"); - if(featureResult.isSuccess()){ - log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult.getData())); - }else{ - log.info("人脸特征提取失败：{}", featureResult.getMessage()); - return; - } - //人脸注册信息 - FaceRegisterInfo faceRegisterInfo = new FaceRegisterInfo(); - //设置人脸注册的自定义元数据，本例中使用 JSON 格式存储用户信息 - JSONObject metadataJson = new JSONObject(); - metadataJson.put("name", "iu"); - metadataJson.put("age", "25"); - faceRegisterInfo.setMetadata(metadataJson.toJSONString()); - //可自定义 ID，若未设置则自动生成。 - //faceRegisterInfo.setId("00001"); - //人脸注册，返回人脸库ID - R registerResult = faceModel.register(faceRegisterInfo, featureResult.getData()); - if(registerResult.isSuccess()){ - log.info("注册成功：ID-{}", registerResult.getData()); - }else{ - log.info("注册失败：{}", registerResult.getMessage()); - } - log.info("====================人脸更新=========================="); - FaceRegisterInfo updateInfo = new FaceRegisterInfo(); - //设置人脸注册的自定义元数据，本例中使用 JSON 格式存储用户信息 - JSONObject metadataJsonUpdate = new JSONObject(); - metadataJsonUpdate.put("name", "iu_update"); - metadataJsonUpdate.put("age", "25"); - updateInfo.setMetadata(metadataJsonUpdate.toJSONString()); - //更新必须设置ID,只有 - updateInfo.setId(registerResult.getData()); - faceModel.upsertFace(updateInfo, "src/main/resources/face/iu_2.jpg"); - log.info("更新人脸成功"); - log.info("====================人脸查询=========================="); - //特征提取（提取分数最高人脸特征）,适用于单人脸场景 - R featureResult2 = faceModel.extractTopFaceFeature("src/main/resources/face/iu_3.jpg"); - if(featureResult2.isSuccess()){ - log.info("人脸特征提取成功：{}", JSONObject.toJSONString(featureResult2.getData())); - }else{ - log.info("人脸特征提取失败：{}", featureResult2.getMessage()); - return; - } - FaceSearchParams faceSearchParams = new FaceSearchParams(); - faceSearchParams.setTopK(1); - faceSearchParams.setThreshold(0.62f); - //等待加载人脸库结束 - while (!faceModel.isLoadFaceCompleted()) { - Thread.sleep(50); // 避免 CPU 占用过高 - } - List faceSearchResults = faceModel.search(featureResult2.getData(), faceSearchParams); - log.info("人脸查询结果：{}", JSONArray.toJSONString(faceSearchResults)); - log.info("====================人脸删除=========================="); - faceModel.removeRegister(registerResult.getData()); - log.info("人脸删除成功"); - } - catch (Exception e){ - e.printStackTrace(); - } - } - - - -} diff --git a/examples/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java b/examples/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java deleted file mode 100644 index 68ff626..0000000 --- a/examples/src/main/java/smartai/examples/face/liveness/LivenessDetDemo.java +++ /dev/null @@ -1,284 +0,0 @@ -package smartai.examples.face.liveness; - -import cn.smartjavaai.common.entity.DetectionInfo; -import cn.smartjavaai.common.entity.DetectionRectangle; -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.entity.FaceInfo; -import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.common.enums.LivenessStatus; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.face.config.LivenessConfig; -import cn.smartjavaai.face.constant.LivenessConstant; -import cn.smartjavaai.face.enums.FaceModelEnum; -import cn.smartjavaai.face.enums.LivenessModelEnum; -import cn.smartjavaai.face.exception.FaceException; -import cn.smartjavaai.face.factory.FaceModelFactory; -import cn.smartjavaai.face.factory.LivenessModelFactory; -import cn.smartjavaai.face.model.facerec.FaceModel; -import cn.smartjavaai.face.model.liveness.LivenessDetModel; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.bytedeco.javacv.FFmpegFrameGrabber; -import org.bytedeco.javacv.Frame; -import org.bytedeco.javacv.Java2DFrameUtils; -import org.junit.Test; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.File; -import java.nio.file.Paths; -import java.util.List; - -/** - * 静态活体检测demo - * 模型下载地址：https://pan.baidu.com/s/10l22x5fRz_gwLr8EAHa1Jg?pwd=1234 提取码: 1234 - * @author dwj - * @date 2025/5/1 - */ -@Slf4j -public class LivenessDetDemo { - - - - /** - * 图片活体检测（多人脸） - */ - @Test - public void testLivenessDetect(){ - LivenessConfig config = new LivenessConfig(); - config.setModelEnum(LivenessModelEnum.SEETA_FACE6_MODEL); - config.setDevice(DeviceEnum.GPU); - //需替换为实际模型存储路径 - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - //人脸清晰度阈值,可选,默认0.3，活体识别时，如果清晰度低的话，就会直接返回FUZZY，清晰度满足阈值，则判断真实度 - config.setFaceClarityThreshold(LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD); - //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 - config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); - LivenessDetModel livenessDetModel = LivenessModelFactory.getInstance().getModel(config); - DetectionResponse livenessStatusList = livenessDetModel.detect("src/main/resources/double_person.png"); - log.info("活体检测结果：{}", JSONObject.toJSONString(livenessStatusList)); - } - - /** - * 图片活体检测（分数最高人脸） - */ - @Test - public void testLivenessDetect2(){ - LivenessConfig config = new LivenessConfig(); - config.setModelEnum(LivenessModelEnum.SEETA_FACE6_MODEL); - //需替换为实际模型存储路径 - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - //人脸清晰度阈值,可选,默认0.3，活体识别时，如果清晰度低的话，就会直接返回FUZZY，清晰度满足阈值，则判断真实度 - config.setFaceClarityThreshold(LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD); - //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 - config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); - LivenessDetModel livenessDetModel = LivenessModelFactory.getInstance().getModel(config); - LivenessStatus livenessStatus = livenessDetModel.detectTopFace("src/main/resources/double_person.png"); - log.info("活体检测结果：{}", JSONObject.toJSONString(livenessStatus)); - } - - /** - * 图片多人脸活体检测（基于已检测出的人脸区域和关键点） - */ - @Test - public void testLivenessDetect3(){ - //人脸检测 - //需替换为实际模型存储路径 - String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; - FaceModelConfig faceDetectModelConfig = new FaceModelConfig(); - faceDetectModelConfig.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - faceDetectModelConfig.setModelPath(modelPath); - FaceModel faceDetectModel = FaceModelFactory.getInstance().getModel(faceDetectModelConfig); - DetectionResponse detectionResponse = faceDetectModel.detect("src/main/resources/double_person.png"); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse)); - //检测到人脸 - if(detectionResponse != null && detectionResponse.getDetectionInfoList() != null && detectionResponse.getDetectionInfoList().size() > 0){ - //活体检测 - LivenessConfig config = new LivenessConfig(); - config.setModelEnum(LivenessModelEnum.SEETA_FACE6_MODEL); - config.setModelPath(modelPath); - //人脸清晰度阈值,可选,默认0.3，活体识别时，如果清晰度低的话，就会直接返回FUZZY，清晰度满足阈值，则判断真实度 - config.setFaceClarityThreshold(LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD); - //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 - config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); - LivenessDetModel livenessDetModel = LivenessModelFactory.getInstance().getModel(config); - List livenessStatusList = livenessDetModel.detect("src/main/resources/double_person.png",detectionResponse); - log.info("活体检测结果：{}", JSONObject.toJSONString(livenessStatusList)); - } - } - - /** - * 图片单人脸活体检测（基于已检测出的人脸区域和关键点） - */ - @Test - public void testLivenessDetect4(){ - try { - //人脸检测 - //需替换为实际模型存储路径 - String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; - String imagePath = "src/main/resources/double_person.png"; - FaceModelConfig faceDetectModelConfig = new FaceModelConfig(); - faceDetectModelConfig.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - faceDetectModelConfig.setModelPath(modelPath); - FaceModel faceDetectModel = FaceModelFactory.getInstance().getModel(faceDetectModelConfig); - DetectionResponse detectionResponse = faceDetectModel.detect(imagePath); - log.info("人脸检测结果：{}", JSONObject.toJSONString(detectionResponse)); - //检测到人脸 - if(detectionResponse != null && detectionResponse.getDetectionInfoList() != null && detectionResponse.getDetectionInfoList().size() > 0){ - //活体检测 - LivenessConfig config = new LivenessConfig(); - config.setModelEnum(LivenessModelEnum.SEETA_FACE6_MODEL); - config.setModelPath(modelPath); - //人脸清晰度阈值,可选,默认0.3，活体识别时，如果清晰度低的话，就会直接返回FUZZY，清晰度满足阈值，则判断真实度 - config.setFaceClarityThreshold(LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD); - //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 - config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); - LivenessDetModel livenessDetModel = LivenessModelFactory.getInstance().getModel(config); - BufferedImage image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - for (DetectionInfo detectionInfo : detectionResponse.getDetectionInfoList()){ - FaceInfo faceInfo = detectionInfo.getFaceInfo(); - LivenessStatus livenessStatus = livenessDetModel.detect(image, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); - log.info("活体检测结果：{}", JSONObject.toJSONString(livenessStatus)); - } - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 视频活体检测 - */ - @Test - public void testLivenessDetectVideo(){ - LivenessConfig config = new LivenessConfig(); - config.setModelEnum(LivenessModelEnum.SEETA_FACE6_MODEL); - //需替换为实际模型存储路径 - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - //人脸清晰度阈值,可选,默认0.3，活体识别时，如果清晰度低的话，就会直接返回FUZZY，清晰度满足阈值，则判断真实度 - config.setFaceClarityThreshold(LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD); - //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 - config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); - /*视频检测帧数，可选，默认10，输出帧数超过这个number之后，就可以输出识别结果。 - 这个数量相当于多帧识别结果融合的融合的帧数。当输入的帧数超过设定帧数的时候，会采用滑动窗口的方式，返回融合的最近输入的帧融合的识别结果。 - 一般来说，在10以内，帧数越多，结果越稳定，相对性能越好，但是得到结果的延时越高。*/ - config.setFrameCount(LivenessConstant.DEFAULT_FRAME_COUNT); - LivenessDetModel livenessDetModel = LivenessModelFactory.getInstance().getModel(config); - LivenessStatus livenessStatus = livenessDetModel.detectVideo("src/main/resources/girl.mp4"); - log.info("视频活体检测结果：{}", JSONObject.toJSONString(livenessStatus)); - } - - - - - /** - * 视频活体检测（逐帧检测，基于已检测出的人脸区域和关键点） - */ - @Test - public void testLivenessDetectVideo2(){ - LivenessConfig config = new LivenessConfig(); - config.setModelEnum(LivenessModelEnum.SEETA_FACE6_MODEL); - //需替换为实际模型存储路径 - config.setModelPath("C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"); - //人脸清晰度阈值,可选,默认0.3，活体识别时，如果清晰度低的话，就会直接返回FUZZY，清晰度满足阈值，则判断真实度 - config.setFaceClarityThreshold(LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD); - //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 - config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); - /* 视频检测帧数，可选，默认10，输出帧数超过这个number之后，就可以输出识别结果。 - 这个数量相当于多帧识别结果融合的融合的帧数。当输入的帧数超过设定帧数的时候，会采用滑动窗口的方式，返回融合的最近输入的帧融合的识别结果。 - 一般来说，在10以内，帧数越多，结果越稳定，相对性能越好，但是得到结果的延时越高。*/ - config.setFrameCount(LivenessConstant.DEFAULT_FRAME_COUNT); - LivenessDetModel livenessDetModel = LivenessModelFactory.getInstance().getModel(config); - try { - FFmpegFrameGrabber grabber = new FFmpegFrameGrabber("src/main/resources/girl.mp4"); - grabber.start(); - // 获取视频总帧数 - int totalFrames = grabber.getLengthInFrames(); - log.info("视频总帧数：{}，检测帧数：{}", totalFrames, config.getFrameCount()); - //活体检测结果 - LivenessStatus livenessStatus = LivenessStatus.UNKNOWN; - // 逐帧处理视频 - for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) { - // 获取当前帧 - Frame frame = grabber.grabImage(); - if (frame != null) { - BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame); - LivenessStatus livenessStatusFrame = livenessDetModel.detectVideoByFrame(bufferedImage); - //满足检测帧数之后停止检测 - if(livenessStatusFrame != LivenessStatus.DETECTING){ - livenessStatus = livenessStatusFrame; - } - } - } - log.info("视频活体检测结果：{}", JSONObject.toJSONString(livenessStatus)); - grabber.stop(); - } catch (FFmpegFrameGrabber.Exception e) { - throw new FaceException(e); - } - } - - /** - * 视频活体检测（逐帧检测） - */ - @Test - public void testLivenessDetectVideo3(){ - //获取活体检测模型 - //需替换为实际模型存储路径 - String modelPath = "C:/Users/Administrator/Downloads/sf3.0_models/sf3.0_models"; - LivenessConfig config = new LivenessConfig(); - config.setModelEnum(LivenessModelEnum.SEETA_FACE6_MODEL); - config.setModelPath(modelPath); - //人脸清晰度阈值,可选,默认0.3，活体识别时，如果清晰度低的话，就会直接返回FUZZY，清晰度满足阈值，则判断真实度 - config.setFaceClarityThreshold(LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD); - //人脸活体阈值,可选,默认0.8，超过阈值则认为是真人，低于阈值是非活体 - config.setRealityThreshold(LivenessConstant.DEFAULT_REALITY_THRESHOLD); - /* 视频检测帧数，可选，默认10，输出帧数超过这个number之后，就可以输出识别结果。 - 这个数量相当于多帧识别结果融合的融合的帧数。当输入的帧数超过设定帧数的时候，会采用滑动窗口的方式，返回融合的最近输入的帧融合的识别结果。 - 一般来说，在10以内，帧数越多，结果越稳定，相对性能越好，但是得到结果的延时越高。*/ - config.setFrameCount(LivenessConstant.DEFAULT_FRAME_COUNT); - LivenessDetModel livenessDetModel = LivenessModelFactory.getInstance().getModel(config); - //获取人脸检测模型 - FaceModelConfig faceDetectModelConfig = new FaceModelConfig(); - faceDetectModelConfig.setModelEnum(FaceModelEnum.SEETA_FACE6_MODEL); - faceDetectModelConfig.setModelPath(modelPath); - FaceModel faceDetectModel = FaceModelFactory.getInstance().getModel(faceDetectModelConfig); - try { - FFmpegFrameGrabber grabber = new FFmpegFrameGrabber("src/main/resources/girl.mp4"); - grabber.start(); - // 获取视频总帧数 - int totalFrames = grabber.getLengthInFrames(); - log.info("视频总帧数：{}，检测帧数：{}", totalFrames, config.getFrameCount()); - //活体检测结果 - LivenessStatus livenessStatus = LivenessStatus.UNKNOWN; - // 逐帧处理视频 - for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) { - // 获取当前帧 - Frame frame = grabber.grabImage(); - if (frame != null) { - BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame); - //检测视频帧人脸 - DetectionResponse detectionResponse = faceDetectModel.detect(bufferedImage); - //检测到人脸 - if(detectionResponse != null && detectionResponse.getDetectionInfoList() != null && detectionResponse.getDetectionInfoList().size() > 0){ - DetectionRectangle detectionRectangle = detectionResponse.getDetectionInfoList().get(0).getDetectionRectangle(); - FaceInfo faceInfo = detectionResponse.getDetectionInfoList().get(0).getFaceInfo(); - //使用人脸检测结果 活体检测 - LivenessStatus livenessStatusFrame = livenessDetModel.detectVideoByFrame(bufferedImage, detectionRectangle, faceInfo.getKeyPoints()); - //满足检测帧数之后停止检测 - if(livenessStatusFrame != LivenessStatus.DETECTING){ - livenessStatus = livenessStatusFrame; - } - }else{ - log.info("未检测到人脸"); - } - } - } - log.info("视频活体检测结果：{}", JSONObject.toJSONString(livenessStatus)); - grabber.stop(); - } catch (FFmpegFrameGrabber.Exception e) { - throw new FaceException(e); - } - } - - -} diff --git a/examples/src/main/java/smartai/examples/objectdetection/ObjectDetection.java b/examples/src/main/java/smartai/examples/objectdetection/ObjectDetection.java deleted file mode 100644 index df383ad..0000000 --- a/examples/src/main/java/smartai/examples/objectdetection/ObjectDetection.java +++ /dev/null @@ -1,148 +0,0 @@ -package smartai.examples.objectdetection; - -import ai.djl.Application; -import ai.djl.MalformedModelException; -import ai.djl.modality.cv.Image; -import ai.djl.modality.cv.ImageFactory; -import ai.djl.modality.cv.output.*; -import ai.djl.modality.cv.output.Rectangle; -import ai.djl.repository.zoo.Criteria; -import ai.djl.repository.zoo.ModelNotFoundException; -import ai.djl.repository.zoo.ModelZoo; -import ai.djl.repository.zoo.ZooModel; -import ai.djl.training.util.ProgressBar; -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.objectdetection.config.DetectorModelConfig; -import cn.smartjavaai.objectdetection.enums.DetectorModelEnum; -import cn.smartjavaai.objectdetection.exception.DetectionException; -import cn.smartjavaai.objectdetection.model.DetectorModel; -import cn.smartjavaai.objectdetection.model.ObjectDetectionModelFactory; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; - -import javax.imageio.ImageIO; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -/** - * 目标检测模型demo - * 支持功能：目标检测 - * 模型下载地址：https://pan.baidu.com/s/1MeQ0oHGl8hneicUIUVJjbg?pwd=1234 提取码: 1234 - * @author dwj - * @date 2025/4/11 - */ -@Slf4j -public class ObjectDetection { - - /** - * 使用默认模型检测：YOLO11N - */ - @Test - public void objectDetection(){ - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/object_detection.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } - - /** - * 指定模型检测(19种模型可选) - */ - @Test - public void objectDetection2(){ - DetectorModelConfig config = new DetectorModelConfig(); - config.setModelEnum(DetectorModelEnum.SSD_300_RESNET50);//检测模型，目前支持19种预置模型 - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } - - /** - * 人脸检测并绘制检测结果 - */ - @Test - public void objectDetectionAndDraw(){ - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - detectorModel.detectAndDraw("src/main/resources/object_detection.jpg","output/object_detection_detected.png"); - } - - /** - * 人脸检测并绘制检测结果,返回BufferedImage - */ - @Test - public void objectDetectionAndDraw2(){ - try { - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(); - BufferedImage image = null; - String imagePath = "src/main/resources/object_detection.jpg"; - image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - //可以根据后续业务场景使用detectedImage - BufferedImage detectedImage = detectorModel.detectAndDraw(image); - Assert.assertNotNull("detectedImage null", detectedImage); - } catch (IOException e) { - throw new RuntimeException(e); - } - - } - - /** - * GPU 目标检测 - */ - @Test - public void gpuObjectDetection(){ - DetectorModelConfig config = new DetectorModelConfig(); - config.setModelEnum(DetectorModelEnum.YOLO11N);//检测模型，目前支持19种模型 - config.setDevice(DeviceEnum.GPU); - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - DetectionResponse detectionResponse = detectorModel.detect("src/main/resources/dog_bike_car.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detectionResponse)); - } - - - /** - * 使用yolo官方模型检测物品识别 - */ - @Test - public void objectDetectionWithOfficialModel(){ - DetectorModelConfig config = new DetectorModelConfig(); - config.setThreshold(0.3f); - //也支持YoloV8：YOLOV8_OFFICIAL 模型可以从文档中提供的地址下载 - config.setModelEnum(DetectorModelEnum.YOLOV12_OFFICIAL);//检测模型，目前支持19种模型 - // 指定模型路径，需要更改为自己的模型路径 - config.setModelPath("E:\\ai\\models\\yolo12m\\yolov12m.onnx"); - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - //一定要将yolo官方的类别文件：synset.txt（文档中下载）放在模型同目录下，否则报错 - DetectionResponse detect = detectorModel.detect("E:\\ai\\testimage\\1.jpg"); - log.info("目标检测结果：{}", JSONObject.toJSONString(detect)); - detectorModel.detectAndDraw("E:\\ai\\testimage\\1.jpg","E:\\ai\\outimage\\11.png"); - } - - /** - * 使用自己训练的模型检测 - */ - @Test - public void objectDetectionWithCustomModel(){ - DetectorModelConfig config = new DetectorModelConfig(); - //也支持YoloV8：YOLOV8_CUSTOM 模型需要自己训练，训练教程可以查看文档 - config.setModelEnum(DetectorModelEnum.YOLOV12_CUSTOM);//自定义YOLOV12模型 - // 指定模型路径，需要更改为自己的模型路径 - config.setModelPath("/Users/xxx/Documents/develop/fire_model/best.onnx"); - DetectorModel detectorModel = ObjectDetectionModelFactory.getInstance().getModel(config); - //一定要将类别文件：synset.txt 放在模型同目录下，否则报错(具体请参看文档) - detectorModel.detectAndDraw("/Users/xxx/Downloads/test.jpg","output/test_detected.jpg"); - } - - -} diff --git a/examples/src/main/resources/double_person.png b/examples/src/main/resources/double_person.png deleted file mode 100644 index b6856fb..0000000 Binary files a/examples/src/main/resources/double_person.png and /dev/null differ diff --git a/examples/src/main/resources/girl.mp4 b/examples/src/main/resources/girl.mp4 deleted file mode 100644 index 5120f9c..0000000 Binary files a/examples/src/main/resources/girl.mp4 and /dev/null differ diff --git a/examples/src/main/resources/kana1.jpg b/examples/src/main/resources/kana1.jpg deleted file mode 100644 index ef364e0..0000000 Binary files a/examples/src/main/resources/kana1.jpg and /dev/null differ diff --git a/examples/src/main/resources/kana2.jpg b/examples/src/main/resources/kana2.jpg deleted file mode 100644 index 59c9e52..0000000 Binary files a/examples/src/main/resources/kana2.jpg and /dev/null differ diff --git a/examples/translation-example/.gitignore b/examples/translation-example/.gitignore new file mode 100644 index 0000000..93dbf83 --- /dev/null +++ b/examples/translation-example/.gitignore @@ -0,0 +1,7 @@ +.idea +.idea/ +target +log +*.iml +/.settings/ +/logging.file_IS_UNDEFINED/ diff --git a/examples/translation-example/README.md b/examples/translation-example/README.md new file mode 100644 index 0000000..657a51d --- /dev/null +++ b/examples/translation-example/README.md @@ -0,0 +1,40 @@ +# 机器翻译示例 + + +## 📁 项目结构 + +``` + +└── main + ├── java + │ └── smartai + │ └── examples + │ └── nlp + │ └── translation 机器翻译 + │ └── TranslationDemo.java + └── resources + └── logback.xml + + +``` +--- + + +## 🚀 快速开始 + +1. 克隆项目到本地： + +2. 导入项目至 IntelliJ IDEA。 + +3. 根据需要修改模型路径（见各 demo 中注释）。 + +4. 运行对应的 JUnit 测试类方法即可体验各项功能。 + +--- + +## 📄 文档 + +有关完整使用说明，请查阅 SmartJavaAI 官方文档： +[http://doc.smartjavaai.cn](http://doc.smartjavaai.cn) + +--- diff --git a/examples/translation-example/pom.xml b/examples/translation-example/pom.xml new file mode 100644 index 0000000..6823205 --- /dev/null +++ b/examples/translation-example/pom.xml @@ -0,0 +1,201 @@ + + + 4.0.0 + + cn.smartjavaai + examples + 1.0.0-SNAPSHOT + + + 11 + 11 + UTF-8 + 1.0.19 + + smartai.examples.nlp.translation.TranslationDemo + + 1.5.10 + + macosx-arm64 + linux-x86_64 + linux-arm64 + windows-x86_64 + + + win-x86_64 + linux-x86_64 + linux-aarch64 + osx-aarch64 + + + + + + cn.smartjavaai + smartjavaai-bom + ${smartjavaai.version} + pom + + import + + + + + + + + commons-cli + commons-cli + 1.9.0 + + + commons-io + commons-io + 2.17.0 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.1 + + + org.testng + testng + 7.10.2 + test + + + + + ch.qos.logback + logback-classic + 1.2.3 + + + org.slf4j + slf4j-api + 1.7.30 + + + + com.alibaba + fastjson + 1.2.83 + + + + junit + junit + 4.13.2 + + + + + cn.smartjavaai + smartjavaai-translate + + + + + ai.djl.pytorch + pytorch-jni + 2.5.1-0.32.0 + runtime + + + + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.windows-x86_64} + 2.5.1 + runtime + + + + + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.linux-x86_64} + 2.5.1 + runtime + + + + + + ai.djl.pytorch + pytorch-native-cpu + ${djl.platform.osx-aarch64} + 2.5.1 + runtime + + + + + + + ai.djl.pytorch + pytorch-native-cpu-precxx11 + ${djl.platform.linux-aarch64} + 2.5.1 + runtime + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.0 + + + package + shade + + false + + + + ${exec.mainClass} + + + + + + + + + + + + aliyunmaven + 阿里云公共仓库 + https://maven.aliyun.com/repository/public + + true + + + false + + + + + + + + + + + diff --git a/examples/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java b/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java similarity index 97% rename from examples/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java rename to examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java index 9e27a39..ed2bcca 100644 --- a/examples/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java +++ b/examples/translation-example/src/main/java/smartai/examples/nlp/translation/TranslationDemo.java @@ -17,9 +17,8 @@ import org.junit.Test; * 翻译Demo * 支持 Meta AI 开源的 NLLB-200 模型，实现 200 多种语言之间的高质量互译。 * NLLB-200官网地址：https://github.com/facebookresearch/fairseq/tree/nllb - * 模型下载地址：https://pan.baidu.com/s/1_AD5QGQ6f6uOajJ-kW20rg?pwd=1234 提取码: 1234 + * 模型下载地址：https://pan.baidu.com/s/1wf7btnb4cyBFv7DB7baHnw?pwd=1234 提取码: 1234 * @author dwj - * @date 2025/6/17 */ @Slf4j public class TranslationDemo { diff --git a/examples/translation-example/src/main/resources/META-INF/MANIFEST.MF b/examples/translation-example/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000..91b424f --- /dev/null +++ b/examples/translation-example/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Main-Class: smartai.examples.face.SeetaFace6LinuxDemo + diff --git a/examples/translation-example/src/main/resources/logback.xml b/examples/translation-example/src/main/resources/logback.xml new file mode 100644 index 0000000..809ebab --- /dev/null +++ b/examples/translation-example/src/main/resources/logback.xml @@ -0,0 +1,14 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{36}) - %msg%n + + + + + + + diff --git a/pom.xml b/pom.xml index 79b7855..6df95c2 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ SmartJavaAI cn.smartjavaai smartjavaai-parent - 1.0.17 + 1.0.19 pom SmartJavaAI diff --git a/smartjavaai-all/pom.xml b/smartjavaai-all/pom.xml index c5d52ed..3bd7b2d 100644 --- a/smartjavaai-all/pom.xml +++ b/smartjavaai-all/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.17 + 1.0.19 smartjavaai-all - 1.0.17 + 1.0.19 ${project.artifactId} SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/smartjavaai-bom/pom.xml b/smartjavaai-bom/pom.xml index 0382c55..22c1a01 100644 --- a/smartjavaai-bom/pom.xml +++ b/smartjavaai-bom/pom.xml @@ -6,10 +6,10 @@ cn.smartjavaai smartjavaai-parent - 1.0.17 + 1.0.19 - 1.0.17 + 1.0.19 smartjavaai-bom smartjavaai-bom 统一版本管理的 BOM 包，同时支持 import 和全量依赖 diff --git a/smartjavaai-common/pom.xml b/smartjavaai-common/pom.xml index 8b6d6e2..8767b3d 100644 --- a/smartjavaai-common/pom.xml +++ b/smartjavaai-common/pom.xml @@ -6,7 +6,7 @@ cn.smartjavaai smartjavaai-parent - 1.0.17 + 1.0.19 smartjavaai-common diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java index 7fbfcf7..4142637 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionInfo.java @@ -1,5 +1,6 @@ package cn.smartjavaai.common.entity; +import cn.smartjavaai.common.entity.face.FaceInfo; import lombok.Data; /** diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionRectangle.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionRectangle.java index 4b53aee..a4bc898 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionRectangle.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/DetectionRectangle.java @@ -1,11 +1,7 @@ package cn.smartjavaai.common.entity; -import cn.smartjavaai.common.enums.GenderType; -import cn.smartjavaai.common.enums.LivenessStatus; import lombok.Data; -import java.util.List; - /** * 检测结果-矩形区域 * @author dwj diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/ObjectDetInfo.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/ObjectDetInfo.java index 3a5cec7..8cfd236 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/ObjectDetInfo.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/ObjectDetInfo.java @@ -1,6 +1,5 @@ package cn.smartjavaai.common.entity; -import cn.smartjavaai.common.enums.LivenessStatus; import lombok.Data; /** diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/R.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/R.java index 9a316af..30cc3d3 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/R.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/R.java @@ -18,6 +18,12 @@ public class R { private T data; + public static R ok() { + R r = new R<>(); + r.code = 0; + r.message = "成功"; + return r; + } public static R ok(T data) { R r = new R<>(); @@ -50,6 +56,7 @@ public class R { FILE_NOT_FOUND(2, "图像文件不存在"), NO_FACE_DETECTED(3, "未检测到人脸"), PARAM_ERROR(4, "参数错误"), + INVALID_VIDEO(5, "视频无效"), Unknown(-1, "未知错误"); private final int code; diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/ExpressionResult.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/ExpressionResult.java new file mode 100644 index 0000000..8cac05d --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/ExpressionResult.java @@ -0,0 +1,44 @@ +package cn.smartjavaai.common.entity.face; + +import ai.djl.modality.Classifications; +import cn.smartjavaai.common.enums.face.FacialExpression; +import cn.smartjavaai.common.enums.face.LivenessStatus; +import lombok.Data; + +/** + * 人脸表情识别结果 + * @author dwj + */ +@Data +public class ExpressionResult { + + /** + * 表情 + */ + private FacialExpression expression; + + /** + * 分数 + */ + private float score; + + /** + * 完整结果 + */ + private Classifications classifications; + + public ExpressionResult() { + } + + + public ExpressionResult(FacialExpression expression, float score) { + this.expression = expression; + this.score = score; + } + + public ExpressionResult(FacialExpression expression, float score, Classifications classifications) { + this.expression = expression; + this.score = score; + this.classifications = classifications; + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceAttribute.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceAttribute.java similarity index 79% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceAttribute.java rename to smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceAttribute.java index d27a833..a51e168 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceAttribute.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceAttribute.java @@ -1,7 +1,7 @@ -package cn.smartjavaai.common.entity; +package cn.smartjavaai.common.entity.face; -import cn.smartjavaai.common.enums.EyeStatus; -import cn.smartjavaai.common.enums.GenderType; +import cn.smartjavaai.common.enums.face.EyeStatus; +import cn.smartjavaai.common.enums.face.GenderType; import lombok.Data; /** diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceInfo.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceInfo.java similarity index 73% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceInfo.java rename to smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceInfo.java index e3d9261..9dde659 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceInfo.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceInfo.java @@ -1,6 +1,6 @@ -package cn.smartjavaai.common.entity; +package cn.smartjavaai.common.entity.face; -import cn.smartjavaai.common.enums.LivenessStatus; +import cn.smartjavaai.common.entity.Point; import lombok.Data; import java.util.List; @@ -26,7 +26,7 @@ public class FaceInfo { /** * 活体检测结果 */ - private LivenessStatus livenessStatus; + private LivenessResult livenessStatus; /** * 人脸查询结果 @@ -38,6 +38,11 @@ public class FaceInfo { */ private float[] feature; + /** + * 表情检测结果 + */ + private ExpressionResult expressionResult; + public FaceInfo() { } @@ -45,14 +50,16 @@ public class FaceInfo { this.keyPoints = keyPoints; } - public FaceInfo(List keyPoints, FaceAttribute faceAttribute, LivenessStatus livenessStatus) { + public FaceInfo(List keyPoints, FaceAttribute faceAttribute, LivenessResult livenessStatus) { this.keyPoints = keyPoints; this.faceAttribute = faceAttribute; this.livenessStatus = livenessStatus; } - public FaceInfo(FaceAttribute faceAttribute, LivenessStatus livenessStatus) { + public FaceInfo(FaceAttribute faceAttribute, LivenessResult livenessStatus) { this.faceAttribute = faceAttribute; this.livenessStatus = livenessStatus; } + + } diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceSearchResult.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceSearchResult.java similarity index 93% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceSearchResult.java rename to smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceSearchResult.java index c0f6aab..fc4ee24 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/FaceSearchResult.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/FaceSearchResult.java @@ -1,4 +1,4 @@ -package cn.smartjavaai.common.entity; +package cn.smartjavaai.common.entity.face; import lombok.Data; /** diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/HeadPose.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/HeadPose.java similarity index 94% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/HeadPose.java rename to smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/HeadPose.java index e873c06..dd2c006 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/HeadPose.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/HeadPose.java @@ -1,4 +1,4 @@ -package cn.smartjavaai.common.entity; +package cn.smartjavaai.common.entity.face; import lombok.Data; diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/LivenessResult.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/LivenessResult.java new file mode 100644 index 0000000..360f28e --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/face/LivenessResult.java @@ -0,0 +1,29 @@ +package cn.smartjavaai.common.entity.face; + +import cn.smartjavaai.common.enums.face.LivenessStatus; +import lombok.Data; + +/** + * 活体检测结果 + * @author dwj + * @date 2025/6/27 + */ +@Data +public class LivenessResult { + + private LivenessStatus status; + + private float score; + + public LivenessResult() { + } + + public LivenessResult(LivenessStatus status, float score) { + this.status = status; + this.score = score; + } + + public LivenessResult(LivenessStatus status) { + this.status = status; + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/EyeStatus.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/EyeStatus.java similarity index 94% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/EyeStatus.java rename to smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/EyeStatus.java index d3f03b4..e0d9656 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/EyeStatus.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/EyeStatus.java @@ -1,4 +1,4 @@ -package cn.smartjavaai.common.enums; +package cn.smartjavaai.common.enums.face; /** * 眼睛状态 diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/FacialExpression.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/FacialExpression.java new file mode 100644 index 0000000..9058df4 --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/FacialExpression.java @@ -0,0 +1,42 @@ +package cn.smartjavaai.common.enums.face; + +/** + * 人脸表情枚举 + * @author dwj + */ +public enum FacialExpression { + + ANGRY("angry", "愤怒"), + DISGUST("disgust", "厌恶"), + FEAR("fear", "害怕"), + HAPPY("happy", "高兴"), + SAD("sad", "伤心"), + SURPRISE("surprise", "惊讶"), + NEUTRAL("neutral", "中性"); + + private final String label; + private final String description; + + FacialExpression(String label, String description) { + this.label = label; + this.description = description; + } + + public String getLabel() { + return label; + } + + public String getDescription() { + return description; + } + + public static FacialExpression fromLabel(String label) { + for (FacialExpression facialExpression : FacialExpression.values()) { + if (facialExpression.getLabel().equals(label)) { + return facialExpression; + } + } + throw new IllegalArgumentException("Invalid facial expression label: " + label); + } + +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/GenderType.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/GenderType.java similarity index 94% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/GenderType.java rename to smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/GenderType.java index 7265f20..19f064a 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/GenderType.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/GenderType.java @@ -1,4 +1,4 @@ -package cn.smartjavaai.common.enums; +package cn.smartjavaai.common.enums.face; /** * 性别枚举 diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/LivenessStatus.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/LivenessStatus.java similarity index 95% rename from smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/LivenessStatus.java rename to smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/LivenessStatus.java index ff2416e..50801d9 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/LivenessStatus.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/enums/face/LivenessStatus.java @@ -1,4 +1,4 @@ -package cn.smartjavaai.common.enums; +package cn.smartjavaai.common.enums.face; /** * 活体检测结果 diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java new file mode 100644 index 0000000..ec0b109 --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/preprocess/BufferedImagePreprocessor.java @@ -0,0 +1,160 @@ +package cn.smartjavaai.common.preprocess; + +import cn.smartjavaai.common.entity.DetectionRectangle; + +import java.awt.*; +import java.awt.image.BufferedImage; + +/** + * 图片预处理 + * @author dwj + * @date 2025/6/27 + */ +public class BufferedImagePreprocessor { + + private BufferedImage image; + private DetectionRectangle rect; + private float extendRatio = 1; + private int targetSize = 128; + private int centerCropSize = 80; + + private Color paddingColor = new Color(127, 127, 127); // 默认灰色 + private boolean enableSquarePadding = true; + private boolean enableScaling = true; + private boolean enableCenterCrop = false; + + + + public BufferedImagePreprocessor(BufferedImage image, DetectionRectangle rect) { + this.image = image; + this.rect = rect; + } + + public BufferedImagePreprocessor setExtendRatio(float ratio) { + this.extendRatio = ratio; + return this; + } + + public BufferedImagePreprocessor setTargetSize(int size) { + this.targetSize = size; + return this; + } + + public BufferedImagePreprocessor setCenterCropSize(int size) { + this.centerCropSize = size; + return this; + } + + public BufferedImagePreprocessor enableSquarePadding(boolean enable) { + this.enableSquarePadding = enable; + return this; + } + + public BufferedImagePreprocessor enableScaling(boolean enable) { + this.enableScaling = enable; + return this; + } + + public BufferedImagePreprocessor enableCenterCrop(boolean enable) { + this.enableCenterCrop = enable; + return this; + } + + public BufferedImagePreprocessor setPaddingColor(Color color) { + this.paddingColor = color; + return this; + } + + + public BufferedImage process() { + // Step 1: 基于检测框扩展 + BufferedImage cropped = cropAndExtend(); + + // Step 2: 补正方形 + 背景填充 + BufferedImage squared = enableSquarePadding ? squarePadding(cropped) : cropped; + + // Step 3: 缩放 + BufferedImage scaled = enableScaling ? scaleToTarget(squared) : squared; + + // Step 4: CenterCrop + BufferedImage finalResult = enableCenterCrop ? centerCrop(scaled) : scaled; + + return finalResult; + } + + /** + * 检测框扩展及裁剪 + * @return + */ + private BufferedImage cropAndExtend() { + int x = rect.x; + int y = rect.y; + int width = rect.width; + int height = rect.height; + + int extendX = Math.round(width * extendRatio); + int extendY = Math.round(height * extendRatio); + + // 计算扩展后的边界 (确保不超出图像范围) + int left = Math.max(0, x - extendX); + int right = Math.min(image.getWidth(), x + width + extendX); + int top = Math.max(0, y - extendY); + int bottom = Math.min(image.getHeight(), y + height + extendY); + + // 动态计算最大可用扩展区域 + int origRoiWidth = right - left; + int origRoiHeight = bottom - top; + int longSide = Math.max(origRoiWidth, origRoiHeight); + + // 计算可扩展空间（不超出原图边界） + int extendLeft = Math.min(left, (longSide - origRoiWidth) / 2); + int extendRight = Math.min(image.getWidth() - right, (longSide - origRoiWidth + 1) / 2); + int extendTop = Math.min(top, (longSide - origRoiHeight) / 2); + int extendBottom = Math.min(image.getHeight() - bottom, (longSide - origRoiHeight + 1) / 2); + + // 计算实际扩展后的区域 + int expandedLeft = left - extendLeft; + int expandedRight = right + extendRight; + int expandedTop = top - extendTop; + int expandedBottom = bottom + extendBottom; + + int expandedWidth = expandedRight - expandedLeft; + int expandedHeight = expandedBottom - expandedTop; + + return image.getSubimage(expandedLeft, expandedTop, expandedWidth, expandedHeight); + } + + /** + * 填充正方形 + * @param src + * @return + */ + private BufferedImage squarePadding(BufferedImage src) { + int longSide = Math.max(src.getWidth(), src.getHeight()); + BufferedImage squared = new BufferedImage(longSide, longSide, BufferedImage.TYPE_3BYTE_BGR); + Graphics2D g = squared.createGraphics(); + g.setColor(paddingColor); + g.fillRect(0, 0, longSide, longSide); + int xOffset = (longSide - src.getWidth()) / 2; + int yOffset = (longSide - src.getHeight()) / 2; + g.drawImage(src, xOffset, yOffset, null); + g.dispose(); + return squared; + } + + private BufferedImage scaleToTarget(BufferedImage src) { + Image scaled = src.getScaledInstance(targetSize, targetSize, Image.SCALE_SMOOTH); + BufferedImage result = new BufferedImage(targetSize, targetSize, BufferedImage.TYPE_3BYTE_BGR); + Graphics2D g = result.createGraphics(); + g.drawImage(scaled, 0, 0, null); + g.dispose(); + return result; + } + + private BufferedImage centerCrop(BufferedImage src) { + int startX = (src.getWidth() - centerCropSize) / 2; + int startY = (src.getHeight() - centerCropSize) / 2; + return src.getSubimage(startX, startY, centerCropSize, centerCropSize); + } + +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ArrayUtils.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ArrayUtils.java new file mode 100644 index 0000000..e1d7b7c --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ArrayUtils.java @@ -0,0 +1,39 @@ +package cn.smartjavaai.common.utils; + +/** + * 数组工具类 + * @author dwj + * @date 2025/6/27 + */ +public class ArrayUtils { + + /** + * 求和并找到最大值的索引 + * @param arr1 + * @param arr2 + * @return + */ + public static int sumAndFindMaxIndex(float[] arr1, float[] arr2, int length) { + float[] sum = new float[length]; + + // 处理可能为null的情况，null当作全0数组处理 + for (int i = 0; i < length; i++) { + float v1 = (arr1 != null && arr1.length > i) ? arr1[i] : 0f; + float v2 = (arr2 != null && arr2.length > i) ? arr2[i] : 0f; + sum[i] = v1 + v2; + } + + // 找最大值索引 + int maxIndex = 0; + float maxValue = sum[0]; + for (int i = 1; i < length; i++) { + if (sum[i] > maxValue) { + maxValue = sum[i]; + maxIndex = i; + } + } + + // 返回最大值的索引 + return maxIndex; + } +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/Base64ImageUtils.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/Base64ImageUtils.java new file mode 100644 index 0000000..9771cc8 --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/Base64ImageUtils.java @@ -0,0 +1,38 @@ +package cn.smartjavaai.common.utils; + +import cn.hutool.core.codec.Base64; + +/** + * + * @author dwj + * @date 2025/6/28 + */ +public class Base64ImageUtils { + + + /** + * 将 Base64 字符串（可带头部）转图片 + */ + public static byte[] base64ToImage(String base64Str){ + String cleanBase64 = stripBase64Header(base64Str); + return Base64.decode(cleanBase64); + } + + /** + * 检查 Base64 字符串是否带有 Data URI 头部 + */ + public static boolean hasBase64Header(String base64Str) { + return base64Str != null && base64Str.startsWith("data:") && base64Str.contains(";base64,"); + } + + /** + * 去除 Base64 字符串的 Data URI 头部 + */ + public static String stripBase64Header(String base64Str) { + if (hasBase64Header(base64Str)) { + return base64Str.substring(base64Str.indexOf(",") + 1); + } + return base64Str; + } + +} diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java index 3ac5485..e30fafb 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/ImageUtils.java @@ -332,5 +332,44 @@ public class ImageUtils { } + /** + * 画检测框(有倾斜角)和文本 + * + * @param image + * @param box + * @param text + */ + public static void drawImageRectWithText(BufferedImage image, DetectionRectangle box, String text, Color color) { + // 将绘制图像转换为Graphics2D + Graphics2D graphics = (Graphics2D) image.getGraphics(); + try { + graphics.setColor(Color.RED);// 边框颜色 + graphics.setStroke(new BasicStroke(2)); // 线宽2像素 + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿 + int stroke = 2; + graphics.setColor(color);// 边框颜色 + graphics.drawRect(box.getX(), box.getY(), box.getWidth(), box.getHeight()); + drawText(graphics, text, box.getX(), box.getY(), stroke, 4); + graphics.dispose(); + } finally { + graphics.dispose(); + } + } + + public static void drawText(Graphics2D g, String text, int x, int y, int stroke, int padding) { + FontMetrics metrics = g.getFontMetrics(); + x += stroke / 2; + y += stroke / 2; + int width = metrics.stringWidth(text) + padding * 2 - stroke / 2; + int height = metrics.getHeight() + metrics.getDescent(); + int ascent = metrics.getAscent(); + java.awt.Rectangle background = new java.awt.Rectangle(x, y, width, height); + g.fill(background); + g.setPaint(Color.WHITE); + g.drawString(text, x + padding, y + ascent); + } + + } diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java index 96ee873..35c8692 100644 --- a/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/utils/OpenCVUtils.java @@ -8,6 +8,7 @@ import org.opencv.core.Point; import org.opencv.core.Scalar; import org.opencv.imgproc.Imgproc; +import java.awt.*; import java.awt.image.BufferedImage; import java.awt.image.DataBufferByte; @@ -111,7 +112,14 @@ public class OpenCVUtils { public static Mat image2Mat(BufferedImage img) { int width = img.getWidth(); int height = img.getHeight(); - byte[] data = ((DataBufferByte) img.getRaster().getDataBuffer()).getData(); + + // 强制转换为 TYPE_3BYTE_BGR，自动去除透明通道 + BufferedImage convertedImg = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR); + Graphics2D g2d = convertedImg.createGraphics(); + g2d.drawImage(img, 0, 0, null); + g2d.dispose(); + + byte[] data = ((DataBufferByte) convertedImg.getRaster().getDataBuffer()).getData(); Mat mat = new Mat(height, width, CvType.CV_8UC3); mat.put(0, 0, data); return mat; diff --git a/smartjavaai-face/pom.xml b/smartjavaai-face/pom.xml index b2fb569..e5980da 100644 --- a/smartjavaai-face/pom.xml +++ b/smartjavaai-face/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.17 + 1.0.19 smartjavaai-face - 1.0.17 + 1.0.19 smartjavaai-face SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceDetConfig.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceDetConfig.java new file mode 100644 index 0000000..3b44a04 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceDetConfig.java @@ -0,0 +1,77 @@ +package cn.smartjavaai.face.config; + +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.constant.FaceDetectConstant; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +/** + * 人脸检测模型配置 + * @author dwj + */ +@Data +public class FaceDetConfig { + + /** + * 人脸检测模型枚举 + */ + private FaceDetModelEnum modelEnum; + + /** + * 置信度阈值 + */ + private double confidenceThreshold = FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD; + + + /** + * 非极大抑制阈值 作用：消除重叠检测框，保留最优结果 + */ + private double nmsThresh = FaceDetectConstant.NMS_THRESHOLD; + + /** + * 模型路径 + */ + private String modelPath; + + /** + * 设备类型 + */ + private DeviceEnum device; + + /** + * 个性化配置（按模型类型动态解析） + */ + private Map customParams = new HashMap<>(); + + + public FaceDetConfig() { + } + + public FaceDetConfig(FaceDetModelEnum modelEnum) { + this.modelEnum = modelEnum; + } + + public FaceDetConfig(FaceDetModelEnum modelEnum, String modelPath) { + this.modelEnum = modelEnum; + this.modelPath = modelPath; + } + + public T getCustomParam(String key, Class clazz) { + Object value = customParams.get(key); + if (value == null) return null; + return clazz.cast(value); + } + + /** + * 添加个性化配置项 + */ + public void putCustomParam(String key, Object value) { + if (customParams == null) { + customParams = new HashMap<>(); + } + customParams.put(key, value); + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceExpressionConfig.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceExpressionConfig.java new file mode 100644 index 0000000..38cefa5 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceExpressionConfig.java @@ -0,0 +1,51 @@ +package cn.smartjavaai.face.config; + +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.enums.ExpressionModelEnum; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.facerec.FaceRecModel; +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author dwj + * @date 2025/7/1 + */ +@Data +public class FaceExpressionConfig { + + /** + * 模型枚举 + */ + private ExpressionModelEnum modelEnum = ExpressionModelEnum.DensNet121; + + /** + * 模型路径 + */ + private String modelPath; + + /** + * 设备类型 + */ + private DeviceEnum device; + + /** + * 人脸检测模型 + */ + private FaceDetModel detectModel; + + /** + * 是否对齐人脸 + */ + private boolean align = true; + + /** + * 是否裁剪人脸 + */ + private boolean cropFace = true; + + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceExtractConfig.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceExtractConfig.java deleted file mode 100644 index c4ff178..0000000 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceExtractConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -package cn.smartjavaai.face.config; - -import cn.smartjavaai.face.model.facerec.FaceModel; -import lombok.Data; - -/** - * 人脸特征提取配置 - * @author dwj - * @date 2025/4/24 - */ -@Data -public class FaceExtractConfig { - - /** - * 是否裁剪人脸 - */ - private boolean cropFace = true; - - /** - * 是否对齐人脸 - */ - private boolean align = false; - - /** - * 人脸检测模型 - */ - private FaceModel detectModel; - - public FaceExtractConfig() { - } - - public FaceExtractConfig(boolean cropFace, boolean align, FaceModel detectModel) { - this.cropFace = cropFace; - this.align = align; - this.detectModel = detectModel; - } - - -} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceModelConfig.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceRecConfig.java similarity index 51% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceModelConfig.java rename to smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceRecConfig.java index 67717fa..48fbbb8 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceModelConfig.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/FaceRecConfig.java @@ -2,22 +2,26 @@ package cn.smartjavaai.face.config; import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.face.constant.FaceDetectConstant; -import cn.smartjavaai.face.enums.FaceModelEnum; -import cn.smartjavaai.face.enums.VectorDBType; +import cn.smartjavaai.face.enums.FaceRecModelEnum; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.facerec.FaceRecModel; import cn.smartjavaai.face.vector.config.VectorDBConfig; import lombok.Data; +import java.util.HashMap; +import java.util.Map; + /** * 人脸检测识别模型配置 * @author dwj */ @Data -public class FaceModelConfig { +public class FaceRecConfig { /** * 人脸模型枚举 */ - private FaceModelEnum modelEnum; + private FaceRecModelEnum modelEnum; /** * 置信度阈值 @@ -49,16 +53,6 @@ public class FaceModelConfig { */ private DeviceEnum device; - /** - * gpu设备ID 当device为GPU时生效 - */ - private int gpuId = 0; - - /** - * 人脸特征提取配置 - */ - private FaceExtractConfig extractConfig; - /** * 向量数据库配置 @@ -70,15 +64,52 @@ public class FaceModelConfig { */ private boolean isAutoLoadFace = true; - public FaceModelConfig() { + + /** + * 是否裁剪人脸 + */ + private boolean cropFace = true; + + /** + * 是否对齐人脸 + */ + private boolean align = false; + + /** + * 人脸检测模型 + */ + private FaceDetModel detectModel; + + /** + * 个性化配置（按模型类型动态解析） + */ + private Map customParams = new HashMap<>(); + + public FaceRecConfig() { } - public FaceModelConfig(FaceModelEnum modelEnum) { + public FaceRecConfig(FaceRecModelEnum modelEnum) { this.modelEnum = modelEnum; } - public FaceModelConfig(FaceModelEnum modelEnum, String modelPath) { + public FaceRecConfig(FaceRecModelEnum modelEnum, String modelPath) { this.modelEnum = modelEnum; this.modelPath = modelPath; } + + public T getCustomParam(String key, Class clazz) { + Object value = customParams.get(key); + if (value == null) return null; + return clazz.cast(value); + } + + /** + * 添加个性化配置项 + */ + public void putCustomParam(String key, Object value) { + if (customParams == null) { + customParams = new HashMap<>(); + } + customParams.put(key, value); + } } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/LivenessConfig.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/LivenessConfig.java index 6115fc3..6085a90 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/LivenessConfig.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/LivenessConfig.java @@ -1,12 +1,15 @@ package cn.smartjavaai.face.config; import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.face.constant.FaceDetectConstant; import cn.smartjavaai.face.constant.LivenessConstant; -import cn.smartjavaai.face.enums.FaceModelEnum; import cn.smartjavaai.face.enums.LivenessModelEnum; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.facerec.FaceRecModel; import lombok.Data; +import java.util.HashMap; +import java.util.Map; + /** * 活体检测模型配置 * @author dwj @@ -30,25 +33,26 @@ public class LivenessConfig { private DeviceEnum device; /** - * gpu设备ID 当device为GPU时生效 + * 人脸检测模型 */ - private int gpuId = 0; + private FaceDetModel detectModel; /** - * 人脸清晰度阈值 + * 个性化配置（按模型类型动态解析） */ - private float faceClarityThreshold = LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD; + private Map customParams = new HashMap<>(); - /** - * 活体阈值 - */ - private float realityThreshold = LivenessConstant.DEFAULT_REALITY_THRESHOLD; /** * 视频检测帧数 */ private int frameCount = LivenessConstant.DEFAULT_FRAME_COUNT; + /** + * 真人阈值 + */ + private Float realityThreshold; + public LivenessConfig() { } @@ -64,4 +68,21 @@ public class LivenessConfig { public LivenessConfig(String modelPath) { this.modelPath = modelPath; } + + // 可选封装方法，便于类型转换和调用 + public T getCustomParam(String key, Class clazz) { + Object value = customParams.get(key); + if (value == null) return null; + return clazz.cast(value); + } + + /** + * 添加个性化配置项 + */ + public void putCustomParam(String key, Object value) { + if (customParams == null) { + customParams = new HashMap<>(); + } + customParams.put(key, value); + } } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/QualityConfig.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/QualityConfig.java new file mode 100644 index 0000000..2ccd1dd --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/config/QualityConfig.java @@ -0,0 +1,52 @@ +package cn.smartjavaai.face.config; + +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.constant.LivenessConstant; +import cn.smartjavaai.face.enums.LivenessModelEnum; +import cn.smartjavaai.face.enums.QualityModelEnum; +import lombok.Data; + +/** + * 质量评估配置 + * @author dwj + */ +@Data +public class QualityConfig { + + /** + * 活体检测模型枚举 + */ + private QualityModelEnum modelEnum = QualityModelEnum.SEETA_FACE6_MODEL; + + /** + * 模型路径 + */ + private String modelPath; + + /** + * 设备类型 + */ + private DeviceEnum device; + + /** + * gpu设备ID 当device为GPU时生效 + */ + private int gpuId = 0; + + + public QualityConfig() { + } + + public QualityConfig(QualityModelEnum modelEnum) { + this.modelEnum = modelEnum; + } + + public QualityConfig(QualityModelEnum modelEnum, String modelPath) { + this.modelEnum = modelEnum; + this.modelPath = modelPath; + } + + public QualityConfig(String modelPath) { + this.modelPath = modelPath; + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/FaceNetConstant.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/FaceNetConstant.java new file mode 100644 index 0000000..3305372 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/FaceNetConstant.java @@ -0,0 +1,14 @@ +package cn.smartjavaai.face.constant; + +/** + * FaceNet人脸模型常量 + * @author dwj + */ +public class FaceNetConstant { + + /** + * 模型下载地址 + */ + public static final String MODEL_URL = "https://resources.djl.ai/test-models/pytorch/face_feature.zip"; + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/MiniVisionConstant.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/MiniVisionConstant.java new file mode 100644 index 0000000..d4463ee --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/MiniVisionConstant.java @@ -0,0 +1,14 @@ +package cn.smartjavaai.face.constant; + +/** + * MiniVision模型常量 + * @author dwj + * @date 2025/7/3 + */ +public class MiniVisionConstant { + + /** + * 真人阈值 + */ + public static final Float REALITY_THRESHOLD = 0.5f; +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/RetinaFaceConstant.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/RetinaFaceConstant.java new file mode 100644 index 0000000..de02817 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/RetinaFaceConstant.java @@ -0,0 +1,28 @@ +package cn.smartjavaai.face.constant; + +/** + * RetinaFace人脸检测模型常量 + * @author dwj + * @date 2025/7/2 + */ +public class RetinaFaceConstant { + + /** + * 特征图层的基础缩放比例 + */ + public static final int[][] scales = {{16, 32}, {64, 128}, {256, 512}}; + /** + * 特征图相对于原图的采样步长 + */ + public static final int[] steps = {8, 16, 32}; + /** + * 缩放系数 + */ + public static final double[] variance = {0.1f, 0.2f}; + + /** + * 模型下载地址 + */ + public static final String MODEL_URL = "https://resources.djl.ai/test-models/pytorch/retinaface.zip"; + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/UltraLightFastGenericFaceConstant.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/UltraLightFastGenericFaceConstant.java new file mode 100644 index 0000000..f24837a --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/constant/UltraLightFastGenericFaceConstant.java @@ -0,0 +1,27 @@ +package cn.smartjavaai.face.constant; + +/** + * UltraLightFastGenericFace人脸检测模型常量 + * @author dwj + */ +public class UltraLightFastGenericFaceConstant { + + /** + * 特征图层的基础缩放比例 + */ + public static final int[][] scales = {{10, 16, 24}, {32, 48}, {64, 96}, {128, 192, 256}}; + /** + * 特征图相对于原图的采样步长 + */ + public static final int[] steps = {8, 16, 32, 64}; + /** + * 缩放系数 + */ + public static final double[] variance = {0.1f, 0.2f}; + + /** + * 模型下载地址 + */ + public static final String MODEL_URL = "https://resources.djl.ai/test-models/pytorch/ultranet.zip"; + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/entity/FaceQualityResult.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/entity/FaceQualityResult.java new file mode 100644 index 0000000..242acb8 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/entity/FaceQualityResult.java @@ -0,0 +1,29 @@ +package cn.smartjavaai.face.entity; + +import cn.smartjavaai.face.enums.QualityGrade; +import lombok.Data; + +/** + * 质量评估结果 + * @author dwj + * @date 2025/6/23 + */ +@Data +public class FaceQualityResult { + + /** + * 评估得分 + */ + private float score; + + private QualityGrade grade; + + public FaceQualityResult() { + } + + public FaceQualityResult(float score, QualityGrade grade) { + this.score = score; + this.grade = grade; + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/entity/FaceQualitySummary.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/entity/FaceQualitySummary.java new file mode 100644 index 0000000..12ad0b0 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/entity/FaceQualitySummary.java @@ -0,0 +1,23 @@ +package cn.smartjavaai.face.entity; + +import lombok.Data; + +import java.util.Map; + +/** + * 人脸质量检测汇总结果 + * @author dwj + * @date 2025/6/27 + */ +@Data +public class FaceQualitySummary { + + private FaceQualityResult brightness; // 亮度 + private FaceQualityResult clarity; // 清晰度 + private FaceQualityResult completeness; // 完整度 + private FaceQualityResult pose; // 姿态 + private FaceQualityResult resolution; // 分辨率 + + private Map extraResults; // 额外检测结果 + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/ExpressionModelEnum.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/ExpressionModelEnum.java new file mode 100644 index 0000000..ebe689f --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/ExpressionModelEnum.java @@ -0,0 +1,38 @@ +package cn.smartjavaai.face.enums; + +/** + * 表情识别模型枚举 + * @author dwj + */ +public enum ExpressionModelEnum { + + DensNet121("DensNet121"), + + FrEmotion("FrEmotion"); + + + private final String modelClassName; + + ExpressionModelEnum(String modelClassName) { + this.modelClassName = modelClassName; + } + + public String getModelClassName() { + return modelClassName; + } + + /** + * 根据名称获取枚举 (忽略大小写和下划线变体) + */ + public static ExpressionModelEnum fromName(String name) { + String formatted = name.trim().toUpperCase().replaceAll("[-_]", ""); + for (ExpressionModelEnum model : values()) { + if (model.name().replaceAll("_", "").equals(formatted)) { + return model; + } + } + throw new IllegalArgumentException("未知模型名称: " + name); + } + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceModelEnum.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceDetModelEnum.java similarity index 75% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceModelEnum.java rename to smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceDetModelEnum.java index 38521ca..fb4fce3 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceModelEnum.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceDetModelEnum.java @@ -1,20 +1,18 @@ package cn.smartjavaai.face.enums; /** - * 人脸模型枚举 + * 人脸检测模型枚举 * @author dwj - * @date 2025/4/10 */ -public enum FaceModelEnum { +public enum FaceDetModelEnum { RETINA_FACE("RetinaFaceModel"), ULTRA_LIGHT_FAST_GENERIC_FACE("UltraLightFastGenericFaceModel"), - FACENET_MODEL("FaceNetModel"), SEETA_FACE6_MODEL("SeetaFace6Model"); private final String modelClassName; - FaceModelEnum(String modelClassName) { + FaceDetModelEnum(String modelClassName) { this.modelClassName = modelClassName; } @@ -25,9 +23,9 @@ public enum FaceModelEnum { /** * 根据名称获取枚举 (忽略大小写和下划线变体) */ - public static FaceModelEnum fromName(String name) { + public static FaceDetModelEnum fromName(String name) { String formatted = name.trim().toUpperCase().replaceAll("[-_]", ""); - for (FaceModelEnum model : values()) { + for (FaceDetModelEnum model : values()) { if (model.name().replaceAll("_", "").equals(formatted)) { return model; } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceRecModelEnum.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceRecModelEnum.java new file mode 100644 index 0000000..20f446f --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/FaceRecModelEnum.java @@ -0,0 +1,39 @@ +package cn.smartjavaai.face.enums; + +/** + * 人脸识别模型枚举 + * @author dwj + */ +public enum FaceRecModelEnum { + + FACENET_MODEL("FaceNetModel"), + SEETA_FACE6_MODEL("SeetaFace6Model"), + INSIGHT_FACE_IRSE50_MODEL("InsightFaceIRSE50Model"), + INSIGHT_FACE_MOBILE_FACENET_MODEL("InsightFaceMobilefacenetModel"), + ELASTIC_FACE_MODEL("ElasticFaceModel"); + + private final String modelClassName; + + FaceRecModelEnum(String modelClassName) { + this.modelClassName = modelClassName; + } + + public String getModelClassName() { + return modelClassName; + } + + /** + * 根据名称获取枚举 (忽略大小写和下划线变体) + */ + public static FaceRecModelEnum fromName(String name) { + String formatted = name.trim().toUpperCase().replaceAll("[-_]", ""); + for (FaceRecModelEnum model : values()) { + if (model.name().replaceAll("_", "").equals(formatted)) { + return model; + } + } + throw new IllegalArgumentException("未知模型名称: " + name); + } + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/LivenessModelEnum.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/LivenessModelEnum.java index 0de2af3..a1a005a 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/LivenessModelEnum.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/LivenessModelEnum.java @@ -7,7 +7,14 @@ package cn.smartjavaai.face.enums; */ public enum LivenessModelEnum { - SEETA_FACE6_MODEL("SeetaFace6Model"); + // SeetaFace6 + SEETA_FACE6_MODEL("SeetaFace6Model"), + + // MiniVision + MINI_VISION_MODEL("MiniVisionModel"), + + //阿里通义实验室 + IIC_FL_MODEL("IicFlModel"); private final String modelClassName; diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/QualityGrade.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/QualityGrade.java new file mode 100644 index 0000000..a558e36 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/QualityGrade.java @@ -0,0 +1,14 @@ +package cn.smartjavaai.face.enums; + +/** + * 质量等级枚举 + * @author dwj + * @date 2025/6/23 + */ +public enum QualityGrade { + + LOW,//Quality level is low + MEDIUM,//Quality level is medium + HIGH,//Quality level is high + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/QualityModelEnum.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/QualityModelEnum.java new file mode 100644 index 0000000..d538147 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/enums/QualityModelEnum.java @@ -0,0 +1,36 @@ +package cn.smartjavaai.face.enums; + +/** + * 质量评估模型枚举 + * @author dwj + * @date 2025/4/10 + */ +public enum QualityModelEnum { + + SEETA_FACE6_MODEL("SeetaFace6Model"); + + private final String modelClassName; + + QualityModelEnum(String modelClassName) { + this.modelClassName = modelClassName; + } + + public String getModelClassName() { + return modelClassName; + } + + /** + * 根据名称获取枚举 (忽略大小写和下划线变体) + */ + public static QualityModelEnum fromName(String name) { + String formatted = name.trim().toUpperCase().replaceAll("[-_]", ""); + for (QualityModelEnum model : values()) { + if (model.name().replaceAll("_", "").equals(formatted)) { + return model; + } + } + throw new IllegalArgumentException("未知模型名称: " + name); + } + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/ExpressionModelFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/ExpressionModelFactory.java new file mode 100644 index 0000000..39e89b8 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/ExpressionModelFactory.java @@ -0,0 +1,101 @@ +package cn.smartjavaai.face.factory; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.face.config.FaceExpressionConfig; +import cn.smartjavaai.face.enums.ExpressionModelEnum; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.model.expression.CommonEmotionModel; +import cn.smartjavaai.face.model.expression.ExpressionModel; +import cn.smartjavaai.face.model.liveness.MiniVisionLivenessModel; +import cn.smartjavaai.face.model.liveness.Seetaface6LivenessModel; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 表情识别模型工厂 + * @author dwj + */ +@Slf4j +public class ExpressionModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile ExpressionModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + public static ExpressionModelFactory getInstance() { + if (instance == null) { + synchronized (ExpressionModelFactory.class) { + if (instance == null) { + instance = new ExpressionModelFactory(); + } + } + } + return instance; + } + + + + /** + * 注册模型 + * @param expressionModelEnum + * @param clazz + */ + private static void registerModel(ExpressionModelEnum expressionModelEnum, Class clazz) { + registry.put(expressionModelEnum, clazz); + } + + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public ExpressionModel getModel(FaceExpressionConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new FaceException("未配置活体检测模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createFaceModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private ExpressionModel createFaceModel(FaceExpressionConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new FaceException("Unsupported algorithm"); + } + ExpressionModel model = null; + try { + model = (ExpressionModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new FaceException(e); + } + model.loadModel(config); + return model; + } + + + // 初始化默认算法 + static { + registerModel(ExpressionModelEnum.DensNet121, CommonEmotionModel.class); + registerModel(ExpressionModelEnum.FrEmotion, CommonEmotionModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceModelFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceDetModelFactory.java similarity index 59% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceModelFactory.java rename to smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceDetModelFactory.java index 9d46fe9..f12f77f 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceModelFactory.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceDetModelFactory.java @@ -1,10 +1,13 @@ package cn.smartjavaai.face.factory; import cn.smartjavaai.common.config.Config; -import cn.smartjavaai.face.config.FaceModelConfig; +import cn.smartjavaai.face.config.FaceDetConfig; import cn.smartjavaai.face.constant.FaceDetectConstant; -import cn.smartjavaai.face.enums.FaceModelEnum; +import cn.smartjavaai.face.enums.FaceDetModelEnum; import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.model.facedect.CommonFaceDetModel; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel; import cn.smartjavaai.face.model.facerec.*; import lombok.extern.slf4j.Slf4j; @@ -13,29 +16,29 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; /** - * 人脸检测识别模型工厂 + * 人脸检测模型工厂 * @author dwj */ @Slf4j -public class FaceModelFactory { +public class FaceDetModelFactory { // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 - private static volatile FaceModelFactory instance; + private static volatile FaceDetModelFactory instance; - private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); /** * 模型注册表 */ - private static final Map> registry = + private static final Map> registry = new ConcurrentHashMap<>(); - public static FaceModelFactory getInstance() { + public static FaceDetModelFactory getInstance() { if (instance == null) { - synchronized (FaceModelFactory.class) { + synchronized (FaceDetModelFactory.class) { if (instance == null) { - instance = new FaceModelFactory(); + instance = new FaceDetModelFactory(); } } } @@ -49,7 +52,7 @@ public class FaceModelFactory { * @param name * @param clazz */ - private static void registerAlgorithm(String name, Class clazz) { + private static void registerAlgorithm(String name, Class clazz) { registry.put(name.toLowerCase(), clazz); } @@ -59,12 +62,12 @@ public class FaceModelFactory { * @param config * @return */ - public FaceModel getModel(FaceModelConfig config) { + public FaceDetModel getModel(FaceDetConfig config) { if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ throw new FaceException("未配置人脸模型"); } return modelMap.computeIfAbsent(config.getModelEnum().name(), k -> { - return createFaceModel(config); + return createFaceDetModel(config); }); } @@ -72,10 +75,10 @@ public class FaceModelFactory { * 获取默认模型 * @return */ - public FaceModel getModel() { + public FaceDetModel getModel() { // 初始化默认配置 - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.RETINA_FACE); + FaceDetConfig config = new FaceDetConfig(); + config.setModelEnum(FaceDetModelEnum.RETINA_FACE); config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD); config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); return getModel(config); @@ -86,14 +89,14 @@ public class FaceModelFactory { * @param config * @return */ - private FaceModel createFaceModel(FaceModelConfig config) { + private FaceDetModel createFaceDetModel(FaceDetConfig config) { Class clazz = registry.get(config.getModelEnum().getModelClassName().toLowerCase()); if(clazz == null){ throw new FaceException("Unsupported algorithm"); } - FaceModel algorithm = null; + FaceDetModel algorithm = null; try { - algorithm = (FaceModel) clazz.newInstance(); + algorithm = (FaceDetModel) clazz.newInstance(); } catch (InstantiationException | IllegalAccessException e) { throw new FaceException(e); } @@ -106,10 +109,10 @@ public class FaceModelFactory { * 获取轻量级人脸模型 * @return */ - public FaceModel getLightFaceModel() { + public FaceDetModel getLightFaceDetModel() { // 初始化默认配置 - FaceModelConfig config = new FaceModelConfig(); - config.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE); + FaceDetConfig config = new FaceDetConfig(); + config.setModelEnum(FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE); config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD); config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD); return getModel(config); @@ -118,11 +121,9 @@ public class FaceModelFactory { // 初始化默认算法 static { - registerAlgorithm("retinafacemodel", RetinaFaceModel.class); - registerAlgorithm("ultralightfastgenericfacemodel", UltraLightFastGenericFaceModel.class); - //人脸特征提取 - registerAlgorithm("facenetmodel", FaceNetModel.class); - registerAlgorithm("seetaface6model", SeetaFace6Model.class); + registerAlgorithm("retinafacemodel", CommonFaceDetModel.class); + registerAlgorithm("ultralightfastgenericfacemodel", CommonFaceDetModel.class); + registerAlgorithm("seetaface6model", SeetaFace6FaceDetModel.class); log.debug("缓存目录：{}", Config.getCachePath()); } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceQualityModelFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceQualityModelFactory.java new file mode 100644 index 0000000..ac0d902 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceQualityModelFactory.java @@ -0,0 +1,97 @@ +package cn.smartjavaai.face.factory; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.face.config.QualityConfig; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.model.quality.FaceQualityModel; +import cn.smartjavaai.face.model.quality.Seetaface6QualityModel; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 质量评估模型工厂 + * @author dwj + */ +@Slf4j +public class FaceQualityModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile FaceQualityModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + public static FaceQualityModelFactory getInstance() { + if (instance == null) { + synchronized (FaceQualityModelFactory.class) { + if (instance == null) { + instance = new FaceQualityModelFactory(); + } + } + } + return instance; + } + + + + /** + * 注册模型 + * @param name + * @param clazz + */ + private static void registerModel(String name, Class clazz) { + registry.put(name.toLowerCase(), clazz); + } + + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public FaceQualityModel getModel(QualityConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new FaceException("未配置质量评估模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum().name(), k -> { + return createFaceModel(config); + }); + } + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private FaceQualityModel createFaceModel(QualityConfig config) { + Class clazz = registry.get(config.getModelEnum().getModelClassName().toLowerCase()); + if(clazz == null){ + throw new FaceException("Unsupported algorithm"); + } + FaceQualityModel model = null; + try { + model = (FaceQualityModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new FaceException(e); + } + model.loadModel(config); + return model; + } + + + // 初始化默认算法 + static { + registerModel("seetaface6model", Seetaface6QualityModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceRecModelFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceRecModelFactory.java new file mode 100644 index 0000000..502cb70 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/FaceRecModelFactory.java @@ -0,0 +1,103 @@ +package cn.smartjavaai.face.factory; + +import cn.smartjavaai.common.config.Config; +import cn.smartjavaai.face.config.FaceRecConfig; +import cn.smartjavaai.face.constant.FaceDetectConstant; +import cn.smartjavaai.face.enums.FaceRecModelEnum; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.model.facerec.*; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 人脸检测识别模型工厂 + * @author dwj + */ +@Slf4j +public class FaceRecModelFactory { + + // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 + private static volatile FaceRecModelFactory instance; + + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + + /** + * 模型注册表 + */ + private static final Map> registry = + new ConcurrentHashMap<>(); + + + public static FaceRecModelFactory getInstance() { + if (instance == null) { + synchronized (FaceRecModelFactory.class) { + if (instance == null) { + instance = new FaceRecModelFactory(); + } + } + } + return instance; + } + + + + /** + * 注册模型 + * @param recModelEnum + * @param clazz + */ + private static void registerAlgorithm(FaceRecModelEnum recModelEnum, Class clazz) { + registry.put(recModelEnum, clazz); + } + + + /** + * 获取模型（通过配置） + * @param config + * @return + */ + public FaceRecModel getModel(FaceRecConfig config) { + if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ + throw new FaceException("未配置人脸模型"); + } + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { + return createFaceModel(config); + }); + } + + + /** + * 使用ModelConfig创建模型 + * @param config + * @return + */ + private FaceRecModel createFaceModel(FaceRecConfig config) { + Class clazz = registry.get(config.getModelEnum()); + if(clazz == null){ + throw new FaceException("Unsupported model"); + } + FaceRecModel algorithm = null; + try { + algorithm = (FaceRecModel) clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new FaceException(e); + } + algorithm.loadModel(config); + return algorithm; + } + + + // 初始化默认算法 + static { + registerAlgorithm(FaceRecModelEnum.FACENET_MODEL, CommonFaceRecModel.class); + registerAlgorithm(FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL, CommonFaceRecModel.class); + registerAlgorithm(FaceRecModelEnum.INSIGHT_FACE_MOBILE_FACENET_MODEL, CommonFaceRecModel.class); + registerAlgorithm(FaceRecModelEnum.ELASTIC_FACE_MODEL, CommonFaceRecModel.class); + registerAlgorithm(FaceRecModelEnum.SEETA_FACE6_MODEL, SeetaFace6FaceRecModel.class); + log.debug("缓存目录：{}", Config.getCachePath()); + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/LivenessModelFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/LivenessModelFactory.java index 2a066cc..6d2525a 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/LivenessModelFactory.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/factory/LivenessModelFactory.java @@ -1,13 +1,12 @@ package cn.smartjavaai.face.factory; import cn.smartjavaai.common.config.Config; -import cn.smartjavaai.face.config.FaceModelConfig; import cn.smartjavaai.face.config.LivenessConfig; -import cn.smartjavaai.face.constant.FaceDetectConstant; -import cn.smartjavaai.face.enums.FaceModelEnum; +import cn.smartjavaai.face.enums.LivenessModelEnum; import cn.smartjavaai.face.exception.FaceException; -import cn.smartjavaai.face.model.facerec.*; +import cn.smartjavaai.face.model.liveness.CommonLivenessModel; import cn.smartjavaai.face.model.liveness.LivenessDetModel; +import cn.smartjavaai.face.model.liveness.MiniVisionLivenessModel; import cn.smartjavaai.face.model.liveness.Seetaface6LivenessModel; import lombok.extern.slf4j.Slf4j; @@ -25,12 +24,12 @@ public class LivenessModelFactory { // 使用 volatile 和双重检查锁定来确保线程安全的单例模式 private static volatile LivenessModelFactory instance; - private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap modelMap = new ConcurrentHashMap<>(); /** * 模型注册表 */ - private static final Map> registry = + private static final Map> registry = new ConcurrentHashMap<>(); @@ -49,11 +48,11 @@ public class LivenessModelFactory { /** * 注册模型 - * @param name + * @param livenessModelEnum * @param clazz */ - private static void registerModel(String name, Class clazz) { - registry.put(name.toLowerCase(), clazz); + private static void registerModel(LivenessModelEnum livenessModelEnum, Class clazz) { + registry.put(livenessModelEnum, clazz); } @@ -66,7 +65,7 @@ public class LivenessModelFactory { if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){ throw new FaceException("未配置活体检测模型"); } - return modelMap.computeIfAbsent(config.getModelEnum().name(), k -> { + return modelMap.computeIfAbsent(config.getModelEnum(), k -> { return createFaceModel(config); }); } @@ -77,7 +76,7 @@ public class LivenessModelFactory { * @return */ private LivenessDetModel createFaceModel(LivenessConfig config) { - Class clazz = registry.get(config.getModelEnum().getModelClassName().toLowerCase()); + Class clazz = registry.get(config.getModelEnum()); if(clazz == null){ throw new FaceException("Unsupported algorithm"); } @@ -94,7 +93,9 @@ public class LivenessModelFactory { // 初始化默认算法 static { - registerModel("seetaface6model", Seetaface6LivenessModel.class); + registerModel(LivenessModelEnum.SEETA_FACE6_MODEL, Seetaface6LivenessModel.class); + registerModel(LivenessModelEnum.MINI_VISION_MODEL, MiniVisionLivenessModel.class); + registerModel(LivenessModelEnum.IIC_FL_MODEL, CommonLivenessModel.class); log.debug("缓存目录：{}", Config.getCachePath()); } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/attribute/FaceAttributeModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/attribute/FaceAttributeModel.java index b3778a3..df00753 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/attribute/FaceAttributeModel.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/attribute/FaceAttributeModel.java @@ -2,10 +2,9 @@ package cn.smartjavaai.face.model.attribute; import cn.smartjavaai.common.entity.DetectionRectangle; import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.entity.FaceAttribute; +import cn.smartjavaai.common.entity.face.FaceAttribute; import cn.smartjavaai.common.entity.Point; import cn.smartjavaai.face.config.FaceAttributeConfig; -import cn.smartjavaai.common.enums.GenderType; import java.awt.image.BufferedImage; import java.util.List; @@ -14,7 +13,7 @@ import java.util.List; * 人脸属性识别模型 * @author dwj */ -public interface FaceAttributeModel { +public interface FaceAttributeModel extends AutoCloseable{ /** * 加载模型 diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java index 263a9b7..095d8f2 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java @@ -1,13 +1,16 @@ package cn.smartjavaai.face.model.attribute; import cn.smartjavaai.common.entity.*; +import cn.smartjavaai.common.entity.face.FaceAttribute; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.entity.face.HeadPose; import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.common.enums.EyeStatus; +import cn.smartjavaai.common.enums.face.EyeStatus; import cn.smartjavaai.common.utils.FileUtils; import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.common.utils.PoolUtils; import cn.smartjavaai.face.config.FaceAttributeConfig; -import cn.smartjavaai.common.enums.GenderType; +import cn.smartjavaai.common.enums.face.GenderType; import cn.smartjavaai.face.context.PredictorContext; import cn.smartjavaai.face.exception.FaceException; import cn.smartjavaai.face.seetaface.NativeLoader; @@ -468,6 +471,28 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel { } } - - + @Override + public void close() throws Exception { + if(Objects.nonNull(faceDetectorPool)){ + faceDetectorPool.close(); + } + if(Objects.nonNull(genderPredictorPool)){ + genderPredictorPool.close(); + } + if(Objects.nonNull(faceLandmarkerPool)){ + faceLandmarkerPool.close(); + } + if(Objects.nonNull(agePredictorPool)){ + agePredictorPool.close(); + } + if(Objects.nonNull(eyeStateDetectorPool)){ + eyeStateDetectorPool.close(); + } + if(Objects.nonNull(maskDetectorPool)){ + maskDetectorPool.close(); + } + if(Objects.nonNull(poseEstimatorPool)){ + poseEstimatorPool.close(); + } + } } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/CommonEmotionModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/CommonEmotionModel.java new file mode 100644 index 0000000..0d36451 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/CommonEmotionModel.java @@ -0,0 +1,364 @@ +package cn.smartjavaai.face.model.expression; + +import ai.djl.Device; +import ai.djl.MalformedModelException; +import ai.djl.inference.Predictor; +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.ndarray.NDManager; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.entity.*; +import cn.smartjavaai.common.entity.face.ExpressionResult; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.entity.face.LivenessResult; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.enums.face.FacialExpression; +import cn.smartjavaai.common.pool.PredictorFactory; +import cn.smartjavaai.common.utils.Base64ImageUtils; +import cn.smartjavaai.common.utils.FileUtils; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.face.config.FaceExpressionConfig; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.model.expression.criterial.EmotionCriteriaFactory; +import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator; +import cn.smartjavaai.face.preprocess.DJLImagePreprocessor; +import cn.smartjavaai.face.utils.FaceUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.ObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.opencv.face.Face; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * 通用人脸表情识别模型 + * @author dwj + */ +@Slf4j +public class CommonEmotionModel implements ExpressionModel{ + + + private FaceExpressionConfig config; + + private ZooModel model; + + private ObjectPool> predictorPool; + + @Override + public void loadModel(FaceExpressionConfig config) { + if(Objects.isNull(config)){ + throw new FaceException("config为null"); + } + if(StringUtils.isBlank(config.getModelPath())){ + throw new FaceException("modelPath为空"); + } + + this.config = config; + + Criteria criteria = EmotionCriteriaFactory.createCriteria(config); + try { + model = criteria.loadModel(); + this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); + } catch (IOException | ModelNotFoundException | MalformedModelException e) { + throw new FaceException("DenseNetEmotionModel模型加载失败", e); + } + } + + public Classifications detectCore(Image image, DetectionRectangle faceDetectionRectangle, List keyPoints){ + Predictor predictor = null; + try (NDManager manager = model.getNDManager().newSubManager()){ + predictor = predictorPool.borrowObject(); + DJLImagePreprocessor imagePreprocessor = new DJLImagePreprocessor(image, manager); + Image faceImg = image; + if(config.isAlign()){ + //仿射变换 + faceImg = imagePreprocessor.enableAffine(FaceUtils.facePoints(keyPoints), 512, 512) + .process(); + return predictor.predict(faceImg); + }else{ + if(config.isCropFace()){ + //裁剪 + faceImg = imagePreprocessor.enableCrop(faceDetectionRectangle) + .process(); + } + } + return predictor.predict(faceImg); + } catch (Exception e) { + throw new FaceException("表情识别异常", e); + }finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + } + + } + + @Override + public R detect(String imagePath) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detect(image); + } + + @Override + public R detect(BufferedImage image) { + if(Objects.isNull(config.getDetectModel())){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型"); + } + R faceDetectionResponse = config.getDetectModel().detect(image); + if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image)); + for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){ + return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点"); + } + Classifications classifications = detectCore(djlImage, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + Classifications.Classification bestClass = classifications.best(); + FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName()); + ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability()); + result.setClassifications(classifications); + faceInfo.setExpressionResult(result); + } + return faceDetectionResponse; + } + + @Override + public R detect(byte[] imageData) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detect(ImageIO.read(new ByteArrayInputStream(imageData))); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R detectBase64(String base64Image) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detect(imageData); + } + + @Override + public R> detect(String imagePath, DetectionResponse faceDetectionResponse) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detect(image, faceDetectionResponse); + } + + @Override + public R> detect(byte[] imageData, DetectionResponse faceDetectionResponse) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R> detect(BufferedImage image, DetectionResponse faceDetectionResponse) { + if(!ImageUtils.isImageValid(image)){ + R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){ + R.fail(R.Status.NO_FACE_DETECTED); + } + Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image)); + List expressionResults = new ArrayList<>(); + for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){ + return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点"); + } + Classifications classifications = detectCore(djlImage, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints()); + Classifications.Classification bestClass = classifications.best(); + FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName()); + ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability()); + result.setClassifications(classifications); + expressionResults.add(result); + } + return R.ok(expressionResults); + } + + @Override + public R> detectBase64(String base64Image, DetectionResponse faceDetectionResponse) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detect(imageData, faceDetectionResponse); + } + + @Override + public R detect(String imagePath, DetectionRectangle faceDetectionRectangle, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detect(image, faceDetectionRectangle, keyPoints); + } + + @Override + public R detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle, keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List keyPoints) { + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image)); + Classifications classifications = detectCore(djlImage, faceDetectionRectangle, keyPoints); + Classifications.Classification bestClass = classifications.best(); + FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName()); + ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability()); + result.setClassifications(classifications); + return R.ok(result); + + } + + @Override + public R detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List keyPoints) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detect(imageData, faceDetectionRectangle, keyPoints); + } + + @Override + public R detectTopFace(BufferedImage image) { + if(Objects.isNull(config.getDetectModel())){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型"); + } + R faceDetectionResponse = config.getDetectModel().detect(image); + if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + DetectionInfo detectionInfo = faceDetectionResponse.getData().getDetectionInfoList().get(0); + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){ + return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点"); + } + return detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints()); + } + + @Override + public R detectTopFace(String imagePath) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detectTopFace(image); + } + + @Override + public R detectTopFace(byte[] imageData) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData))); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R detectTopFaceBase64(String base64Image) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detectTopFace(imageData); + } + + + + @Override + public void close() { + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/ExpressionModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/ExpressionModel.java new file mode 100644 index 0000000..2821677 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/ExpressionModel.java @@ -0,0 +1,195 @@ +package cn.smartjavaai.face.model.expression; + +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.Point; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.entity.face.ExpressionResult; +import cn.smartjavaai.face.config.FaceExpressionConfig; + +import java.awt.image.BufferedImage; +import java.util.List; + +/** + * @author dwj + * @date 2025/7/1 + */ +public interface ExpressionModel extends AutoCloseable{ + + + /** + * 加载模型 + * @param config + */ + void loadModel(FaceExpressionConfig config); // 加载模型 + + + + /** + * 表情识别(多人脸) + * @param imagePath 图片路径 + * @return + */ + default R detect(String imagePath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 表情识别(多人脸) + * @param image BufferedImage + * @return + */ + default R detect(BufferedImage image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 表情识别(多人脸) + * @param imageData 图片字节流 + * @return + */ + default R detect(byte[] imageData){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 表情识别(多人脸) + * @param base64Image + * @return + */ + default R detectBase64(String base64Image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 表情识别(多人脸) + * @param imagePath 图片路径 + * @param faceDetectionResponse 人脸检测结果 + * @return + */ + default R> detect(String imagePath, DetectionResponse faceDetectionResponse){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 表情识别(多人脸) + * @param imageData 图片数据 + * @param faceDetectionResponse 人脸检测结果 + * @return + */ + default R> detect(byte[] imageData,DetectionResponse faceDetectionResponse){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 表情识别(多人脸) + * @param image BufferedImage + * @param faceDetectionResponse 人脸检测结果 + * @return + */ + default R> detect(BufferedImage image,DetectionResponse faceDetectionResponse){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 表情识别(多人脸) + * @param base64Image + * @param faceDetectionResponse 人脸检测结果 + * @return + */ + default R> detectBase64(String base64Image,DetectionResponse faceDetectionResponse){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 表情识别(单人脸) + * @param imagePath 图片路径 + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detect(String imagePath, DetectionRectangle faceDetectionRectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 表情识别(单人脸) + * @param imageData + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + + + /** + * 表情识别(单人脸) + * @param image BufferedImage + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 表情识别(单人脸) + * @param base64Image + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 表情识别(分数最高人脸) + * @param image + * @return + */ + default R detectTopFace(BufferedImage image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 表情识别(分数最高人脸) + * @param imagePath + * @return + */ + default R detectTopFace(String imagePath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 表情识别(分数最高人脸) + * @param imageData + * @return + */ + default R detectTopFace(byte[] imageData){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 表情识别(分数最高人脸) + * @param base64Image + * @return + */ + default R detectTopFaceBase64(String base64Image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/criterial/EmotionCriteriaFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/criterial/EmotionCriteriaFactory.java new file mode 100644 index 0000000..26d8cd8 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/criterial/EmotionCriteriaFactory.java @@ -0,0 +1,58 @@ +package cn.smartjavaai.face.model.expression.criterial; + +import ai.djl.Device; +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.config.FaceExpressionConfig; +import cn.smartjavaai.face.enums.ExpressionModelEnum; +import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator; +import cn.smartjavaai.face.model.expression.translator.FrEmotionTranslator; +import org.apache.commons.lang3.StringUtils; + +import java.nio.file.Paths; +import java.util.Objects; + +/** + * 人脸表情识别 Criteria构建工厂 + * @author dwj + * @date 2025/5/14 + */ +public class EmotionCriteriaFactory { + + public static Criteria createCriteria(FaceExpressionConfig config) { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); + } + Criteria criteria = null; + if(config.getModelEnum() == ExpressionModelEnum.DensNet121){ + //开源项目地址：https://github.com/sajjjadayobi/FaceLib + //初始化 检测Criteria + criteria = + Criteria.builder() + .optEngine("PyTorch") + .setTypes(Image.class, Classifications.class) + .optModelPath(Paths.get(config.getModelPath())) + .optTranslator(new DenseNetEmotionTranslator(224)) + .optProgress(new ProgressBar()) + .optDevice(device) + .build(); + }else if (config.getModelEnum() == ExpressionModelEnum.FrEmotion){ + //初始化 检测Criteria + criteria = + Criteria.builder() + .optEngine("OnnxRuntime") + .setTypes(ai.djl.modality.cv.Image.class, Classifications.class) + .optModelPath(Paths.get(config.getModelPath())) + .optTranslator(new FrEmotionTranslator(224)) + .optProgress(new ProgressBar()) + .build(); + } + return criteria; + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/translator/DenseNetEmotionTranslator.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/translator/DenseNetEmotionTranslator.java new file mode 100644 index 0000000..4b7c62e --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/translator/DenseNetEmotionTranslator.java @@ -0,0 +1,63 @@ +package cn.smartjavaai.face.model.expression.translator; + +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.DataType; +import ai.djl.ndarray.types.Shape; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; + +import java.util.Arrays; +import java.util.List; + +/** + * @author dwj + * @date 2025/6/30 + */ +public class DenseNetEmotionTranslator implements Translator { + + private final List labels = Arrays.asList("angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"); + + private int imageSize = 224; + + public DenseNetEmotionTranslator(int imageSize) { + this.imageSize = imageSize; + } + + @Override + public Classifications processOutput(TranslatorContext ctx, NDList list) { + NDArray output = list.singletonOrThrow(); + output = output.softmax(1); + return new Classifications(labels, output); + } + + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + // 直接转换为灰度NDArray + NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR); + // 调整大小 + Shape shape = array.getShape(); + long height = shape.get(0); + long width = shape.get(1); + if (height != imageSize || width != imageSize) { + array = NDImageUtils.resize(array, imageSize, imageSize); + } + array = NDImageUtils.resize(array, imageSize, imageSize); + array = array.transpose(2, 0, 1); + array = array.expandDims(0); + // 归一化 + array = array.toType(DataType.FLOAT32, false).div(255.0f); + return new NDList(array); + } + + @Override + public Batchifier getBatchifier() { + return null; + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/translator/FrEmotionTranslator.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/translator/FrEmotionTranslator.java new file mode 100644 index 0000000..3c1aab2 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/expression/translator/FrEmotionTranslator.java @@ -0,0 +1,61 @@ +package cn.smartjavaai.face.model.expression.translator; + +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.DataType; +import ai.djl.ndarray.types.Shape; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; + +import java.util.Arrays; +import java.util.List; + +/** + * @author dwj + * @date 2025/6/30 + */ +public class FrEmotionTranslator implements Translator { + + private final List labels = Arrays.asList("angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"); + + private int imageSize = 224; + + public FrEmotionTranslator(int imageSize) { + this.imageSize = imageSize; + } + + @Override + public Classifications processOutput(TranslatorContext ctx, NDList list) { + NDArray output = list.singletonOrThrow(); + output = output.softmax(1); + return new Classifications(labels, output); + } + + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR); + // 调整大小 + Shape shape = array.getShape(); + long height = shape.get(0); + long width = shape.get(1); + if (height != imageSize || width != imageSize) { + array = NDImageUtils.resize(array, imageSize, imageSize); + } + array = array.transpose(2, 0, 1); // 变成 (3, 224, 224) + array = array.expandDims(0); + // 归一化 + array = array.toType(DataType.FLOAT32, false).div(255.0f); + return new NDList(array); + } + + @Override + public Batchifier getBatchifier() { + return null; + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/RetinaFaceModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java similarity index 59% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/RetinaFaceModel.java rename to smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java index 99fb61a..9e97ce2 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/RetinaFaceModel.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java @@ -1,7 +1,7 @@ -package cn.smartjavaai.face.model.facerec; +package cn.smartjavaai.face.model.facedect; -import ai.djl.Device; import ai.djl.MalformedModelException; +import ai.djl.engine.Engine; import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; @@ -9,89 +9,59 @@ import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; -import ai.djl.training.util.ProgressBar; import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.entity.R; import cn.smartjavaai.common.pool.PredictorFactory; +import cn.smartjavaai.common.utils.Base64ImageUtils; import cn.smartjavaai.common.utils.FileUtils; import cn.smartjavaai.common.utils.ImageUtils; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.face.constant.FaceDetectConstant; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.face.config.FaceDetConfig; import cn.smartjavaai.face.exception.FaceException; -import cn.smartjavaai.face.translator.FaceDetectionTranslator; +import cn.smartjavaai.face.model.facedect.criterial.FaceDetCriteriaFactory; import cn.smartjavaai.face.utils.FaceUtils; -import cn.smartjavaai.face.utils.OpenCVUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.impl.GenericObjectPool; - import javax.imageio.ImageIO; import java.awt.image.BufferedImage; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Objects; /** - * RetinaFace实现 + * DJL通用人脸检测模型实现 * @author dwj */ @Slf4j -public class RetinaFaceModel implements FaceModel, AutoCloseable{ - +public class CommonFaceDetModel implements FaceDetModel{ private ObjectPool> predictorPool; private ZooModel model; - /** - * 特征图层的基础缩放比例 - */ - public static final int[][] scales = {{16, 32}, {64, 128}, {256, 512}}; - /** - * 特征图相对于原图的采样步长 - */ - public static final int[] steps = {8, 16, 32}; - /** - * 缩放系数 - */ - public static final double[] variance = {0.1f, 0.2f}; - /** * 加载模型 * @param config */ @Override - public void loadModel(FaceModelConfig config){ - Device device = null; - if(!Objects.isNull(config.getDevice())){ - device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); - } - FaceDetectionTranslator translator = - new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), variance, FaceDetectConstant.MAX_FACE_LIMIT, scales, steps); - Criteria criteria = - Criteria.builder() - .setTypes(Image.class, DetectedObjects.class) - .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : "https://resources.djl.ai/test-models/pytorch/retinaface.zip") - // Load model from local file, e.g: - .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) - .optModelName("retinaface") // specify model file prefix - .optTranslator(translator) - .optDevice(device) - .optProgress(new ProgressBar()) - .optEngine("PyTorch") // Use PyTorch engine - .build(); + public void loadModel(FaceDetConfig config){ + Criteria criteria = FaceDetCriteriaFactory.createCriteria(config); try { model = criteria.loadModel(); - // 创建池子：每个线程独享 Predictor this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); - log.info("当前设备: " + model.getNDManager().getDevice()); + log.debug("当前设备: " + model.getNDManager().getDevice()); + log.debug("当前引擎: " + Engine.getInstance().getEngineName()); } catch (IOException | ModelNotFoundException | MalformedModelException e) { - throw new FaceException("模型加载失败", e); + throw new FaceException("人脸检测模型加载失败", e); } } @@ -104,9 +74,9 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{ * @throws Exception */ @Override - public DetectionResponse detect(String imagePath){ + public R detect(String imagePath){ if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); + return R.fail(R.Status.FILE_NOT_FOUND); } Image img = null; try { @@ -115,7 +85,7 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{ throw new FaceException("无效的图片", e); } DetectedObjects detection = detect(img); - return FaceUtils.convertToDetectionResponse(detection,img); + return R.ok(FaceUtils.convertToDetectionResponse(detection,img)); } /** @@ -125,14 +95,14 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{ * @throws Exception */ @Override - public DetectionResponse detect(InputStream imageInputStream){ + public R detect(InputStream imageInputStream){ if(Objects.isNull(imageInputStream)){ - throw new FaceException("图像输入流无效"); + return R.fail(R.Status.INVALID_IMAGE); } try { Image img = ImageFactory.getInstance().fromInputStream(imageInputStream); DetectedObjects detection = detect(img); - return FaceUtils.convertToDetectionResponse(detection,img); + return R.ok(FaceUtils.convertToDetectionResponse(detection,img)); } catch (IOException e) { throw new FaceException("无效图片输入流", e); } @@ -140,19 +110,19 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{ } @Override - public DetectionResponse detect(BufferedImage image) { + public R detect(BufferedImage image) { if(!ImageUtils.isImageValid(image)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image)); DetectedObjects detection = detect(img); - return FaceUtils.convertToDetectionResponse(detection,img); + return R.ok(FaceUtils.convertToDetectionResponse(detection,img)); } @Override - public DetectionResponse detect(byte[] imageData) { + public R detect(byte[] imageData) { if(Objects.isNull(imageData)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } try { return detect(ImageIO.read(new ByteArrayInputStream(imageData))); @@ -162,34 +132,44 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{ } @Override - public void detectAndDraw(String imagePath, String outputPath) { + public R detectBase64(String base64Image) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detect(imageData); + } + + @Override + public R detectAndDraw(String imagePath, String outputPath) { if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); + return R.fail(R.Status.FILE_NOT_FOUND); } try { Image img = ImageFactory.getInstance().fromFile(Paths.get(imagePath)); DetectedObjects detectedObjects = detect(img); if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ - throw new FaceException("未识别到人脸"); + return R.fail(R.Status.NO_FACE_DETECTED); } img.drawBoundingBoxes(detectedObjects); Path output = Paths.get(outputPath); log.debug("Saving to {}", output.toAbsolutePath().toString()); img.save(Files.newOutputStream(output), "png"); + return R.ok(); } catch (IOException e) { throw new FaceException(e); } } @Override - public BufferedImage detectAndDraw(BufferedImage sourceImage) { + public R detectAndDraw(BufferedImage sourceImage) { if(!ImageUtils.isImageValid(sourceImage)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(sourceImage)); DetectedObjects detectedObjects = detect(img); if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ - throw new FaceException("未识别到人脸"); + return R.fail(R.Status.NO_FACE_DETECTED); } img.drawBoundingBoxes(detectedObjects); try { @@ -198,7 +178,7 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{ img.save(outputStream, "png"); // 将字节流转换为 BufferedImage byte[] imageBytes = outputStream.toByteArray(); - return ImageIO.read(new ByteArrayInputStream(imageBytes)); + return R.ok(ImageIO.read(new ByteArrayInputStream(imageBytes))); } catch (IOException e) { throw new FaceException("导出图片失败", e); } @@ -209,13 +189,13 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{ * @param image * @return */ - private DetectedObjects detect(Image image){ + public DetectedObjects detect(Image image){ Predictor predictor = null; try { predictor = predictorPool.borrowObject(); return predictor.predict(image); } catch (Exception e) { - throw new FaceException("目标检测错误", e); + throw new FaceException("人脸检测错误", e); }finally { if (predictor != null) { try { @@ -235,8 +215,19 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{ @Override public void close() { - if (predictorPool != null) { - predictorPool.close(); + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); } } } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java new file mode 100644 index 0000000..ea6e4ea --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java @@ -0,0 +1,88 @@ +package cn.smartjavaai.face.model.facedect; + +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.face.config.FaceDetConfig; + +import java.awt.image.BufferedImage; +import java.io.InputStream; + +/** + * 人脸检测模型 + * @author dwj + */ +public interface FaceDetModel extends AutoCloseable{ + + /** + * 加载模型 + * @param config + */ + void loadModel(FaceDetConfig config); // 加载模型 + + + /** + * 人脸检测 + * @param imagePath 图片路径 + * @return + */ + default R detect(String imagePath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸检测 + * @param imageInputStream 图片输入流 + * @return + */ + default R detect(InputStream imageInputStream){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸检测 + * @param image BufferedImage + * @return + */ + default R detect(BufferedImage image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸检测 + * @param imageData + * @return + */ + default R detect(byte[] imageData){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸检测 + * @param base64Image + * @return + */ + default R detectBase64(String base64Image) { + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 检测并绘制人脸 + * @param imagePath 图片输入路径（包含文件名称） + * @param outputPath 图片输出路径（包含文件名称） + */ + default R detectAndDraw(String imagePath, String outputPath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 检测并绘制人脸 + * @param sourceImage + * @return + */ + default R detectAndDraw(BufferedImage sourceImage){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java new file mode 100644 index 0000000..362680d --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java @@ -0,0 +1,236 @@ +package cn.smartjavaai.face.model.facedect; + +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.Base64ImageUtils; +import cn.smartjavaai.common.utils.FileUtils; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.seetaface.NativeLoader; +import cn.smartjavaai.face.utils.FaceUtils; +import com.seeta.pool.*; +import com.seeta.sdk.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.*; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * SeetaFace6 人脸检测模型 + * @author dwj + */ +@Slf4j +public class SeetaFace6FaceDetModel implements FaceDetModel{ + private FaceDetConfig config; + + private FaceDetectorPool faceDetectorPool; + private FaceLandmarkerPool faceLandmarkerPool; + + + @Override + public void loadModel(FaceDetConfig config) { + this.config = config; + if(StringUtils.isBlank(config.getModelPath())){ + throw new FaceException("modelPath is null"); + } + //加载依赖库 + NativeLoader.loadNativeLibraries(config.getDevice()); + log.debug("Loading seetaFace6 library successfully."); + String[] faceDetectorModelPath = {config.getModelPath() + File.separator + "face_detector.csta"}; + String[] faceLandmarkerModelPath = {config.getModelPath() + File.separator + "face_landmarker_pts5.csta"}; + SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO; + int gpuId = 0; + if(Objects.nonNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU; + Integer gpuIdValue = config.getCustomParam("gpuId", Integer.class); + if(Objects.nonNull(gpuIdValue) && device == SeetaDevice.SEETA_DEVICE_GPU){ + gpuId = gpuIdValue; + } + } + try { + SeetaModelSetting faceDetectorPoolSetting = new SeetaModelSetting(gpuId, faceDetectorModelPath, device); + SeetaConfSetting faceDetectorPoolConfSetting = new SeetaConfSetting(faceDetectorPoolSetting); + + SeetaModelSetting faceLandmarkerPoolSetting = new SeetaModelSetting(gpuId, faceLandmarkerModelPath, device); + SeetaConfSetting faceLandmarkerPoolConfSetting = new SeetaConfSetting(faceLandmarkerPoolSetting); + + this.faceDetectorPool = new FaceDetectorPool(faceDetectorPoolConfSetting); + this.faceLandmarkerPool = new FaceLandmarkerPool(faceLandmarkerPoolConfSetting); + } catch (FileNotFoundException e) { + throw new FaceException(e); + } + + } + + @Override + public R detect(String imagePath) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detect(image); + } + + @Override + public R detect(InputStream imageInputStream) { + if(Objects.isNull(imageInputStream)){ + return R.fail(R.Status.INVALID_IMAGE); + } + BufferedImage image = null; + try { + image = ImageIO.read(imageInputStream); + } catch (IOException e) { + throw new FaceException("无效图片输入流", e); + } + return detect(image); + } + + @Override + public R detect(BufferedImage image) { + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + FaceDetector predictor = null; + FaceLandmarker faceLandmarker = null; + try { + predictor = faceDetectorPool.borrowObject(); + faceLandmarker = faceLandmarkerPool.borrowObject(); + SeetaRect[] seetaResult = predictor.Detect(imageData); + List seetaPointFSList = new ArrayList(); + for(SeetaRect seetaRect : seetaResult){ + //提取人脸的5点人脸标识 + SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()]; + faceLandmarker.mark(imageData, seetaRect, pointFS); + seetaPointFSList.add(pointFS); + } + return R.ok(FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList)); + } catch (Exception e) { + throw new FaceException("目标检测错误", e); + }finally { + if (predictor != null) { + try { + faceDetectorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + if (faceLandmarker != null) { + try { + faceLandmarkerPool.returnObject(faceLandmarker); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + @Override + public R detect(byte[] imageData) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detect(ImageIO.read(new ByteArrayInputStream(imageData))); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R detectBase64(String base64Image) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detect(imageData); + } + + @Override + public R detectAndDraw(String imagePath, String outputPath) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + try { + //创建保存路径 + Path imageOutputPath = Paths.get(outputPath); + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + R result = detect(image); + if(result.getCode() != R.Status.SUCCESS.getCode()){ + return R.fail(result.getCode(), result.getMessage()); + } + if(Objects.isNull(result.getData()) || Objects.isNull(result.getData().getDetectionInfoList()) || result.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + //绘制人脸框 + FaceUtils.drawBoundingBoxes(image, result.getData(), imageOutputPath.toAbsolutePath().toString()); + return R.ok(); + } catch (IOException e) { + throw new FaceException(e); + } + } + + @Override + public R detectAndDraw(BufferedImage sourceImage) { + if(!ImageUtils.isImageValid(sourceImage)){ + return R.fail(R.Status.INVALID_IMAGE); + } + R result = detect(sourceImage); + if(result.getCode() != R.Status.SUCCESS.getCode()){ + return R.fail(result.getCode(), result.getMessage()); + } + if(Objects.isNull(result.getData()) || Objects.isNull(result.getData().getDetectionInfoList()) || result.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + //绘制人脸框 + try { + return R.ok(FaceUtils.drawBoundingBoxes(sourceImage, result.getData())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + + + + @Override + public void close() throws Exception { + try { + if (faceDetectorPool != null) { + faceDetectorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (faceLandmarkerPool != null) { + faceLandmarkerPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/criterial/FaceDetCriteriaFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/criterial/FaceDetCriteriaFactory.java new file mode 100644 index 0000000..a0f236a --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facedect/criterial/FaceDetCriteriaFactory.java @@ -0,0 +1,69 @@ +package cn.smartjavaai.face.model.facedect.criterial; + +import ai.djl.Device; +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.config.FaceExpressionConfig; +import cn.smartjavaai.face.constant.FaceDetectConstant; +import cn.smartjavaai.face.constant.RetinaFaceConstant; +import cn.smartjavaai.face.constant.UltraLightFastGenericFaceConstant; +import cn.smartjavaai.face.enums.ExpressionModelEnum; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator; +import cn.smartjavaai.face.model.expression.translator.FrEmotionTranslator; +import cn.smartjavaai.face.translator.FaceDetectionTranslator; +import org.apache.commons.lang3.StringUtils; + +import java.nio.file.Paths; +import java.util.Objects; + +/** + * 人脸检测 Criteria构建工厂 + * @author dwj + */ +public class FaceDetCriteriaFactory { + + public static Criteria createCriteria(FaceDetConfig config) { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); + } + Criteria criteria = null; + if(config.getModelEnum() == FaceDetModelEnum.RETINA_FACE){ + FaceDetectionTranslator translator = + new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), RetinaFaceConstant.variance, FaceDetectConstant.MAX_FACE_LIMIT, RetinaFaceConstant.scales, RetinaFaceConstant.steps); + criteria = + Criteria.builder() + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : RetinaFaceConstant.MODEL_URL) + // Load model from local file, e.g: + .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) + .optModelName("retinaface") // specify model file prefix + .optTranslator(translator) + .optDevice(device) + .optProgress(new ProgressBar()) + .optEngine("PyTorch") // Use PyTorch engine + .build(); + }else if (config.getModelEnum() == FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE){ + FaceDetectionTranslator translator = + new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), UltraLightFastGenericFaceConstant.variance, FaceDetectConstant.MAX_FACE_LIMIT, UltraLightFastGenericFaceConstant.scales, UltraLightFastGenericFaceConstant.steps); + criteria = + Criteria.builder() + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : UltraLightFastGenericFaceConstant.MODEL_URL) + .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) + .optTranslator(translator) + .optProgress(new ProgressBar()) + .optDevice(device) + .optEngine("PyTorch") // Use PyTorch engine + .build(); + } + return criteria; + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceNetModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/CommonFaceRecModel.java similarity index 70% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceNetModel.java rename to smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/CommonFaceRecModel.java index 4ddabf3..749825e 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceNetModel.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/CommonFaceRecModel.java @@ -1,32 +1,35 @@ package cn.smartjavaai.face.model.facerec; -import ai.djl.Device; import ai.djl.MalformedModelException; +import ai.djl.engine.Engine; import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.ImageFactory; -import ai.djl.ndarray.NDArray; import ai.djl.ndarray.NDManager; -import ai.djl.opencv.OpenCVImageFactory; import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelNotFoundException; import ai.djl.repository.zoo.ZooModel; -import ai.djl.training.util.ProgressBar; +import cn.hutool.core.lang.UUID; +import cn.hutool.core.lang.generator.UUIDGenerator; import cn.smartjavaai.common.entity.*; -import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.entity.face.FaceSearchResult; import cn.smartjavaai.common.pool.PredictorFactory; import cn.smartjavaai.common.utils.FileUtils; import cn.smartjavaai.common.utils.ImageUtils; -import cn.smartjavaai.face.config.FaceExtractConfig; -import cn.smartjavaai.face.config.FaceModelConfig; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.config.FaceRecConfig; import cn.smartjavaai.face.constant.FaceDetectConstant; import cn.smartjavaai.face.entity.FaceRegisterInfo; import cn.smartjavaai.face.entity.FaceSearchParams; -import cn.smartjavaai.face.enums.FaceModelEnum; +import cn.smartjavaai.face.enums.FaceDetModelEnum; import cn.smartjavaai.face.enums.SimilarityType; import cn.smartjavaai.face.exception.FaceException; -import cn.smartjavaai.face.factory.FaceModelFactory; -import cn.smartjavaai.face.translator.FaceFeatureTranslator; +import cn.smartjavaai.face.factory.FaceDetModelFactory; +import cn.smartjavaai.face.model.facedect.FaceDetModel; +import cn.smartjavaai.face.model.facerec.criterial.FaceRecCriteriaFactory; +import cn.smartjavaai.face.preprocess.DJLImagePreprocessor; import cn.smartjavaai.face.utils.*; import cn.smartjavaai.face.vector.config.MilvusConfig; import cn.smartjavaai.face.vector.config.SQLiteConfig; @@ -37,31 +40,24 @@ import cn.smartjavaai.face.vector.entity.FaceVector; import cn.smartjavaai.face.vector.exception.VectorDBException; import io.milvus.param.MetricType; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.impl.GenericObjectPool; -import org.opencv.core.Mat; import javax.imageio.ImageIO; import java.awt.image.BufferedImage; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * FaceNet 人脸特征提取模型 * @author dwj */ @Slf4j -public class FaceNetModel implements FaceModel, AutoCloseable{ +public class CommonFaceRecModel implements FaceRecModel{ /** * 特征维度 @@ -79,23 +75,13 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ private ZooModel model; - private FaceModelConfig config; + private FaceRecConfig config; /** * 是否归一化相似度 */ public static final boolean NORMALIZE_SIMILARITY = true; - - public static final List mean = - Arrays.asList( - 127.5f / 255.0f, - 127.5f / 255.0f, - 127.5f / 255.0f, - 128.0f / 255.0f, - 128.0f / 255.0f, - 128.0f / 255.0f); - private VectorDBClient vectorDBClient; @@ -104,42 +90,21 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ * @param config */ @Override - public void loadModel(FaceModelConfig config) { + public void loadModel(FaceRecConfig config) { if(Objects.isNull(config)){ throw new FaceException("config为null"); } - if(Objects.isNull(config.getExtractConfig())){ - config.setExtractConfig(getDefaultConfig()); - }else{ - if(Objects.isNull(config.getExtractConfig().getDetectModel())){ - throw new FaceException("请设置人脸检测模型"); - } - } - Device device = null; - if(!Objects.isNull(config.getDevice())){ - device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); + if(Objects.isNull(config.getDetectModel())){ + config.setDetectModel(getDefaultDetModel()); } this.config = config; - String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); - Criteria faceFeatureCriteria = - Criteria.builder() - .setTypes(Image.class, float[].class) - .optModelName("face_feature") // specify model file prefix - .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : - "https://resources.djl.ai/test-models/pytorch/face_feature.zip") - .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) - .optTranslator(new FaceFeatureTranslator()) - .optArgument("normalize", normalize) - .optDevice(device) - .optEngine("PyTorch") // Use PyTorch engine - .optProgress(new ProgressBar()) - .build(); - + Criteria faceFeatureCriteria = FaceRecCriteriaFactory.createCriteria(config); try { model = faceFeatureCriteria.loadModel(); // 创建池子：每个线程独享 Predictor this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); - log.info("当前设备: " + model.getNDManager().getDevice()); + log.debug("当前设备: " + model.getNDManager().getDevice()); + log.debug("当前引擎: " + Engine.getInstance().getEngineName()); } catch (IOException | ModelNotFoundException | MalformedModelException e) { throw new FaceException("模型加载失败", e); } @@ -184,14 +149,13 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ } } - private float[] featureExtraction(Image image){ - image.getWrappedImage(); + public float[] featureExtraction(Image image){ Predictor predictor = null; try { predictor = predictorPool.borrowObject(); return predictor.predict(image); } catch (Exception e) { - throw new FaceException("目标检测错误", e); + throw new FaceException("人脸特征提取错误", e); }finally { if (predictor != null) { try { @@ -218,7 +182,7 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ @Override public float calculSimilar(float[] feature1, float[] feature2) { //默认返回归一化结果 - return SimilarityUtil.calculate(feature1, feature2, SimilarityType.IP, NORMALIZE_SIMILARITY); + return SimilarityUtil.calculate(feature1, feature2, SimilarityType.IP, true); } /** @@ -228,105 +192,103 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ * @return */ @Override - public float featureComparison(String imagePath1, String imagePath2) { + public R featureComparison(String imagePath1, String imagePath2) { if(!FileUtils.isFileExists(imagePath1) || !FileUtils.isFileExists(imagePath2)){ - throw new FaceException("图像文件不存在"); + return R.fail(R.Status.FILE_NOT_FOUND); } - R feature1 = extractTopFaceFeature(imagePath1); - if (!feature1.isSuccess()){ - throw new FaceException(feature1.getMessage()); + // 将图片路径转换为 BufferedImage + BufferedImage image1 = null; + BufferedImage image2 = null; + try { + image1 = ImageIO.read(new File(Paths.get(imagePath1).toAbsolutePath().toString())); + image2 = ImageIO.read(new File(Paths.get(imagePath2).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); } - R feature2 = extractTopFaceFeature(imagePath2); - if (!feature2.isSuccess()){ - throw new FaceException(feature2.getMessage()); - } - float ret = calculSimilar(feature1.getData(), feature2.getData()); - return ret; + return featureComparison(image1, image2); } @Override - public float featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2) { + public R featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2) { if(!ImageUtils.isImageValid(sourceImage1) || !ImageUtils.isImageValid(sourceImag2)){ throw new FaceException("图像无效"); } R feature1 = extractTopFaceFeature(sourceImage1); if (!feature1.isSuccess()){ - throw new FaceException(feature1.getMessage()); + return R.fail(feature1.getCode(), feature1.getMessage()); } R feature2 = extractTopFaceFeature(sourceImag2); if (!feature2.isSuccess()){ - throw new FaceException(feature2.getMessage()); + return R.fail(feature2.getCode(), feature2.getMessage()); } float ret = calculSimilar(feature1.getData(), feature2.getData()); - return ret; + return R.ok(ret); } @Override - public float featureComparison(byte[] imageData1, byte[] imageData2) { + public R featureComparison(byte[] imageData1, byte[] imageData2) { if(Objects.isNull(imageData1) || Objects.isNull(imageData2)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } - R feature1 = extractTopFaceFeature(imageData1); - if (!feature1.isSuccess()){ - throw new FaceException(feature1.getMessage()); + try { + BufferedImage bufferedImage1 = ImageIO.read(new ByteArrayInputStream(imageData1)); + BufferedImage bufferedImage2 = ImageIO.read(new ByteArrayInputStream(imageData2)); + return featureComparison(bufferedImage1, bufferedImage2); + } catch (IOException e) { + throw new FaceException("错误的图像", e); } - R feature2 = extractTopFaceFeature(imageData2); - if (!feature2.isSuccess()){ - throw new FaceException(feature2.getMessage()); - } - float ret = calculSimilar(feature1.getData(), feature2.getData()); - return ret; } /** - * 获取默认特征提取配置 + * 获取默认人脸检测模型 * @return */ - private FaceExtractConfig getDefaultConfig() { - FaceExtractConfig config = new FaceExtractConfig(); - FaceModelConfig detectModelConfig = new FaceModelConfig(); - detectModelConfig.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE); + private FaceDetModel getDefaultDetModel() { + FaceDetConfig detectModelConfig = new FaceDetConfig(); + detectModelConfig.setModelEnum(FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE); detectModelConfig.setConfidenceThreshold(0.98); - log.debug("创建默认检测模型：ULTRA_LIGHT_FAST_GENERIC_FACE"); - FaceModel detectModel = FaceModelFactory.getInstance().getModel(detectModelConfig); - log.debug("创建检测模型完毕"); - config.setDetectModel(detectModel); - return config; + log.debug("创建默认人脸检测模型：ULTRA_LIGHT_FAST_GENERIC_FACE"); + FaceDetModel detectModel = FaceDetModelFactory.getInstance().getModel(detectModelConfig); + return detectModel; } @Override public R extractFeatures(BufferedImage image) { - DetectionResponse detectedResult = config.getExtractConfig().getDetectModel().detect(image); - if(Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getDetectionInfoList()) || detectedResult.getDetectionInfoList().isEmpty()){ + R detectedResult = config.getDetectModel().detect(image); + if(!detectedResult.isSuccess()){ + return detectedResult; + } + if(Objects.isNull(detectedResult.getData()) || Objects.isNull(detectedResult.getData().getDetectionInfoList()) || detectedResult.getData().getDetectionInfoList().isEmpty()){ return R.fail(R.Status.NO_FACE_DETECTED); } Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image)); - NDManager manager = NDManager.newBaseManager(); - for (DetectionInfo detectionInfo : detectedResult.getDetectionInfoList()){ - DetectionRectangle rectangle = detectionInfo.getDetectionRectangle(); - FaceInfo faceInfo = detectionInfo.getFaceInfo(); - float[] features = null; - //裁剪人脸 - Image subImage = djlImage.getSubImage(rectangle.getX(), rectangle.getY() , rectangle.getWidth() , rectangle.getHeight()); - //人脸对齐 - if(config.getExtractConfig().isAlign()){ - //获取子图中人脸关键点坐标 - double[][] pointsArray = FaceUtils.facePoints(detectionInfo.getFaceInfo().getKeyPoints()); - NDArray srcPoints = manager.create(pointsArray); - NDArray dstPoints = FaceUtils.faceTemplate512x512(manager); - // 5点仿射变换 - Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints); - Mat mat = FaceAlignUtils.warpAffine((Mat) djlImage.getWrappedImage(), affine_matrix); - Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat); - features = featureExtraction(alignedImg); - }else{ - //不对齐人脸 + try (NDManager manager = model.getNDManager().newSubManager()) { + DJLImagePreprocessor djlImagePreprocessor = new DJLImagePreprocessor(djlImage, manager); + for (DetectionInfo detectionInfo : detectedResult.getData().getDetectionInfoList()){ + DetectionRectangle rectangle = detectionInfo.getDetectionRectangle(); + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + float[] features = null; + Image subImage = djlImage; + //人脸对齐 + if(config.isAlign()){ + //人脸对齐 + double[][] pointsArray = FaceUtils.facePoints(faceInfo.getKeyPoints()); + djlImagePreprocessor.enableCrop(rectangle).enableAffine(pointsArray, 96, 112); + subImage = djlImagePreprocessor.process(); + }else{ + //裁剪 + djlImagePreprocessor.enableCrop(rectangle); + if(config.isCropFace()){ + subImage = djlImagePreprocessor.process(); + } + } + features = featureExtraction(subImage); + faceInfo.setFeature(features); } - faceInfo.setFeature(features); } - return R.ok(detectedResult); + return detectedResult; } @@ -359,37 +321,36 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ @Override public R extractTopFaceFeature(BufferedImage image) { + R detectedResult = config.getDetectModel().detect(image); + if(!detectedResult.isSuccess()){ + return R.fail(detectedResult.getCode(), detectedResult.getMessage()); + } + if(Objects.isNull(detectedResult.getData()) || Objects.isNull(detectedResult.getData().getDetectionInfoList()) || detectedResult.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image)); float[] features = null; - if(config.getExtractConfig().isCropFace()){ - DetectionResponse detectedResult = config.getExtractConfig().getDetectModel().detect(image); - if(Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getDetectionInfoList()) || detectedResult.getDetectionInfoList().isEmpty()){ - return R.fail(R.Status.NO_FACE_DETECTED); - } + try (NDManager manager = model.getNDManager().newSubManager()) { + DJLImagePreprocessor djlImagePreprocessor = new DJLImagePreprocessor(djlImage, manager); //只取第一个人脸 - DetectionInfo detectionInfo = detectedResult.getDetectionInfoList().get(0); + DetectionInfo detectionInfo = detectedResult.getData().getDetectionInfoList().get(0); DetectionRectangle rectangle = detectionInfo.getDetectionRectangle(); - //裁剪人脸 - Image subImage = djlImage.getSubImage(rectangle.getX(), rectangle.getY() , rectangle.getWidth() , rectangle.getHeight()); + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + Image subImage = djlImage; //人脸对齐 - if(config.getExtractConfig().isAlign()){ - NDManager manager = NDManager.newBaseManager(); - //获取子图中人脸关键点坐标 - double[][] pointsArray = FaceUtils.facePoints(detectionInfo.getFaceInfo().getKeyPoints()); - NDArray srcPoints = manager.create(pointsArray); - NDArray dstPoints = FaceUtils.faceTemplate512x512(manager); - // 5点仿射变换 - Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints); - Mat mat = FaceAlignUtils.warpAffine((Mat) djlImage.getWrappedImage(), affine_matrix); - Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat); - features = featureExtraction(alignedImg); + if(config.isAlign()){ + //人脸对齐 + double[][] pointsArray = FaceUtils.facePoints(faceInfo.getKeyPoints()); + djlImagePreprocessor.enableCrop(rectangle).enableAffine(pointsArray, 96, 112); + subImage = djlImagePreprocessor.process(); }else{ - //不对齐人脸 - features = featureExtraction(subImage); + //裁剪 + djlImagePreprocessor.enableCrop(rectangle); + if(config.isCropFace()){ + subImage = djlImagePreprocessor.process(); + } } - }else{ - //不裁剪人脸直接提取特征 - features = featureExtraction(djlImage); + features = featureExtraction(subImage); } return Objects.isNull(features) ? R.fail(R.Status.Unknown) : R.ok(features); } @@ -481,10 +442,10 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ @Override public R register(FaceRegisterInfo faceRegisterInfo, float[] feature) { if(vectorDBClient == null){ - throw new VectorDBException("向量数据库未初始化成功"); + return R.fail(1000, "向量数据库未初始化成功"); } if(Objects.isNull(feature)){ - throw new FaceException("人脸特征为空"); + return R.fail(R.Status.PARAM_ERROR.getCode(), "人脸特征为空"); } FaceVector faceVector = new FaceVector(); if(faceRegisterInfo != null){ @@ -603,7 +564,7 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ @Override public R> searchByTopFace(BufferedImage sourceImage, FaceSearchParams params) { if(vectorDBClient == null){ - throw new VectorDBException("向量数据库未初始化成功"); + return R.fail(1000, "向量数据库未初始化成功"); } //提取最大人脸特征 R featureResponse = extractTopFaceFeature(sourceImage); @@ -704,12 +665,22 @@ public class FaceNetModel implements FaceModel, AutoCloseable{ @Override public void close() { - if (predictorPool != null) { - predictorPool.close(); + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); } - if(Objects.nonNull(vectorDBClient)){ - vectorDBClient.close(); + + try { + if(Objects.nonNull(vectorDBClient)){ + vectorDBClient.close(); + } + } catch (Exception e) { + log.warn("关闭 vectorDBClient 失败", e); } + } @Override diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceRecModel.java similarity index 83% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceModel.java rename to smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceRecModel.java index 54b9349..b353323 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceModel.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/FaceRecModel.java @@ -2,10 +2,10 @@ package cn.smartjavaai.face.model.facerec; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; -import cn.smartjavaai.face.config.FaceModelConfig; +import cn.smartjavaai.face.config.FaceRecConfig; import cn.smartjavaai.face.entity.FaceRegisterInfo; import cn.smartjavaai.face.entity.FaceSearchParams; -import cn.smartjavaai.common.entity.FaceSearchResult; +import cn.smartjavaai.common.entity.face.FaceSearchResult; import java.awt.image.BufferedImage; import java.io.InputStream; @@ -15,69 +15,15 @@ import java.util.List; * 人脸识别模型 * @author dwj */ -public interface FaceModel { +public interface FaceRecModel extends AutoCloseable{ /** * 加载模型 * @param config */ - void loadModel(FaceModelConfig config); // 加载模型 + void loadModel(FaceRecConfig config); // 加载模型 - /** - * 人脸检测 - * @param imagePath 图片路径 - * @return - */ - default DetectionResponse detect(String imagePath){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - - /** - * 人脸检测 - * @param imageInputStream 图片输入流 - * @return - */ - default DetectionResponse detect(InputStream imageInputStream){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - - /** - * 人脸检测 - * @param image BufferedImage - * @return - */ - default DetectionResponse detect(BufferedImage image){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - - /** - * 人脸检测 - * @param imageData - * @return - */ - default DetectionResponse detect(byte[] imageData){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - - /** - * 检测并绘制人脸 - * @param imagePath 图片输入路径（包含文件名称） - * @param outputPath 图片输出路径（包含文件名称） - */ - default void detectAndDraw(String imagePath, String outputPath){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - - /** - * 检测并绘制人脸 - * @param sourceImage - * @return - */ - default BufferedImage detectAndDraw(BufferedImage sourceImage){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - /** * 计算相似度 * @param feature1 图1特征 @@ -94,7 +40,7 @@ public interface FaceModel { * @param imagePath2 图2路径 * @return */ - default float featureComparison(String imagePath1, String imagePath2){ + default R featureComparison(String imagePath1, String imagePath2){ throw new UnsupportedOperationException("默认不支持该功能"); } @@ -104,7 +50,7 @@ public interface FaceModel { * @param sourceImag2 图2BufferedImage * @return */ - default float featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2){ + default R featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2){ throw new UnsupportedOperationException("默认不支持该功能"); } @@ -115,7 +61,7 @@ public interface FaceModel { * @param imageData2 * @return */ - default float featureComparison(byte[] imageData1, byte[] imageData2){ + default R featureComparison(byte[] imageData1, byte[] imageData2){ throw new UnsupportedOperationException("默认不支持该功能"); } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/SeetaFace6Model.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/SeetaFace6FaceRecModel.java similarity index 67% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/SeetaFace6Model.java rename to smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/SeetaFace6FaceRecModel.java index 104434f..1624b64 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/SeetaFace6Model.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/SeetaFace6FaceRecModel.java @@ -3,10 +3,11 @@ package cn.smartjavaai.face.model.facerec; import cn.smartjavaai.common.entity.DetectionInfo; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.entity.face.FaceInfo; import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.common.utils.FileUtils; import cn.smartjavaai.common.utils.ImageUtils; -import cn.smartjavaai.face.config.FaceModelConfig; +import cn.smartjavaai.face.config.FaceRecConfig; import cn.smartjavaai.face.constant.FaceDetectConstant; import cn.smartjavaai.face.entity.FaceRegisterInfo; import cn.smartjavaai.face.entity.FaceResult; @@ -18,7 +19,7 @@ import cn.smartjavaai.face.vector.config.MilvusConfig; import cn.smartjavaai.face.vector.config.SQLiteConfig; import cn.smartjavaai.face.vector.core.VectorDBClient; import cn.smartjavaai.face.vector.core.VectorDBFactory; -import cn.smartjavaai.common.entity.FaceSearchResult; +import cn.smartjavaai.common.entity.face.FaceSearchResult; import cn.smartjavaai.face.vector.entity.FaceVector; import cn.smartjavaai.face.vector.exception.VectorDBException; import com.seeta.pool.*; @@ -26,7 +27,6 @@ import com.seeta.sdk.*; import cn.smartjavaai.face.seetaface.NativeLoader; import io.milvus.param.MetricType; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import javax.imageio.ImageIO; @@ -46,7 +46,7 @@ import java.util.Objects; */ @SuppressWarnings("AliMissingOverrideAnnotation") @Slf4j -public class SeetaFace6Model implements FaceModel , AutoCloseable{ +public class SeetaFace6FaceRecModel implements FaceRecModel{ /** * 特征维度 @@ -54,7 +54,7 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ private static final int DIMENSION = 1024; - private FaceModelConfig config; + private FaceRecConfig config; private FaceDetectorPool faceDetectorPool; private FaceRecognizerPool faceRecognizerPool; @@ -77,7 +77,7 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ @Override - public void loadModel(FaceModelConfig config) { + public void loadModel(FaceRecConfig config) { this.config = config; if(StringUtils.isBlank(config.getModelPath())){ throw new FaceException("modelPath is null"); @@ -92,8 +92,9 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ int gpuId = 0; if(Objects.nonNull(config.getDevice())){ device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU; - if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){ - gpuId = config.getGpuId(); + Integer gpuIdValue = config.getCustomParam("gpuId", Integer.class); + if(Objects.nonNull(gpuIdValue) && device == SeetaDevice.SEETA_DEVICE_GPU){ + gpuId = gpuIdValue; } } try { @@ -156,204 +157,8 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ } catch (FileNotFoundException e) { throw new FaceException(e); } - } - @Override - public DetectionResponse detect(String imagePath) { - if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); - } - // 将图片路径转换为 BufferedImage - BufferedImage image = null; - try { - image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - } catch (IOException e) { - throw new FaceException("无效图片路径", e); - } - return detect(image); - } - - @Override - public DetectionResponse detect(InputStream imageInputStream) { - if(Objects.isNull(imageInputStream)){ - throw new FaceException("图像输入流无效"); - } - BufferedImage image = null; - try { - image = ImageIO.read(imageInputStream); - } catch (IOException e) { - throw new FaceException("无效图片输入流", e); - } - return detect(image); - } - - @Override - public DetectionResponse detect(BufferedImage image) { - if(!ImageUtils.isImageValid(image)){ - throw new FaceException("图像无效"); - } - SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); - imageData.data = ImageUtils.getMatrixBGR(image); - FaceDetector predictor = null; - FaceLandmarker faceLandmarker = null; - try { - predictor = faceDetectorPool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); - SeetaRect[] seetaResult = predictor.Detect(imageData); - List seetaPointFSList = new ArrayList(); - for(SeetaRect seetaRect : seetaResult){ - //提取人脸的5点人脸标识 - SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData, seetaRect, pointFS); - seetaPointFSList.add(pointFS); - } - return FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList); - } catch (Exception e) { - throw new FaceException("目标检测错误", e); - }finally { - if (predictor != null) { - try { - faceDetectorPool.returnObject(predictor); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - if (faceLandmarker != null) { - try { - faceLandmarkerPool.returnObject(faceLandmarker); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - } - } - - @Override - public DetectionResponse detect(byte[] imageData) { - if(Objects.isNull(imageData)){ - throw new FaceException("图像无效"); - } - try { - return detect(ImageIO.read(new ByteArrayInputStream(imageData))); - } catch (IOException e) { - throw new FaceException("错误的图像", e); - } - } - - @Override - public void detectAndDraw(String imagePath, String outputPath) { - if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); - } - try { - //创建保存路径 - Path imageOutputPath = Paths.get(outputPath); - BufferedImage image = null; - try { - image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); - } catch (IOException e) { - throw new FaceException("无效图片路径", e); - } - DetectionResponse result = detect(image); - if(Objects.isNull(result) || Objects.isNull(result.getDetectionInfoList()) || result.getDetectionInfoList().isEmpty()){ - throw new FaceException("未识别到人脸"); - } - //绘制人脸框 - FaceUtils.drawBoundingBoxes(image, result, imageOutputPath.toAbsolutePath().toString()); - } catch (IOException e) { - throw new FaceException(e); - } - } - - @Override - public BufferedImage detectAndDraw(BufferedImage sourceImage) { - if(!ImageUtils.isImageValid(sourceImage)){ - throw new FaceException("图像无效"); - } - DetectionResponse detectedObjects = detect(sourceImage); - if(Objects.isNull(detectedObjects) || Objects.isNull(detectedObjects.getDetectionInfoList()) || detectedObjects.getDetectionInfoList().isEmpty()){ - throw new FaceException("未识别到人脸"); - } - //绘制人脸框 - try { - return FaceUtils.drawBoundingBoxes(sourceImage, detectedObjects); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * 获取5点坐标,循序依次为，左眼中心、右眼中心、鼻尖、左嘴角和右嘴角 - * @param imageData - * @return - */ - private SeetaPointF[] getMaskPoint(SeetaImageData imageData) { - FaceDetector faceDetector = null; - FaceLandmarker faceLandmarker = null; - try { - faceDetector = faceDetectorPool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); - //检测人脸 - SeetaRect[] seetaResult = faceDetector.Detect(imageData); - if(Objects.isNull(seetaResult) || seetaResult.length == 0){ - throw new FaceException("未检测到人脸"); - } - //提取第一个人脸的5点人脸标识 - SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData, seetaResult[0], pointFS); - return pointFS; - } catch (FaceException e) { - throw e; - } catch (Exception e) { - throw new FaceException("目标检测错误", e); - }finally { - if (faceDetector != null) { - try { - faceDetectorPool.returnObject(faceDetector); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - if (faceLandmarker != null) { - try { - faceLandmarkerPool.returnObject(faceLandmarker); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - } - } - - /** - * 裁剪人脸 - * @param imageData - * @return - */ - private SeetaImageData getMaxCropFace(SeetaImageData imageData){ - FaceRecognizer faceRecognizer = null; - try { - faceRecognizer = faceRecognizerPool.borrowObject(); - //提取第一个人脸的5点人脸标识 - SeetaPointF[] pointFS = getMaskPoint(imageData); - //裁剪人脸 - SeetaImageData cropImageData = new SeetaImageData(faceRecognizer.GetCropFaceWidthV2(), faceRecognizer.GetCropFaceHeightV2(), faceRecognizer.GetCropFaceChannelsV2()); - faceRecognizer.CropFaceV2(imageData, pointFS, cropImageData); - return cropImageData; - } catch (Exception e) { - throw new FaceException(e); - }finally { - if (faceRecognizer != null) { - try { - faceRecognizerPool.returnObject(faceRecognizer); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - } - } - - @Override public float calculSimilar(float[] feature1, float[] feature2) { if(Objects.isNull(feature1) || Objects.isNull(feature2)){ @@ -377,7 +182,7 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ } @Override - public float featureComparison(String imagePath1, String imagePath2) { + public R featureComparison(String imagePath1, String imagePath2) { if(!FileUtils.isFileExists(imagePath1) || !FileUtils.isFileExists(imagePath2)){ throw new FaceException("图像文件不存在"); } @@ -395,96 +200,26 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ @Override - public float featureComparison(BufferedImage image1, BufferedImage image2) { + public R featureComparison(BufferedImage image1, BufferedImage image2) { if(!ImageUtils.isImageValid(image1) || !ImageUtils.isImageValid(image2)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } - SeetaImageData imageData1 = new SeetaImageData(image1.getWidth(), image1.getHeight(), 3); - imageData1.data = ImageUtils.getMatrixBGR(image1); - - SeetaImageData imageData2 = new SeetaImageData(image2.getWidth(), image2.getHeight(), 3); - imageData2.data = ImageUtils.getMatrixBGR(image2); - - - FaceRecognizer faceRecognizer = null; - FaceDatabase faceDatabase = null; - FaceLandmarker faceLandmarker = null; - FaceDetector faceDetector = null; - try { - faceRecognizer = faceRecognizerPool.borrowObject(); - faceDatabase = faceDatabasePool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); - faceDetector = faceDetectorPool.borrowObject(); - - //检测人脸 - SeetaRect[] seetaResult = faceDetector.Detect(imageData1); - if(Objects.isNull(seetaResult) || seetaResult.length == 0){ - throw new FaceException("未检测到人脸"); - } - //提取第一个人脸的5点人脸标识 - SeetaPointF[] pointFS1 = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData1, seetaResult[0], pointFS1); - - //裁剪人脸 -// SeetaImageData cropImageData1 = new SeetaImageData(faceRecognizer.GetCropFaceWidthV2(), faceRecognizer.GetCropFaceHeightV2(), faceRecognizer.GetCropFaceChannelsV2()); -// faceRecognizer.CropFaceV2(imageData1, pointFS1, cropImageData1); - - //图片2：检测人脸 - SeetaRect[] seetaResult2 = faceDetector.Detect(imageData2); - if(Objects.isNull(seetaResult2) || seetaResult2.length == 0){ - throw new FaceException("未检测到人脸"); - } - //图片2：提取第一个人脸的5点人脸标识 - SeetaPointF[] pointFS2 = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData2, seetaResult2[0], pointFS2); - - //图片2：裁剪人脸 -// SeetaImageData cropImageData2 = new SeetaImageData(faceRecognizer.GetCropFaceWidthV2(), faceRecognizer.GetCropFaceHeightV2(), faceRecognizer.GetCropFaceChannelsV2()); -// faceRecognizer.CropFaceV2(imageData2, pointFS2, cropImageData2); -// return faceDatabase.CompareByCroppedFace(cropImageData1, cropImageData2); - return faceDatabase.Compare(imageData1, pointFS1, imageData2, pointFS2); - } catch (FaceException e) { - throw e; - } catch (Exception e) { - throw new FaceException(e); - }finally { - if (faceDetector != null) { - try { - faceDetectorPool.returnObject(faceDetector); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - if (faceLandmarker != null) { - try { - faceLandmarkerPool.returnObject(faceLandmarker); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - if (faceRecognizer != null) { - try { - faceRecognizerPool.returnObject(faceRecognizer); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - if (faceDatabase != null) { - try { - faceDatabasePool.returnObject(faceDatabase); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } + R feature1 = extractTopFaceFeature(image1); + if(!feature1.isSuccess()){ + return R.fail(feature1.getCode(), feature1.getMessage()); } - + R feature2 = extractTopFaceFeature(image2); + if(!feature2.isSuccess()){ + return R.fail(feature2.getCode(), feature2.getMessage()); + } + return R.ok(calculSimilar(feature1.getData(), feature2.getData())); } @Override - public float featureComparison(byte[] imageData1, byte[] imageData2) { + public R featureComparison(byte[] imageData1, byte[] imageData2) { if(Objects.isNull(imageData1) || Objects.isNull(imageData2)){ throw new FaceException("图像无效"); } @@ -842,37 +577,64 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ if(!ImageUtils.isImageValid(image)){ return R.fail(R.Status.INVALID_IMAGE); } - List featureList = new ArrayList(); - List seetaPointFSList = new ArrayList(); - SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); - imageData.data = ImageUtils.getMatrixBGR(image); FaceDetector faceDetector = null; FaceLandmarker faceLandmarker = null; FaceRecognizer faceRecognizer = null; try { - faceDetector = faceDetectorPool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); faceRecognizer = faceRecognizerPool.borrowObject(); - //检测人脸 - SeetaRect[] seetaResult = faceDetector.Detect(imageData); - if(Objects.isNull(seetaResult) || seetaResult.length == 0){ - return R.fail(R.Status.NO_FACE_DETECTED); - } - for(SeetaRect seetaRect : seetaResult){ - //提取人脸的5点人脸标识 - SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData, seetaRect, pointFS); - //提取特征 - float[] features = new float[faceRecognizer.GetExtractFeatureSize()]; - //CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐 - boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features); - if(!isSuccess){ - log.warn("人脸特征提取失败"); + //默认使用Seetaface6检测模型 + if(Objects.isNull(config.getDetectModel())){ + List featureList = new ArrayList(); + List seetaPointFSList = new ArrayList(); + faceDetector = faceDetectorPool.borrowObject(); + faceLandmarker = faceLandmarkerPool.borrowObject(); + //检测人脸 + SeetaRect[] seetaResult = faceDetector.Detect(imageData); + if(Objects.isNull(seetaResult) || seetaResult.length == 0){ + return R.fail(R.Status.NO_FACE_DETECTED); } - featureList.add(features); - seetaPointFSList.add(pointFS); + for(SeetaRect seetaRect : seetaResult){ + //提取人脸的5点人脸标识 + SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()]; + faceLandmarker.mark(imageData, seetaRect, pointFS); + //提取特征 + float[] features = new float[faceRecognizer.GetExtractFeatureSize()]; + //CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐 + boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features); + if(!isSuccess){ + return R.fail(R.Status.Unknown.getCode(), "人脸特征提取失败"); + } + featureList.add(features); + seetaPointFSList.add(pointFS); + } + return R.ok(FaceUtils.featuresConvertToResponse(seetaResult, seetaPointFSList, featureList)); + }else{ + R detectResponse = config.getDetectModel().detect(image); + if(!detectResponse.isSuccess()){ + return detectResponse; + } + if(Objects.isNull(detectResponse.getData()) || Objects.isNull(detectResponse.getData().getDetectionInfoList()) || detectResponse.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + for(DetectionInfo detectionInfo : detectResponse.getData().getDetectionInfoList()){ + //提取特征 + float[] features = new float[faceRecognizer.GetExtractFeatureSize()]; + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){ + return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点"); + } + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(faceInfo.getKeyPoints()); + //CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐 + boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features); + if(!isSuccess){ + return R.fail(R.Status.Unknown.getCode(), "人脸特征提取失败"); + } + faceInfo.setFeature(features); + } + return detectResponse; } - return R.ok(FaceUtils.featuresConvertToResponse(seetaResult, seetaPointFSList, featureList)); } catch (FaceException e) { throw e; } catch (Exception e) { @@ -915,17 +677,31 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ FaceLandmarker faceLandmarker = null; FaceRecognizer faceRecognizer = null; try { - faceDetector = faceDetectorPool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); faceRecognizer = faceRecognizerPool.borrowObject(); - //检测人脸 - SeetaRect[] seetaResult = faceDetector.Detect(imageData); - if(Objects.isNull(seetaResult) || seetaResult.length == 0){ - return R.fail(R.Status.NO_FACE_DETECTED); - } //提取人脸的5点人脸标识 - SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData, seetaResult[0], pointFS); + SeetaPointF[] pointFS = null; + //默认使用Seetaface6检测模型 + if(Objects.isNull(config.getDetectModel())){ + faceDetector = faceDetectorPool.borrowObject(); + faceLandmarker = faceLandmarkerPool.borrowObject(); + //检测人脸 + SeetaRect[] seetaResult = faceDetector.Detect(imageData); + if(Objects.isNull(seetaResult) || seetaResult.length == 0){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + pointFS = new SeetaPointF[faceLandmarker.number()]; + faceLandmarker.mark(imageData, seetaResult[0], pointFS); + }else{ + R detectResponse = config.getDetectModel().detect(image); + if(!detectResponse.isSuccess()){ + return R.fail(detectResponse.getCode(), detectResponse.getMessage()); + } + if(Objects.isNull(detectResponse.getData()) || Objects.isNull(detectResponse.getData().getDetectionInfoList()) || detectResponse.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + DetectionInfo detectionInfo = detectResponse.getData().getDetectionInfoList().get(0); + pointFS = FaceUtils.convertToSeetaPointF(detectionInfo.getFaceInfo().getKeyPoints()); + } //提取特征 features = new float[faceRecognizer.GetExtractFeatureSize()]; //CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐 @@ -1058,7 +834,7 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{ @Override public void upsertFace(FaceRegisterInfo faceRegisterInfo, byte[] imageData) { - FaceModel.super.upsertFace(faceRegisterInfo, imageData); + FaceRecModel.super.upsertFace(faceRegisterInfo, imageData); } @Override diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/UltraLightFastGenericFaceModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/UltraLightFastGenericFaceModel.java deleted file mode 100644 index 75388e2..0000000 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/UltraLightFastGenericFaceModel.java +++ /dev/null @@ -1,234 +0,0 @@ -package cn.smartjavaai.face.model.facerec; - -import ai.djl.Device; -import ai.djl.MalformedModelException; -import ai.djl.inference.Predictor; -import ai.djl.modality.cv.Image; -import ai.djl.modality.cv.ImageFactory; -import ai.djl.modality.cv.output.DetectedObjects; -import ai.djl.repository.zoo.Criteria; -import ai.djl.repository.zoo.ModelNotFoundException; -import ai.djl.repository.zoo.ZooModel; -import ai.djl.training.util.ProgressBar; -import cn.smartjavaai.common.entity.DetectionResponse; -import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.common.pool.PredictorFactory; -import cn.smartjavaai.common.utils.FileUtils; -import cn.smartjavaai.common.utils.ImageUtils; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.face.constant.FaceDetectConstant; -import cn.smartjavaai.face.exception.FaceException; -import cn.smartjavaai.face.translator.FaceDetectionTranslator; -import cn.smartjavaai.face.utils.FaceUtils; -import cn.smartjavaai.face.utils.OpenCVUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.pool2.ObjectPool; -import org.apache.commons.pool2.impl.GenericObjectPool; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.*; -import java.nio.file.Paths; -import java.util.Objects; - -/** - * @author dwj - */ -@Slf4j -public class UltraLightFastGenericFaceModel implements FaceModel, AutoCloseable{ - - - private ObjectPool> predictorPool; - - /** - * 特征图层的基础缩放比例 - */ - private static final int[][] scales = {{10, 16, 24}, {32, 48}, {64, 96}, {128, 192, 256}}; - /** - * 特征图相对于原图的采样步长 - */ - private static final int[] steps = {8, 16, 32, 64}; - /** - * 缩放系数 - */ - private static final double[] variance = {0.1f, 0.2f}; - - - private ZooModel model; - - - - - /** - * 加载模型 - * @param config - */ - @Override - public void loadModel(FaceModelConfig config) { - Device device = null; - if(!Objects.isNull(config.getDevice())){ - device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); - } - FaceDetectionTranslator translator = - new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), variance, FaceDetectConstant.MAX_FACE_LIMIT, scales, steps); - Criteria criteria = - Criteria.builder() - .setTypes(Image.class, DetectedObjects.class) - .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : "https://resources.djl.ai/test-models/pytorch/ultranet.zip") - .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) - .optTranslator(translator) - .optProgress(new ProgressBar()) - .optDevice(device) - .optEngine("PyTorch") // Use PyTorch engine - .build(); - try { - model = criteria.loadModel(); - // 创建池子：每个线程独享 Predictor - this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); - log.info("当前设备: " + model.getNDManager().getDevice()); - } catch (IOException | ModelNotFoundException | MalformedModelException e) { - throw new FaceException("模型加载失败", e); - } - } - - /** - * 检测人脸 - * @param imagePath 图片路径 - * @return - * @throws Exception - */ - @Override - public DetectionResponse detect(String imagePath){ - if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); - } - Image img = null; - try { - img = ImageFactory.getInstance().fromFile(Paths.get(imagePath)); - } catch (IOException e) { - throw new FaceException("无效的图片", e); - } - DetectedObjects detection = detect(img); - return FaceUtils.convertToDetectionResponse(detection,img); - } - - /** - * 检测人脸 - * @param imageInputStream 图片流 - * @return - * @throws Exception - */ - @Override - public DetectionResponse detect(InputStream imageInputStream){ - try { - Image img = ImageFactory.getInstance().fromInputStream(imageInputStream); - DetectedObjects detection = detect(img); - return FaceUtils.convertToDetectionResponse(detection,img); - } catch (IOException e) { - throw new FaceException("无效图片输入流", e); - } - - } - - @Override - public DetectionResponse detect(BufferedImage image) { - Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image)); - DetectedObjects detection = detect(img); - return FaceUtils.convertToDetectionResponse(detection,img); - } - - @Override - public DetectionResponse detect(byte[] imageData) { - if(Objects.isNull(imageData)){ - throw new FaceException("图像无效"); - } - try { - return detect(ImageIO.read(new ByteArrayInputStream(imageData))); - } catch (IOException e) { - throw new FaceException("错误的图像", e); - } - } - - @Override - public void detectAndDraw(String imagePath, String outputPath) { - if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); - } - try { - Image img = ImageFactory.getInstance().fromFile(Paths.get(imagePath)); - DetectedObjects detectedObjects = detect(img); - if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ - throw new FaceException("未识别到人脸"); - } - img.drawBoundingBoxes(detectedObjects); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 调用 save 方法将 Image 写入字节流 - img.save(new FileOutputStream(Paths.get(outputPath).toAbsolutePath().toString()), "png"); - } catch (IOException e) { - throw new FaceException(e); - } - } - - @Override - public BufferedImage detectAndDraw(BufferedImage sourceImage) { - if(!ImageUtils.isImageValid(sourceImage)){ - throw new FaceException("图像无效"); - } - Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(sourceImage)); - DetectedObjects detectedObjects = detect(img); - if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){ - throw new FaceException("未识别到人脸"); - } - img.drawBoundingBoxes(detectedObjects); - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 调用 save 方法将 Image 写入字节流 - img.save(outputStream, "png"); - // 将字节流转换为 BufferedImage - byte[] imageBytes = outputStream.toByteArray(); - return ImageIO.read(new ByteArrayInputStream(imageBytes)); - } catch (IOException e) { - throw new FaceException("导出图片失败", e); - } - } - - /** - * 人脸检测 - * @param image - * @return - */ - private DetectedObjects detect(Image image){ - Predictor predictor = null; - try { - predictor = predictorPool.borrowObject(); - return predictor.predict(image); - } catch (Exception e) { - throw new FaceException("目标检测错误", e); - }finally { - if (predictor != null) { - try { - predictorPool.returnObject(predictor); //归还 - } catch (Exception e) { - log.warn("归还Predictor失败", e); - try { - predictor.close(); // 归还失败才销毁 - } catch (Exception ex) { - log.error("关闭Predictor失败", ex); - } - } - } - } - } - - - - - - @Override - public void close() { - if (predictorPool != null) { - predictorPool.close(); - } - } -} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/criterial/FaceRecCriteriaFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/criterial/FaceRecCriteriaFactory.java new file mode 100644 index 0000000..be2e88a --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/criterial/FaceRecCriteriaFactory.java @@ -0,0 +1,111 @@ +package cn.smartjavaai.face.model.facerec.criterial; + +import ai.djl.Device; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.transform.Normalize; +import ai.djl.modality.cv.transform.Resize; +import ai.djl.modality.cv.transform.ToTensor; +import ai.djl.modality.cv.translator.ImageFeatureExtractor; +import ai.djl.modality.cv.translator.ImageFeatureExtractorFactory; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Translator; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.config.FaceDetConfig; +import cn.smartjavaai.face.config.FaceRecConfig; +import cn.smartjavaai.face.constant.FaceDetectConstant; +import cn.smartjavaai.face.constant.FaceNetConstant; +import cn.smartjavaai.face.constant.RetinaFaceConstant; +import cn.smartjavaai.face.constant.UltraLightFastGenericFaceConstant; +import cn.smartjavaai.face.enums.FaceDetModelEnum; +import cn.smartjavaai.face.enums.FaceRecModelEnum; +import cn.smartjavaai.face.model.facerec.translator.FaceFeatureTranslator; +import cn.smartjavaai.face.model.facerec.translator.FaceNetRecTranslator; +import cn.smartjavaai.face.translator.FaceDetectionTranslator; +import org.apache.commons.lang3.StringUtils; + +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * 人脸识别 Criteria构建工厂 + * @author dwj + */ +public class FaceRecCriteriaFactory { + + public static Criteria createCriteria(FaceRecConfig config) { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); + } + Criteria criteria = null; + if(config.getModelEnum() == FaceRecModelEnum.FACENET_MODEL){ + criteria = + Criteria.builder() + .setTypes(Image.class, float[].class) + .optModelName("face_feature") // specify model file prefix + .optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : + FaceNetConstant.MODEL_URL) + .optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null) + .optTranslator(new FaceNetRecTranslator()) + .optDevice(device) + .optEngine("PyTorch") // Use PyTorch engine + .optProgress(new ProgressBar()) + .build(); + }else if (config.getModelEnum() == FaceRecModelEnum.INSIGHT_FACE_MOBILE_FACENET_MODEL){ + if(StringUtils.isBlank(config.getModelPath())){ + throw new RuntimeException("请指定模型路径"); + } + List mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f); + String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); + criteria = Criteria.builder() + .setTypes(Image.class, float[].class) + .optModelPath(Paths.get(config.getModelPath())) +// .optTranslatorFactory(new ImageFeatureExtractorFactory()) +// .optArgument("normalize", normalize) +// .optArgument("resize", "112,112") + .optTranslator(new FaceFeatureTranslator()) + .optEngine("PyTorch") // Use PyTorch engine + .optProgress(new ProgressBar()) + .build(); + }else if (config.getModelEnum() == FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL){ + if(StringUtils.isBlank(config.getModelPath())){ + throw new RuntimeException("请指定模型路径"); + } + List mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f); + String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); + criteria = Criteria.builder() + .setTypes(Image.class, float[].class) + .optModelPath(Paths.get(config.getModelPath())) +// .optTranslatorFactory(new ImageFeatureExtractorFactory()) +// .optArgument("normalize", normalize) +// .optArgument("resize", "112,112") + .optTranslator(new FaceFeatureTranslator()) + .optEngine("PyTorch") // Use PyTorch engine + .optProgress(new ProgressBar()) + .build(); + }else if (config.getModelEnum() == FaceRecModelEnum.ELASTIC_FACE_MODEL){ + if(StringUtils.isBlank(config.getModelPath())){ + throw new RuntimeException("请指定模型路径"); + } + List mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f); + String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(",")); + criteria = Criteria.builder() + .setTypes(Image.class, float[].class) + .optModelPath(Paths.get(config.getModelPath())) +// .optTranslatorFactory(new ImageFeatureExtractorFactory()) +// .optArgument("normalize", normalize) +// .optArgument("resize", "112,112") + .optTranslator(new FaceFeatureTranslator()) + .optEngine("PyTorch") // Use PyTorch engine + .optProgress(new ProgressBar()) + .build(); + } + return criteria; + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/translator/FaceFeatureTranslator.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/translator/FaceFeatureTranslator.java similarity index 82% rename from smartjavaai-face/src/main/java/cn/smartjavaai/face/translator/FaceFeatureTranslator.java rename to smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/translator/FaceFeatureTranslator.java index 53a130a..52c1e9e 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/translator/FaceFeatureTranslator.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/translator/FaceFeatureTranslator.java @@ -1,4 +1,4 @@ -package cn.smartjavaai.face.translator; +package cn.smartjavaai.face.model.facerec.translator; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.transform.Normalize; @@ -18,6 +18,8 @@ import ai.djl.translate.TranslatorContext; */ public final class FaceFeatureTranslator implements Translator { + + public FaceFeatureTranslator() { } @@ -28,12 +30,14 @@ public final class FaceFeatureTranslator implements Translator { public NDList processInput(TranslatorContext ctx, Image input) { NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR); Pipeline pipeline = new Pipeline(); + if(input.getWidth() != 112 || input.getHeight() != 112){ + pipeline.add(new Resize(112)); + } pipeline - .add(new Resize(180)) .add(new ToTensor()) .add(new Normalize( - new float[]{127.5f / 255.0f, 127.5f / 255.0f, 127.5f / 255.0f}, - new float[]{128.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f})); + new float[]{0.5F, 0.5F, 0.5F}, + new float[]{0.5F, 0.5F, 0.5F})); return pipeline.transform(new NDList(array)); } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/translator/FaceNetRecTranslator.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/translator/FaceNetRecTranslator.java new file mode 100644 index 0000000..5c21867 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/facerec/translator/FaceNetRecTranslator.java @@ -0,0 +1,57 @@ +package cn.smartjavaai.face.model.facerec.translator; + +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.transform.Normalize; +import ai.djl.modality.cv.transform.Resize; +import ai.djl.modality.cv.transform.ToTensor; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Pipeline; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; + +/** + * facenet人脸特征提取Translator + * @author dwj + * @date 2025/3/31 + */ +public final class FaceNetRecTranslator implements Translator { + + + + public FaceNetRecTranslator() { + } + + /** + * {@inheritDoc} + */ + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR); + Pipeline pipeline = new Pipeline(); + pipeline + //.add(new Resize(112)) + .add(new ToTensor()) + .add(new Normalize( + new float[]{0.5F, 0.5F, 0.5F}, + new float[]{0.5F, 0.5F, 0.5F})); + + return pipeline.transform(new NDList(array)); + } + + /** + * {@inheritDoc} + */ + @Override + public float[] processOutput(TranslatorContext ctx, NDList list) { + NDArray embedding = list.singletonOrThrow(); + embedding = embedding.div(embedding.norm()); // L2归一化 + return embedding.toFloatArray(); + } + + @Override + public Batchifier getBatchifier() { + return Batchifier.STACK; + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java new file mode 100644 index 0000000..d72ed8b --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java @@ -0,0 +1,421 @@ +package cn.smartjavaai.face.model.liveness; + +import ai.djl.Device; +import ai.djl.MalformedModelException; +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import ai.djl.util.JsonUtils; +import cn.smartjavaai.common.entity.DetectionInfo; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.DetectionResponse; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.entity.face.LivenessResult; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.enums.face.LivenessStatus; +import cn.smartjavaai.common.pool.PredictorFactory; +import cn.smartjavaai.common.preprocess.BufferedImagePreprocessor; +import cn.smartjavaai.common.utils.*; +import cn.smartjavaai.face.config.LivenessConfig; +import cn.smartjavaai.face.constant.MiniVisionConstant; +import cn.smartjavaai.face.enums.LivenessModelEnum; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.model.liveness.criterial.LivenessCriteriaFactory; +import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator; +import com.seeta.sdk.FaceAntiSpoofing; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.ObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.bytedeco.javacv.FFmpegFrameGrabber; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.Java2DFrameUtils; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.file.Paths; +import java.util.*; + +/** + * 通用活体检测模型 + * @author dwj + */ +@Slf4j +public class CommonLivenessModel implements LivenessDetModel{ + + protected ObjectPool> predictorPool; + + protected LivenessConfig config; + + protected ZooModel model; + @Override + public void loadModel(LivenessConfig config) { + if(Objects.isNull(config)){ + throw new FaceException("config为null"); + } + if(StringUtils.isBlank(config.getModelPath())){ + throw new FaceException("modelPath不能为空"); + } + this.config = config; + //设置真人阈值 + Float realityThreshold = Objects.isNull(config.getRealityThreshold()) ? MiniVisionConstant.REALITY_THRESHOLD : config.getRealityThreshold(); + this.config.setRealityThreshold(realityThreshold); + Criteria criteria = LivenessCriteriaFactory.createCriteria(config); + try { + model = criteria.loadModel(); + this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); + } catch (IOException | ModelNotFoundException | MalformedModelException e) { + throw new FaceException("阿里通义实验室活体检测模型加载失败", e); + } + } + + + @Override + public R detect(BufferedImage image, DetectionRectangle faceDetectionRectangle) { + if(!ImageUtils.isImageValid(image)){ + throw new FaceException("图像无效"); + } + Predictor predictor = null; + try { + predictor = predictorPool.borrowObject(); + //预处理图片 + BufferedImage processedImage = image; + if(config.getModelEnum() == LivenessModelEnum.IIC_FL_MODEL){ + processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle) + .setExtendRatio(96f / 112f) + .enableSquarePadding(true) + .enableScaling(true) + .setTargetSize(128) + .enableCenterCrop(true) + .setCenterCropSize(112) + .process(); + } + Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage)); + Float result = predictor.predict(djlImage); + if(result >= config.getRealityThreshold()){ + return R.ok(new LivenessResult(LivenessStatus.LIVE, result)); + }else{ + float nonLiveScore = BigDecimal.ONE.subtract(new BigDecimal(result)).floatValue(); + return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore)); + } + } catch (Exception e) { + throw new FaceException("活体检测错误", e); + }finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + } + } + + @Override + public R detect(String imagePath, DetectionRectangle faceDetectionRectangle) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detect(image, faceDetectionRectangle); + } + + @Override + public R detect(byte[] imageData, DetectionRectangle faceDetectionRectangle) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detect(imageData, faceDetectionRectangle); + } + + @Override + public R> detect(String imagePath, DetectionResponse faceDetectionResponse) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detect(image, faceDetectionResponse); + } + + @Override + public R> detect(byte[] imageData, DetectionResponse faceDetectionResponse) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R> detect(BufferedImage image, DetectionResponse faceDetectionResponse) { + if(!ImageUtils.isImageValid(image)){ + R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){ + R.fail(R.Status.NO_FACE_DETECTED); + } + List livenessStatusList = new ArrayList(); + for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){ + R result = detect(image, detectionInfo.getDetectionRectangle()); + if(!result.isSuccess()){ + return R.fail(result.getCode(), result.getMessage()); + } + livenessStatusList.add(result.getData()); + } + return R.ok(livenessStatusList); + } + + @Override + public R> detectBase64(String base64Image, DetectionResponse faceDetectionResponse) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detect(imageData, faceDetectionResponse); + } + + + @Override + public R detectTopFace(BufferedImage image) { + if(Objects.isNull(config.getDetectModel())){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型"); + } + R faceDetectionResponse = config.getDetectModel().detect(image); + if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + return detect(image, faceDetectionResponse.getData().getDetectionInfoList().get(0).getDetectionRectangle()); + } + + @Override + public R detectTopFace(String imagePath) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detectTopFace(image); + } + + @Override + public R detectTopFace(byte[] imageData) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData))); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R detectTopFaceBase64(String base64Image) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detectTopFace(imageData); + } + + @Override + public R detect(String imagePath) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return detect(image); + } + + @Override + public R detect(BufferedImage image) { + if(Objects.isNull(config.getDetectModel())){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型"); + } + R faceDetectionResponse = config.getDetectModel().detect(image); + if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){ + R result = detect(image, detectionInfo.getDetectionRectangle()); + if(!result.isSuccess()){ + return R.fail(result.getCode(), result.getMessage()); + } + if(Objects.isNull(detectionInfo.getFaceInfo())){ + detectionInfo.setFaceInfo(new FaceInfo()); + } + detectionInfo.getFaceInfo().setLivenessStatus(result.getData()); + } + return faceDetectionResponse; + } + + @Override + public R detect(byte[] imageData) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detect(ImageIO.read(new ByteArrayInputStream(imageData))); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R detectBase64(String base64Image) { + if(StringUtils.isBlank(base64Image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + byte[] imageData = Base64ImageUtils.base64ToImage(base64Image); + return detect(imageData); + } + + @Override + public R detectVideo(InputStream videoInputStream) { + if(Objects.isNull(videoInputStream)){ + return R.fail(R.Status.INVALID_VIDEO); + } + return detectVideo(new FFmpegFrameGrabber(videoInputStream)); + } + + @Override + public R detectVideo(String videoPath) { + if(!FileUtils.isFileExists(videoPath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + return detectVideo(new FFmpegFrameGrabber(videoPath)); + } + + private R detectVideo(FFmpegFrameGrabber grabber) { + try { + //滑动窗口 + Deque scoreWindow = new ArrayDeque<>(); + grabber.start(); + // 获取视频总帧数 + int totalFrames = grabber.getLengthInFrames(); + log.debug("视频总帧数：{}，检测帧数：{}", totalFrames, config.getFrameCount()); + if(totalFrames < config.getFrameCount()){ + return R.fail(10001, "视频帧数低于检测帧数"); + } + // 逐帧处理视频 + for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) { + // 获取当前帧 + Frame frame = grabber.grabImage(); + if (frame != null) { + BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame); + R livenessStatus = detectTopFace(bufferedImage); + if(!livenessStatus.isSuccess()){ + log.debug("第" + frameIndex + "帧处理失败：" + livenessStatus.getMessage()); + continue; + }else{ + log.debug("第" + frameIndex + "帧活体检测结果：" + JsonUtils.toJson(livenessStatus)); + float liveScore = 0; + if(livenessStatus.getData().getStatus() == LivenessStatus.LIVE){ + liveScore = livenessStatus.getData().getScore(); + }else{ + liveScore = BigDecimal.ONE.subtract(BigDecimal.valueOf(livenessStatus.getData().getScore())).floatValue(); + } + scoreWindow.add(liveScore); + } + // 如果累计检测帧数 >= 配置值，开始判断 + if (scoreWindow.size() >= config.getFrameCount()) { + float avgScore = (float) scoreWindow.stream() + .mapToDouble(Float::doubleValue) + .average() + .orElse(0.0); + log.debug("滑动窗口平均得分: {}", avgScore); + if (avgScore >= config.getRealityThreshold()) { + grabber.stop(); + return R.ok(new LivenessResult(LivenessStatus.LIVE, avgScore)); + } else { + grabber.stop(); + float nonLiveScore = BigDecimal.ONE.subtract(BigDecimal.valueOf(avgScore)).floatValue(); + return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore)); + } + } + } + } + grabber.stop(); + if(scoreWindow.size() < config.getFrameCount()){ + return R.fail(1000, "有效帧数量不足，无法完成活体检测"); + } + } catch (Exception e) { + throw new FaceException(e); + } + return R.fail(R.Status.Unknown); + } + + @Override + public void close() throws Exception { + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/LivenessDetModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/LivenessDetModel.java index 755cc8c..c0c4f49 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/LivenessDetModel.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/LivenessDetModel.java @@ -3,8 +3,9 @@ package cn.smartjavaai.face.model.liveness; import cn.smartjavaai.common.entity.DetectionRectangle; import cn.smartjavaai.common.entity.DetectionResponse; import cn.smartjavaai.common.entity.Point; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.entity.face.LivenessResult; import cn.smartjavaai.face.config.LivenessConfig; -import cn.smartjavaai.common.enums.LivenessStatus; import java.awt.image.BufferedImage; import java.io.InputStream; @@ -14,7 +15,7 @@ import java.util.List; * 活体检测模型 * @author dwj */ -public interface LivenessDetModel { +public interface LivenessDetModel extends AutoCloseable{ /** * 加载模型 @@ -28,7 +29,7 @@ public interface LivenessDetModel { * @param imagePath 图片路径 * @return */ - default DetectionResponse detect(String imagePath){ + default R detect(String imagePath){ throw new UnsupportedOperationException("默认不支持该功能"); } @@ -37,7 +38,7 @@ public interface LivenessDetModel { * @param image BufferedImage * @return */ - default DetectionResponse detect(BufferedImage image){ + default R detect(BufferedImage image){ throw new UnsupportedOperationException("默认不支持该功能"); } @@ -46,29 +47,31 @@ public interface LivenessDetModel { * @param imageData 图片字节流 * @return */ - default DetectionResponse detect(byte[] imageData){ + default R detect(byte[] imageData){ throw new UnsupportedOperationException("默认不支持该功能"); } + + /** + * 活体检测(多人脸) + * @param base64Image + * @return + */ + default R detectBase64(String base64Image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** * 活体检测(多人脸) * @param imagePath 图片路径 * @param faceDetectionResponse 人脸检测结果 * @return */ - default List detect(String imagePath, DetectionResponse faceDetectionResponse){ + default R> detect(String imagePath, DetectionResponse faceDetectionResponse){ throw new UnsupportedOperationException("默认不支持该功能"); } - /** - * 活体检测(单人脸) - * @param imagePath 图片路径 - * @param faceDetectionRectangle 人脸检测结果-人脸框 - * @return - */ - default LivenessStatus detect(String imagePath, DetectionRectangle faceDetectionRectangle, List keyPoints){ - throw new UnsupportedOperationException("默认不支持该功能"); - } /** * 活体检测(多人脸) @@ -76,38 +79,116 @@ public interface LivenessDetModel { * @param faceDetectionResponse 人脸检测结果 * @return */ - default List detect(byte[] imageData,DetectionResponse faceDetectionResponse){ + default R> detect(byte[] imageData,DetectionResponse faceDetectionResponse){ throw new UnsupportedOperationException("默认不支持该功能"); } - /** - * 活体检测(单人脸) - * @param imageData - * @param faceDetectionRectangle 人脸检测结果-人脸框 - * @return - */ - default LivenessStatus detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List keyPoints){ - throw new UnsupportedOperationException("默认不支持该功能"); - } - - /** * 活体检测(多人脸) * @param image BufferedImage * @param faceDetectionResponse 人脸检测结果 * @return */ - default List detect(BufferedImage image,DetectionResponse faceDetectionResponse){ + default R> detect(BufferedImage image,DetectionResponse faceDetectionResponse){ throw new UnsupportedOperationException("默认不支持该功能"); } + /** + * 活体检测(多人脸) + * @param base64Image + * @param faceDetectionResponse 人脸检测结果 + * @return + */ + default R> detectBase64(String base64Image,DetectionResponse faceDetectionResponse){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 活体检测(单人脸) + * @param imagePath 图片路径 + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detect(String imagePath, DetectionRectangle faceDetectionRectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 活体检测(单人脸) + * @param imageData + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + + /** * 活体检测(单人脸) * @param image BufferedImage * @param faceDetectionRectangle 人脸检测结果-人脸框 * @return */ - default LivenessStatus detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List keyPoints){ + default R detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 活体检测(单人脸) + * @param base64Image + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 活体检测(单人脸) + * @param imagePath 图片路径 + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detect(String imagePath, DetectionRectangle faceDetectionRectangle){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 活体检测(单人脸) + * @param imageData + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detect(byte[] imageData, DetectionRectangle faceDetectionRectangle){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + + + /** + * 活体检测(单人脸) + * @param image BufferedImage + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detect(BufferedImage image, DetectionRectangle faceDetectionRectangle){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 活体检测(单人脸) + * @param base64Image + * @param faceDetectionRectangle 人脸检测结果-人脸框 + * @return + */ + default R detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle){ throw new UnsupportedOperationException("默认不支持该功能"); } @@ -117,7 +198,7 @@ public interface LivenessDetModel { * @param image * @return */ - default LivenessStatus detectTopFace(BufferedImage image){ + default R detectTopFace(BufferedImage image){ throw new UnsupportedOperationException("默认不支持该功能"); } @@ -127,7 +208,7 @@ public interface LivenessDetModel { * @param imagePath * @return */ - default LivenessStatus detectTopFace(String imagePath){ + default R detectTopFace(String imagePath){ throw new UnsupportedOperationException("默认不支持该功能"); } @@ -136,20 +217,31 @@ public interface LivenessDetModel { * @param imageData * @return */ - default LivenessStatus detectTopFace(byte[] imageData){ + default R detectTopFace(byte[] imageData){ throw new UnsupportedOperationException("默认不支持该功能"); } + /** + * 活体检测(分数最高人脸) + * @param base64Image + * @return + */ + default R detectTopFaceBase64(String base64Image){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** * 视频活体检测(逐帧检测) * @param frameImage * @param faceDetectionRectangle * @return */ - default LivenessStatus detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List keyPoints){ - throw new UnsupportedOperationException("默认不支持该功能"); - } +// default R detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List keyPoints){ +// throw new UnsupportedOperationException("默认不支持该功能"); +// } /** * 视频活体检测(逐帧检测) @@ -157,34 +249,34 @@ public interface LivenessDetModel { * @param faceDetectionRectangle * @return */ - default LivenessStatus detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List keyPoints){ - throw new UnsupportedOperationException("默认不支持该功能"); - } +// default R detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List keyPoints){ +// throw new UnsupportedOperationException("默认不支持该功能"); +// } /** * 视频活体检测(逐帧检测) * @param frameImageData * @return */ - default LivenessStatus detectVideoByFrame(byte[] frameImageData){ - throw new UnsupportedOperationException("默认不支持该功能"); - } +// default R detectVideoByFrame(byte[] frameImageData){ +// throw new UnsupportedOperationException("默认不支持该功能"); +// } /** * 视频活体检测(逐帧检测) * @param frameImageData * @return */ - default LivenessStatus detectVideoByFrame(BufferedImage frameImageData){ - throw new UnsupportedOperationException("默认不支持该功能"); - } +// default R detectVideoByFrame(BufferedImage frameImageData){ +// throw new UnsupportedOperationException("默认不支持该功能"); +// } /** * 视频活体检测 * @param videoInputStream * @return */ - default LivenessStatus detectVideo(InputStream videoInputStream){ + default R detectVideo(InputStream videoInputStream){ throw new UnsupportedOperationException("默认不支持该功能"); } @@ -193,7 +285,7 @@ public interface LivenessDetModel { * @param videoPath * @return */ - default LivenessStatus detectVideo(String videoPath){ + default R detectVideo(String videoPath){ throw new UnsupportedOperationException("默认不支持该功能"); } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java new file mode 100644 index 0000000..fadb52b --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java @@ -0,0 +1,265 @@ +package cn.smartjavaai.face.model.liveness; + +import ai.djl.Device; +import ai.djl.MalformedModelException; +import ai.djl.inference.Predictor; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.ImageFactory; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.entity.*; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.entity.face.LivenessResult; +import cn.smartjavaai.common.enums.face.LivenessStatus; +import cn.smartjavaai.common.pool.PredictorFactory; +import cn.smartjavaai.common.preprocess.BufferedImagePreprocessor; +import cn.smartjavaai.common.utils.ArrayUtils; +import cn.smartjavaai.common.utils.Base64ImageUtils; +import cn.smartjavaai.common.utils.FileUtils; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.face.config.LivenessConfig; +import cn.smartjavaai.face.constant.MiniVisionConstant; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator; +import cn.smartjavaai.common.utils.OpenCVUtils; +import com.seeta.sdk.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.pool2.ObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.bytedeco.javacv.FFmpegFrameGrabber; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.Java2DFrameUtils; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.file.Paths; +import java.util.*; + +/** + * 小视科技 活体检测模型 + * @author dwj + * @date 2025/6/27 + */ +@Slf4j +public class MiniVisionLivenessModel extends CommonLivenessModel{ + + /** + * 个性化参数：seModelPath + */ + private static final String SE_MODEL_PATH_KEY = "seModelPath"; + + private ObjectPool> predictorPool; + + private ObjectPool> sePredictorPool; + + + /** + * 模型策略 + */ + private ModelStrategy modelStrategy; + + private ZooModel model; + + private ZooModel seModel; + + @Override + public void loadModel(LivenessConfig config) { + if(Objects.isNull(config)){ + throw new FaceException("config为null"); + } + String seModelPath = config.getCustomParam(SE_MODEL_PATH_KEY, String.class); + if(StringUtils.isBlank(config.getModelPath()) && StringUtils.isBlank(seModelPath)){ + throw new FaceException("modelPath 和 seModelPath 至少有一个不能为空"); + } + this.config = config; + //设置真人阈值 + Float realityThreshold = Objects.isNull(config.getRealityThreshold()) ? MiniVisionConstant.REALITY_THRESHOLD : config.getRealityThreshold(); + this.config.setRealityThreshold(realityThreshold); + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); + } + if(StringUtils.isNotBlank(config.getModelPath()) && StringUtils.isBlank(seModelPath)){ + //2.7_80x80_MiniFASNetV2 + modelStrategy = ModelStrategy.MINIFASNET_V2; + }else if (StringUtils.isBlank(config.getModelPath()) && StringUtils.isNotBlank(seModelPath)){ + //4_0_0_80x80_MiniFASNetV1SE + modelStrategy = ModelStrategy.MINIFASNET_V1_SE; + }else{ + //融合 + modelStrategy = ModelStrategy.FUSION; + } + + if(modelStrategy == ModelStrategy.MINIFASNET_V2 || modelStrategy == ModelStrategy.FUSION){ + //初始化 检测Criteria + Criteria criteria = + Criteria.builder() + .optEngine("OnnxRuntime") + .setTypes(Image.class, float[].class) + .optModelPath(Paths.get(config.getModelPath())) + .optTranslator(new MiniVisionTranslator()) + .optProgress(new ProgressBar()) + .build(); + try { + model = criteria.loadModel(); + this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); + } catch (IOException | ModelNotFoundException | MalformedModelException e) { + throw new FaceException("MiniFASNetV2模型加载失败", e); + } + + } + + if(modelStrategy == ModelStrategy.MINIFASNET_V1_SE || modelStrategy == ModelStrategy.FUSION){ + //初始化 检测Criteria + Criteria seCriteria = + Criteria.builder() + .optEngine("OnnxRuntime") + .setTypes(Image.class, float[].class) + .optModelPath(Paths.get(seModelPath)) + .optTranslator(new MiniVisionTranslator()) + .optProgress(new ProgressBar()) + .build(); + try { + seModel = seCriteria.loadModel(); + this.sePredictorPool = new GenericObjectPool<>(new PredictorFactory<>(seModel)); + } catch (IOException | ModelNotFoundException | MalformedModelException e) { + throw new FaceException("MiniFASNetV1SE模型加载失败", e); + } + } + } + + + @Override + public R detect(BufferedImage image, DetectionRectangle faceDetectionRectangle) { + if(!ImageUtils.isImageValid(image)){ + throw new FaceException("图像无效"); + } + Predictor predictor = null; + Predictor sePredictor = null; + try { + float[] result = null; + float[] seResult = null; + if(Objects.nonNull(predictorPool)){ + //预处理图片 + BufferedImage processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle) + .setExtendRatio(2.7f) + .enableSquarePadding(true) + .enableScaling(true) + .setTargetSize(80) + .process(); + predictor = predictorPool.borrowObject(); + Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage)); + result = predictor.predict(djlImage); + } + if(Objects.nonNull(sePredictorPool)){ + //预处理图片 + BufferedImage processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle) + .setExtendRatio(4) + .enableSquarePadding(true) + .enableScaling(true) + .setTargetSize(80) + .process(); + Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage)); + sePredictor = sePredictorPool.borrowObject(); + seResult = sePredictor.predict(djlImage); + } + if(Objects.isNull(result) && Objects.isNull(seResult)){ + throw new FaceException("活体检测错误"); + } + //计算结果 + int maxIndex = ArrayUtils.sumAndFindMaxIndex(result, seResult, 3); + BigDecimal score = Objects.isNull(result) ? BigDecimal.ZERO : BigDecimal.valueOf(result[maxIndex]); + BigDecimal seScore = Objects.isNull(seResult) ? BigDecimal.ZERO : BigDecimal.valueOf(seResult[maxIndex]); + BigDecimal avgSocre = score.add(seScore).divide(BigDecimal.valueOf(2), 2, RoundingMode.HALF_UP); + if(maxIndex == 1){ + if(avgSocre.floatValue() >= config.getRealityThreshold()){ + return R.ok(new LivenessResult(LivenessStatus.LIVE, avgSocre.floatValue())); + }else{ + float nonLiveScore = BigDecimal.ONE.subtract(avgSocre).floatValue(); + return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore)); + } + }else{ + return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, avgSocre.floatValue())); + } + } catch (Exception e) { + throw new FaceException("活体检测错误", e); + }finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + if (sePredictor != null) { + try { + sePredictorPool.returnObject(sePredictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + sePredictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } + } + } + + @Override + public void close() throws Exception { + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + + try { + if (sePredictorPool != null) { + sePredictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 sePredictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + try { + if (seModel != null) { + seModel.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + } + + /** + * 模型策略 + */ + protected enum ModelStrategy { + MINIFASNET_V2, + MINIFASNET_V1_SE, + FUSION // 融合模型 + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java index 41be082..130426d 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java @@ -1,11 +1,14 @@ package cn.smartjavaai.face.model.liveness; import cn.smartjavaai.common.entity.*; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.entity.face.LivenessResult; import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.common.utils.FileUtils; import cn.smartjavaai.common.utils.ImageUtils; import cn.smartjavaai.face.config.LivenessConfig; -import cn.smartjavaai.common.enums.LivenessStatus; +import cn.smartjavaai.common.enums.face.LivenessStatus; +import cn.smartjavaai.face.constant.LivenessConstant; import cn.smartjavaai.face.exception.FaceException; import cn.smartjavaai.face.seetaface.NativeLoader; import cn.smartjavaai.face.utils.FaceUtils; @@ -38,12 +41,14 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ private FaceAntiSpoofingPool faceAntiSpoofingPool; private FaceLandmarkerPool faceLandmarkerPool; + private LivenessConfig config; @Override public void loadModel(LivenessConfig config) { if(StringUtils.isBlank(config.getModelPath())){ throw new FaceException("modelPath is null"); } + this.config = config; //加载依赖库 NativeLoader.loadNativeLibraries(config.getDevice()); log.debug("Loading seetaFace6 library successfully."); @@ -54,8 +59,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ int gpuId = 0; if(Objects.nonNull(config.getDevice())){ device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU; - if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){ - gpuId = config.getGpuId(); + Integer gpuIdValue = config.getCustomParam("gpuId", Integer.class); + if(Objects.nonNull(gpuIdValue) && device == SeetaDevice.SEETA_DEVICE_GPU){ + gpuId = gpuIdValue; } } @@ -72,36 +78,93 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ this.faceDetectorPool = new FaceDetectorPool(faceDetectorPoolConfSetting); this.faceAntiSpoofingPool = new FaceAntiSpoofingPool(faceAntiSpoofingPoolConfSetting); this.faceLandmarkerPool = new FaceLandmarkerPool(faceLandmarkerPoolConfSetting); - FaceAntiSpoofing faceAntiSpoofing = null; - //设置参数 - try { - faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); - if(config.getFaceClarityThreshold() > 0 && config.getRealityThreshold() > 0){ - faceAntiSpoofing.SetThreshold(config.getFaceClarityThreshold(), config.getRealityThreshold()); - } - if(config.getFrameCount() > 0){ - faceAntiSpoofing.SetVideoFrameCount(config.getFrameCount()); - } - } catch (Exception e) { - throw new FaceException(e); - } finally { - if (faceAntiSpoofing != null) { - try { - faceAntiSpoofingPool.returnObject(faceAntiSpoofing); - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - } + + //初始化模型参数 + initConfig(); } catch (FileNotFoundException e) { throw new FaceException(e); } } + /** + * 初始化模型参数 + */ + private void initConfig(){ + FaceAntiSpoofing faceAntiSpoofing = null; + //设置参数 + try { + //人脸清晰度阈值 + float faceClarityThreshold = LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD; + //活体阈值 + float realityThreshold = LivenessConstant.DEFAULT_REALITY_THRESHOLD; + Float faceClarityThresholdValue = config.getCustomParam("faceClarityThreshold", Float.class); + if(Objects.nonNull(faceClarityThresholdValue)){ + faceClarityThreshold = faceClarityThresholdValue; + } + Float realityThresholdValue = config.getRealityThreshold(); + if(Objects.nonNull(realityThresholdValue)){ + realityThreshold = realityThresholdValue; + } + faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); + faceAntiSpoofing.SetThreshold(faceClarityThreshold, realityThreshold); + faceAntiSpoofing.SetVideoFrameCount(config.getFrameCount()); + } catch (Exception e) { + throw new FaceException(e); + } finally { + if (faceAntiSpoofing != null) { + try { + faceAntiSpoofingPool.returnObject(faceAntiSpoofing); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + + private R detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List keyPoints, boolean isImage) { + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(faceDetectionRectangle)){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + if(keyPoints == null || keyPoints.isEmpty()){ + return R.fail(1002,"人脸关键点keyPoints为空"); + } + FaceAntiSpoofing.Status status = null; + FaceAntiSpoofing faceAntiSpoofing = null; + try { + faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaRect seetaRect = FaceUtils.convertToSeetaRect(faceDetectionRectangle); + SeetaPointF[] landmarks = FaceUtils.convertToSeetaPointF(keyPoints); + //检测图片 + if(isImage){ + status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks); + }else{ + //检测视频 + status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks); + } + return R.ok(new LivenessResult(FaceUtils.convertToLivenessStatus(status))); + } catch (Exception e) { + throw new FaceException("活体检测错误", e); + } finally { + if (faceAntiSpoofing != null) { + try { + faceAntiSpoofingPool.returnObject(faceAntiSpoofing); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + @Override - public DetectionResponse detect(String imagePath) { + public R detect(String imagePath) { if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); + return R.fail(R.Status.FILE_NOT_FOUND); } // 将图片路径转换为 BufferedImage BufferedImage image = null; @@ -114,9 +177,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } @Override - public DetectionResponse detect(byte[] imageData) { + public R detect(byte[] imageData) { if(Objects.isNull(imageData)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } try { return detect(ImageIO.read(new ByteArrayInputStream(imageData))); @@ -126,36 +189,54 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } @Override - public DetectionResponse detect(BufferedImage image) { + public R detect(BufferedImage image) { if(!ImageUtils.isImageValid(image)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } - FaceAntiSpoofing.Status status = null; FaceAntiSpoofing faceAntiSpoofing = null; FaceLandmarker faceLandmarker = null; FaceDetector detectPredictor = null; - List seetaPointFSList = new ArrayList(); - List livenessStatusList = new ArrayList(); try { - detectPredictor = faceDetectorPool.borrowObject(); - faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); - SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); - imageData.data = ImageUtils.getMatrixBGR(image); - //检测人脸 - SeetaRect[] seetaResult = detectPredictor.Detect(imageData); - if(Objects.isNull(seetaResult)){ - throw new FaceException("无人脸数据"); + //默认使用Seetaface6检测模型 + if(Objects.isNull(config.getDetectModel())){ + faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); + detectPredictor = faceDetectorPool.borrowObject(); + faceLandmarker = faceLandmarkerPool.borrowObject(); + List seetaPointFSList = new ArrayList(); + List livenessStatusList = new ArrayList(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + //检测人脸 + SeetaRect[] seetaResult = detectPredictor.Detect(imageData); + if(Objects.isNull(seetaResult)){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + for(SeetaRect seetaRect : seetaResult){ + SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()]; + faceLandmarker.mark(imageData, seetaRect, landmarks); + seetaPointFSList.add(landmarks); + //检测图片 + FaceAntiSpoofing.Status status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks); + livenessStatusList.add(FaceUtils.convertToLivenessStatus(status)); + } + return R.ok(FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList, livenessStatusList)); + }else{ + R faceDetectionResponse = config.getDetectModel().detect(image); + if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){ + R result = detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints()); + if(!result.isSuccess()){ + return R.fail(result.getCode(), result.getMessage()); + } + if(Objects.isNull(detectionInfo.getFaceInfo())){ + detectionInfo.setFaceInfo(new FaceInfo()); + } + detectionInfo.getFaceInfo().setLivenessStatus(result.getData()); + } + return faceDetectionResponse; } - for(SeetaRect seetaRect : seetaResult){ - SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData, seetaRect, landmarks); - seetaPointFSList.add(landmarks); - //检测图片 - status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks); - livenessStatusList.add(FaceUtils.convertToLivenessStatus(status)); - } - return FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList, livenessStatusList); } catch (Exception e) { throw new FaceException("活体检测错误", e); } finally { @@ -184,9 +265,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } @Override - public List detect(String imagePath, DetectionResponse faceDetectionResponse) { + public R> detect(String imagePath, DetectionResponse faceDetectionResponse) { if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); + return R.fail(R.Status.FILE_NOT_FOUND); } // 将图片路径转换为 BufferedImage BufferedImage image = null; @@ -198,10 +279,48 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ return detect(image, faceDetectionResponse); } + @Override - public LivenessStatus detect(String imagePath, DetectionRectangle faceDetectionRectangle, List keyPoints) { + public R> detect(BufferedImage image, DetectionResponse faceDetectionResponse) { + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + List livenessStatusList = new ArrayList(); + for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){ + FaceInfo faceInfo = detectionInfo.getFaceInfo(); + if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){ + return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点"); + } + R result = detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints()); + if(!result.isSuccess()){ + return R.fail(result.getCode(), result.getMessage()); + } + livenessStatusList.add(result.getData()); + } + return R.ok(livenessStatusList); + } + + + + @Override + public R> detect(byte[] imageData, DetectionResponse faceDetectionResponse) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R detect(String imagePath, DetectionRectangle faceDetectionRectangle, List keyPoints) { if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); + return R.fail(R.Status.FILE_NOT_FOUND); } // 将图片路径转换为 BufferedImage BufferedImage image = null; @@ -213,144 +332,15 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ return detect(image, faceDetectionRectangle, keyPoints); } - - - - private List detect(BufferedImage image, DetectionResponse faceDetectionResponse,boolean isImage) { - if(!ImageUtils.isImageValid(image)){ - throw new FaceException("图像无效"); - } - if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){ - throw new FaceException("无人脸数据"); - } - FaceAntiSpoofing.Status status = null; - FaceAntiSpoofing faceAntiSpoofing = null; - FaceLandmarker faceLandmarker = null; - List livenessStatusList = new ArrayList(); - try { - for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){ - faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); - SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); - imageData.data = ImageUtils.getMatrixBGR(image); - SeetaRect seetaRect = FaceUtils.convertToSeetaRect(detectionInfo.getDetectionRectangle()); - SeetaPointF[] landmarks = null; - FaceInfo faceInfo = detectionInfo.getFaceInfo(); - //如果没有人脸标识，则提取人脸标识 - if(faceInfo == null || faceInfo.getKeyPoints() == null || faceInfo.getKeyPoints().isEmpty()){ - //提取人脸的5点人脸标识 - landmarks = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData, seetaRect, landmarks); - }else{ - landmarks = FaceUtils.convertToSeetaPointF(faceInfo.getKeyPoints()); - } - //检测图片 - if(isImage){ - status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks); - }else{ - //检测视频 - status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks); - } - livenessStatusList.add(FaceUtils.convertToLivenessStatus(status)); - } - } catch (Exception e) { - throw new FaceException("活体检测错误", e); - } finally { - if (faceAntiSpoofing != null) { - try { - faceAntiSpoofingPool.returnObject(faceAntiSpoofing); - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - if (faceLandmarker != null) { - try { - faceLandmarkerPool.returnObject(faceLandmarker); - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - } - return livenessStatusList; - } - @Override - public List detect(BufferedImage image, DetectionResponse faceDetectionResponse) { - return detect(image, faceDetectionResponse, true); - } - - private LivenessStatus detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List keyPoints, boolean isImage) { - if(!ImageUtils.isImageValid(image)){ - throw new FaceException("图像无效"); - } - if(Objects.isNull(faceDetectionRectangle)){ - throw new FaceException("无人脸数据"); - } - FaceAntiSpoofing.Status status = null; - FaceAntiSpoofing faceAntiSpoofing = null; - FaceLandmarker faceLandmarker = null; - try { - faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); - SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); - imageData.data = ImageUtils.getMatrixBGR(image); - SeetaRect seetaRect = FaceUtils.convertToSeetaRect(faceDetectionRectangle); - SeetaPointF[] landmarks = null; - if(keyPoints == null || keyPoints.isEmpty()){ - throw new FaceException("人脸关键点keyPoints为空"); - } - landmarks = FaceUtils.convertToSeetaPointF(keyPoints); - //检测图片 - if(isImage){ - status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks); - }else{ - //检测视频 - status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks); - } - return FaceUtils.convertToLivenessStatus(status); - } catch (Exception e) { - throw new FaceException("活体检测错误", e); - } finally { - if (faceAntiSpoofing != null) { - try { - faceAntiSpoofingPool.returnObject(faceAntiSpoofing); - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - if (faceLandmarker != null) { - try { - faceLandmarkerPool.returnObject(faceLandmarker); - } catch (Exception e) { - log.warn("归还Predictor失败", e); - } - } - } - } - - @Override - public LivenessStatus detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List keyPoints) { + public R detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List keyPoints) { return detect(image, faceDetectionRectangle, keyPoints, true); } - - @Override - public List detect(byte[] imageData, DetectionResponse faceDetectionResponse) { + public R detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List keyPoints) { if(Objects.isNull(imageData)){ - throw new FaceException("图像无效"); - } - try { - return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse); - } catch (IOException e) { - throw new FaceException("错误的图像", e); - } - } - - @Override - public LivenessStatus detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List keyPoints) { - if(Objects.isNull(imageData)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } try { return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle, keyPoints); @@ -361,9 +351,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ @Override - public LivenessStatus detectTopFace(String imagePath) { + public R detectTopFace(String imagePath) { if(!FileUtils.isFileExists(imagePath)){ - throw new FaceException("图像文件不存在"); + return R.fail(R.Status.FILE_NOT_FOUND); } // 将图片路径转换为 BufferedImage BufferedImage image = null; @@ -376,34 +366,44 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } - private LivenessStatus detectTopFace(BufferedImage image, boolean isImage) { + private R detectTopFace(BufferedImage image, boolean isImage) { if(!ImageUtils.isImageValid(image)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } - FaceAntiSpoofing.Status status = null; FaceAntiSpoofing faceAntiSpoofing = null; FaceLandmarker faceLandmarker = null; FaceDetector detectPredictor = null; try { - detectPredictor = faceDetectorPool.borrowObject(); - faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); - faceLandmarker = faceLandmarkerPool.borrowObject(); - SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); - imageData.data = ImageUtils.getMatrixBGR(image); - //检测人脸 - SeetaRect[] seetaResult = detectPredictor.Detect(imageData); - if(Objects.isNull(seetaResult)){ - throw new FaceException("无人脸数据"); - } - SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()]; - faceLandmarker.mark(imageData, seetaResult[0], landmarks); - //检测图片 - if(isImage){ - status = faceAntiSpoofing.Predict(imageData, seetaResult[0], landmarks); + //默认使用Seetaface6检测模型 + if(Objects.isNull(config.getDetectModel())){ + detectPredictor = faceDetectorPool.borrowObject(); + faceLandmarker = faceLandmarkerPool.borrowObject(); + faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + //检测人脸 + SeetaRect[] seetaResult = detectPredictor.Detect(imageData); + if(Objects.isNull(seetaResult)){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()]; + faceLandmarker.mark(imageData, seetaResult[0], landmarks); + FaceAntiSpoofing.Status status = null; + //检测图片 + if(isImage){ + status = faceAntiSpoofing.Predict(imageData, seetaResult[0], landmarks); + }else{ + status = faceAntiSpoofing.PredictVideo(imageData, seetaResult[0], landmarks); + } + return R.ok(new LivenessResult(FaceUtils.convertToLivenessStatus(status))); }else{ - status = faceAntiSpoofing.PredictVideo(imageData, seetaResult[0], landmarks); + R faceDetectionResponse = config.getDetectModel().detect(image); + if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){ + return R.fail(R.Status.NO_FACE_DETECTED); + } + DetectionInfo detectionInfo = faceDetectionResponse.getData().getDetectionInfoList().get(0); + return detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints(), isImage); } - return FaceUtils.convertToLivenessStatus(status); } catch (Exception e) { throw new FaceException("活体检测错误", e); } finally { @@ -433,14 +433,14 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ @Override - public LivenessStatus detectTopFace(BufferedImage image) { + public R detectTopFace(BufferedImage image) { return detectTopFace(image, true); } @Override - public LivenessStatus detectTopFace(byte[] imageData) { + public R detectTopFace(byte[] imageData) { if(Objects.isNull(imageData)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } try { return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData))); @@ -450,18 +450,16 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } - @Override - public LivenessStatus detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List keyPoints) { + public R detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List keyPoints) { if(!ImageUtils.isImageValid(frameImage)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } return detect(frameImage,faceDetectionRectangle, keyPoints,false); } - @Override - public LivenessStatus detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List keyPoints) { + public R detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List keyPoints) { if(Objects.isNull(frameData)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } try { return detect(ImageIO.read(new ByteArrayInputStream(frameData)), faceDetectionRectangle, keyPoints, false); @@ -470,10 +468,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } } - @Override - public LivenessStatus detectVideoByFrame(byte[] frameImageData) { + public R detectVideoByFrame(byte[] frameImageData) { if(Objects.isNull(frameImageData)){ - throw new FaceException("图像无效"); + return R.fail(R.Status.INVALID_IMAGE); } try { return detectVideoByFrame(ImageIO.read(new ByteArrayInputStream(frameImageData))); @@ -482,13 +479,12 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } } - @Override - public LivenessStatus detectVideoByFrame(BufferedImage frameImageData) { + public R detectVideoByFrame(BufferedImage frameImageData) { return detectTopFace(frameImageData, false); } @Override - public LivenessStatus detectVideo(InputStream videoInputStream) { + public R detectVideo(InputStream videoInputStream) { if(Objects.isNull(videoInputStream)){ throw new FaceException("视频无效"); } @@ -496,24 +492,26 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } @Override - public LivenessStatus detectVideo(String videoPath) { + public R detectVideo(String videoPath) { if(!FileUtils.isFileExists(videoPath)){ throw new FaceException("视频文件不存在"); } return detectVideo(new FFmpegFrameGrabber(videoPath)); } - private LivenessStatus detectVideo(FFmpegFrameGrabber grabber) { + private R detectVideo(FFmpegFrameGrabber grabber) { FaceAntiSpoofing faceAntiSpoofing = null; try { faceAntiSpoofing = faceAntiSpoofingPool.borrowObject(); + //重置视频 + faceAntiSpoofing.ResetVideo(); grabber.start(); // 获取视频总帧数 int totalFrames = grabber.getLengthInFrames(); int videoFrameCountConfig = faceAntiSpoofing.GetVideoFrameCount(); log.debug("视频总帧数：{}，检测帧数：{}", totalFrames, videoFrameCountConfig); if(totalFrames < videoFrameCountConfig){ - throw new FaceException("视频帧数低于检测帧数"); + return R.fail(1001, "视频帧数低于检测帧数"); } // 逐帧处理视频 for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) { @@ -521,9 +519,13 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ Frame frame = grabber.grabImage(); if (frame != null) { BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame); - LivenessStatus livenessStatus = detectVideoByFrame(bufferedImage); + R livenessStatus = detectVideoByFrame(bufferedImage); + if(!livenessStatus.isSuccess()){ + log.debug("第" + frameIndex + "帧处理失败：" + livenessStatus.getMessage()); + continue; + } //满足检测帧数之后停止检测 - if(livenessStatus != LivenessStatus.DETECTING){ + if(livenessStatus.getData().getStatus() != LivenessStatus.DETECTING){ return livenessStatus; } } @@ -542,6 +544,31 @@ public class Seetaface6LivenessModel implements LivenessDetModel{ } } } - return LivenessStatus.UNKNOWN; + return R.fail(1000, "有效帧数量不足，无法完成活体检测"); + } + + @Override + public void close() throws Exception { + try { + if (faceDetectorPool != null) { + faceDetectorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (faceAntiSpoofingPool != null) { + faceAntiSpoofingPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (faceLandmarkerPool != null) { + faceLandmarkerPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } } } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/criterial/LivenessCriteriaFactory.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/criterial/LivenessCriteriaFactory.java new file mode 100644 index 0000000..086797b --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/criterial/LivenessCriteriaFactory.java @@ -0,0 +1,50 @@ +package cn.smartjavaai.face.model.liveness.criterial; + +import ai.djl.Device; +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.translator.ImageFeatureExtractorFactory; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.face.config.FaceRecConfig; +import cn.smartjavaai.face.config.LivenessConfig; +import cn.smartjavaai.face.constant.FaceNetConstant; +import cn.smartjavaai.face.enums.FaceRecModelEnum; +import cn.smartjavaai.face.enums.LivenessModelEnum; +import cn.smartjavaai.face.model.facerec.translator.FaceNetRecTranslator; +import cn.smartjavaai.face.model.liveness.translator.IicFrTranslator; +import org.apache.commons.lang3.StringUtils; + +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * 活体检测 Criteria构建工厂 + * @author dwj + */ +public class LivenessCriteriaFactory { + + public static Criteria createCriteria(LivenessConfig config) { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); + } + Criteria criteria = null; + if(config.getModelEnum() == LivenessModelEnum.IIC_FL_MODEL){ + criteria = Criteria.builder() + .optEngine("OnnxRuntime") + .setTypes(ai.djl.modality.cv.Image.class, Float.class) + .optModelPath(Paths.get(config.getModelPath())) + .optTranslator(new IicFrTranslator()) + .optProgress(new ProgressBar()) + .optDevice(device) + .build(); + } + return criteria; + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/translator/IicFrTranslator.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/translator/IicFrTranslator.java new file mode 100644 index 0000000..e7e1b5f --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/translator/IicFrTranslator.java @@ -0,0 +1,42 @@ +package cn.smartjavaai.face.model.liveness.translator; + +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.DataType; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; + +import java.util.Arrays; + + +public class IicFrTranslator implements Translator { + + + @Override + public Float processOutput(TranslatorContext ctx, NDList list) { + NDArray prob = list.singletonOrThrow(); + return prob.toFloatArray()[1]; + } + + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + NDManager manager = ctx.getNDManager(); + NDArray array = input.toNDArray(manager, Image.Flag.COLOR); + array = array.transpose(2, 0, 1); + array = array.expandDims(0); + // 归一化 + array = array.toType(DataType.FLOAT32, false).div(255.0f); + return new NDList(array); + + } + + @Override + public Batchifier getBatchifier() { + return null; + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/translator/MiniVisionTranslator.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/translator/MiniVisionTranslator.java new file mode 100644 index 0000000..958aaf0 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/liveness/translator/MiniVisionTranslator.java @@ -0,0 +1,45 @@ +package cn.smartjavaai.face.model.liveness.translator; + +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.transform.Pad; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.translate.*; + +import java.util.Arrays; + +/** + * minivision translator + * @author dwj + * @date 2025/6/27 + */ +public class MiniVisionTranslator implements Translator { + + + @Override + public float[] processOutput(TranslatorContext ctx, NDList list) { + NDArray prob = list.singletonOrThrow(); + NDArray softmax = prob.softmax(-1); + return softmax.toFloatArray(); + } + + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR); + array = array.toType(ai.djl.ndarray.types.DataType.FLOAT32, false); + // 调整数据布局: HWC -> CHW + array = array.transpose(2, 0, 1); + // 添加batch维度 (NCHW) + array = array.expandDims(0); + return new NDList(array); + + } + + @Override + public Batchifier getBatchifier() { + return null; + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/FaceQualityModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/FaceQualityModel.java new file mode 100644 index 0000000..301cc38 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/FaceQualityModel.java @@ -0,0 +1,229 @@ +package cn.smartjavaai.face.model.quality; + +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.entity.Point; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.face.config.QualityConfig; +import cn.smartjavaai.face.entity.FaceQualitySummary; +import cn.smartjavaai.face.entity.FaceQualityResult; + +import java.awt.image.BufferedImage; +import java.util.List; + +/** + * 质量评估模型 + * @author dwj + * @date 2025/6/23 + */ +public interface FaceQualityModel extends AutoCloseable{ + + /** + * 加载模型 + * @param config + */ + void loadModel(QualityConfig config); + + + /** + * 亮度评估 + * @param image + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateBrightness(BufferedImage image, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 亮度评估 + * @param imagePath + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateBrightness(String imagePath, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 亮度评估 + * @param imageData + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateBrightness(byte[] imageData, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 清晰度评估 + * @param image + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateClarity(BufferedImage image, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 清晰度评估 + * @param imagePath + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateClarity(String imagePath, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 清晰度评估 + * @param imageData + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateClarity(byte[] imageData, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 完整度评估 + * @param image + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateCompleteness(BufferedImage image, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 完整度评估 + * @param imagePath + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateCompleteness(String imagePath, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 完整度评估 + * @param imageData + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateCompleteness(byte[] imageData, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸姿态评估 + * @param image + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluatePose(BufferedImage image, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸姿态评估 + * @param imagePath + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluatePose(String imagePath, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸姿态评估 + * @param imageData + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluatePose(byte[] imageData, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸分辨率评估 + * @param image + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateResolution(BufferedImage image, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸分辨率评估 + * @param imagePath + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateResolution(String imagePath, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 人脸分辨率评估 + * @param imageData + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateResolution(byte[] imageData, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 评估所有 + * @param imagePath + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateAll(String imagePath, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + /** + * 评估所有 + * @param image + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateAll(BufferedImage image, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + + /** + * 评估所有 + * @param imageData + * @param rectangle + * @param keyPoints + * @return + */ + default R evaluateAll(byte[] imageData, DetectionRectangle rectangle, List keyPoints){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/Seetaface6QualityModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/Seetaface6QualityModel.java new file mode 100644 index 0000000..f8d9dfe --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/Seetaface6QualityModel.java @@ -0,0 +1,717 @@ +package cn.smartjavaai.face.model.quality; + +import cn.smartjavaai.common.entity.*; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.common.utils.FileUtils; +import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.common.utils.PoolUtils; +import cn.smartjavaai.face.config.QualityConfig; +import cn.smartjavaai.face.entity.FaceQualityResult; +import cn.smartjavaai.face.entity.FaceQualitySummary; +import cn.smartjavaai.face.enums.QualityGrade; +import cn.smartjavaai.face.exception.FaceException; +import cn.smartjavaai.face.seetaface.ClarityDLResult; +import cn.smartjavaai.face.seetaface.NativeLoader; +import cn.smartjavaai.face.utils.FaceUtils; +import com.seeta.pool.*; +import com.seeta.sdk.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; +import java.util.Objects; + +/** + * seetaface6 质量评估模型 + * @author dwj + * @date 2025/4/30 + */ +@Slf4j +public class Seetaface6QualityModel implements FaceQualityModel { + + private QualityConfig config; + + + /** + * 人脸亮度评估器池 + */ + private QualityOfBrightnessPool qualityOfBrightnessPool; + + /** + * 人脸清晰度评估器池 + */ + private QualityOfClarityPool qualityOfClarityPool; + + /** + * 人脸清晰度评估器池(深度学习) + */ + private QualityOfLBNPool qualityOfLBNPool; + + /** + * 人脸完整度评估器池 + */ + private QualityOfIntegrityPool qualityOfIntegrityPool; + + /** + * 人脸姿态评估器池 + */ + private QualityOfPosePool qualityOfPosePool; + + /** + * 人脸姿态评估器池(深度学习) + */ + private QualityOfPoseExPool qualityOfPoseExPool; + + /** + * 人脸分辨率评估器池 + */ + private QualityOfResolutionPool qualityOfResolutionPool; + + + @Override + public void loadModel(QualityConfig config) { + DeviceEnum device = DeviceEnum.CPU; + if(Objects.nonNull(config.getDevice())){ + device = config.getDevice(); + } + //加载依赖库 + NativeLoader.loadNativeLibraries(device); + this.config = config; + log.debug("Loading seetaFace6 library successfully."); + } + + + @Override + public R evaluateBrightness(BufferedImage image, DetectionRectangle rectangle, List keyPoints) { + //参数检查 + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(rectangle)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空"); + } + if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空"); + } + QualityOfBrightness qualityOfBrightness = null; + try { + if(Objects.isNull(this.qualityOfBrightnessPool)){ + this.qualityOfBrightnessPool = new QualityOfBrightnessPool(new SeetaConfSetting()); + } + qualityOfBrightness = qualityOfBrightnessPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle); + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints); + float[] scores = new float[1]; + QualityOfBrightness.QualityLevel level = qualityOfBrightness.check(imageData, seetaRect, pointFS, scores); + FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name())); + return R.ok(result); + } catch (Exception e) { + throw new FaceException("亮度评估错误", e); + } finally { + if (qualityOfBrightness != null) { + try { + qualityOfBrightnessPool.returnObject(qualityOfBrightness); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + @Override + public R evaluateBrightness(String imagePath, DetectionRectangle rectangle, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return evaluateBrightness(image, rectangle, keyPoints); + } + + @Override + public R evaluateBrightness(byte[] imageData, DetectionRectangle rectangle, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return evaluateBrightness(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R evaluateClarity(BufferedImage image, DetectionRectangle rectangle, List keyPoints) { + //参数检查 + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(rectangle)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空"); + } + if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空"); + } + QualityOfClarity qualityOfClarity = null; + try { + if(Objects.isNull(this.qualityOfClarityPool)){ + this.qualityOfClarityPool = new QualityOfClarityPool(new SeetaConfSetting()); + } + qualityOfClarity = qualityOfClarityPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle); + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints); + float[] scores = new float[1]; + QualityOfClarity.QualityLevel level = qualityOfClarity.check(imageData, seetaRect, pointFS, scores); + FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name())); + return R.ok(result); + } catch (Exception e) { + throw new FaceException("清晰度评估错误", e); + } finally { + if (qualityOfClarity != null) { + try { + qualityOfClarityPool.returnObject(qualityOfClarity); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + @Override + public R evaluateClarity(String imagePath, DetectionRectangle rectangle, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return evaluateClarity(image, rectangle, keyPoints); + } + + @Override + public R evaluateClarity(byte[] imageData, DetectionRectangle rectangle, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return evaluateClarity(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R evaluateCompleteness(BufferedImage image, DetectionRectangle rectangle, List keyPoints) { + //参数检查 + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(rectangle)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空"); + } + if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空"); + } + QualityOfIntegrity qualityOfIntegrity = null; + try { + if(Objects.isNull(this.qualityOfIntegrityPool)){ + this.qualityOfIntegrityPool = new QualityOfIntegrityPool(new SeetaConfSetting()); + } + qualityOfIntegrity = qualityOfIntegrityPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle); + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints); + float[] scores = new float[1]; + QualityOfIntegrity.QualityLevel level = qualityOfIntegrity.check(imageData, seetaRect, pointFS, scores); + FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name())); + return R.ok(result); + } catch (Exception e) { + throw new FaceException("完整度评估错误", e); + } finally { + if (qualityOfIntegrity != null) { + try { + qualityOfIntegrityPool.returnObject(qualityOfIntegrity); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + @Override + public R evaluateCompleteness(String imagePath, DetectionRectangle rectangle, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return evaluateCompleteness(image, rectangle, keyPoints); + } + + @Override + public R evaluateCompleteness(byte[] imageData, DetectionRectangle rectangle, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return evaluateCompleteness(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R evaluatePose(BufferedImage image, DetectionRectangle rectangle, List keyPoints) { + //参数检查 + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(rectangle)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空"); + } + if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空"); + } + QualityOfPose qualityOfPose = null; + try { + if(Objects.isNull(this.qualityOfPosePool)){ + this.qualityOfPosePool = new QualityOfPosePool(new SeetaConfSetting()); + } + qualityOfPose = qualityOfPosePool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle); + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints); + float[] scores = new float[1]; + QualityOfPose.QualityLevel level = qualityOfPose.check(imageData, seetaRect, pointFS, scores); + FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name())); + return R.ok(result); + } catch (Exception e) { + throw new FaceException("姿态评估错误", e); + } finally { + if (qualityOfPose != null) { + try { + qualityOfPosePool.returnObject(qualityOfPose); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + @Override + public R evaluatePose(String imagePath, DetectionRectangle rectangle, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return evaluatePose(image, rectangle, keyPoints); + } + + @Override + public R evaluatePose(byte[] imageData, DetectionRectangle rectangle, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return evaluatePose(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public R evaluateResolution(BufferedImage image, DetectionRectangle rectangle, List keyPoints) { + //参数检查 + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(rectangle)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空"); + } + if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空"); + } + QualityOfResolution qualityOfResolution = null; + try { + if(Objects.isNull(this.qualityOfResolutionPool)){ + this.qualityOfResolutionPool = new QualityOfResolutionPool(new SeetaConfSetting()); + } + qualityOfResolution = qualityOfResolutionPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle); + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints); + float[] scores = new float[1]; + QualityOfResolution.QualityLevel level = qualityOfResolution.check(imageData, seetaRect, pointFS, scores); + FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name())); + return R.ok(result); + } catch (Exception e) { + throw new FaceException("姿态评估错误", e); + } finally { + if (qualityOfResolution != null) { + try { + qualityOfResolutionPool.returnObject(qualityOfResolution); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + @Override + public R evaluateResolution(String imagePath, DetectionRectangle rectangle, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return evaluateResolution(image, rectangle, keyPoints); + } + + @Override + public R evaluateResolution(byte[] imageData, DetectionRectangle rectangle, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return evaluateResolution(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + + public R evaluateClarityWithDL(BufferedImage image, List keyPoints) { + //参数检查 + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空"); + } + QualityOfLBN qualityOfLBN = null; + try { + if(Objects.isNull(this.qualityOfLBNPool)){ + if(Objects.isNull(config)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "缺少必要配置（QualityConfig），请在调用前初始化模型配置"); + } + if(StringUtils.isBlank(config.getModelPath())){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "QualityConfig中，modelPath为空"); + } + SeetaConfSetting setting = getClarityMLSetting(); + this.qualityOfLBNPool = new QualityOfLBNPool(setting); + } + qualityOfLBN = qualityOfLBNPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints); + int[] light = new int[1]; + int[] blur = new int[1]; + int[] noise = new int[1]; + qualityOfLBN.Detect(imageData, pointFS, light, blur, noise); + return R.ok(new ClarityDLResult(light, blur, noise)); + } catch (Exception e) { + throw new FaceException("亮度评估错误", e); + } finally { + if (qualityOfLBN != null) { + try { + qualityOfLBNPool.returnObject(qualityOfLBN); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + + public R evaluateClarityWithDL(String imagePath, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return evaluateClarityWithDL(image, keyPoints); + } + + public R evaluateClarityWithDL(byte[] imageData, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return evaluateClarityWithDL(ImageIO.read(new ByteArrayInputStream(imageData)), keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + public R evaluatePoseWithDL(BufferedImage image, DetectionRectangle rectangle, List keyPoints) { + //参数检查 + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(rectangle)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空"); + } + if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空"); + } + QualityOfPoseEx qualityOfPoseEx = null; + try { + if(Objects.isNull(this.qualityOfPoseExPool)){ + if(Objects.isNull(config)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "缺少必要配置（QualityConfig），请在调用前初始化模型配置"); + } + if(StringUtils.isBlank(config.getModelPath())){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "QualityConfig中，modelPath为空"); + } + SeetaConfSetting setting = getPoseMLSetting(); + this.qualityOfPoseExPool = new QualityOfPoseExPool(setting); + } + qualityOfPoseEx = qualityOfPoseExPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints); + SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle); + float[] scores = new float[1]; + QualityOfPoseEx.QualityLevel level = qualityOfPoseEx.check(imageData, seetaRect, pointFS, scores); + FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name())); + return R.ok(result); + } catch (Exception e) { + throw new FaceException("亮度评估错误", e); + } finally { + if (qualityOfPoseEx != null) { + try { + qualityOfPoseExPool.returnObject(qualityOfPoseEx); + } catch (Exception e) { + log.warn("归还Predictor失败", e); + } + } + } + } + + public R evaluatePoseWithDL(String imagePath, DetectionRectangle rectangle, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return evaluatePoseWithDL(image, rectangle, keyPoints); + } + + public R evaluatePoseWithDL(byte[] imageData, DetectionRectangle rectangle, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return evaluatePoseWithDL(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + /** + * 获取清晰度模型配置(深度学习) + * @return + */ + private SeetaConfSetting getClarityMLSetting() throws FileNotFoundException { + String[] modelPath = {config.getModelPath() + File.separator + "quality_lbn.csta"}; + SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO; + int gpuId = 0; + if(Objects.nonNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU; + if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){ + gpuId = config.getGpuId(); + } + } + SeetaConfSetting setting = new SeetaConfSetting(new SeetaModelSetting(gpuId, modelPath, device)); + return setting; + } + + /** + * 获取人脸姿态模型配置(深度学习) + * @return + */ + private SeetaConfSetting getPoseMLSetting() throws FileNotFoundException { + String[] modelPath = {config.getModelPath() + File.separator + "pose_estimation.csta"}; + SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO; + int gpuId = 0; + if(Objects.nonNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU; + if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){ + gpuId = config.getGpuId(); + } + } + SeetaConfSetting setting = new SeetaConfSetting(new SeetaModelSetting(gpuId, modelPath, device)); + return setting; + } + + @Override + public R evaluateAll(String imagePath, DetectionRectangle rectangle, List keyPoints) { + if(!FileUtils.isFileExists(imagePath)){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + // 将图片路径转换为 BufferedImage + BufferedImage image = null; + try { + image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString())); + } catch (IOException e) { + throw new FaceException("无效图片路径", e); + } + return evaluateAll(image, rectangle, keyPoints); + } + + @Override + public R evaluateAll(BufferedImage image, DetectionRectangle rectangle, List keyPoints) { + //参数检查 + if(!ImageUtils.isImageValid(image)){ + return R.fail(R.Status.INVALID_IMAGE); + } + if(Objects.isNull(rectangle)){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空"); + } + if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){ + return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空"); + } + QualityOfBrightness qualityOfBrightness = null; + QualityOfClarity qualityOfClarity = null; + QualityOfIntegrity qualityOfIntegrity = null; + QualityOfPose qualityOfPose = null; + QualityOfResolution qualityOfResolution = null; + try { + if(Objects.isNull(this.qualityOfBrightnessPool)){ + this.qualityOfBrightnessPool = new QualityOfBrightnessPool(new SeetaConfSetting()); + } + if(Objects.isNull(this.qualityOfClarityPool)){ + this.qualityOfClarityPool = new QualityOfClarityPool(new SeetaConfSetting()); + } + if(Objects.isNull(this.qualityOfIntegrityPool)){ + this.qualityOfIntegrityPool = new QualityOfIntegrityPool(new SeetaConfSetting()); + } + if(Objects.isNull(this.qualityOfPosePool)){ + this.qualityOfPosePool = new QualityOfPosePool(new SeetaConfSetting()); + } + if(Objects.isNull(this.qualityOfResolutionPool)){ + this.qualityOfResolutionPool = new QualityOfResolutionPool(new SeetaConfSetting()); + } + FaceQualitySummary summary = new FaceQualitySummary(); + qualityOfBrightness = qualityOfBrightnessPool.borrowObject(); + qualityOfClarity = qualityOfClarityPool.borrowObject(); + qualityOfIntegrity = qualityOfIntegrityPool.borrowObject(); + qualityOfPose = qualityOfPosePool.borrowObject(); + qualityOfResolution = qualityOfResolutionPool.borrowObject(); + SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3); + imageData.data = ImageUtils.getMatrixBGR(image); + SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle); + SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints); + float[] scoresBrightness = new float[1]; + QualityOfBrightness.QualityLevel level = qualityOfBrightness.check(imageData, seetaRect, pointFS, scoresBrightness); + summary.setBrightness(new FaceQualityResult(scoresBrightness[0], QualityGrade.valueOf(level.name()))); + float[] scoresClarity = new float[1]; + QualityOfClarity.QualityLevel clarityLevel = qualityOfClarity.check(imageData, seetaRect, pointFS, scoresClarity); + summary.setClarity(new FaceQualityResult(scoresClarity[0], QualityGrade.valueOf(clarityLevel.name()))); + float[] scoresIntegrity = new float[1]; + QualityOfIntegrity.QualityLevel integrityLevel = qualityOfIntegrity.check(imageData, seetaRect, pointFS, scoresIntegrity); + summary.setCompleteness(new FaceQualityResult(scoresIntegrity[0], QualityGrade.valueOf(integrityLevel.name()))); + float[] scoresPose = new float[1]; + QualityOfPose.QualityLevel poseLevel = qualityOfPose.check(imageData, seetaRect, pointFS, scoresPose); + summary.setPose(new FaceQualityResult(scoresPose[0], QualityGrade.valueOf(poseLevel.name()))); + float[] scoresResolution = new float[1]; + QualityOfResolution.QualityLevel resolutionLevel = qualityOfResolution.check(imageData, seetaRect, pointFS, scoresResolution); + summary.setResolution(new FaceQualityResult(scoresResolution[0], QualityGrade.valueOf(resolutionLevel.name()))); + return R.ok(summary); + } catch (Exception e) { + throw new FaceException("亮度评估错误", e); + } finally { + PoolUtils.returnToPool(qualityOfBrightnessPool, qualityOfBrightness); + PoolUtils.returnToPool(qualityOfClarityPool, qualityOfClarity); + PoolUtils.returnToPool(qualityOfIntegrityPool, qualityOfIntegrity); + PoolUtils.returnToPool(qualityOfPosePool, qualityOfPose); + PoolUtils.returnToPool(qualityOfResolutionPool, qualityOfResolution); + } + } + + @Override + public R evaluateAll(byte[] imageData, DetectionRectangle rectangle, List keyPoints) { + if(Objects.isNull(imageData)){ + return R.fail(R.Status.INVALID_IMAGE); + } + try { + return evaluateAll(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints); + } catch (IOException e) { + throw new FaceException("错误的图像", e); + } + } + + @Override + public void close() throws Exception { + if(Objects.nonNull(qualityOfBrightnessPool)){ + qualityOfBrightnessPool.close(); + } + if(Objects.nonNull(qualityOfClarityPool)){ + qualityOfClarityPool.close(); + } + if(Objects.nonNull(qualityOfLBNPool)){ + qualityOfLBNPool.close(); + } + if(Objects.nonNull(qualityOfIntegrityPool)){ + qualityOfIntegrityPool.close(); + } + if(Objects.nonNull(qualityOfPosePool)){ + qualityOfPosePool.close(); + } + if(Objects.nonNull(qualityOfPoseExPool)){ + qualityOfPoseExPool.close(); + } + if(Objects.nonNull(qualityOfResolutionPool)){ + qualityOfResolutionPool.close(); + } + } +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/preprocess/DJLImagePreprocessor.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/preprocess/DJLImagePreprocessor.java new file mode 100644 index 0000000..de62d4b --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/preprocess/DJLImagePreprocessor.java @@ -0,0 +1,94 @@ +package cn.smartjavaai.face.preprocess; + +import ai.djl.modality.cv.Image; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDManager; +import ai.djl.opencv.OpenCVImageFactory; +import cn.smartjavaai.common.entity.DetectionRectangle; +import cn.smartjavaai.common.utils.OpenCVUtils; +import cn.smartjavaai.face.utils.FaceAlignUtils; +import cn.smartjavaai.face.utils.FaceUtils; +import org.opencv.core.Mat; + +import java.util.Objects; + +/** + * 图片预处理 + * @author dwj + * @date 2025/6/27 + */ +public class DJLImagePreprocessor { + + private Image image; + + private NDManager manager; + + // 裁剪参数 + private boolean enableCrop = false; + private DetectionRectangle cropRect; + + // 仿射变换参数 + private boolean enableAffine = false; + private double[][] keyPoints; + private int affineTargetWidth; + private int affineTargetHeight; + + public DJLImagePreprocessor(Image image, NDManager manager) { + this.image = image; + this.manager = manager; + } + + // 启用裁剪 + public DJLImagePreprocessor enableCrop(DetectionRectangle rect) { + this.enableCrop = true; + this.cropRect = rect; + return this; + } + + // 启用仿射变换 + public DJLImagePreprocessor enableAffine(double[][] keyPoints, int targetWidth, int targetHeight) { + if(Objects.isNull(keyPoints)){ + throw new IllegalArgumentException("keyPoints must be not null"); + } + this.enableAffine = true; + this.affineTargetWidth = targetWidth; + this.affineTargetHeight = targetHeight; + this.keyPoints = keyPoints; + return this; + } + + // 处理流程 + public Image process() { + Image result = image; + + if (enableCrop) { + result = result.getSubImage(cropRect.x, cropRect.y, cropRect.width, cropRect.height); + } + + if (enableAffine) { + result = warpAffine(keyPoints, affineTargetWidth, affineTargetHeight); + } + + return result; + } + + // 仿射变换 + private Image warpAffine(double[][] keyPoints, int width, int height) { + NDArray srcPoints = manager.create(keyPoints); + NDArray dstPoints = null; + if(width == 512 && height == 512){ + dstPoints = FaceUtils.faceTemplate512x512(manager); + }else if(width == 112 && height == 112){ + dstPoints = FaceUtils.faceTemplate112x112(manager); + }else if(width == 96 && height == 112){ + dstPoints = FaceUtils.faceTemplate96x112(manager); + } + // 5点仿射变换 + Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints); + Mat mat = FaceAlignUtils.warpAffine((Mat) image.getWrappedImage(), affine_matrix, width, height); + Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat); + return alignedImg; + } + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/seetaface/ClarityDLResult.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/seetaface/ClarityDLResult.java new file mode 100644 index 0000000..dfeddc3 --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/seetaface/ClarityDLResult.java @@ -0,0 +1,24 @@ +package cn.smartjavaai.face.seetaface; + +import com.seeta.sdk.QualityOfLBN; +import lombok.Data; + +/** + * Seetaface6 清晰度（深度学习）结果 + * @author dwj + * @date 2025/6/25 + */ +@Data +public class ClarityDLResult { + + private QualityOfLBN.LIGHTSTATE lightstate; + private QualityOfLBN.BLURSTATE blurstate; + private QualityOfLBN.NOISESTATE noisestate; + + public ClarityDLResult(int[] light, int[] blur, int[] noise) { + this.lightstate = QualityOfLBN.LIGHTSTATE.values()[light[0]]; + this.blurstate = QualityOfLBN.BLURSTATE.values()[blur[0]]; + this.noisestate = QualityOfLBN.NOISESTATE.values()[noise[0]]; + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/seetaface/NativeLoader.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/seetaface/NativeLoader.java index b4156b9..3a4e541 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/seetaface/NativeLoader.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/seetaface/NativeLoader.java @@ -7,7 +7,6 @@ import cn.hutool.system.OsInfo; import cn.hutool.system.SystemUtil; import cn.smartjavaai.common.config.Config; import cn.smartjavaai.common.enums.DeviceEnum; -import cn.smartjavaai.face.config.FaceModelConfig; import cn.smartjavaai.face.exception.FaceException; import com.seeta.sdk.util.DllItem; import com.seeta.sdk.util.LoadNativeCore; diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceAlignUtils.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceAlignUtils.java index 0809e11..ed22020 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceAlignUtils.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceAlignUtils.java @@ -5,6 +5,7 @@ import ai.djl.ndarray.NDArray; import ai.djl.ndarray.NDManager; import ai.djl.opencv.OpenCVImageFactory; import cn.smartjavaai.common.utils.ImageUtils; +import cn.smartjavaai.common.utils.OpenCVUtils; import com.seeta.sdk.SeetaImageData; import com.seeta.sdk.SeetaPointF; import org.opencv.core.Mat; @@ -39,7 +40,8 @@ public class FaceAlignUtils { public static Mat warpAffine(Mat src, Mat rot_mat, int width, int height) { Mat rot = new Mat(); Size size = new Size(width, height); - Imgproc.warpAffine(src, rot, rot_mat, size); + Scalar scalar = new Scalar(135, 133, 132); + Imgproc.warpAffine(src, rot, rot_mat, size,0, 0, scalar); return rot; } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceUtils.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceUtils.java index 9e65043..35d61df 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceUtils.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/FaceUtils.java @@ -7,11 +7,14 @@ import ai.djl.ndarray.NDArray; import ai.djl.ndarray.NDManager; import cn.smartjavaai.common.entity.*; import cn.smartjavaai.common.entity.Point; -import cn.smartjavaai.common.enums.EyeStatus; -import cn.smartjavaai.common.enums.GenderType; +import cn.smartjavaai.common.entity.face.FaceAttribute; +import cn.smartjavaai.common.entity.face.FaceInfo; +import cn.smartjavaai.common.entity.face.HeadPose; +import cn.smartjavaai.common.entity.face.LivenessResult; +import cn.smartjavaai.common.enums.face.EyeStatus; +import cn.smartjavaai.common.enums.face.GenderType; import cn.smartjavaai.common.utils.ImageUtils; -import cn.smartjavaai.face.config.FaceModelConfig; -import cn.smartjavaai.common.enums.LivenessStatus; +import cn.smartjavaai.common.enums.face.LivenessStatus; import cn.smartjavaai.face.exception.FaceException; import com.seeta.sdk.*; @@ -340,6 +343,44 @@ public class FaceUtils { return points; } + /** + * 112x112的目标点 - Target point of 112x112 + * standard 5 landmarks for FFHQ faces with 112x112 + * + * @param manager + * @return + */ + public static NDArray faceTemplate112x112(NDManager manager) { + double[][] coord5point = { + {30.29459953, 51.69630051}, // 112x112的目标点 - Target point of 512x512 + {65.53179932, 51.50139999}, + {48.02519989, 71.73660278}, + {33.54930115, 87}, + {62.72990036, 87} + }; + NDArray points = manager.create(coord5point); + return points; + } + + /** + * 96x112的目标点 - Target point of 96x112 + * standard 5 landmarks for FFHQ faces with 96x112 + * + * @param manager + * @return + */ + public static NDArray faceTemplate96x112(NDManager manager) { + double[][] coord5point = { + {30.29459953, 51.69630051}, + {65.53179932, 51.50139999}, + {48.02519989, 71.73660278}, + {33.54930115, 92.3655014}, + {62.72990036, 92.20410156} + }; + NDArray points = manager.create(coord5point); + return points; + } + /** * bgr转图片 * @return 图片 @@ -507,7 +548,7 @@ public class FaceUtils { .map(p -> new Point(p.x, p.y)) .collect(Collectors.toList()); FaceInfo faceInfo = new FaceInfo(keyPoints); - faceInfo.setLivenessStatus(livenessStatusList.get(i)); + faceInfo.setLivenessStatus(new LivenessResult(livenessStatusList.get(i))); DetectionInfo detectionInfo = new DetectionInfo(rectangle, 0, faceInfo); detectionInfoList.add(detectionInfo); } diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/OpenCVUtils.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/OpenCVUtils.java deleted file mode 100644 index 878c5a3..0000000 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/OpenCVUtils.java +++ /dev/null @@ -1,137 +0,0 @@ -package cn.smartjavaai.face.utils; - -import ai.djl.ndarray.NDArray; -import ai.djl.ndarray.NDManager; -import org.opencv.core.CvType; -import org.opencv.core.Mat; -import org.opencv.core.Point; -import org.opencv.core.Scalar; -import org.opencv.imgproc.Imgproc; - -import java.awt.image.BufferedImage; -import java.awt.image.DataBufferByte; - -/** - * OpenCV 工具类 - */ -public class OpenCVUtils { - /** - * canny算法，边缘检测 - * - * @param src - * @return - */ - public static Mat canny(Mat src) { - Mat mat = src.clone(); - Imgproc.Canny(src, mat, 100, 200); - return mat; - } - - /** - * 画线 - * - * @param mat - * @param point1 - * @param point2 - */ - public static void line(Mat mat, Point point1, Point point2) { - Imgproc.line(mat, point1, point2, new Scalar(255, 255, 255), 1); - } - - /** - * NDArray to opencv_core.Mat - * - * @param manager - * @param srcPoints - * @param dstPoints - * @return - */ - public static Mat toOpenCVMat(NDManager manager, NDArray srcPoints, NDArray dstPoints) { - NDArray svdMat = SVDUtils.transformationFromPoints(manager, srcPoints, dstPoints); - double[] doubleArray = svdMat.toDoubleArray(); - Mat newSvdMat = new Mat(2, 3, CvType.CV_64F); - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 3; j++) { - newSvdMat.put(i, j, doubleArray[i * 3 + j]); - } - } - return newSvdMat; - } - - /** - * double[][] points array to Mat - * @param points - * @return - */ - public static Mat toOpenCVMat(double[][] points) { - Mat mat = new Mat(5, 2, CvType.CV_64F); - for (int i = 0; i < 5; i++) { - for (int j = 0; j < 2; j++) { - mat.put(i, j, points[i * 5 + j]); - } - } - return mat; - } - - /** - * 变换矩阵的逆矩阵 - * - * @param src - * @return - */ - public static Mat invertAffineTransform(Mat src) { - Mat dst = src.clone(); - Imgproc.invertAffineTransform(src, dst); - return dst; - } - - /** - * Mat to BufferedImage - * - * @param mat - * @return - */ - public static BufferedImage mat2Image(Mat mat) { - int width = mat.width(); - int height = mat.height(); - byte[] data = new byte[width * height * (int) mat.elemSize()]; - Imgproc.cvtColor(mat, mat, 4); - mat.get(0, 0, data); - BufferedImage ret = new BufferedImage(width, height, 5); - ret.getRaster().setDataElements(0, 0, width, height, data); - return ret; - } - - /** - * BufferedImage to Mat - * - * @param img - * @return - */ - public static Mat image2Mat(BufferedImage img) { - int width = img.getWidth(); - int height = img.getHeight(); - int channels; - - // 获取图像类型 - int imageType = img.getType(); - - // 判断是3通道还是4通道 - if (imageType == BufferedImage.TYPE_3BYTE_BGR) { - channels = 3; - } else if (imageType == BufferedImage.TYPE_4BYTE_ABGR || imageType == BufferedImage.TYPE_4BYTE_ABGR_PRE) { - channels = 4; - } else { - // 如果不是已知格式，强制转换为 3 通道 BGR - BufferedImage convertedImg = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR); - convertedImg.getGraphics().drawImage(img, 0, 0, null); - img = convertedImg; - channels = 3; - } - - byte[] data = ((DataBufferByte) img.getRaster().getDataBuffer()).getData(); - Mat mat = new Mat(height, width, CvType.CV_8UC(channels)); - mat.put(0, 0, data); - return mat; - } -} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/Seetaface6Utils.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/Seetaface6Utils.java new file mode 100644 index 0000000..a4c4c3b --- /dev/null +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/utils/Seetaface6Utils.java @@ -0,0 +1,13 @@ +package cn.smartjavaai.face.utils; + +import cn.smartjavaai.face.enums.QualityGrade; + +/** + * Seetaface6工具类 + * @author dwj + * @date 2025/6/24 + */ +public class Seetaface6Utils { + + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java index d9b311d..f71eb9e 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java @@ -6,7 +6,7 @@ import cn.smartjavaai.face.utils.FaceUtils; import cn.smartjavaai.face.vector.config.MilvusConfig; import cn.smartjavaai.face.vector.constant.VectorDBConstants; import cn.smartjavaai.face.vector.entity.FaceVector; -import cn.smartjavaai.common.entity.FaceSearchResult; +import cn.smartjavaai.common.entity.face.FaceSearchResult; import cn.smartjavaai.face.vector.exception.VectorDBException; import io.milvus.client.MilvusServiceClient; import io.milvus.grpc.*; diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java index 67803b5..098520e 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java @@ -4,11 +4,10 @@ import cn.hutool.core.util.IdUtil; import cn.smartjavaai.common.config.Config; import cn.smartjavaai.face.dao.FaceDao; import cn.smartjavaai.face.entity.FaceSearchParams; -import cn.smartjavaai.face.utils.FaceUtils; import cn.smartjavaai.face.utils.SimilarityUtil; import cn.smartjavaai.face.vector.config.SQLiteConfig; import cn.smartjavaai.face.vector.entity.FaceVector; -import cn.smartjavaai.common.entity.FaceSearchResult; +import cn.smartjavaai.common.entity.face.FaceSearchResult; import cn.smartjavaai.face.vector.exception.VectorDBException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -24,7 +23,8 @@ import java.util.stream.Collectors; public class SQLiteClient implements VectorDBClient { private final FaceDao faceDao; - private final List memoryIndex = new CopyOnWriteArrayList<>(); + //private final List memoryIndex = new CopyOnWriteArrayList<>(); + private final ConcurrentHashMap memoryIndex = new ConcurrentHashMap<>(); private int featureDimension; // 维度 private final ExecutorService executor = Executors.newFixedThreadPool(4); @@ -138,7 +138,7 @@ public class SQLiteClient implements VectorDBClient { // 从数据库中删除 boolean isSuccess = faceDao.deleteFace(ids.toArray(new String[0])); // 从内存中删除 - memoryIndex.removeIf(v -> ids.contains(v.getId())); + ids.forEach(memoryIndex::remove); if(!isSuccess){ throw new VectorDBException("删除失败"); } @@ -156,7 +156,7 @@ public class SQLiteClient implements VectorDBClient { return Collections.emptyList(); } // 并行计算相似度 - List> futures = memoryIndex.stream() + List> futures = memoryIndex.values().stream() .map(vector -> CompletableFuture.supplyAsync(() -> { float similarity = SimilarityUtil.calculate(queryVector, vector.getVector(), config.getSimilarityType(), faceSearchParams.getNormalizeSimilarity()); return similarity >= faceSearchParams.getThreshold() ? @@ -219,7 +219,7 @@ public class SQLiteClient implements VectorDBClient { } private void addToMemoryIndex(FaceVector faceVector) { - memoryIndex.add(faceVector); + memoryIndex.put(faceVector.getId(), faceVector); } private void clearAllData() { diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/VectorDBClient.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/VectorDBClient.java index cdab49a..91ab325 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/VectorDBClient.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/vector/core/VectorDBClient.java @@ -2,7 +2,7 @@ package cn.smartjavaai.face.vector.core; import cn.smartjavaai.face.entity.FaceSearchParams; import cn.smartjavaai.face.vector.entity.FaceVector; -import cn.smartjavaai.common.entity.FaceSearchResult; +import cn.smartjavaai.common.entity.face.FaceSearchResult; import cn.smartjavaai.face.vector.exception.VectorDBException; import java.util.List; diff --git a/smartjavaai-objectdetection/pom.xml b/smartjavaai-objectdetection/pom.xml index 842044a..a3c550a 100644 --- a/smartjavaai-objectdetection/pom.xml +++ b/smartjavaai-objectdetection/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.17 + 1.0.19 smartjavaai-objectdetection - 1.0.17 + 1.0.19 smartjavaai-objectdetection SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/config/DetectorModelConfig.java b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/config/DetectorModelConfig.java index 2a68a4f..36e2624 100644 --- a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/config/DetectorModelConfig.java +++ b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/config/DetectorModelConfig.java @@ -5,6 +5,9 @@ import cn.smartjavaai.objectdetection.constant.DetectorConstant; import cn.smartjavaai.objectdetection.enums.DetectorModelEnum; import lombok.Data; +import java.util.HashMap; +import java.util.Map; + /** * 目标检测模型参数配置 * @@ -41,6 +44,11 @@ public class DetectorModelConfig { */ private int maxBox; + /** + * 个性化配置 + */ + private Map customParams = new HashMap<>(); + public DetectorModelConfig() { } @@ -52,4 +60,20 @@ public class DetectorModelConfig { public DetectorModelConfig(DetectorModelEnum modelEnum) { this.modelEnum = modelEnum; } + + public T getCustomParam(String key, Class clazz) { + Object value = customParams.get(key); + if (value == null) return null; + return clazz.cast(value); + } + + /** + * 添加个性化配置项 + */ + public void putCustomParam(String key, Object value) { + if (customParams == null) { + customParams = new HashMap<>(); + } + customParams.put(key, value); + } } diff --git a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/criteria/DJLModelCriteriaBuilder.java b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/criteria/DJLModelCriteriaBuilder.java index 4554866..bb64684 100644 --- a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/criteria/DJLModelCriteriaBuilder.java +++ b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/criteria/DJLModelCriteriaBuilder.java @@ -33,6 +33,7 @@ public class DJLModelCriteriaBuilder implements CriteriaBuilderStrategy { .optArgument("threshold", config.getThreshold() > 0 ? config.getThreshold() : DetectorConstant.DEFAULT_THRESHOLD) .optModelUrls(DJL_MODEL_PREFIX + config.getModelEnum().getModelUri()) .optDevice(device) + //.optOption("ortDevice", "TensorRT") .optProgress(new ProgressBar()) .build(); return criteria; diff --git a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/criteria/YoloCriteriaBuilder.java b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/criteria/YoloCriteriaBuilder.java index 11605ee..7e4f3c0 100644 --- a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/criteria/YoloCriteriaBuilder.java +++ b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/criteria/YoloCriteriaBuilder.java @@ -1,14 +1,19 @@ package cn.smartjavaai.objectdetection.criteria; +import ai.djl.Device; import ai.djl.modality.cv.Image; import ai.djl.modality.cv.output.DetectedObjects; import ai.djl.modality.cv.translator.YoloV8TranslatorFactory; import ai.djl.repository.zoo.Criteria; import ai.djl.training.util.ProgressBar; +import cn.smartjavaai.common.enums.DeviceEnum; import cn.smartjavaai.objectdetection.constant.DetectorConstant; import cn.smartjavaai.objectdetection.config.DetectorModelConfig; import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; /** * YOLO模型Criteria 构建器 @@ -18,16 +23,25 @@ import java.nio.file.Paths; public class YoloCriteriaBuilder implements CriteriaBuilderStrategy { @Override public Criteria buildCriteria(DetectorModelConfig config) { + Device device = null; + if(!Objects.isNull(config.getDevice())){ + device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(); + } + + Map customParams = getDefaultConfig(); + // 合并用户自定义参数（如有重复，覆盖默认默认值） + if (config.getCustomParams() != null) { + customParams.putAll(config.getCustomParams()); + } + Criteria.Builder criteriaBuilder = Criteria.builder() .setTypes(Image.class, DetectedObjects.class) //.optModelUrls("/Users/wenjie/Documents/develop/face_model/yolo") .optModelPath(Paths.get(config.getModelPath())) .optEngine("OnnxRuntime") - .optArgument("width", 640) //将输入图像的宽度缩放为 640 像素 - .optArgument("height", 640) - .optArgument("resize", true) - .optArgument("toTensor", true) - .optArgument("applyRatio", true) + //.optOption("ortDevice", "TensorRT") + .optArguments(customParams) + .optDevice(device) .optTranslatorFactory(new YoloV8TranslatorFactory()) .optProgress(new ProgressBar()) .optArgument("threshold", config.getThreshold() > 0 ? config.getThreshold() : DetectorConstant.DEFAULT_THRESHOLD); @@ -37,4 +51,15 @@ public class YoloCriteriaBuilder implements CriteriaBuilderStrategy { Criteria criteria = criteriaBuilder.build(); return criteria; } + + public Map getDefaultConfig(){ + Map arguments = new HashMap<>(); + // 添加默认参数 + arguments.put("width", 640); + arguments.put("height", 640); + arguments.put("resize", true); + arguments.put("toTensor", true); + arguments.put("applyRatio", true); + return arguments; + } } diff --git a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/model/DetectorModel.java b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/model/DetectorModel.java index 299bf02..bd517b0 100644 --- a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/model/DetectorModel.java +++ b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/model/DetectorModel.java @@ -41,10 +41,6 @@ public class DetectorModel implements AutoCloseable{ private ZooModel model; - //private Predictor predictor; - - - private ObjectPool> predictorPool; public void loadModel(DetectorModelConfig config){ @@ -193,12 +189,23 @@ public class DetectorModel implements AutoCloseable{ /** - * 显式释放资源（必须调用！） + * 显式释放资源 */ @Override public void close() { - if (predictorPool != null) { - predictorPool.close(); + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); } } } diff --git a/smartjavaai-ocr/pom.xml b/smartjavaai-ocr/pom.xml index bfd930e..24fca8e 100644 --- a/smartjavaai-ocr/pom.xml +++ b/smartjavaai-ocr/pom.xml @@ -6,7 +6,7 @@ cn.smartjavaai smartjavaai-parent - 1.0.17 + 1.0.19 smartjavaai-ocr @@ -20,7 +20,7 @@ - 1.0.17 + 1.0.19 smartjavaai-ocr SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/detect/OcrCommonDetModel.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/detect/OcrCommonDetModel.java index 124ce3d..b28f232 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/detect/OcrCommonDetModel.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/detect/OcrCommonDetModel.java @@ -11,7 +11,7 @@ import java.util.List; * OCR 通用检测模型 * @author dwj */ -public interface OcrCommonDetModel { +public interface OcrCommonDetModel extends AutoCloseable{ /** * 加载模型 diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/detect/PpOCRV5DetModel.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/detect/PpOCRV5DetModel.java index efef89f..a09fee4 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/detect/PpOCRV5DetModel.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/detect/PpOCRV5DetModel.java @@ -53,6 +53,8 @@ public class PpOCRV5DetModel implements OcrCommonDetModel { private ObjectPool> detPredictorPool; + private ZooModel detectionModel; + private OcrDetModelConfig config; @Override @@ -77,7 +79,7 @@ public class PpOCRV5DetModel implements OcrCommonDetModel { .build(); try{ - ZooModel detectionModel = ModelZoo.loadModel(detCriteria); + detectionModel = ModelZoo.loadModel(detCriteria); // 创建池子：每个线程独享 Predictor this.detPredictorPool = new GenericObjectPool<>(new PredictorFactory<>(detectionModel)); log.debug("当前设备: " + detectionModel.getNDManager().getDevice()); @@ -198,4 +200,22 @@ public class PpOCRV5DetModel implements OcrCommonDetModel { throw new OcrException("导出图片失败", e); } } + + @Override + public void close() throws Exception { + try { + if (detPredictorPool != null) { + detPredictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (detectionModel != null) { + detectionModel.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + } } diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/direction/OcrDirectionModel.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/direction/OcrDirectionModel.java index a1d2620..c2f45c2 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/direction/OcrDirectionModel.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/direction/OcrDirectionModel.java @@ -17,7 +17,7 @@ import java.util.List; * OCR 文本方向分类模型 * @author dwj */ -public interface OcrDirectionModel { +public interface OcrDirectionModel extends AutoCloseable{ /** * 加载模型 diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/direction/PPOCRMobileV2Model.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/direction/PPOCRMobileV2Model.java index 73da37d..37c4e2d 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/direction/PPOCRMobileV2Model.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/direction/PPOCRMobileV2Model.java @@ -64,6 +64,8 @@ public class PPOCRMobileV2Model implements OcrDirectionModel { private OcrCommonDetModel detModel; + private ZooModel model; + @Override public void loadModel(DirectionModelConfig config){ @@ -86,7 +88,7 @@ public class PPOCRMobileV2Model implements OcrDirectionModel { .optProgress(new ProgressBar()) .build(); try{ - ZooModel model = ModelZoo.loadModel(criteria); + model = ModelZoo.loadModel(criteria); // 创建池子：每个线程独享 Predictor this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model)); log.debug("当前设备: " + model.getNDManager().getDevice()); @@ -100,6 +102,7 @@ public class PPOCRMobileV2Model implements OcrDirectionModel { OcrDetModelConfig detModelConfig = new OcrDetModelConfig(); detModelConfig.setModelEnum(config.getDetModelEnum()); detModelConfig.setDetModelPath(config.getDetModelPath()); + detModelConfig.setDevice(config.getDevice()); detModel = OcrModelFactory.getInstance().getDetModel(detModelConfig); } } @@ -215,6 +218,19 @@ public class PPOCRMobileV2Model implements OcrDirectionModel { } } catch (Exception e) { throw new OcrException("OCR检测错误", e); + }finally { + if (predictor != null) { + try { + predictorPool.returnObject(predictor); //归还 + } catch (Exception e) { + log.warn("归还Predictor失败", e); + try { + predictor.close(); // 归还失败才销毁 + } catch (Exception ex) { + log.error("关闭Predictor失败", ex); + } + } + } } return ocrItemList; } @@ -287,4 +303,29 @@ public class PPOCRMobileV2Model implements OcrDirectionModel { throw new OcrException("导出图片失败", e); } } + + @Override + public void close() throws Exception { + try { + if (predictorPool != null) { + predictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (model != null) { + model.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + try { + if (detModel != null) { + detModel.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + } } diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/OcrCommonRecModel.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/OcrCommonRecModel.java index 03b023b..73dd78d 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/OcrCommonRecModel.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/OcrCommonRecModel.java @@ -13,7 +13,7 @@ import java.util.List; * OCR 通用识别模型 * @author dwj */ -public interface OcrCommonRecModel { +public interface OcrCommonRecModel extends AutoCloseable{ /** * 加载模型 diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/PpOCRV5RecModel.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/PpOCRV5RecModel.java index fe1d809..423b0a8 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/PpOCRV5RecModel.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/PpOCRV5RecModel.java @@ -67,6 +67,8 @@ public class PpOCRV5RecModel implements OcrCommonRecModel { private OcrDirectionModel directionModel; + private ZooModel recognitionModel; + @Override public void loadModel(OcrRecModelConfig config){ if(StringUtils.isBlank(config.getRecModelPath())){ @@ -88,7 +90,7 @@ public class PpOCRV5RecModel implements OcrCommonRecModel { .optDevice(device) .build(); try{ - ZooModel recognitionModel = ModelZoo.loadModel(recCriteria); + recognitionModel = ModelZoo.loadModel(recCriteria); this.recPredictorPool = new GenericObjectPool<>(new PredictorFactory<>(recognitionModel)); log.debug("当前设备: " + recognitionModel.getNDManager().getDevice()); log.debug("当前引擎: " + Engine.getInstance().getEngineName()); @@ -102,6 +104,7 @@ public class PpOCRV5RecModel implements OcrCommonRecModel { OcrDetModelConfig detModelConfig = new OcrDetModelConfig(); detModelConfig.setModelEnum(config.getDetModelEnum()); detModelConfig.setDetModelPath(config.getDetModelPath()); + detModelConfig.setDevice(config.getDevice()); detModel = OcrModelFactory.getInstance().getDetModel(detModelConfig); } @@ -110,6 +113,7 @@ public class PpOCRV5RecModel implements OcrCommonRecModel { DirectionModelConfig directionModelConfig = new DirectionModelConfig(); directionModelConfig.setModelEnum(config.getDirectionModelEnum()); directionModelConfig.setModelPath(config.getDirectionModelPath()); + directionModelConfig.setDevice(config.getDevice()); directionModel = OcrModelFactory.getInstance().getDirectionModel(directionModelConfig); } } @@ -320,4 +324,36 @@ public class PpOCRV5RecModel implements OcrCommonRecModel { throw new OcrException("导出图片失败", e); } } + + @Override + public void close() throws Exception { + try { + if (recPredictorPool != null) { + recPredictorPool.close(); + } + } catch (Exception e) { + log.warn("关闭 predictorPool 失败", e); + } + try { + if (detModel != null) { + detModel.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + try { + if (directionModel != null) { + directionModel.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + try { + if (recognitionModel != null) { + recognitionModel.close(); + } + } catch (Exception e) { + log.warn("关闭 model 失败", e); + } + } } diff --git a/smartjavaai-translate/pom.xml b/smartjavaai-translate/pom.xml index 931c7bb..7438e4b 100644 --- a/smartjavaai-translate/pom.xml +++ b/smartjavaai-translate/pom.xml @@ -20,7 +20,7 @@ - 1.0.17 + 1.0.19 smartjavaai-translate SmartJavaAI https://github.com/geekwenjie/SmartJavaAI