ZF/SmartJavaAI
1
0
Fork 0
mirror of https://github.com/geekwenjie/SmartJavaAI.git synced 2026-09-15 22:57:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

临时提交

Browse Source
This commit is contained in:
dengwenjie
2025-08-29 10:30:35 +08:00
parent 8bf620a330
commit 86ea7eb03e
364 changed files with 8572 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
face/pom.xml Normal file
View File

@@ -0,0 +1,160 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>cn.smartjavaai</groupId>
<artifactId>smartjavaai-parent</artifactId>
<version>1.0.24</version>
</parent>
<artifactId>face</artifactId>
<version>1.0.24</version>
<name>face</name>
<description>SmartJavaAI</description>
<url>https://github.com/geekwenjie/SmartJavaAI</url>
<licenses>
<license>
<name>MIT License</name>
<url>https://opensource.org/licenses/MIT</url>
</license>
</licenses>
<properties>
<!-- <maven.compiler.source>11</maven.compiler.source>-->
<!-- <maven.compiler.target>11</maven.compiler.target>-->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.test.skip>true</maven.test.skip>
<javacv.version>1.5.8</javacv.version>
<javacv.ffmpeg.version>5.1.2-1.5.8</javacv.ffmpeg.version>
</properties>
<dependencies>
<dependency>
<groupId>cn.smartjavaai</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.gitee.dengwenjie</groupId>
<artifactId>seeta-sdk-platform</artifactId>
<version>1.2.2</version>
</dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.36.0.3</version>
</dependency>
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
<version>2.5.7</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.sonatype.central</groupId>
<artifactId>central-publishing-maven-plugin</artifactId>
<version>0.4.0</version>
<extensions>true</extensions>
<configuration>
<publishingServerId>dengwenjie</publishingServerId>
<tokenAuth>true</tokenAuth>
<deploymentName>${project.groupId}:${project.artifactId}:${project.version}</deploymentName>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<!-- <javadocExecutable>${java.home}/bin/javadoc</javadocExecutable>-->
<doclint>none</doclint>
<additionalJOptions>
<additionalJOption>-Xdoclint:none</additionalJOption>
</additionalJOptions>
</configuration>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<!-- 必须添加：SCM信息 -->
<scm>
<connection>scm:git:git://github.com/geekwenjie/SmartJavaAI.git</connection>
<developerConnection>scm:git:ssh://github.com/geekwenjie/SmartJavaAI.git</developerConnection>
<url>http://github.com/geekwenjie/SmartJavaAI/tree/master</url>
</scm>
<distributionManagement>
<snapshotRepository>
<id>dengwenjie</id>
<url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
</snapshotRepository>
<repository>
<id>dengwenjie</id>
<url>https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
<developers>
<developer>
<name>dengwenjie</name>
<email>775747758@qq.com</email>
<roles>
<role>Project Manager</role>
<role>Architect</role>
</roles>
</developer>
</developers>
</project>

68
face/src/main/java/cn/smartjavaai/face/config/FaceAttributeConfig.java Normal file
View File

@@ -0,0 +1,68 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.config.ModelConfig;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.enums.FaceAttributeModelEnum;
import lombok.Data;
/**
* 人脸属性识别模型配置
* @author dwj
*/
@Data
public class FaceAttributeConfig extends ModelConfig {
/**
* 人脸属性识别模型枚举
*/
private FaceAttributeModelEnum modelEnum = FaceAttributeModelEnum.SEETA_FACE6_MODEL;
/**
* 模型路径
*/
private String modelPath;
/**
* 是否启用年龄检测
*/
private boolean enableAge = true;
/**
* 是否启用性别检测
*/
private boolean enableGender = true;
/**
* 是否启用人脸姿态检测
*/
private boolean enableHeadPose = true;
/**
* 是否启用眼睛状态检测
*/
private boolean enableEyeStatus = true;
/**
* 是否启用口罩检测
*/
private boolean enableMask = true;
public FaceAttributeConfig() {
}
public FaceAttributeConfig(FaceAttributeModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public FaceAttributeConfig(FaceAttributeModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
public FaceAttributeConfig(String modelPath) {
this.modelPath = modelPath;
}
}

53
face/src/main/java/cn/smartjavaai/face/config/FaceDetConfig.java Normal file
View File

@@ -0,0 +1,53 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.config.ModelConfig;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* 人脸检测模型配置
* @author dwj
*/
@Data
public class FaceDetConfig extends ModelConfig {
/**
* 人脸检测模型枚举
*/
private FaceDetModelEnum modelEnum;
/**
* 置信度阈值
*/
private double confidenceThreshold;
/**
* 非极大抑制阈值 作用：消除重叠检测框，保留最优结果
*/
private double nmsThresh = FaceDetectConstant.NMS_THRESHOLD;
/**
* 模型路径
*/
private String modelPath;
public FaceDetConfig() {
}
public FaceDetConfig(FaceDetModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public FaceDetConfig(FaceDetModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
}

47
face/src/main/java/cn/smartjavaai/face/config/FaceExpressionConfig.java Normal file
View File

@@ -0,0 +1,47 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.config.ModelConfig;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.enums.ExpressionModelEnum;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.FaceRecModel;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* @author dwj
* @date 2025/7/1
*/
@Data
public class FaceExpressionConfig extends ModelConfig {
/**
* 模型枚举
*/
private ExpressionModelEnum modelEnum = ExpressionModelEnum.DensNet121;
/**
* 模型路径
*/
private String modelPath;
/**
* 人脸检测模型
*/
private FaceDetModel detectModel;
/**
* 是否对齐人脸
*/
private boolean align = true;
/**
* 是否裁剪人脸
*/
private boolean cropFace = true;
}

73
face/src/main/java/cn/smartjavaai/face/config/FaceRecConfig.java Normal file
View File

@@ -0,0 +1,73 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.config.ModelConfig;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.FaceRecModel;
import cn.smartjavaai.face.vector.config.VectorDBConfig;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* 人脸检测识别模型配置
* @author dwj
*/
@Data
public class FaceRecConfig extends ModelConfig {
/**
* 人脸模型枚举
*/
private FaceRecModelEnum modelEnum;
/**
* 模型路径
*/
private String modelPath;
/**
* 向量数据库配置
*/
private VectorDBConfig vectorDBConfig;
/**
* 是否自动加载人脸到内存
*/
private boolean isAutoLoadFace = true;
/**
* 是否裁剪人脸
*/
private boolean cropFace = true;
/**
* 是否对齐人脸
*/
private boolean align = false;
/**
* 人脸检测模型
*/
private FaceDetModel detectModel;
public FaceRecConfig() {
}
public FaceRecConfig(FaceRecModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public FaceRecConfig(FaceRecModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
}

70
face/src/main/java/cn/smartjavaai/face/config/LivenessConfig.java Normal file
View File

@@ -0,0 +1,70 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.config.ModelConfig;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.constant.LivenessConstant;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.FaceRecModel;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* 活体检测模型配置
* @author dwj
*/
@Data
public class LivenessConfig extends ModelConfig {
/**
* 活体检测模型枚举
*/
private LivenessModelEnum modelEnum = LivenessModelEnum.SEETA_FACE6_MODEL;
/**
* 模型路径
*/
private String modelPath;
/**
* 人脸检测模型
*/
private FaceDetModel detectModel;
/**
* 视频检测帧数
*/
private int frameCount = LivenessConstant.DEFAULT_FRAME_COUNT;
/**
* 视频检测最大帧数
*/
private int maxVideoDetectFrames = LivenessConstant.DEFAULT_MAX_VIDEO_DETECT_FRAMES;
/**
* 真人阈值
*/
private Float realityThreshold;
public LivenessConfig() {
}
public LivenessConfig(LivenessModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public LivenessConfig(LivenessModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
public LivenessConfig(String modelPath) {
this.modelPath = modelPath;
}
}

43
face/src/main/java/cn/smartjavaai/face/config/QualityConfig.java Normal file
View File

@@ -0,0 +1,43 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.config.ModelConfig;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.constant.LivenessConstant;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.enums.QualityModelEnum;
import lombok.Data;
/**
* 质量评估配置
* @author dwj
*/
@Data
public class QualityConfig extends ModelConfig {
/**
* 活体检测模型枚举
*/
private QualityModelEnum modelEnum = QualityModelEnum.SEETA_FACE6_MODEL;
/**
* 模型路径
*/
private String modelPath;
public QualityConfig() {
}
public QualityConfig(QualityModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public QualityConfig(QualityModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
public QualityConfig(String modelPath) {
this.modelPath = modelPath;
}
}

36
face/src/main/java/cn/smartjavaai/face/constant/FaceDetectConstant.java Normal file
View File

@@ -0,0 +1,36 @@
package cn.smartjavaai.face.constant;
/**
* 人脸检测常量
* @author dwj
*/
public class FaceDetectConstant {
/**
* 置信度阈值
*/
public static final float DEFAULT_CONFIDENCE_THRESHOLD = 0.85F;
/**
* 每张特征图保留的最大候选框数量
*/
public static final int MAX_FACE_LIMIT = 5000;
/**
* nms阈值:控制重叠框的合并程度
*/
public static final float NMS_THRESHOLD = 0.45F;
/**
* 默认相似度阈值
*/
public static final float SEETAFACE_DEFAULT_SIMILARITY_THRESHOLD = 0.85F;
/**
* 默认相似度阈值
*/
public static final float FACENET_DEFAULT_SIMILARITY_THRESHOLD = 0.8F;
}

14
face/src/main/java/cn/smartjavaai/face/constant/FaceNetConstant.java Normal file
View File

@@ -0,0 +1,14 @@
package cn.smartjavaai.face.constant;
/**
* FaceNet人脸模型常量
* @author dwj
*/
public class FaceNetConstant {
/**
* 模型下载地址
*/
public static final String MODEL_URL = "https://resources.djl.ai/test-models/pytorch/face_feature.zip";
}

32
face/src/main/java/cn/smartjavaai/face/constant/LivenessConstant.java Normal file
View File

@@ -0,0 +1,32 @@
package cn.smartjavaai.face.constant;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import lombok.Data;
/**
* 活体检测常量
* @author dwj
*/
public class LivenessConstant {
/**
* 默认人脸清晰度阈值
*/
public static final float DEFAULT_FACE_CLARITY_THRESHOLD = 0.3F;
/**
* 默认活体阈值
*/
public static final float DEFAULT_REALITY_THRESHOLD = 0.8F;
/**
* 视频默认检测帧数
*/
public static final int DEFAULT_FRAME_COUNT = 10;
/**
* 视频默认最大检测帧数
*/
public static final int DEFAULT_MAX_VIDEO_DETECT_FRAMES = Integer.MAX_VALUE;
}

14
face/src/main/java/cn/smartjavaai/face/constant/MiniVisionConstant.java Normal file
View File

@@ -0,0 +1,14 @@
package cn.smartjavaai.face.constant;
/**
* MiniVision模型常量
* @author dwj
* @date 2025/7/3
*/
public class MiniVisionConstant {
/**
* 真人阈值
*/
public static final Float REALITY_THRESHOLD = 0.5f;
}

24
face/src/main/java/cn/smartjavaai/face/constant/RetinaFaceConstant.java Normal file
View File

@@ -0,0 +1,24 @@
package cn.smartjavaai.face.constant;
/**
* RetinaFace人脸检测模型常量
* @author dwj
* @date 2025/7/2
*/
public class RetinaFaceConstant {
/**
* 特征图层的基础缩放比例
*/
public static final int[][] scales = {{16, 32}, {64, 128}, {256, 512}};
/**
* 特征图相对于原图的采样步长
*/
public static final int[] steps = {8, 16, 32};
/**
* 缩放系数
*/
public static final double[] variance = {0.1f, 0.2f};
}

27
face/src/main/java/cn/smartjavaai/face/constant/UltraLightFastGenericFaceConstant.java Normal file
View File

@@ -0,0 +1,27 @@
package cn.smartjavaai.face.constant;
/**
* UltraLightFastGenericFace人脸检测模型常量
* @author dwj
*/
public class UltraLightFastGenericFaceConstant {
/**
* 特征图层的基础缩放比例
*/
public static final int[][] scales = {{10, 16, 24}, {32, 48}, {64, 96}, {128, 192, 256}};
/**
* 特征图相对于原图的采样步长
*/
public static final int[] steps = {8, 16, 32, 64};
/**
* 缩放系数
*/
public static final double[] variance = {0.1f, 0.2f};
/**
* 模型下载地址
*/
public static final String MODEL_URL = "https://resources.djl.ai/test-models/pytorch/ultranet.zip";
}

17
face/src/main/java/cn/smartjavaai/face/context/PredictorContext.java Normal file
View File

@@ -0,0 +1,17 @@
package cn.smartjavaai.face.context;
import com.seeta.sdk.*;
/**
* @author dwj
* @date 2025/5/8
*/
public class PredictorContext {
public GenderPredictor genderPredictor;
public AgePredictor agePredictor;
public EyeStateDetector eyeStateDetector;
public MaskDetector maskDetector;
public PoseEstimator poseEstimator;
}

189
face/src/main/java/cn/smartjavaai/face/dao/FaceDao.java Normal file
View File

@@ -0,0 +1,189 @@
package cn.smartjavaai.face.dao;
import cn.smartjavaai.face.sqllite.RowMapper;
import cn.smartjavaai.face.sqllite.SqliteHelper;
import cn.smartjavaai.face.utils.VectorUtils;
import cn.smartjavaai.face.vector.entity.FaceVector;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.collections4.CollectionUtils;
import java.lang.reflect.InvocationTargetException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
* 人脸库持久层
* @author dwj
*/
@Slf4j
public class FaceDao {
private static final String FACE_TABLE_NAME = "face";
private static final String SCHEMA_RESOURCE = "db/schema.sql";
private static final ConcurrentHashMap<String, FaceDao> INSTANCES = new ConcurrentHashMap<>();
private final String dbFilePath;
/**
* 获取FaceDao实例（单例模式）
* @param dbFilePath 数据库文件路径
* @return FaceDao实例
*/
public static FaceDao getInstance(String dbFilePath) {
return INSTANCES.computeIfAbsent(dbFilePath, path -> new FaceDao(path));
}
/**
* 私有构造函数
* @param dbFilePath 数据库文件路径
*/
private FaceDao(String dbFilePath) {
this.dbFilePath = dbFilePath;
try {
SqliteHelper sqliteHelper = SqliteHelper.getInstance(dbFilePath);
//自动创建数据库+表
sqliteHelper.initializeDatabase(FACE_TABLE_NAME, SCHEMA_RESOURCE);
} catch (SQLException e) {
throw new RuntimeException(e);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
/**
* 插入或更新人脸向量
* @param faceVector 人脸向量
* @throws SQLException SQL异常
* @throws ClassNotFoundException 类未找到异常
*/
public void insertOrUpdate(FaceVector faceVector) throws SQLException, ClassNotFoundException {
SqliteHelper sqliteHelper = SqliteHelper.getInstance(dbFilePath);
Map<String, Object> params = new HashMap<>();
params.put("id", faceVector.getId());
params.put("vector", VectorUtils.toByteArray(faceVector.getVector()));
params.put("metadata", faceVector.getMetadata());
sqliteHelper.executeInsertOrUpdate(FACE_TABLE_NAME, params);
}
/**
* 使用index查询key
* @param id 人脸ID
* @return 人脸向量
* @throws SQLException SQL异常
* @throws ClassNotFoundException 类未找到异常
*/
public FaceVector findById(String id) throws SQLException, ClassNotFoundException {
SqliteHelper sqliteHelper = SqliteHelper.getInstance(dbFilePath);
String sql = "select \"id\",\"vector\",\"metadata\" from " + FACE_TABLE_NAME + " where \"id\" = '" + id + "'";
List<FaceVector> faceVectors = sqliteHelper.executeQuery(sql, new RowMapper<FaceVector>() {
@Override
public FaceVector mapRow(ResultSet rs, int id) throws SQLException {
FaceVector face = new FaceVector();
face.setId(rs.getString("id"));
face.setVector(VectorUtils.toFloatArray(rs.getBytes("vector")));
face.setMetadata(rs.getString("metadata"));
return face;
}
});
return CollectionUtils.isNotEmpty(faceVectors) ? faceVectors.get(0) : null;
}
/**
* 删除全部
* @return 删除的行数
* @throws SQLException SQL异常
* @throws ClassNotFoundException 类未找到异常
*/
public long deleteAll() throws SQLException, ClassNotFoundException {
SqliteHelper sqliteHelper = SqliteHelper.getInstance(dbFilePath);
long rows = sqliteHelper.executeUpdate("delete from " + FACE_TABLE_NAME);
return rows;
}
/**
* 使用id数组查询
* @param ids ID数组
* @return 人脸向量列表
* @throws SQLException SQL异常
* @throws ClassNotFoundException 类未找到异常
*/
public List<FaceVector> findByIds(String... ids) throws SQLException, ClassNotFoundException {
// 使用 Stream API
String inKeys = Arrays.stream(ids)
.map(s -> "'" + s + "'")
.reduce((s1, s2) -> s1 + "," + s2)
.orElse("");
String sql = "select \"id\",\"vector\",\"metadata\" from " + FACE_TABLE_NAME + " where \"id\" in (" + inKeys + ")";
log.debug("sql：{}", sql);
SqliteHelper sqliteHelper = SqliteHelper.getInstance(dbFilePath);
List<FaceVector> faceVectors = sqliteHelper.executeQuery(sql, new RowMapper<FaceVector>() {
@Override
public FaceVector mapRow(ResultSet rs, int id) throws SQLException {
FaceVector face = new FaceVector();
face.setId(rs.getString("id"));
face.setVector(VectorUtils.toFloatArray(rs.getBytes("vector")));
face.setMetadata(rs.getString("metadata"));
return face;
}
});
return faceVectors;
}
/**
* 删除人脸
* @param ids ID数组
* @return 是否成功
* @throws SQLException SQL异常
* @throws ClassNotFoundException 类未找到异常
*/
public boolean deleteFace(String... ids) throws SQLException, ClassNotFoundException {
String inKeys = Arrays.stream(ids)
.map(s -> "'" + s + "'")
.reduce((s1, s2) -> s1 + "," + s2)
.orElse("");
SqliteHelper sqliteHelper = SqliteHelper.getInstance(dbFilePath);
String sql = "delete from " + FACE_TABLE_NAME + " where \"id\" in (" + inKeys + ")";
int rows = sqliteHelper.executeUpdate(sql);
log.debug("删除了{}行数据", rows);
return rows == ids.length;
}
/**
* 分页查询人脸
* @param pageNo 页码
* @param pageSize 每页大小
* @return 人脸向量列表
* @throws SQLException SQL异常
* @throws ClassNotFoundException 类未找到异常
*/
public List<FaceVector> findFace(int pageNo, int pageSize) throws SQLException, ClassNotFoundException {
long offset = (pageNo - 1) * pageSize;
String sql = "select \"id\",\"vector\",\"metadata\" from " + FACE_TABLE_NAME +
" limit " + offset + "," + pageSize;
SqliteHelper sqliteHelper = SqliteHelper.getInstance(dbFilePath);
return sqliteHelper.executeQuery(sql, new RowMapper<FaceVector>() {
@Override
public FaceVector mapRow(ResultSet rs, int id) throws SQLException {
FaceVector face = new FaceVector();
face.setId(rs.getString("id"));
face.setVector(VectorUtils.toFloatArray(rs.getBytes("vector")));
face.setMetadata(rs.getString("metadata"));
return face;
}
});
}
/**
* 关闭所有实例
*/
public static void closeAll() {
INSTANCES.clear();
log.debug("所有FaceDao实例已关闭");
}
}

29
face/src/main/java/cn/smartjavaai/face/entity/FaceQualityResult.java Normal file
View File

@@ -0,0 +1,29 @@
package cn.smartjavaai.face.entity;
import cn.smartjavaai.face.enums.QualityGrade;
import lombok.Data;
/**
* 质量评估结果
* @author dwj
* @date 2025/6/23
*/
@Data
public class FaceQualityResult {
/**
* 评估得分
*/
private float score;
private QualityGrade grade;
public FaceQualityResult() {
}
public FaceQualityResult(float score, QualityGrade grade) {
this.score = score;
this.grade = grade;
}
}

23
face/src/main/java/cn/smartjavaai/face/entity/FaceQualitySummary.java Normal file
View File

@@ -0,0 +1,23 @@
package cn.smartjavaai.face.entity;
import lombok.Data;
import java.util.Map;
/**
* 人脸质量检测汇总结果
* @author dwj
* @date 2025/6/27
*/
@Data
public class FaceQualitySummary {
private FaceQualityResult brightness; // 亮度
private FaceQualityResult clarity; // 清晰度
private FaceQualityResult completeness; // 完整度
private FaceQualityResult pose; // 姿态
private FaceQualityResult resolution; // 分辨率
private Map<String, Object> extraResults; // 额外检测结果
}

30
face/src/main/java/cn/smartjavaai/face/entity/FaceRegisterInfo.java Normal file
View File

@@ -0,0 +1,30 @@
package cn.smartjavaai.face.entity;
import lombok.Data;
/**
* 人脸注册信息
* @author dwj
* @date 2025/5/29
*/
@Data
public class FaceRegisterInfo {
/**
* 向量ID
*/
private String id;
/**
* 元数据，可以存储人脸相关的其他信息（JSON格式）
*/
private String metadata;
public FaceRegisterInfo(String id, String metadata) {
this.id = id;
this.metadata = metadata;
}
public FaceRegisterInfo() {
}
}

22
face/src/main/java/cn/smartjavaai/face/entity/FaceResult.java Normal file
View File

@@ -0,0 +1,22 @@
package cn.smartjavaai.face.entity;
import lombok.Data;
/**
* 人脸查询结果
* @author dwj
*/
@Data
public class FaceResult {
private String key;
private float similar;
public FaceResult() {
}
public FaceResult(String key, float similar) {
this.key = key;
this.similar = similar;
}
}

43
face/src/main/java/cn/smartjavaai/face/entity/FaceSearchParams.java Normal file
View File

@@ -0,0 +1,43 @@
package cn.smartjavaai.face.entity;
import lombok.Data;
/**
* 人脸查询参数
* @author dwj
* @date 2025/5/30
*/
@Data
public class FaceSearchParams {
/**
* 搜索结果数量
*/
private Integer topK = 1;
/**
* 搜索阈值
*/
private Float threshold;
/**
* 是否对查询结果进行归一化
*/
private Boolean normalizeSimilarity;
public FaceSearchParams() {
}
public FaceSearchParams(Integer topK, Float threshold) {
this.topK = topK;
this.threshold = threshold;
}
public FaceSearchParams(Integer topK, Float threshold, Boolean normalizeSimilarity) {
this.topK = topK;
this.threshold = threshold;
this.normalizeSimilarity = normalizeSimilarity;
}
}

38
face/src/main/java/cn/smartjavaai/face/enums/ExpressionModelEnum.java Normal file
View File

@@ -0,0 +1,38 @@
package cn.smartjavaai.face.enums;
/**
* 表情识别模型枚举
* @author dwj
*/
public enum ExpressionModelEnum {
DensNet121("DensNet121"),
FrEmotion("FrEmotion");
private final String modelClassName;
ExpressionModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
public String getModelClassName() {
return modelClassName;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static ExpressionModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (ExpressionModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

36
face/src/main/java/cn/smartjavaai/face/enums/FaceAttributeModelEnum.java Normal file
View File

@@ -0,0 +1,36 @@
package cn.smartjavaai.face.enums;
/**
* 人脸属性识别模型枚举
* @author dwj
* @date 2025/4/10
*/
public enum FaceAttributeModelEnum {
SEETA_FACE6_MODEL("SeetaFace6Model");
private final String modelClassName;
FaceAttributeModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
public String getModelClassName() {
return modelClassName;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static FaceAttributeModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (FaceAttributeModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

93
face/src/main/java/cn/smartjavaai/face/enums/FaceDetModelEnum.java Normal file
View File

@@ -0,0 +1,93 @@
package cn.smartjavaai.face.enums;
import lombok.Data;
/**
* 人脸检测模型枚举
* @author dwj
*/
public enum FaceDetModelEnum {
RETINA_FACE("PyTorch",0,0, "https://resources.djl.ai/test-models/pytorch/retinaface.zip"),
RETINA_FACE_ONNX("OnnxRuntime",0,0, null),
RETINA_FACE_640_ONNX("OnnxRuntime",640,640, null),
RETINA_FACE_320_ONNX("OnnxRuntime",320,320, null),
RETINA_FACE_720_1280_ONNX("OnnxRuntime",720,1280, null),
RETINA_FACE_MOBILE_ONNX("OnnxRuntime",0,0, null),
RETINA_FACE_MOBILE_320_ONNX("OnnxRuntime",320,320, null),
RETINA_FACE_MOBILE_640_ONNX("OnnxRuntime",640,640, null),
RETINA_FACE_MOBILE_720_1280_ONNX("OnnxRuntime",720,1080, null),
ULTRA_LIGHT_FAST_GENERIC_FACE("PyTorch",0,0, "https://resources.djl.ai/test-models/pytorch/ultranet.zip"),
SEETA_FACE6_MODEL(null,0,0, null),
YOLOV8_FACE("OnnxRuntime",640,640, null),
YOLOV5_FACE_640("OnnxRuntime", 640,640, null),
YOLOV5_FACE_320("OnnxRuntime",320,320, null),
SCRFD_160("OnnxRuntime",160,160, null),
SCRFD_320("OnnxRuntime",320,320, null),
SCRFD_640("OnnxRuntime",640,640, null),
SCRFD_1280("OnnxRuntime",1280,1280, null),
MTCNN("OnnxRuntime",1280,1280, null);
/**
* 模型输入尺寸：宽
*/
private final int inputWidth;
/**
* 模型输入尺寸：高
*/
private final int inputHeight;
/**
* 模型地址
*/
private final String modelUrl;
/**
* 模型引擎
*/
private final String engine;
FaceDetModelEnum(String engine, int inputWidth, int inputHeight, String modelUrl) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
this.modelUrl = modelUrl;
this.engine = engine;
}
public int getInputWidth() {
return inputWidth;
}
public int getInputHeight() {
return inputHeight;
}
public String getModelUrl() {
return modelUrl;
}
public String getEngine() {
return engine;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static FaceDetModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (FaceDetModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

40
face/src/main/java/cn/smartjavaai/face/enums/FaceRecModelEnum.java Normal file
View File

@@ -0,0 +1,40 @@
package cn.smartjavaai.face.enums;
/**
* 人脸识别模型枚举
* @author dwj
*/
public enum FaceRecModelEnum {
FACENET_MODEL("FaceNetModel"),
SEETA_FACE6_MODEL("SeetaFace6Model"),
SEETA_FACE6_LIGHT_MODEL("SeetaFace6Model"),
INSIGHT_FACE_IRSE50_MODEL("InsightFaceIRSE50Model"),
INSIGHT_FACE_MOBILE_FACENET_MODEL("InsightFaceMobilefacenetModel"),
ELASTIC_FACE_MODEL("ElasticFaceModel");
private final String modelClassName;
FaceRecModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
public String getModelClassName() {
return modelClassName;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static FaceRecModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (FaceRecModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

13
face/src/main/java/cn/smartjavaai/face/enums/IdStrategy.java Normal file
View File

@@ -0,0 +1,13 @@
package cn.smartjavaai.face.enums;
/**
* ID生成策略
* @author dwj
* @date 2025/5/29
*/
public enum IdStrategy {
AUTO, // 自动生成
CUSTOM // 用户自定义 ID（由 config.idValue 指定）
}

43
face/src/main/java/cn/smartjavaai/face/enums/LivenessModelEnum.java Normal file
View File

@@ -0,0 +1,43 @@
package cn.smartjavaai.face.enums;
/**
* 活体检测模型枚举
* @author dwj
* @date 2025/4/10
*/
public enum LivenessModelEnum {
// SeetaFace6
SEETA_FACE6_MODEL("SeetaFace6Model"),
// MiniVision
MINI_VISION_MODEL("MiniVisionModel"),
//阿里通义实验室
IIC_FL_MODEL("IicFlModel");
private final String modelClassName;
LivenessModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
public String getModelClassName() {
return modelClassName;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static LivenessModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (LivenessModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

14
face/src/main/java/cn/smartjavaai/face/enums/QualityGrade.java Normal file
View File

@@ -0,0 +1,14 @@
package cn.smartjavaai.face.enums;
/**
* 质量等级枚举
* @author dwj
* @date 2025/6/23
*/
public enum QualityGrade {
LOW,//Quality level is low
MEDIUM,//Quality level is medium
HIGH,//Quality level is high
}

36
face/src/main/java/cn/smartjavaai/face/enums/QualityModelEnum.java Normal file
View File

@@ -0,0 +1,36 @@
package cn.smartjavaai.face.enums;
/**
* 质量评估模型枚举
* @author dwj
* @date 2025/4/10
*/
public enum QualityModelEnum {
SEETA_FACE6_MODEL("SeetaFace6Model");
private final String modelClassName;
QualityModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
public String getModelClassName() {
return modelClassName;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static QualityModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (QualityModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

13
face/src/main/java/cn/smartjavaai/face/enums/SimilarityType.java Normal file
View File

@@ -0,0 +1,13 @@
package cn.smartjavaai.face.enums;
/**
* @author dwj
* @date 2025/5/31
*/
public enum SimilarityType {
IP, // 内积 (Inner Product)
L2, // 欧氏距离 (Euclidean Distance)
COSINE // 余弦相似度 (Cosine Similarity)
}

27
face/src/main/java/cn/smartjavaai/face/enums/VectorDBType.java Normal file
View File

@@ -0,0 +1,27 @@
package cn.smartjavaai.face.enums;
/**
* 向量数据库类型枚举
* @author dwj
* @date 2025/5/29
*/
public enum VectorDBType {
/**
* Sqlite,非专用向量库的备用方案
*/
SQLITE,
/**
* Milvus向量数据库
*/
MILVUS;
/**
* 未来可以添加其他向量数据库类型
*/
// FAISS,
// ELASTICSEARCH,
// PINECONE
}

30
face/src/main/java/cn/smartjavaai/face/exception/FaceException.java Normal file
View File

@@ -0,0 +1,30 @@
package cn.smartjavaai.face.exception;
/**
* 人脸检测异常
* @author dwj
* @date 2025/4/4
*/
public class FaceException extends RuntimeException{
public FaceException() {
super();
}
public FaceException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public FaceException(String message, Throwable cause) {
super(message, cause);
}
public FaceException(String message) {
super(message);
}
public FaceException(Throwable cause) {
super(cause);
}
}

104
face/src/main/java/cn/smartjavaai/face/factory/ExpressionModelFactory.java Normal file
View File

@@ -0,0 +1,104 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import cn.smartjavaai.face.enums.ExpressionModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.expression.CommonEmotionModel;
import cn.smartjavaai.face.model.expression.ExpressionModel;
import cn.smartjavaai.face.model.liveness.MiniVisionLivenessModel;
import cn.smartjavaai.face.model.liveness.Seetaface6LivenessModel;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 表情识别模型工厂
* @author dwj
*/
@Slf4j
public class ExpressionModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile ExpressionModelFactory instance;
/**
* 模型缓存
*/
private static final ConcurrentHashMap<ExpressionModelEnum, ExpressionModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<ExpressionModelEnum, Class<? extends ExpressionModel>> registry =
new ConcurrentHashMap<>();
public static ExpressionModelFactory getInstance() {
if (instance == null) {
synchronized (ExpressionModelFactory.class) {
if (instance == null) {
instance = new ExpressionModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param expressionModelEnum
* @param clazz
*/
private static void registerModel(ExpressionModelEnum expressionModelEnum, Class<? extends ExpressionModel> clazz) {
registry.put(expressionModelEnum, clazz);
}
/**
* 获取模型（通过配置）
* @param config
* @return
*/
public ExpressionModel getModel(FaceExpressionConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置活体检测模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private ExpressionModel createFaceModel(FaceExpressionConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
ExpressionModel model = null;
try {
model = (ExpressionModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
model.loadModel(config);
return model;
}
// 初始化默认算法
static {
registerModel(ExpressionModelEnum.DensNet121, CommonEmotionModel.class);
registerModel(ExpressionModelEnum.FrEmotion, CommonEmotionModel.class);
log.debug("缓存目录：{}", Config.getCachePath());
}
}

97
face/src/main/java/cn/smartjavaai/face/factory/FaceAttributeModelFactory.java Normal file
View File

@@ -0,0 +1,97 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.FaceAttributeConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.attribute.Seetaface6FaceAttributeModel;
import lombok.extern.slf4j.Slf4j;
import cn.smartjavaai.face.model.attribute.FaceAttributeModel;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 人脸属性检测模型工厂
* @author dwj
*/
@Slf4j
public class FaceAttributeModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile FaceAttributeModelFactory instance;
private static final ConcurrentHashMap<String, FaceAttributeModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<String, Class<? extends FaceAttributeModel>> registry =
new ConcurrentHashMap<>();
public static FaceAttributeModelFactory getInstance() {
if (instance == null) {
synchronized (FaceAttributeModelFactory.class) {
if (instance == null) {
instance = new FaceAttributeModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param name
* @param clazz
*/
private static void registerModel(String name, Class<? extends FaceAttributeModel> clazz) {
registry.put(name.toLowerCase(), clazz);
}
/**
* 获取模型（通过配置）
* @param config
* @return
*/
public FaceAttributeModel getModel(FaceAttributeConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置活体检测模型");
}
return modelMap.computeIfAbsent(config.getModelEnum().name(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建算法
* @param config
* @return
*/
private FaceAttributeModel createFaceModel(FaceAttributeConfig config) {
Class<?> clazz = registry.get(config.getModelEnum().getModelClassName().toLowerCase());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
FaceAttributeModel model = null;
try {
model = (FaceAttributeModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
model.loadModel(config);
return model;
}
// 初始化默认算法
static {
registerModel("seetaface6model", Seetaface6FaceAttributeModel.class);
log.debug("缓存目录：{}", Config.getCachePath());
}
}

133
face/src/main/java/cn/smartjavaai/face/factory/FaceDetModelFactory.java Normal file
View File

@@ -0,0 +1,133 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.facedect.CommonFaceDetModel;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel;
import cn.smartjavaai.face.model.facerec.*;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 人脸检测模型工厂
* @author dwj
*/
@Slf4j
public class FaceDetModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile FaceDetModelFactory instance;
private static final ConcurrentHashMap<FaceDetModelEnum, FaceDetModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<FaceDetModelEnum, Class<? extends FaceDetModel>> registry =
new ConcurrentHashMap<>();
public static FaceDetModelFactory getInstance() {
if (instance == null) {
synchronized (FaceDetModelFactory.class) {
if (instance == null) {
instance = new FaceDetModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param faceDetModelEnum
* @param clazz
*/
private static void registerAlgorithm(FaceDetModelEnum faceDetModelEnum, Class<? extends FaceDetModel> clazz) {
registry.put(faceDetModelEnum, clazz);
}
/**
* 获取模型（通过配置）
* @param config
* @return
*/
public FaceDetModel getModel(FaceDetConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置人脸模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
});
}
/**
* 获取默认模型
* @return
*/
public FaceDetModel getModel() {
// 初始化默认配置
FaceDetConfig config = new FaceDetConfig();
config.setModelEnum(FaceDetModelEnum.RETINA_FACE);
config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);
config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);
return getModel(config);
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private FaceDetModel createFaceDetModel(FaceDetConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
FaceDetModel algorithm = null;
try {
algorithm = (FaceDetModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
algorithm.loadModel(config);
return algorithm;
}
/**
* 获取轻量级人脸模型
* @return
*/
public FaceDetModel getLightFaceDetModel() {
// 初始化默认配置
FaceDetConfig config = new FaceDetConfig();
config.setModelEnum(FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);
config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);
config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);
return getModel(config);
}
// 初始化默认算法
static {
registerAlgorithm(FaceDetModelEnum.RETINA_FACE, CommonFaceDetModel.class);
registerAlgorithm(FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE, CommonFaceDetModel.class);
registerAlgorithm(FaceDetModelEnum.SEETA_FACE6_MODEL, SeetaFace6FaceDetModel.class);
registerAlgorithm(FaceDetModelEnum.YOLOV8_FACE, CommonFaceDetModel.class);
registerAlgorithm(FaceDetModelEnum.YOLOV5_FACE_640, CommonFaceDetModel.class);
registerAlgorithm(FaceDetModelEnum.YOLOV5_FACE_320, CommonFaceDetModel.class);
log.debug("缓存目录：{}", Config.getCachePath());
}
}

97
face/src/main/java/cn/smartjavaai/face/factory/FaceQualityModelFactory.java Normal file
View File

@@ -0,0 +1,97 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.QualityConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.quality.FaceQualityModel;
import cn.smartjavaai.face.model.quality.Seetaface6QualityModel;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 质量评估模型工厂
* @author dwj
*/
@Slf4j
public class FaceQualityModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile FaceQualityModelFactory instance;
private static final ConcurrentHashMap<String, FaceQualityModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<String, Class<? extends FaceQualityModel>> registry =
new ConcurrentHashMap<>();
public static FaceQualityModelFactory getInstance() {
if (instance == null) {
synchronized (FaceQualityModelFactory.class) {
if (instance == null) {
instance = new FaceQualityModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param name
* @param clazz
*/
private static void registerModel(String name, Class<? extends FaceQualityModel> clazz) {
registry.put(name.toLowerCase(), clazz);
}
/**
* 获取模型（通过配置）
* @param config
* @return
*/
public FaceQualityModel getModel(QualityConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置质量评估模型");
}
return modelMap.computeIfAbsent(config.getModelEnum().name(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private FaceQualityModel createFaceModel(QualityConfig config) {
Class<?> clazz = registry.get(config.getModelEnum().getModelClassName().toLowerCase());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
FaceQualityModel model = null;
try {
model = (FaceQualityModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
model.loadModel(config);
return model;
}
// 初始化默认算法
static {
registerModel("seetaface6model", Seetaface6QualityModel.class);
log.debug("缓存目录：{}", Config.getCachePath());
}
}

104
face/src/main/java/cn/smartjavaai/face/factory/FaceRecModelFactory.java Normal file
View File

@@ -0,0 +1,104 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.facerec.*;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 人脸检测识别模型工厂
* @author dwj
*/
@Slf4j
public class FaceRecModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile FaceRecModelFactory instance;
private static final ConcurrentHashMap<FaceRecModelEnum, FaceRecModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<FaceRecModelEnum, Class<? extends FaceRecModel>> registry =
new ConcurrentHashMap<>();
public static FaceRecModelFactory getInstance() {
if (instance == null) {
synchronized (FaceRecModelFactory.class) {
if (instance == null) {
instance = new FaceRecModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param recModelEnum
* @param clazz
*/
private static void registerAlgorithm(FaceRecModelEnum recModelEnum, Class<? extends FaceRecModel> clazz) {
registry.put(recModelEnum, clazz);
}
/**
* 获取模型（通过配置）
* @param config
* @return
*/
public FaceRecModel getModel(FaceRecConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置人脸模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private FaceRecModel createFaceModel(FaceRecConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new FaceException("Unsupported model");
}
FaceRecModel algorithm = null;
try {
algorithm = (FaceRecModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
algorithm.loadModel(config);
return algorithm;
}
// 初始化默认算法
static {
registerAlgorithm(FaceRecModelEnum.FACENET_MODEL, CommonFaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL, CommonFaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.INSIGHT_FACE_MOBILE_FACENET_MODEL, CommonFaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.ELASTIC_FACE_MODEL, CommonFaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.SEETA_FACE6_MODEL, SeetaFace6FaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.SEETA_FACE6_LIGHT_MODEL, SeetaFace6FaceRecModel.class);
log.debug("缓存目录：{}", Config.getCachePath());
}
}

102
face/src/main/java/cn/smartjavaai/face/factory/LivenessModelFactory.java Normal file
View File

@@ -0,0 +1,102 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.liveness.CommonLivenessModel;
import cn.smartjavaai.face.model.liveness.LivenessDetModel;
import cn.smartjavaai.face.model.liveness.MiniVisionLivenessModel;
import cn.smartjavaai.face.model.liveness.Seetaface6LivenessModel;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 活体检测模型工厂
* @author dwj
*/
@Slf4j
public class LivenessModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile LivenessModelFactory instance;
private static final ConcurrentHashMap<LivenessModelEnum, LivenessDetModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<LivenessModelEnum, Class<? extends LivenessDetModel>> registry =
new ConcurrentHashMap<>();
public static LivenessModelFactory getInstance() {
if (instance == null) {
synchronized (LivenessModelFactory.class) {
if (instance == null) {
instance = new LivenessModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param livenessModelEnum
* @param clazz
*/
private static void registerModel(LivenessModelEnum livenessModelEnum, Class<? extends LivenessDetModel> clazz) {
registry.put(livenessModelEnum, clazz);
}
/**
* 获取模型（通过配置）
* @param config
* @return
*/
public LivenessDetModel getModel(LivenessConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置活体检测模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private LivenessDetModel createFaceModel(LivenessConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
LivenessDetModel model = null;
try {
model = (LivenessDetModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
model.loadModel(config);
return model;
}
// 初始化默认算法
static {
registerModel(LivenessModelEnum.SEETA_FACE6_MODEL, Seetaface6LivenessModel.class);
registerModel(LivenessModelEnum.MINI_VISION_MODEL, MiniVisionLivenessModel.class);
registerModel(LivenessModelEnum.IIC_FL_MODEL, CommonLivenessModel.class);
log.debug("缓存目录：{}", Config.getCachePath());
}
}

179
face/src/main/java/cn/smartjavaai/face/model/attribute/FaceAttributeModel.java Normal file
View File

@@ -0,0 +1,179 @@
package cn.smartjavaai.face.model.attribute;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.face.FaceAttribute;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.face.config.FaceAttributeConfig;
import java.awt.image.BufferedImage;
import java.util.List;
/**
* 人脸属性识别模型
* @author dwj
*/
public interface FaceAttributeModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(FaceAttributeConfig config); // 加载模型
/**
* 人脸属性识别(多人脸)
* @param imagePath 图片路径
* @return
*/
default DetectionResponse detect(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(多人脸)
* @param image BufferedImage
* @return
*/
default DetectionResponse detect(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(多人脸)
* @param imageData 图片字节流
* @return
*/
default DetectionResponse detect(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(多人脸)
* @param imagePath 图片路径
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default List<FaceAttribute> detect(String imagePath, DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(单人脸)
* @param imagePath 图片路径
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default FaceAttribute detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(多人脸)
* @param imageData 图片数据
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default List<FaceAttribute> detect(byte[] imageData,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(单人脸)
* @param imageData
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default FaceAttribute detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(多人脸)
* @param image BufferedImage
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default List<FaceAttribute> detect(BufferedImage image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(单人脸)
* @param image BufferedImage
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default FaceAttribute detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(分数最高人脸)
* @param image
* @return
*/
default FaceAttribute detectTopFace(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(分数最高人脸)
* @param imagePath
* @return
*/
default FaceAttribute detectTopFace(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(分数最高人脸)
* @param imageData
* @return
*/
default FaceAttribute detectTopFace(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(裁剪后的人脸)
* @param image
* @return
*/
default FaceAttribute detectCropedFace(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(裁剪后的人脸)
* @param imagePath
* @return
*/
default FaceAttribute detectCropedFace(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸属性识别(裁剪后的人脸)
* @param imageData
* @return
*/
default FaceAttribute detectCropedFace(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

541
face/src/main/java/cn/smartjavaai/face/model/attribute/Seetaface6FaceAttributeModel.java Normal file
View File

@@ -0,0 +1,541 @@
package cn.smartjavaai.face.model.attribute;
import ai.djl.engine.Engine;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceAttribute;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.HeadPose;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.enums.face.EyeStatus;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.PoolUtils;
import cn.smartjavaai.face.config.FaceAttributeConfig;
import cn.smartjavaai.common.enums.face.GenderType;
import cn.smartjavaai.face.context.PredictorContext;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.pool.*;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* seetaface6 人脸属性识别模型
* @author dwj
* @date 2025/4/30
*/
@Slf4j
public class Seetaface6FaceAttributeModel implements FaceAttributeModel {
private FaceDetectorPool faceDetectorPool;
private GenderPredictorPool genderPredictorPool;
private FaceLandmarkerPool faceLandmarkerPool;
private AgePredictorPool agePredictorPool;
private EyeStateDetectorPool eyeStateDetectorPool;
private MaskDetectorPool maskDetectorPool;
private PoseEstimatorPool poseEstimatorPool;
private FaceAttributeConfig config;
@Override
public void loadModel(FaceAttributeConfig config) {
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
}
this.config = config;
//加载依赖库
NativeLoader.loadNativeLibraries(config.getDevice());
log.debug("Loading seetaFace6 library successfully.");
String[] faceDetectorModelPath = {config.getModelPath() + File.separator + "face_detector.csta"};
String[] faceLandmarkerModelPath = {config.getModelPath() + File.separator + "face_landmarker_pts5.csta"};
String[] genderPredictorModelPath = {config.getModelPath() + File.separator + "gender_predictor.csta"};
String[] agePredictorModelPath = {config.getModelPath() + File.separator + "age_predictor.csta"};
String[] eyeStateDetectorModelPath = {config.getModelPath() + File.separator + "eye_state.csta"};
String[] maskDetectorModelPath = {config.getModelPath() + File.separator + "mask_detector.csta"};
String[] poseEstimatorModelPath = {config.getModelPath() + File.separator + "pose_estimation.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = 0;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = config.getGpuId();
}
}
try {
SeetaModelSetting faceDetectorPoolSetting = new SeetaModelSetting(gpuId, faceDetectorModelPath, device);
SeetaConfSetting faceDetectorPoolConfSetting = new SeetaConfSetting(faceDetectorPoolSetting);
SeetaModelSetting faceLandmarkerPoolSetting = new SeetaModelSetting(gpuId, faceLandmarkerModelPath, device);
SeetaConfSetting faceLandmarkerPoolConfSetting = new SeetaConfSetting(faceLandmarkerPoolSetting);
SeetaModelSetting genderPredictorPoolSetting = new SeetaModelSetting(gpuId, genderPredictorModelPath, device);
SeetaConfSetting genderPredictorPoolConfSetting = new SeetaConfSetting(genderPredictorPoolSetting);
SeetaModelSetting agePredictorPoolSetting = new SeetaModelSetting(gpuId, agePredictorModelPath, device);
SeetaConfSetting agePredictorPoolConfSetting = new SeetaConfSetting(agePredictorPoolSetting);
SeetaModelSetting eyeStateDetectorPoolSetting = new SeetaModelSetting(gpuId, eyeStateDetectorModelPath, device);
SeetaConfSetting eyeStateDetectorPoolConfSetting = new SeetaConfSetting(eyeStateDetectorPoolSetting);
SeetaModelSetting maskDetectorPoolSetting = new SeetaModelSetting(gpuId, maskDetectorModelPath, device);
SeetaConfSetting maskDetectorPoolConfSetting = new SeetaConfSetting(maskDetectorPoolSetting);
SeetaModelSetting poseEstimatorPoolSetting = new SeetaModelSetting(gpuId, poseEstimatorModelPath, device);
SeetaConfSetting poseEstimatorPoolConfSetting = new SeetaConfSetting(poseEstimatorPoolSetting);
this.faceDetectorPool = new FaceDetectorPool(faceDetectorPoolConfSetting);
this.faceLandmarkerPool = new FaceLandmarkerPool(faceLandmarkerPoolConfSetting);
this.genderPredictorPool = new GenderPredictorPool(genderPredictorPoolConfSetting);
this.agePredictorPool = new AgePredictorPool(agePredictorPoolConfSetting);
this.eyeStateDetectorPool = new EyeStateDetectorPool(eyeStateDetectorPoolConfSetting);
this.maskDetectorPool = new MaskDetectorPool(maskDetectorPoolConfSetting);
this.poseEstimatorPool = new PoseEstimatorPool(poseEstimatorPoolConfSetting);
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
faceDetectorPool.setMaxTotal(predictorPoolSize);
faceLandmarkerPool.setMaxTotal(predictorPoolSize);
genderPredictorPool.setMaxTotal(predictorPoolSize);
agePredictorPool.setMaxTotal(predictorPoolSize);
eyeStateDetectorPool.setMaxTotal(predictorPoolSize);
maskDetectorPool.setMaxTotal(predictorPoolSize);
poseEstimatorPool.setMaxTotal(predictorPoolSize);
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
} catch (FileNotFoundException e) {
throw new FaceException(e);
}
}
@Override
public DetectionResponse detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public DetectionResponse detect(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public DetectionResponse detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
//创建推力器上下文
PredictorContext predictorContext = new PredictorContext();
FaceLandmarker faceLandmarker = null;
FaceDetector detectPredictor = null;
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
List<FaceAttribute> faceAttributeList = new ArrayList<FaceAttribute>();
try {
detectPredictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
predictorContext.genderPredictor = config.isEnableGender() ? genderPredictorPool.borrowObject() : null;
predictorContext.agePredictor = config.isEnableAge() ? agePredictorPool.borrowObject() : null;
predictorContext.maskDetector = config.isEnableMask() ? maskDetectorPool.borrowObject() : null;
predictorContext.eyeStateDetector = config.isEnableEyeStatus() ? eyeStateDetectorPool.borrowObject() : null;
predictorContext.poseEstimator = config.isEnableHeadPose() ? poseEstimatorPool.borrowObject() : null;
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
throw new FaceException("无人脸数据");
}
for(SeetaRect seetaRect : seetaResult){
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, landmarks);
seetaPointFSList.add(landmarks);
//人脸属性检测
FaceAttribute faceAttribute = detect(imageData, seetaRect, landmarks, predictorContext);
faceAttributeList.add(faceAttribute);
}
return FaceUtils.convertToFaceAttributeResponse(seetaResult, seetaPointFSList, faceAttributeList);
} catch (Exception e) {
throw new FaceException("人脸属性检测错误", e);
} finally {
// 统一归还所有 Predictor 到池
PoolUtils.returnToPool(faceDetectorPool, detectPredictor);
PoolUtils.returnToPool(faceLandmarkerPool, faceLandmarker);
PoolUtils.returnToPool(genderPredictorPool, predictorContext.genderPredictor);
PoolUtils.returnToPool(agePredictorPool, predictorContext.agePredictor);
PoolUtils.returnToPool(maskDetectorPool, predictorContext.maskDetector);
PoolUtils.returnToPool(eyeStateDetectorPool, predictorContext.eyeStateDetector);
PoolUtils.returnToPool(poseEstimatorPool, predictorContext.poseEstimator);
}
}
/**
* 单人脸属性检测
* @param imageData
* @param seetaRect
* @param landmarks
* @param predictorContext
* @return
*/
private FaceAttribute detect(SeetaImageData imageData, SeetaRect seetaRect, SeetaPointF[] landmarks, PredictorContext predictorContext){
FaceAttribute faceAttribute = new FaceAttribute();
//性别检测
GenderType genderType = null;
if (config.isEnableGender()){
GenderPredictor.GENDER[] gender = new GenderPredictor.GENDER[1];
boolean isSuccess = predictorContext.genderPredictor.PredictGenderWithCrop(imageData, landmarks, gender);
genderType = isSuccess ? FaceUtils.convertToGenderType(gender[0]) : GenderType.UNKNOWN;
}
//眼睛状态检测
EyeStatus leftEyeStatus = null;
EyeStatus rightEyeStatus = null;
if (config.isEnableEyeStatus()){
EyeStateDetector.EYE_STATE[] eyeState = predictorContext.eyeStateDetector.detect(imageData, landmarks);
leftEyeStatus = FaceUtils.convertToEyeStatus(eyeState[0]);
rightEyeStatus = FaceUtils.convertToEyeStatus(eyeState[1]);
}
//年龄检测
Integer age = 0;
if (config.isEnableAge()){
age = predictorContext.agePredictor.predictAgeWithCrop(imageData, landmarks);
}
//口罩检测
Boolean wearingMask = null;
if (config.isEnableMask()){
float[] score = new float[1];
wearingMask = predictorContext.maskDetector.detect(imageData, seetaRect, score);
}
//姿态检测
if (config.isEnableHeadPose()){
float[] yaw = new float[1];//左右转头（水平旋转）
float[] pitch = new float[1]; //上下抬头/低头（垂直旋转）
float[] roll = new float[1]; //头部左右倾斜（平面旋转）
predictorContext.poseEstimator.Estimate(imageData, seetaRect, yaw, pitch, roll);
faceAttribute.setHeadPose(new HeadPose(yaw[0], pitch[0], roll[0]));
}
faceAttribute.setGenderType(genderType);
faceAttribute.setAge(age);
faceAttribute.setLeftEyeStatus(leftEyeStatus);
faceAttribute.setRightEyeStatus(rightEyeStatus);
faceAttribute.setWearingMask(wearingMask);
return faceAttribute;
}
@Override
public List<FaceAttribute> detect(String imagePath, DetectionResponse faceDetectionResponse) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionResponse);
}
@Override
public List<FaceAttribute> detect(byte[] imageData, DetectionResponse faceDetectionResponse) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public List<FaceAttribute> detect(BufferedImage image, DetectionResponse faceDetectionResponse) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){
throw new FaceException("无人脸数据");
}
//创建推力器上下文
PredictorContext predictorContext = new PredictorContext();
FaceLandmarker faceLandmarker = null;
List<FaceAttribute> faceAttributeList = new ArrayList<FaceAttribute>();
try {
faceLandmarker = faceLandmarkerPool.borrowObject();
predictorContext.genderPredictor = config.isEnableGender() ? genderPredictorPool.borrowObject() : null;
predictorContext.agePredictor = config.isEnableAge() ? agePredictorPool.borrowObject() : null;
predictorContext.maskDetector = config.isEnableMask() ? maskDetectorPool.borrowObject() : null;
predictorContext.eyeStateDetector = config.isEnableEyeStatus() ? eyeStateDetectorPool.borrowObject() : null;
predictorContext.poseEstimator = config.isEnableHeadPose() ? poseEstimatorPool.borrowObject() : null;
for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(detectionInfo.getDetectionRectangle());
SeetaPointF[] landmarks = null;
FaceInfo faceInfo = detectionInfo.getFaceInfo();
//如果没有人脸标识，则提取人脸标识
if(faceInfo == null || faceInfo.getKeyPoints() == null || faceInfo.getKeyPoints().isEmpty()){
//提取人脸的5点人脸标识
landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, landmarks);
}else{
landmarks = FaceUtils.convertToSeetaPointF(faceInfo.getKeyPoints());
}
//人脸属性检测
FaceAttribute faceAttribute = detect(imageData, seetaRect, landmarks, predictorContext);
faceAttributeList.add(faceAttribute);
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
PoolUtils.returnToPool(genderPredictorPool, predictorContext.genderPredictor);
PoolUtils.returnToPool(agePredictorPool, predictorContext.agePredictor);
PoolUtils.returnToPool(maskDetectorPool, predictorContext.maskDetector);
PoolUtils.returnToPool(eyeStateDetectorPool, predictorContext.eyeStateDetector);
PoolUtils.returnToPool(poseEstimatorPool, predictorContext.poseEstimator);
}
return faceAttributeList;
}
@Override
public FaceAttribute detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionRectangle, keyPoints);
}
@Override
public FaceAttribute detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public FaceAttribute detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
if(Objects.isNull(faceDetectionRectangle)){
throw new FaceException("无人脸数据");
}
//创建推力器上下文
PredictorContext predictorContext = new PredictorContext();
try {
predictorContext.genderPredictor = config.isEnableGender() ? genderPredictorPool.borrowObject() : null;
predictorContext.agePredictor = config.isEnableAge() ? agePredictorPool.borrowObject() : null;
predictorContext.maskDetector = config.isEnableMask() ? maskDetectorPool.borrowObject() : null;
predictorContext.eyeStateDetector = config.isEnableEyeStatus() ? eyeStateDetectorPool.borrowObject() : null;
predictorContext.poseEstimator = config.isEnableHeadPose() ? poseEstimatorPool.borrowObject() : null;
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(faceDetectionRectangle);
SeetaPointF[] landmarks = null;
if(keyPoints == null || keyPoints.isEmpty()){
throw new FaceException("人脸关键点keyPoints为空");
}
landmarks = FaceUtils.convertToSeetaPointF(keyPoints);
//人脸属性检测
FaceAttribute faceAttribute = detect(imageData, seetaRect, landmarks, predictorContext);
return faceAttribute;
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
PoolUtils.returnToPool(genderPredictorPool, predictorContext.genderPredictor);
PoolUtils.returnToPool(agePredictorPool, predictorContext.agePredictor);
PoolUtils.returnToPool(maskDetectorPool, predictorContext.maskDetector);
PoolUtils.returnToPool(eyeStateDetectorPool, predictorContext.eyeStateDetector);
PoolUtils.returnToPool(poseEstimatorPool, predictorContext.poseEstimator);
}
}
@Override
public FaceAttribute detectTopFace(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detectTopFace(image);
}
@Override
public FaceAttribute detectTopFace(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public FaceAttribute detectTopFace(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
FaceLandmarker faceLandmarker = null;
FaceDetector detectPredictor = null;
//创建推力器上下文
PredictorContext predictorContext = new PredictorContext();
try {
detectPredictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
predictorContext.genderPredictor = config.isEnableGender() ? genderPredictorPool.borrowObject() : null;
predictorContext.agePredictor = config.isEnableAge() ? agePredictorPool.borrowObject() : null;
predictorContext.maskDetector = config.isEnableMask() ? maskDetectorPool.borrowObject() : null;
predictorContext.eyeStateDetector = config.isEnableEyeStatus() ? eyeStateDetectorPool.borrowObject() : null;
predictorContext.poseEstimator = config.isEnableHeadPose() ? poseEstimatorPool.borrowObject() : null;
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
throw new FaceException("无人脸数据");
}
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], landmarks);
//人脸属性检测
FaceAttribute faceAttribute = detect(imageData, seetaResult[0], landmarks, predictorContext);
return faceAttribute;
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
if (detectPredictor != null) {
try {
faceDetectorPool.returnObject(detectPredictor);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
PoolUtils.returnToPool(genderPredictorPool, predictorContext.genderPredictor);
PoolUtils.returnToPool(agePredictorPool, predictorContext.agePredictor);
PoolUtils.returnToPool(maskDetectorPool, predictorContext.maskDetector);
PoolUtils.returnToPool(eyeStateDetectorPool, predictorContext.eyeStateDetector);
PoolUtils.returnToPool(poseEstimatorPool, predictorContext.poseEstimator);
}
}
public FaceDetectorPool getFaceDetectorPool() {
return faceDetectorPool;
}
public GenderPredictorPool getGenderPredictorPool() {
return genderPredictorPool;
}
public FaceLandmarkerPool getFaceLandmarkerPool() {
return faceLandmarkerPool;
}
public AgePredictorPool getAgePredictorPool() {
return agePredictorPool;
}
public EyeStateDetectorPool getEyeStateDetectorPool() {
return eyeStateDetectorPool;
}
public MaskDetectorPool getMaskDetectorPool() {
return maskDetectorPool;
}
public PoseEstimatorPool getPoseEstimatorPool() {
return poseEstimatorPool;
}
@Override
public void close() throws Exception {
if(Objects.nonNull(faceDetectorPool)){
faceDetectorPool.close();
}
if(Objects.nonNull(genderPredictorPool)){
genderPredictorPool.close();
}
if(Objects.nonNull(faceLandmarkerPool)){
faceLandmarkerPool.close();
}
if(Objects.nonNull(agePredictorPool)){
agePredictorPool.close();
}
if(Objects.nonNull(eyeStateDetectorPool)){
eyeStateDetectorPool.close();
}
if(Objects.nonNull(maskDetectorPool)){
maskDetectorPool.close();
}
if(Objects.nonNull(poseEstimatorPool)){
poseEstimatorPool.close();
}
}
}

380
face/src/main/java/cn/smartjavaai/face/model/expression/CommonEmotionModel.java Normal file
View File

@@ -0,0 +1,380 @@
package cn.smartjavaai.face.model.expression;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.ndarray.NDManager;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.ExpressionResult;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.enums.face.FacialExpression;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.expression.criterial.EmotionCriteriaFactory;
import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator;
import cn.smartjavaai.face.preprocess.DJLImagePreprocessor;
import cn.smartjavaai.face.utils.FaceUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.opencv.core.Mat;
import org.opencv.face.Face;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* 通用人脸表情识别模型
* @author dwj
*/
@Slf4j
public class CommonEmotionModel implements ExpressionModel{
private FaceExpressionConfig config;
private ZooModel<Image, Classifications> model;
private GenericObjectPool<Predictor<Image, Classifications>> predictorPool;
@Override
public void loadModel(FaceExpressionConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath为空");
}
this.config = config;
Criteria<Image, Classifications> criteria = EmotionCriteriaFactory.createCriteria(config);
try {
model = criteria.loadModel();
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
predictorPool.setMaxTotal(predictorPoolSize);
log.debug("当前设备: " + model.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("DenseNetEmotionModel模型加载失败", e);
}
}
public Classifications detectCore(Image image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
Predictor<Image, Classifications> predictor = null;
try (NDManager manager = model.getNDManager().newSubManager()){
predictor = predictorPool.borrowObject();
DJLImagePreprocessor imagePreprocessor = new DJLImagePreprocessor(image, manager);
Image faceImg = image;
if(config.isAlign()){
//仿射变换
faceImg = imagePreprocessor.enableAffine(FaceUtils.facePoints(keyPoints), 512, 512)
.process();
return predictor.predict(faceImg);
}else{
if(config.isCropFace()){
//裁剪
faceImg = imagePreprocessor.enableCrop(faceDetectionRectangle)
.process();
}
}
return predictor.predict(faceImg);
} catch (Exception e) {
throw new FaceException("表情识别异常", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
@Override
public R<DetectionResponse> detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(Objects.isNull(config.getDetectModel())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型");
}
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
Classifications classifications = detectCore(djlImage, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints());
Classifications.Classification bestClass = classifications.best();
FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName());
ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability());
result.setClassifications(classifications);
faceInfo.setExpressionResult(result);
}
((Mat)djlImage.getWrappedImage()).release();
return faceDetectionResponse;
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<List<ExpressionResult>> detect(String imagePath, DetectionResponse faceDetectionResponse) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionResponse);
}
@Override
public R<List<ExpressionResult>> detect(byte[] imageData, DetectionResponse faceDetectionResponse) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<List<ExpressionResult>> detect(BufferedImage image, DetectionResponse faceDetectionResponse) {
if(!ImageUtils.isImageValid(image)){
R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){
R.fail(R.Status.NO_FACE_DETECTED);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
List<ExpressionResult> expressionResults = new ArrayList<>();
for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
Classifications classifications = detectCore(djlImage, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints());
Classifications.Classification bestClass = classifications.best();
FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName());
ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability());
result.setClassifications(classifications);
expressionResults.add(result);
}
((Mat)djlImage.getWrappedImage()).release();
return R.ok(expressionResults);
}
@Override
public R<List<ExpressionResult>> detectBase64(String base64Image, DetectionResponse faceDetectionResponse) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData, faceDetectionResponse);
}
@Override
public R<ExpressionResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionRectangle, keyPoints);
}
@Override
public R<ExpressionResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<ExpressionResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
Classifications classifications = detectCore(djlImage, faceDetectionRectangle, keyPoints);
Classifications.Classification bestClass = classifications.best();
FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName());
ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability());
result.setClassifications(classifications);
((Mat)djlImage.getWrappedImage()).release();
return R.ok(result);
}
@Override
public R<ExpressionResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData, faceDetectionRectangle, keyPoints);
}
@Override
public R<ExpressionResult> detectTopFace(BufferedImage image) {
if(Objects.isNull(config.getDetectModel())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型");
}
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
DetectionInfo detectionInfo = faceDetectionResponse.getData().getDetectionInfoList().get(0);
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
return detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints());
}
@Override
public R<ExpressionResult> detectTopFace(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detectTopFace(image);
}
@Override
public R<ExpressionResult> detectTopFace(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<ExpressionResult> detectTopFaceBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detectTopFace(imageData);
}
@Override
public GenericObjectPool<Predictor<Image, Classifications>> getPool() {
return predictorPool;
}
@Override
public void close() {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
}

203
face/src/main/java/cn/smartjavaai/face/model/expression/ExpressionModel.java Normal file
View File

@@ -0,0 +1,203 @@
package cn.smartjavaai.face.model.expression;
import ai.djl.inference.Predictor;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.entity.face.ExpressionResult;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import org.apache.commons.pool2.impl.GenericObjectPool;
import java.awt.image.BufferedImage;
import java.util.List;
/**
* @author dwj
* @date 2025/7/1
*/
public interface ExpressionModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(FaceExpressionConfig config); // 加载模型
/**
* 表情识别(多人脸)
* @param imagePath 图片路径
* @return
*/
default R<DetectionResponse> detect(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param image BufferedImage
* @return
*/
default R<DetectionResponse> detect(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param imageData 图片字节流
* @return
*/
default R<DetectionResponse> detect(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param base64Image
* @return
*/
default R<DetectionResponse> detectBase64(String base64Image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param imagePath 图片路径
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<ExpressionResult>> detect(String imagePath, DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param imageData 图片数据
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<ExpressionResult>> detect(byte[] imageData,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param image BufferedImage
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<ExpressionResult>> detect(BufferedImage image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param base64Image
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<ExpressionResult>> detectBase64(String base64Image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(单人脸)
* @param imagePath 图片路径
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<ExpressionResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(单人脸)
* @param imageData
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<ExpressionResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(单人脸)
* @param image BufferedImage
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<ExpressionResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(单人脸)
* @param base64Image
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<ExpressionResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(分数最高人脸)
* @param image
* @return
*/
default R<ExpressionResult> detectTopFace(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(分数最高人脸)
* @param imagePath
* @return
*/
default R<ExpressionResult> detectTopFace(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(分数最高人脸)
* @param imageData
* @return
*/
default R<ExpressionResult> detectTopFace(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(分数最高人脸)
* @param base64Image
* @return
*/
default R<ExpressionResult> detectTopFaceBase64(String base64Image){
throw new UnsupportedOperationException("默认不支持该功能");
}
default GenericObjectPool<Predictor<Image, Classifications>> getPool(){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

59
face/src/main/java/cn/smartjavaai/face/model/expression/criterial/EmotionCriteriaFactory.java Normal file
View File

@@ -0,0 +1,59 @@
package cn.smartjavaai.face.model.expression.criterial;
import ai.djl.Device;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import cn.smartjavaai.face.enums.ExpressionModelEnum;
import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator;
import cn.smartjavaai.face.model.expression.translator.FrEmotionTranslator;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.Objects;
/**
* 人脸表情识别 Criteria构建工厂
* @author dwj
* @date 2025/5/14
*/
public class EmotionCriteriaFactory {
public static Criteria<Image, Classifications> createCriteria(FaceExpressionConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId());
}
Criteria<Image, Classifications> criteria = null;
if(config.getModelEnum() == ExpressionModelEnum.DensNet121){
//开源项目地址：https://github.com/sajjjadayobi/FaceLib
//初始化 检测Criteria
criteria =
Criteria.builder()
.optEngine("PyTorch")
.setTypes(Image.class, Classifications.class)
.optModelPath(Paths.get(config.getModelPath()))
.optTranslator(new DenseNetEmotionTranslator(224))
.optProgress(new ProgressBar())
.optDevice(device)
.build();
}else if (config.getModelEnum() == ExpressionModelEnum.FrEmotion){
//初始化 检测Criteria
criteria =
Criteria.builder()
.optEngine("OnnxRuntime")
.setTypes(ai.djl.modality.cv.Image.class, Classifications.class)
.optModelPath(Paths.get(config.getModelPath()))
.optTranslator(new FrEmotionTranslator(224))
.optProgress(new ProgressBar())
.optDevice(device)
.build();
}
return criteria;
}
}

63
face/src/main/java/cn/smartjavaai/face/model/expression/translator/DenseNetEmotionTranslator.java Normal file
View File

@@ -0,0 +1,63 @@
package cn.smartjavaai.face.model.expression.translator;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
import java.util.Arrays;
import java.util.List;
/**
* @author dwj
* @date 2025/6/30
*/
public class DenseNetEmotionTranslator implements Translator<Image, Classifications> {
private final List<String> labels = Arrays.asList("angry", "disgust", "fear", "happy", "sad", "surprise", "neutral");
private int imageSize = 224;
public DenseNetEmotionTranslator(int imageSize) {
this.imageSize = imageSize;
}
@Override
public Classifications processOutput(TranslatorContext ctx, NDList list) {
NDArray output = list.singletonOrThrow();
output = output.softmax(1);
return new Classifications(labels, output);
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
// 直接转换为灰度NDArray
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
// 调整大小
Shape shape = array.getShape();
long height = shape.get(0);
long width = shape.get(1);
if (height != imageSize || width != imageSize) {
array = NDImageUtils.resize(array, imageSize, imageSize);
}
array = NDImageUtils.resize(array, imageSize, imageSize);
array = array.transpose(2, 0, 1);
array = array.expandDims(0);
// 归一化
array = array.toType(DataType.FLOAT32, false).div(255.0f);
return new NDList(array);
}
@Override
public Batchifier getBatchifier() {
return null;
}
}

61
face/src/main/java/cn/smartjavaai/face/model/expression/translator/FrEmotionTranslator.java Normal file
View File

@@ -0,0 +1,61 @@
package cn.smartjavaai.face.model.expression.translator;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
import java.util.Arrays;
import java.util.List;
/**
* @author dwj
* @date 2025/6/30
*/
public class FrEmotionTranslator implements Translator<Image, Classifications> {
private final List<String> labels = Arrays.asList("angry", "disgust", "fear", "happy", "sad", "surprise", "neutral");
private int imageSize = 224;
public FrEmotionTranslator(int imageSize) {
this.imageSize = imageSize;
}
@Override
public Classifications processOutput(TranslatorContext ctx, NDList list) {
NDArray output = list.singletonOrThrow();
output = output.softmax(1);
return new Classifications(labels, output);
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
// 调整大小
Shape shape = array.getShape();
long height = shape.get(0);
long width = shape.get(1);
if (height != imageSize || width != imageSize) {
array = NDImageUtils.resize(array, imageSize, imageSize);
}
array = array.transpose(2, 0, 1); // 变成 (3, 224, 224)
array = array.expandDims(0);
// 归一化
array = array.toType(DataType.FLOAT32, false).div(255.0f);
return new NDList(array);
}
@Override
public Batchifier getBatchifier() {
return null;
}
}

270
face/src/main/java/cn/smartjavaai/face/model/facedect/CommonFaceDetModel.java Normal file
View File

@@ -0,0 +1,270 @@
package cn.smartjavaai.face.model.facedect;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.facedect.criterial.FaceDetCriteriaFactory;
import cn.smartjavaai.face.utils.FaceUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.opencv.core.Mat;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
/**
* DJL通用人脸检测模型实现
* @author dwj
*/
@Slf4j
public class CommonFaceDetModel implements FaceDetModel{
private GenericObjectPool<Predictor<Image, DetectedObjects>> predictorPool;
private ZooModel<Image, DetectedObjects> model;
/**
* 加载模型
* @param config
*/
@Override
public void loadModel(FaceDetConfig config){
Criteria<Image, DetectedObjects> criteria = FaceDetCriteriaFactory.createCriteria(config);
try {
model = criteria.loadModel();
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
predictorPool.setMaxTotal(predictorPoolSize);
log.debug("当前设备: " + model.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("人脸检测模型加载失败", e);
}
}
/**
* 检测人脸
* @param imagePath 图片路径
* @return
* @throws Exception
*/
@Override
public R<DetectionResponse> detect(String imagePath){
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
Image img = null;
try {
img = ImageFactory.getInstance().fromFile(Paths.get(imagePath));
DetectedObjects detection = detect(img);
return R.ok(FaceUtils.convertToDetectionResponse(detection,img));
} catch (IOException e) {
throw new FaceException("无效的图片", e);
} finally {
if (img != null) {
((Mat)img.getWrappedImage()).release();
}
}
}
/**
* 检测人脸
* @param imageInputStream 图片流
* @return
* @throws Exception
*/
@Override
public R<DetectionResponse> detect(InputStream imageInputStream){
if(Objects.isNull(imageInputStream)){
return R.fail(R.Status.INVALID_IMAGE);
}
Image img = null;
try {
img = ImageFactory.getInstance().fromInputStream(imageInputStream);
DetectedObjects detection = detect(img);
return R.ok(FaceUtils.convertToDetectionResponse(detection,img));
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
} finally {
if (img != null) {
((Mat)img.getWrappedImage()).release();
}
}
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
Image img = null;
try {
img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
DetectedObjects detection = detect(img);
return R.ok(FaceUtils.convertToDetectionResponse(detection,img));
} catch (Exception e) {
throw new FaceException(e);
} finally {
if (img != null) {
((Mat)img.getWrappedImage()).release();
}
}
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
return detect(new ByteArrayInputStream(imageData));
}
@Override
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<Void> detectAndDraw(String imagePath, String outputPath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
Image img = null;
try {
img = ImageFactory.getInstance().fromFile(Paths.get(imagePath));
DetectedObjects detectedObjects = detect(img);
if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
img.drawBoundingBoxes(detectedObjects);
Path output = Paths.get(outputPath);
log.debug("Saving to {}", output.toAbsolutePath().toString());
img.save(Files.newOutputStream(output), "png");
return R.ok();
} catch (IOException e) {
throw new FaceException(e);
} finally {
if (img != null){
((Mat)img.getWrappedImage()).release();
}
}
}
@Override
public R<BufferedImage> detectAndDraw(BufferedImage sourceImage) {
if(!ImageUtils.isImageValid(sourceImage)){
return R.fail(R.Status.INVALID_IMAGE);
}
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(sourceImage));
DetectedObjects detectedObjects = detect(img);
if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
img.drawBoundingBoxes(detectedObjects);
try {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
// 调用 save 方法将 Image 写入字节流
img.save(outputStream, "png");
// 将字节流转换为 BufferedImage
byte[] imageBytes = outputStream.toByteArray();
return R.ok(ImageIO.read(new ByteArrayInputStream(imageBytes)));
} catch (IOException e) {
throw new FaceException("导出图片失败", e);
} finally {
if (img != null){
((Mat)img.getWrappedImage()).release();
}
}
}
/**
* 人脸检测
* @param image
* @return
*/
public DetectedObjects detect(Image image){
Predictor<Image, DetectedObjects> predictor = null;
try {
predictor = predictorPool.borrowObject();
return predictor.predict(image);
} catch (Exception e) {
throw new FaceException("人脸检测错误", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
@Override
public GenericObjectPool<Predictor<Image, DetectedObjects>> getPool() {
return predictorPool;
}
@Override
public void close() {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
}

96
face/src/main/java/cn/smartjavaai/face/model/facedect/FaceDetModel.java Normal file
View File

@@ -0,0 +1,96 @@
package cn.smartjavaai.face.model.facedect;
import ai.djl.inference.Predictor;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.config.FaceDetConfig;
import org.apache.commons.pool2.impl.GenericObjectPool;
import java.awt.image.BufferedImage;
import java.io.InputStream;
/**
* 人脸检测模型
* @author dwj
*/
public interface FaceDetModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(FaceDetConfig config); // 加载模型
/**
* 人脸检测
* @param imagePath 图片路径
* @return
*/
default R<DetectionResponse> detect(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param imageInputStream 图片输入流
* @return
*/
default R<DetectionResponse> detect(InputStream imageInputStream){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param image BufferedImage
* @return
*/
default R<DetectionResponse> detect(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param imageData
* @return
*/
default R<DetectionResponse> detect(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param base64Image
* @return
*/
default R<DetectionResponse> detectBase64(String base64Image) {
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 检测并绘制人脸
* @param imagePath 图片输入路径（包含文件名称）
* @param outputPath 图片输出路径（包含文件名称）
*/
default R<Void> detectAndDraw(String imagePath, String outputPath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 检测并绘制人脸
* @param sourceImage
* @return
*/
default R<BufferedImage> detectAndDraw(BufferedImage sourceImage){
throw new UnsupportedOperationException("默认不支持该功能");
}
default GenericObjectPool<Predictor<Image, DetectedObjects>> getPool(){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

399
face/src/main/java/cn/smartjavaai/face/model/facedect/MtcnnFaceDetModel.java Normal file
View File

@@ -0,0 +1,399 @@
package cn.smartjavaai.face.model.facedect;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.modality.cv.output.BoundingBox;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import ai.djl.translate.NoopTranslator;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.facedect.criterial.FaceDetCriteriaFactory;
import cn.smartjavaai.face.model.facedect.mtcnn.*;
import cn.smartjavaai.face.utils.FaceUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.opencv.core.Mat;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
/**
* MTCNN 人脸检测模型实现
* @author dwj
*/
@Slf4j
public class MtcnnFaceDetModel implements FaceDetModel{
public ZooModel<NDList, NDList> pNetModel;
public ZooModel<NDList, NDList> rNetModel;
public ZooModel<NDList, NDList> oNetModel;
private GenericObjectPool<Predictor<NDList, NDList>> pnetPredictorPool;
private GenericObjectPool<Predictor<NDList, NDList>> rnetPredictorPool;
private GenericObjectPool<Predictor<NDList, NDList>> onetPredictorPool;
/**
* 加载模型
* @param config
*/
@Override
public void loadModel(FaceDetConfig config){
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
}
Path modelPath = Paths.get(config.getModelPath());
if(!Files.isDirectory(modelPath)){
throw new FaceException("MTCNN 模型需要指定存放模型文件的目录路径");
}
try {
Path pnetPath = modelPath.resolve("pnet_script.pt");
Path rnetPath = modelPath.resolve("rnet_script.pt");
Path onetPath = modelPath.resolve("onet_script.pt");
pNetModel = getModel(pnetPath);
rNetModel = getModel(pnetPath);
oNetModel = getModel(pnetPath);
this.pnetPredictorPool = new GenericObjectPool<>(new PredictorFactory<>(pNetModel));
this.rnetPredictorPool = new GenericObjectPool<>(new PredictorFactory<>(rNetModel));
this.onetPredictorPool = new GenericObjectPool<>(new PredictorFactory<>(oNetModel));
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
pnetPredictorPool.setMaxTotal(predictorPoolSize);
rnetPredictorPool.setMaxTotal(predictorPoolSize);
onetPredictorPool.setMaxTotal(predictorPoolSize);
log.debug("当前设备: " + pNetModel.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("mtcnn人脸检测模型加载失败", e);
}
}
/**
* 加载模型
* @param modelPath
* @throws ModelNotFoundException
* @throws MalformedModelException
* @throws IOException
*/
public ZooModel<NDList, NDList> getModel(Path modelPath) throws ModelNotFoundException, MalformedModelException, IOException {
Criteria<NDList, NDList> criteria =
Criteria.builder()
.setTypes(NDList.class, NDList.class)
.optTranslator(new NoopTranslator())
.optEngine("PyTorch")
.optModelPath(modelPath)
.optProgress(new ProgressBar())
.build();
return criteria.loadModel();
}
/**
* 检测人脸
* @param imagePath 图片路径
* @return
* @throws Exception
*/
@Override
public R<DetectionResponse> detect(String imagePath){
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
Image img = null;
try {
img = ImageFactory.getInstance().fromFile(Paths.get(imagePath));
return detect(img);
} catch (IOException e) {
throw new FaceException("无效的图片", e);
} finally {
if (img != null) {
((Mat)img.getWrappedImage()).release();
}
}
}
/**
* 检测人脸
* @param imageInputStream 图片流
* @return
* @throws Exception
*/
@Override
public R<DetectionResponse> detect(InputStream imageInputStream){
if(Objects.isNull(imageInputStream)){
return R.fail(R.Status.INVALID_IMAGE);
}
Image img = null;
try {
img = ImageFactory.getInstance().fromInputStream(imageInputStream);
return detect(img);
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
} finally {
if (img != null) {
((Mat)img.getWrappedImage()).release();
}
}
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
Image img = null;
try {
img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
return detect(img);
} catch (Exception e) {
throw new FaceException(e);
} finally {
if (img != null) {
((Mat)img.getWrappedImage()).release();
}
}
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
return detect(new ByteArrayInputStream(imageData));
}
@Override
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<Void> detectAndDraw(String imagePath, String outputPath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
Image img = null;
try {
img = ImageFactory.getInstance().fromFile(Paths.get(imagePath));
R<DetectionResponse> detectionResponseR = detect(img);
if(!detectionResponseR.isSuccess()){
return R.fail(detectionResponseR.getCode(), detectionResponseR.getMessage());
}
if(Objects.isNull(detectionResponseR.getData()) ||
CollectionUtils.isEmpty(detectionResponseR.getData().getDetectionInfoList())){
return R.fail(R.Status.NO_FACE_DETECTED);
}
BufferedImage sourceImage = OpenCVUtils.mat2Image((Mat)img.getWrappedImage());
FaceUtils.drawBoundingBoxes(sourceImage, detectionResponseR.getData(), outputPath);
return R.ok();
} catch (IOException e) {
throw new FaceException(e);
} finally {
if (img != null){
((Mat)img.getWrappedImage()).release();
}
}
}
@Override
public R<BufferedImage> detectAndDraw(BufferedImage sourceImage) {
if(!ImageUtils.isImageValid(sourceImage)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
R<DetectionResponse> detectionResponseR = detect(sourceImage);
if(!detectionResponseR.isSuccess()){
return R.fail(detectionResponseR.getCode(), detectionResponseR.getMessage());
}
if(Objects.isNull(detectionResponseR.getData()) ||
CollectionUtils.isEmpty(detectionResponseR.getData().getDetectionInfoList())){
return R.fail(R.Status.NO_FACE_DETECTED);
}
return R.ok(FaceUtils.drawBoundingBoxes(sourceImage, detectionResponseR.getData()));
} catch (IOException e) {
throw new FaceException("导出图片失败", e);
}
}
/**
* 人脸检测
* @param image
* @return
*/
public R<DetectionResponse> detect(Image image){
try (NDManager manager = NDManager.newBaseManager(pNetModel.getNDManager().getDevice())){
List<Double> scales = MtcnnProcess.generateScales(image);
NDArray imgs = MtcnnProcess.processInput(manager, image);
int h = image.getHeight();
int w = image.getWidth();
NDList outputPnet = PNetModel.firstStage(manager, pnetPredictorPool.borrowObject(), imgs, scales, w, h);
NDArray boxes = outputPnet.get(0);
NDArray image_inds = outputPnet.get(1);
NDList pad = MtcnnUtils.pad(boxes, w, h);
NDList outputRnet = RNetModel.secondStage(manager, rnetPredictorPool.borrowObject(), imgs,boxes,pad, image_inds);
NDArray image_indsFiltered = outputRnet.get(0);
NDArray scoresFiltered = outputRnet.get(1);
MtcnnBatchResult oNetResult = ONetModel.thirdStage(manager, onetPredictorPool.borrowObject(), imgs,boxes, w, h, scoresFiltered, image_indsFiltered);
return R.ok(convertToDetectionResponse(oNetResult));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* 转换为FaceDetectedResult
* @param mtcnnBatchResult
* @return
*/
public static DetectionResponse convertToDetectionResponse(MtcnnBatchResult mtcnnBatchResult){
if(Objects.isNull(mtcnnBatchResult) || CollectionUtils.isEmpty(mtcnnBatchResult.boxes)
|| CollectionUtils.isEmpty(mtcnnBatchResult.points)
|| CollectionUtils.isEmpty(mtcnnBatchResult.probs)){
return null;
}
DetectionResponse detectionResponse = new DetectionResponse();
List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
NDArray boxes = mtcnnBatchResult.boxes.get(0);
NDArray probs = mtcnnBatchResult.probs.get(0);
NDArray points = mtcnnBatchResult.points.get(0);
long numBoxes = boxes.getShape().get(0);
for (int i = 0; i < numBoxes; i++) {
float[] boxCoords = boxes.get(i).toFloatArray(); // [x1, y1, x2, y2]
float score = probs.getFloat(i);
NDArray pointND = points.get(i); // shape [5,2]
float[] flatPoints = pointND.toFloatArray(); // 一维长度 10
List<Point> keyPoints = new ArrayList<Point>();
for (int p = 0; p < 5; p++) {
keyPoints.add(new Point(flatPoints[p * 2], flatPoints[p * 2 + 1]));
}
int x = Math.round(boxCoords[0]);
int y = Math.round(boxCoords[1]);
int w = Math.round(boxCoords[2] - boxCoords[0]);
int h = Math.round(boxCoords[3] - boxCoords[1]);
DetectionRectangle rectangle = new DetectionRectangle(x, y, w, h);
FaceInfo faceInfo = new FaceInfo(keyPoints);
DetectionInfo detectionInfo = new DetectionInfo(rectangle, score, faceInfo);
detectionInfoList.add(detectionInfo);
}
detectionResponse.setDetectionInfoList(detectionInfoList);
return detectionResponse;
}
public GenericObjectPool<Predictor<NDList, NDList>> getPnetPredictorPool() {
return pnetPredictorPool;
}
public GenericObjectPool<Predictor<NDList, NDList>> getRnetPredictorPool() {
return rnetPredictorPool;
}
public GenericObjectPool<Predictor<NDList, NDList>> getOnetPredictorPool() {
return onetPredictorPool;
}
@Override
public void close() {
try {
if (pnetPredictorPool != null) {
pnetPredictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (rnetPredictorPool != null) {
rnetPredictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (onetPredictorPool != null) {
onetPredictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (pNetModel != null) {
pNetModel.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
try {
if (pNetModel != null) {
pNetModel.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
try {
if (rNetModel != null) {
rNetModel.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
try {
if (oNetModel != null) {
oNetModel.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
}

252
face/src/main/java/cn/smartjavaai/face/model/facedect/SeetaFace6FaceDetModel.java Normal file
View File

@@ -0,0 +1,252 @@
package cn.smartjavaai.face.model.facedect;
import ai.djl.engine.Engine;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.pool.*;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* SeetaFace6 人脸检测模型
* @author dwj
*/
@Slf4j
public class SeetaFace6FaceDetModel implements FaceDetModel{
private FaceDetConfig config;
private FaceDetectorPool faceDetectorPool;
private FaceLandmarkerPool faceLandmarkerPool;
/**
* 阈值
*/
private static final double THRESHOLD = 0.9d;
@Override
public void loadModel(FaceDetConfig config) {
this.config = config;
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
}
//加载依赖库
NativeLoader.loadNativeLibraries(config.getDevice());
log.debug("Loading seetaFace6 library successfully.");
String[] faceDetectorModelPath = {config.getModelPath() + File.separator + "face_detector.csta"};
String[] faceLandmarkerModelPath = {config.getModelPath() + File.separator + "face_landmarker_pts5.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = config.getGpuId();
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
}
try {
SeetaModelSetting faceDetectorPoolSetting = new SeetaModelSetting(gpuId, faceDetectorModelPath, device);
SeetaConfSetting faceDetectorPoolConfSetting = new SeetaConfSetting(faceDetectorPoolSetting);
SeetaModelSetting faceLandmarkerPoolSetting = new SeetaModelSetting(gpuId, faceLandmarkerModelPath, device);
SeetaConfSetting faceLandmarkerPoolConfSetting = new SeetaConfSetting(faceLandmarkerPoolSetting);
this.faceDetectorPool = new FaceDetectorPool(faceDetectorPoolConfSetting);
this.faceLandmarkerPool = new FaceLandmarkerPool(faceLandmarkerPoolConfSetting);
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
faceDetectorPool.setMaxTotal(predictorPoolSize);
faceLandmarkerPool.setMaxTotal(predictorPoolSize);
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
} catch (FileNotFoundException e) {
throw new FaceException(e);
}
}
@Override
public R<DetectionResponse> detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(InputStream imageInputStream) {
if(Objects.isNull(imageInputStream)){
return R.fail(R.Status.INVALID_IMAGE);
}
BufferedImage image = null;
try {
image = ImageIO.read(imageInputStream);
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector predictor = null;
FaceLandmarker faceLandmarker = null;
try {
predictor = faceDetectorPool.borrowObject();
predictor.set(FaceDetector.Property.PROPERTY_THRESHOLD, config.getConfidenceThreshold() > 0 ? config.getConfidenceThreshold() : THRESHOLD);
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaRect[] seetaResult = predictor.Detect(imageData);
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
seetaPointFSList.add(pointFS);
}
return R.ok(FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList));
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (predictor != null) {
try {
faceDetectorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<Void> detectAndDraw(String imagePath, String outputPath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
try {
//创建保存路径
Path imageOutputPath = Paths.get(outputPath);
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
R<DetectionResponse> result = detect(image);
if(result.getCode() != R.Status.SUCCESS.getCode()){
return R.fail(result.getCode(), result.getMessage());
}
if(Objects.isNull(result.getData()) || Objects.isNull(result.getData().getDetectionInfoList()) || result.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
//绘制人脸框
FaceUtils.drawBoundingBoxes(image, result.getData(), imageOutputPath.toAbsolutePath().toString());
return R.ok();
} catch (IOException e) {
throw new FaceException(e);
}
}
@Override
public R<BufferedImage> detectAndDraw(BufferedImage sourceImage) {
if(!ImageUtils.isImageValid(sourceImage)){
return R.fail(R.Status.INVALID_IMAGE);
}
R<DetectionResponse> result = detect(sourceImage);
if(result.getCode() != R.Status.SUCCESS.getCode()){
return R.fail(result.getCode(), result.getMessage());
}
if(Objects.isNull(result.getData()) || Objects.isNull(result.getData().getDetectionInfoList()) || result.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
//绘制人脸框
try {
return R.ok(FaceUtils.drawBoundingBoxes(sourceImage, result.getData()));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public FaceDetectorPool getFaceDetectorPool() {
return faceDetectorPool;
}
public FaceLandmarkerPool getFaceLandmarkerPool() {
return faceLandmarkerPool;
}
@Override
public void close() throws Exception {
try {
if (faceDetectorPool != null) {
faceDetectorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (faceLandmarkerPool != null) {
faceLandmarkerPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
}
}

107
face/src/main/java/cn/smartjavaai/face/model/facedect/criterial/FaceDetCriteriaFactory.java Normal file
View File

@@ -0,0 +1,107 @@
package cn.smartjavaai.face.model.facedect.criterial;
import ai.djl.Device;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import ai.djl.translate.Translator;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.constant.RetinaFaceConstant;
import cn.smartjavaai.face.constant.UltraLightFastGenericFaceConstant;
import cn.smartjavaai.face.enums.ExpressionModelEnum;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator;
import cn.smartjavaai.face.model.expression.translator.FrEmotionTranslator;
import cn.smartjavaai.face.translator.FaceDetectionTranslator;
import cn.smartjavaai.face.translator.SCRFDFaceTranslator;
import cn.smartjavaai.face.translator.YoloV5FaceTranslator;
import cn.smartjavaai.face.translator.YoloV8FaceTranslator;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
/**
* 人脸检测 Criteria构建工厂
* @author dwj
*/
public class FaceDetCriteriaFactory {
/**
* 创建人脸检测Criteria
* @param config
* @return
*/
public static Criteria<Image, DetectedObjects> createCriteria(FaceDetConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId());
}
Translator<Image, DetectedObjects> translator = getTranslator(config);
if(StringUtils.isBlank(config.getModelEnum().getModelUrl())){
//检查模型路径
if (StringUtils.isBlank(config.getModelPath())){
throw new FaceException("请指定模型路径");
}
}
Criteria<Image, DetectedObjects> criteria =
Criteria.builder()
.setTypes(Image.class, DetectedObjects.class)
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : config.getModelEnum().getModelUrl())
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optTranslator(translator)
.optDevice(device)
.optProgress(new ProgressBar())
.optEngine(config.getModelEnum().getEngine())
.build();
return criteria;
}
/**
* 获取人脸检测Translator
* @param config
* @return
*/
public static Translator<Image, DetectedObjects> getTranslator(FaceDetConfig config) {
Translator<Image, DetectedObjects> translator = null;
if(config.getModelEnum() == FaceDetModelEnum.RETINA_FACE){
translator =
new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), RetinaFaceConstant.variance, FaceDetectConstant.MAX_FACE_LIMIT, RetinaFaceConstant.scales, RetinaFaceConstant.steps);
}else if (config.getModelEnum() == FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE){
translator =
new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), UltraLightFastGenericFaceConstant.variance, FaceDetectConstant.MAX_FACE_LIMIT, UltraLightFastGenericFaceConstant.scales, UltraLightFastGenericFaceConstant.steps);
}else if (config.getModelEnum() == FaceDetModelEnum.YOLOV8_FACE){
Map<String, Object> arguments = new HashMap<>();
arguments.put("width", 640);
arguments.put("height", 640);
arguments.put("resizeShort", 640);
arguments.put("centerFit", true);
translator = YoloV8FaceTranslator.builder(arguments).build();
}else if (config.getModelEnum() == FaceDetModelEnum.YOLOV5_FACE_640
|| config.getModelEnum() == FaceDetModelEnum.YOLOV5_FACE_320){
Map<String, Object> arguments = new HashMap<>();
arguments.put("width", config.getModelEnum().getInputWidth());
arguments.put("height", config.getModelEnum().getInputHeight());
arguments.put("resizeShort", true);
arguments.put("centerFit", true);
translator = YoloV5FaceTranslator.builder(arguments).build();
}else if (config.getModelEnum() == FaceDetModelEnum.SCRFD_160
|| config.getModelEnum() == FaceDetModelEnum.SCRFD_320
|| config.getModelEnum() == FaceDetModelEnum.SCRFD_640
|| config.getModelEnum() == FaceDetModelEnum.SCRFD_1280){
translator =
new SCRFDFaceTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), 100, new int[]{8, 16, 32});
}
return translator;
}
}

24
face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/MtcnnBatchResult.java Normal file
View File

@@ -0,0 +1,24 @@
package cn.smartjavaai.face.model.facedect.mtcnn;
import ai.djl.ndarray.NDArray;
import lombok.Data;
import java.util.List;
/**
* @author dwj
*/
@Data
public class MtcnnBatchResult {
public List<NDArray> boxes;
public List<NDArray> probs;
public List<NDArray> points;
public MtcnnBatchResult(List<NDArray> boxes, List<NDArray> probs, List<NDArray> points) {
this.boxes = boxes;
this.probs = probs;
this.points = points;
}
}

64
face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/MtcnnProcess.java Normal file
View File

@@ -0,0 +1,64 @@
package cn.smartjavaai.face.model.facedect.mtcnn;
import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import java.util.ArrayList;
import java.util.List;
/**
* @author dwj
*/
public class MtcnnProcess {
/**
* 预处理
* @param image
* @return
*/
public static NDArray processInput(NDManager manager, Image image){
// (N, C, H, W)
// Image -> NDArray (H, W, C)
NDArray array = image.toNDArray(manager, Image.Flag.COLOR);
// 增加 batch 维度 (1, H, W, C) ----
array = array.expandDims(0);
// 交换维度 (N, C, H, W)
array = array.transpose(0, 3, 1, 2);
// 转成模型的数据类型
if (!array.getDataType().equals(DataType.FLOAT32)) {
array = array.toType(DataType.FLOAT32, false);
}
return array;
}
/**
* 生成金字塔缩放比例列表
* @param image
* @return
*/
public static List<Double> generateScales(Image image){
long h = image.getHeight();
long w = image.getWidth();
// 计算最小缩放比例
double minsize = 20;
double m = 12.0 / minsize;
double minl = Math.min(h, w) * m;
// 创建金字塔缩放比例列表
double factor = 0.709; // 你原代码的 factor
List<Double> scales = new ArrayList<>();
double scale_i = m;
while (minl >= 12) {
scales.add(scale_i);
scale_i *= factor;
minl *= factor;
}
return scales;
}
}

112
face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/MtcnnUtils.java Normal file
View File

@@ -0,0 +1,112 @@
package cn.smartjavaai.face.model.facedect.mtcnn;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDArrays;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.index.NDIndex;
import ai.djl.ndarray.types.DataType;
/**
* @author dwj
*/
public class MtcnnUtils {
/**
* 矩形框重新映射
* @param bboxA
* @return
*/
public static NDArray rerec(NDArray bboxA) {
// h = y2 - y1
NDArray h = bboxA.get(":, 3").sub(bboxA.get(":, 1"));
// w = x2 - x1
NDArray w = bboxA.get(":, 2").sub(bboxA.get(":, 0"));
// l = max(w, h)
NDArray l = w.maximum(h);
// x1 = x1 + w*0.5 - l*0.5
NDArray x1 = bboxA.get(":, 0").add(w.mul(0.5)).sub(l.mul(0.5));
// y1 = y1 + h*0.5 - l*0.5
NDArray y1 = bboxA.get(":, 1").add(h.mul(0.5)).sub(l.mul(0.5));
// x2, y2
NDArray x2 = x1.add(l);
NDArray y2 = y1.add(l);
// 坐标变成 [N,1] 方便拼接
NDArray coords = NDArrays.concat(
new NDList(
x1.expandDims(1),
y1.expandDims(1),
x2.expandDims(1),
y2.expandDims(1)
),
1
);
// 保留原来的 score（bboxA[:, 4:]）
if (bboxA.getShape().get(1) > 4) {
NDArray rest = bboxA.get(":, 4:");
return NDArrays.concat(new NDList(coords, rest), 1);
} else {
return coords;
}
}
/**
* 限制范围
* @param boxes
* @param w
* @param h
* @return
*/
public static NDList pad(NDArray boxes, int w, int h) {
// 去小数 -> 转 int
boxes = boxes.floor().toType(DataType.INT32, false);
NDArray x = boxes.get(":, 0");
NDArray y = boxes.get(":, 1");
NDArray ex = boxes.get(":, 2");
NDArray ey = boxes.get(":, 3");
// 限制范围
x = x.maximum(1);
y = y.maximum(1);
ex = ex.minimum(w);
ey = ey.minimum(h);
return new NDList(y, ey, x, ex);
}
/**
* bbox regression
* @param boundingbox
* @param reg
* @return
*/
public static NDArray bbreg(NDArray boundingbox, NDArray reg) {
// 如果 reg 是形状 [N,1,H,W]，重塑为 [H,W] 或 [N,H] 这里假设 NCHW
if (reg.getShape().get(1) == 1) {
reg = reg.reshape(reg.getShape().get(2), reg.getShape().get(3));
}
// 确保 float32
boundingbox = boundingbox.toType(DataType.FLOAT32, false);
reg = reg.toType(DataType.FLOAT32, false);
// 计算宽高
NDArray w = boundingbox.get(":, 2").sub(boundingbox.get(":, 0")).add(1);
NDArray h = boundingbox.get(":, 3").sub(boundingbox.get(":, 1")).add(1);
NDArray b1 = boundingbox.get(":, 0").add(reg.get(":, 0").mul(w));
NDArray b2 = boundingbox.get(":, 1").add(reg.get(":, 1").mul(h));
NDArray b3 = boundingbox.get(":, 2").add(reg.get(":, 2").mul(w));
NDArray b4 = boundingbox.get(":, 3").add(reg.get(":, 3").mul(h));
// stack + transpose 对应 Python stack + permute
NDArray newBox = NDArrays.stack(new NDList(b1, b2, b3, b4), 0).transpose();
// 更新 boundingbox[:, :4]
boundingbox.set(new NDIndex(":, 0:4"), newBox);
return boundingbox;
}
}

202
face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/ONetModel.java Normal file
View File

@@ -0,0 +1,202 @@
package cn.smartjavaai.face.model.facedect.mtcnn;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDArrays;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.TranslateException;
import cn.smartjavaai.common.utils.NMSUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author dwj
*/
public class ONetModel {
public static MtcnnBatchResult thirdStage(NDManager manager, Predictor<NDList, NDList> onetPredictor, NDArray imgs, NDArray boxes, int w, int h, NDArray scoresFiltered,NDArray image_indsFiltered) throws TranslateException {
// Third stage
NDArray points = manager.zeros(new Shape(0, 5, 2));
NDList pad = MtcnnUtils.pad(boxes, (int)w, (int)h);
NDArray y = pad.get(0);
NDArray ey = pad.get(1);
NDArray x = pad.get(2);
NDArray ex = pad.get(3);
List<NDArray> crops = new ArrayList<>();
long numFaces = y.size(0);
for (long k = 0; k < numFaces; k++) {
// 检查坐标合法性
if (ey.getInt(k) > (y.getInt(k) - 1) &&
ex.getInt(k) > (x.getInt(k) - 1)) {
// 裁剪 (imageInd, :, y1:ey, x1:ex)
NDArray imgK = imgs.get(
image_indsFiltered.getLong(k) + ", :" +
", " + (y.getInt(k) - 1) + ":" + ey.getInt(k) +
", " + (x.getInt(k) - 1) + ":" + ex.getInt(k)
).expandDims(0); // 加 batch 维
// 缩放到 (24, 24)
// (N, H, W, C)
NDArray transposed = imgK.transpose(0, 2, 3, 1);
transposed = NDImageUtils.resize(transposed, 48, 48, Image.Interpolation.AREA);
// (N, C, H, W)
transposed = transposed.transpose(0, 3, 1, 2);
crops.add(transposed);
}
}
// 合并成一个 batch
NDArray im_data = NDArrays.concat(new NDList(crops), 0);
// 归一化
im_data = im_data.sub(127.5).mul(0.0078125);
// 74 48 48
NDList out = onetPredictor.predict(new NDList(im_data));
NDArray out0 = out.get(0).transpose(1, 0); // permute(1,0)
NDArray out1 = out.get(1).transpose(1, 0);
NDArray out2 = out.get(2).transpose(1, 0);
NDArray score = out1.get(1); // out1[1, :]
points = out1.duplicate();
NDArray ipass = score.gt(0.7); // score > threshold[1]
// ipass 为布尔/0-1张量，长度应等于 points 的第 1 维（这里是 7）
NDArray ipassBool = ipass.toType(DataType.BOOLEAN, false);
long[] colIdx = ipassBool.nonzero().toLongArray(); // 取 True 的列索引
// 把第 1 维换到第 0 维：(10, 7) -> (7, 10)
NDArray moved = points.swapAxes(0, 1);
// 现在按第一维取行即可，相当于选中列
NDArray selected = moved.get(points.getManager().create(colIdx)); // (sel, 10)
// 换回原来的轴顺序：(sel, 10) -> (10, sel)
points = selected.swapAxes(0, 1);
// 筛选 boxes 和 scores
// 先获取布尔索引为 true 的行索引
long[] validIndices = ipass.nonzero().toLongArray();
// 筛选 boxes 对应行
NDArray boxesSelected = boxes.get(manager.create(validIndices)); // 行筛选
// 取前 4 列
boxesSelected = boxesSelected.get(":, 0:4"); // 只保留前 4 列
scoresFiltered = scoresFiltered.get(ipass).reshape(-1, 1); // score[ipass].unsqueeze(1)
boxes = NDArrays.concat(new NDList(boxesSelected, scoresFiltered), 1); // 拼接成 (N,5)
// 筛选 image_inds
image_indsFiltered = image_indsFiltered.get(ipass);
NDArray mv = out0.transpose() // (N, 4)
.get(ipass); // 1-D 花式索引在第 0 维，得到 (k, 4)
System.out.println("----------");
// w_i = boxes[:, 2] - boxes[:, 0] + 1
NDArray w_i = boxes.get(":,2").sub(boxes.get(":,0")).add(1);
// h_i = boxes[:, 3] - boxes[:, 1] + 1
NDArray h_i = boxes.get(":,3").sub(boxes.get(":,1")).add(1);
// points_x = w_i.repeat(5, 1) * points[:5, :] + boxes[:, 0].repeat(5, 1) - 1
NDArray w_repeat = w_i.expandDims(0).repeat(0, 5); // shape: [5, N]
NDArray p_x = points.get("0:5,:").mul(w_repeat)
.add(boxes.get(":,0").expandDims(0).repeat(0, 5))
.sub(1);
// points_y = h_i.repeat(5, 1) * points[5:10, :] + boxes[:, 1].repeat(5, 1) - 1
NDArray h_repeat = h_i.expandDims(0).repeat(0, 5); // shape: [5, N]
NDArray p_y = points.get("5:10,:").mul(h_repeat)
.add(boxes.get(":,1").expandDims(0).repeat(0, 5))
.sub(1);
// points = torch.stack((points_x, points_y)).permute(2, 1, 0)
NDArray pointsStacked = NDArrays.stack(new NDList(p_x, p_y)); // shape: [2, 5, N]
points = pointsStacked.transpose(2, 1, 0); // permute(2, 1, 0) => shape [N, 5, 2]
// boxes = bbreg(boxes, mv)
boxes = MtcnnUtils.bbreg(boxes, mv);
NDArray pick = NMSUtils.batchedNms(boxes.get(":, :4"), boxes.get(":, 4"), image_indsFiltered, 0.7f, manager);
boxes = boxes.get(pick);
image_indsFiltered = image_indsFiltered.get(pick);
points = points.get(pick);
List<NDArray> batchBoxes = new ArrayList<>();
List<NDArray> batchPoints = new ArrayList<>();
for (int b_i = 0; b_i < 1; b_i++) {
// mask: image_inds == b_i
NDArray mask = image_indsFiltered.eq(b_i);
// 只保留当前 batch 的 boxes 和 points
NDArray batchBox = boxes.get(mask);
NDArray batchPoint = points.get(mask);
batchBoxes.add(batchBox);
batchPoints.add(batchPoint);
}
return processBatchBoxes(batchBoxes, batchPoints,true, manager);
}
public static MtcnnBatchResult processBatchBoxes(
List<NDArray> batchBoxes,
List<NDArray> batchPoints,
boolean selectLargest,
NDManager manager) {
List<NDArray> boxesOut = new ArrayList<>();
List<NDArray> probsOut = new ArrayList<>();
List<NDArray> pointsOut = new ArrayList<>();
for (int i = 0; i < batchBoxes.size(); i++) {
NDArray box = batchBoxes.get(i); // shape [num_boxes, ?] 或空 NDArray
NDArray point = batchPoints.get(i); // shape [num_boxes, 5, 2] 或空 NDArray
if (box == null || box.isEmpty()) {
boxesOut.add(null);
probsOut.add(null);
pointsOut.add(null);
continue;
}
NDArray boxesSelected;
NDArray probsSelected;
NDArray pointsSelected;
if (selectLargest) {
// 计算面积 (x2 - x1) * (y2 - y1)
NDArray w = box.get(":,2").sub(box.get(":,0"));
NDArray h = box.get(":,3").sub(box.get(":,1"));
NDArray areas = w.mul(h);
// 按面积降序排序
NDArray order = areas.argSort().flip(0);
boxesSelected = box.get(order);
pointsSelected = point.get(order);
} else {
boxesSelected = box;
pointsSelected = point;
}
// boxes[:, :4]
boxesSelected = boxesSelected.get(":,0:4");
// probs = box[:, 4]
probsSelected = box.get(":,4");
boxesOut.add(boxesSelected);
probsOut.add(probsSelected);
pointsOut.add(pointsSelected);
}
MtcnnBatchResult result = new MtcnnBatchResult(boxesOut, probsOut, pointsOut);
result.boxes = boxesOut;
result.probs = probsOut;
result.points = pointsOut;
return result;
}
}

186
face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/PNetModel.java Normal file
View File

@@ -0,0 +1,186 @@
package cn.smartjavaai.face.model.facedect.mtcnn;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDArrays;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.TranslateException;
import ai.djl.translate.TranslatorContext;
import cn.smartjavaai.common.utils.NMSUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author dwj
*/
public class PNetModel {
/**
* 输入图片
* @param imgs
* @param w
* @param h
* @param scale
* @return
*/
public static NDList processInput(NDArray imgs, int w, int h, double scale){
int newH = (int) (h * scale + 1);
int newW = (int) (w * scale + 1);
// (N, H, W, C)
NDArray transposed = imgs.transpose(0, 2, 3, 1);
transposed = NDImageUtils.resize(transposed, newW, newH, Image.Interpolation.AREA);
// (N, C, H, W)
transposed = transposed.transpose(0, 3, 1, 2);
// 归一化
transposed = transposed.sub(127.5).mul(0.0078125f);
System.out.println("inputPnet: " + transposed.getShape());
return new NDList(transposed);
}
public static NDList processOutput(NDList outputPnet, double scale, NDManager manager){
NDArray reg = outputPnet.get(0); // [B, 4, H, W]
NDArray probs = outputPnet.get(1); // [B, 2, H, W]
List<NDArray> boundingBox = generateBoundingBox(reg, probs.get(":, 1"), (float)scale, 0.6f);
NDArray boxes_scale = boundingBox.get(0); // [N,9]
NDArray imgIndND = boundingBox.get(1); // [N]
NDArray pick = NMSUtils.batchedNms(boxes_scale.get(":,:4"), boxes_scale.get(":,4"), imgIndND, 0.5f, manager);
return new NDList(boxes_scale, imgIndND, pick);
}
public static NDList firstStage(NDManager manager, Predictor<NDList, NDList> pnetPredictor, NDArray imgs, List<Double> scales, int width, int height) throws TranslateException {
// 第一阶段
NDList boxes_list = new NDList();
NDList image_inds_list = new NDList();
NDList scale_picks_list = new NDList();
int offset = 0;
for (double scale : scales) {
NDList inputPnet = processInput(imgs, width, height, scale);
NDList outputPnet = pnetPredictor.predict(inputPnet);
NDList output = processOutput(outputPnet, scale, manager);
NDArray boxes_scale = output.get(0);
NDArray imgIndND = output.get(1);
NDArray pick = output.get(2);
boxes_list.add(boxes_scale);
image_inds_list.add(imgIndND);
scale_picks_list.add(pick.add(offset));
offset += boxes_scale.getShape().get(0);
}
// 270 9
NDArray boxes = NDArrays.concat(boxes_list, 0);
NDArray image_inds = NDArrays.concat(image_inds_list, 0);
NDArray scale_picks = NDArrays.concat(scale_picks_list, 0);
// NMS within each scale + image
boxes = boxes.get(scale_picks); // scalePicksAll 是 NDArrays.concat 后的 INT64 NDArray
image_inds = image_inds.get(scale_picks); // 同样索引
// NMS within each image
NDArray pick = NMSUtils.batchedNms(
boxes.get(":, :4"), // 坐标
boxes.get(":, 4"), // score
image_inds, // 每个框对应的图片编号
0.7f, // IoU 阈值
manager
);
// 8 9
boxes = boxes.get(pick);
image_inds = image_inds.get(pick);
System.out.println(Arrays.toString(boxes.get(0).toFloatArray()));
NDArray regw = boxes.get(":, 2").sub(boxes.get(":, 0"));
NDArray regh = boxes.get(":, 3").sub(boxes.get(":, 1"));
NDArray qq1 = boxes.get(":, 0").add(boxes.get(":, 5").mul(regw));
NDArray qq2 = boxes.get(":, 1").add(boxes.get(":, 6").mul(regh));
NDArray qq3 = boxes.get(":, 2").add(boxes.get(":, 7").mul(regw));
NDArray qq4 = boxes.get(":, 3").add(boxes.get(":, 8").mul(regh));
boxes = NDArrays.stack(new NDList(qq1, qq2, qq3, qq4, boxes.get(":, 4")), 1);
boxes = MtcnnUtils.rerec(boxes);
return new NDList(boxes, image_inds);
}
/**
* 生成候选框，等价于 Python 版 generateBoundingBox
*
* @param reg NDArray [B,4,H,W]，回归偏移量
* @param probs NDArray [B,H,W]，人脸概率
* @param scale 当前金字塔缩放比例
* @param threshold 阈值
* @return 一个包含两个元素的 List：
* 0 -> NDArray bounding boxes [N,9] (x1,y1,x2,y2,score,dx1,dy1,dx2,dy2)
* 1 -> NDArray image_inds [N]
*/
public static List<NDArray> generateBoundingBox(
NDArray reg, NDArray probs, float scale, float threshold) {
float stride = 2f;
float cellSize = 12f;
// mask = probs >= thresh -> [B,H,W]
NDArray mask = probs.gte(threshold);
System.out.println("reg: " + reg.getShape());
System.out.println("probs shape: " + probs.getShape());
System.out.println("scale: " + scale);
System.out.println("mask: " + mask.getShape());
// mask_inds = mask.nonzero() -> [N,3] 每行: (batch, y, x)
NDArray maskInds = mask.nonzero();
System.out.println("maskInds: " + maskInds.getShape());
// image_inds = mask_inds[:, 0]
NDArray imageInds = maskInds.get(":,0");
// yx = mask_inds[:, 1:] [N,2] -> (y, x)
NDArray yx = maskInds.get(":,1:");
// bb = mask_inds[:, 1:].flip(1) Python 是 (y,x) -> (x,y)
NDArray bb = yx.flip(1); // [N,2] (x, y)
// 左上角 (x1, y1) 坐标 q1 = ((stride * bb + 1) / scale).floor()
NDArray q1 = bb.mul(stride).add(1).div(scale).floor();
// 右下角 (x2, y2) 坐标 q2 = ((stride * bb + cellsize) / scale).floor()
NDArray q2 = bb.mul(stride).add(cellSize).div(scale).floor();
Shape probShape = probs.getShape(); // [B,H,W]
long H = probShape.get(1);
long W = probShape.get(2);
NDArray linearIndex = maskInds.get(":,0").mul(H * W)
.add(maskInds.get(":,1").mul(W))
.add(maskInds.get(":,2"));
NDArray scores = probs.reshape(-1).gather(linearIndex, 0);
NDArray regPerm = reg.transpose(1, 0, 2, 3); // [4,B,H,W]
NDArray regFlat = regPerm.reshape(4, -1); // [4, total]
NDArray linearIndexForGather = linearIndex.expandDims(0).repeat(0, 4); // [4, N]
NDArray regPicked = regFlat.gather(linearIndexForGather, 1).transpose(); // [N,4]
NDArray x1 = q1.get(":, 0").expandDims(1);
NDArray y1 = q1.get(":, 1").expandDims(1);
NDArray x2 = q2.get(":, 0").expandDims(1);
NDArray y2 = q2.get(":, 1").expandDims(1);
NDArray boundingBoxes = NDArrays.concat(
new NDList(x1, y1, x2, y2, scores.expandDims(1), regPicked), 1
);
return Arrays.asList(boundingBoxes, imageInds);
}
}

107
face/src/main/java/cn/smartjavaai/face/model/facedect/mtcnn/RNetModel.java Normal file
View File

@@ -0,0 +1,107 @@
package cn.smartjavaai.face.model.facedect.mtcnn;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDArrays;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.translate.TranslateException;
import cn.smartjavaai.common.utils.NMSUtils;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author dwj
*/
@Slf4j
public class RNetModel {
public static NDList secondStage(NDManager manager, Predictor<NDList, NDList> rnetPredictor,NDArray imgs, NDArray boxes, NDList pad, NDArray image_inds) throws TranslateException {
NDArray y = pad.get(0);
NDArray ey = pad.get(1);
NDArray x = pad.get(2);
NDArray ex = pad.get(3);
List<NDArray> crops = new ArrayList<>();
long numFaces = y.size(0);
for (long k = 0; k < numFaces; k++) {
// 检查坐标合法性
if (ey.getInt(k) > (y.getInt(k) - 1) &&
ex.getInt(k) > (x.getInt(k) - 1)) {
// 裁剪 (imageInd, :, y1:ey, x1:ex)
NDArray imgK = imgs.get(
image_inds.getLong(k) + ", :" +
", " + (y.getInt(k) - 1) + ":" + ey.getInt(k) +
", " + (x.getInt(k) - 1) + ":" + ex.getInt(k)
).expandDims(0); // 加 batch 维
// 缩放到 (24, 24)
// (N, H, W, C)
NDArray transposed = imgK.transpose(0, 2, 3, 1);
transposed = NDImageUtils.resize(transposed, 24, 24, Image.Interpolation.AREA);
// (N, C, H, W)
transposed = transposed.transpose(0, 3, 1, 2);
crops.add(transposed);
}
}
if (crops.isEmpty()) {
log.debug("No face detected.");
return null;
}
// 合并成一个 batch
NDArray im_data = NDArrays.concat(new NDList(crops), 0);
// 归一化
im_data = im_data.sub(127.5).mul(0.0078125);
NDList out = rnetPredictor.predict(new NDList(im_data));
// 假设 out 是 NDList，threshold 是 float[]，NMSUtils2.batchedNms 已经有了
NDArray out0 = out.get(0).transpose(1, 0); // permute(1,0)
NDArray out1 = out.get(1).transpose(1, 0);
NDArray score = out1.get(1); // out1[1, :]
NDArray ipass = score.gt(0.7); // score > threshold[1]
// 筛选 boxes 和 scores
// 先获取布尔索引为 true 的行索引
long[] validIndices = ipass.nonzero().toLongArray();
// 筛选 boxes 对应行
NDArray boxesSelected = boxes.get(manager.create(validIndices)); // 行筛选
// 取前 4 列
boxesSelected = boxesSelected.get(":, 0:4"); // 只保留前 4 列
NDArray scoresFiltered = score.get(ipass).reshape(-1, 1); // score[ipass].unsqueeze(1)
boxes = NDArrays.concat(new NDList(boxesSelected, scoresFiltered), 1); // 拼接成 (N,5)
// 筛选 image_inds
NDArray image_indsFiltered = image_inds.get(ipass);
// out0: (4, N)
NDArray mv = out0.transpose() // (N, 4)
.get(ipass); // 1-D 花式索引在第 0 维，得到 (k, 4)
// .transpose(); // 如需要保持 (k, 4) 可省略；如想与 Python 顺序一致可再转置
// NMS
NDArray pick = NMSUtils.batchedNms(boxes.get(":, :4"), boxes.get(":, 4"), image_indsFiltered, 0.7f, manager);
// 最终筛选
boxes = boxes.get(pick);
image_indsFiltered = image_indsFiltered.get(pick);
mv = mv.get(pick);
// 框回归和方形化
boxes = MtcnnUtils.bbreg(boxes, mv);
boxes = MtcnnUtils.rerec(boxes);
if(boxes.size(0) == 0){
log.debug("No face detected.");
return null;
}
return new NDList(image_indsFiltered, scoresFiltered);
}
}

720
face/src/main/java/cn/smartjavaai/face/model/facerec/CommonFaceRecModel.java Normal file
View File

@@ -0,0 +1,720 @@
package cn.smartjavaai.face.model.facerec;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.ndarray.NDManager;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import cn.hutool.core.lang.UUID;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.entity.FaceRegisterInfo;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import cn.smartjavaai.face.enums.SimilarityType;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.factory.FaceDetModelFactory;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.criteria.FaceRecCriteriaFactory;
import cn.smartjavaai.face.preprocess.DJLImagePreprocessor;
import cn.smartjavaai.face.utils.*;
import cn.smartjavaai.face.vector.config.MilvusConfig;
import cn.smartjavaai.face.vector.config.SQLiteConfig;
import cn.smartjavaai.face.vector.constant.VectorDBConstants;
import cn.smartjavaai.face.vector.core.VectorDBClient;
import cn.smartjavaai.face.vector.core.VectorDBFactory;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import io.milvus.param.MetricType;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.opencv.core.Mat;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
/**
* FaceNet 人脸特征提取模型
* @author dwj
*/
@Slf4j
public class CommonFaceRecModel implements FaceRecModel{
/**
* 特征维度
*/
private static final int DIMENSION = 512;
/**
* 是否加载人脸库完毕
*/
private static volatile boolean isLoadCompleted = false;
private GenericObjectPool<Predictor<Image, float[]>> predictorPool;
private ZooModel<Image, float[]> model;
private FaceRecConfig config;
/**
* 是否归一化相似度
*/
public static final boolean NORMALIZE_SIMILARITY = true;
private VectorDBClient vectorDBClient;
/**
* 加载人脸特征提取模型
* @param config
*/
@Override
public void loadModel(FaceRecConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
if(Objects.isNull(config.getDetectModel())){
config.setDetectModel(getDefaultDetModel());
}
this.config = config;
Criteria<Image, float[]> faceFeatureCriteria = FaceRecCriteriaFactory.createCriteria(config);
try {
model = faceFeatureCriteria.loadModel();
// 创建池子：每个线程独享 Predictor
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
predictorPool.setMaxTotal(predictorPoolSize);
log.debug("当前设备: " + model.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("模型加载失败", e);
}
//初始化人脸库
if(config.getVectorDBConfig() != null && config.getVectorDBConfig().getType() != null){
if(config.getVectorDBConfig() instanceof MilvusConfig){
MilvusConfig milvusConfig = ((MilvusConfig) config.getVectorDBConfig());
//设置向量维度
milvusConfig.setDimension(DIMENSION);
//相似度计算方式 为空，设置默认值
if(Objects.isNull(milvusConfig.getMetricType())){
//FaceNet 默认使用内积
milvusConfig.setMetricType(MetricType.IP);
}
}else if (config.getVectorDBConfig() instanceof SQLiteConfig){
SQLiteConfig sqliteConfig = (SQLiteConfig) config.getVectorDBConfig();
if(Objects.isNull(sqliteConfig.getSimilarityType())){
//seetaface6 默认使用内积
sqliteConfig.setSimilarityType(SimilarityType.IP);
}
}
vectorDBClient = VectorDBFactory.createClient(config.getVectorDBConfig());
// 加载人脸数据库
if(config.isAutoLoadFace()){
new Thread(new Runnable() {
@Override
public void run() {
try {
log.debug("start load face...");
vectorDBClient.initialize();
isLoadCompleted = true;
log.debug("Load face success!");
} catch (Exception e) {
e.printStackTrace();
}
}
}).start();
}
}
}
public float[] featureExtraction(Image image){
Predictor<Image, float[]> predictor = null;
try {
predictor = predictorPool.borrowObject();
return predictor.predict(image);
} catch (Exception e) {
throw new FaceException("人脸特征提取错误", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
/**
* 计算相似度，返回归一化结果
* @param feature1 图1特征
* @param feature2 图2特征
* @return
*/
@Override
public float calculSimilar(float[] feature1, float[] feature2) {
//默认返回归一化结果
return SimilarityUtil.calculate(feature1, feature2, SimilarityType.IP, true);
}
/**
* 特征比较
* @param imagePath1 图1路径
* @param imagePath2 图2路径
* @return
*/
@Override
public R<Float> featureComparison(String imagePath1, String imagePath2) {
if(!FileUtils.isFileExists(imagePath1) || !FileUtils.isFileExists(imagePath2)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image1 = null;
BufferedImage image2 = null;
try {
image1 = ImageIO.read(new File(Paths.get(imagePath1).toAbsolutePath().toString()));
image2 = ImageIO.read(new File(Paths.get(imagePath2).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return featureComparison(image1, image2);
}
@Override
public R<Float> featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2) {
if(!ImageUtils.isImageValid(sourceImage1) || !ImageUtils.isImageValid(sourceImag2)){
throw new FaceException("图像无效");
}
R<float[]> feature1 = extractTopFaceFeature(sourceImage1);
if (!feature1.isSuccess()){
return R.fail(feature1.getCode(), feature1.getMessage());
}
R<float[]> feature2 = extractTopFaceFeature(sourceImag2);
if (!feature2.isSuccess()){
return R.fail(feature2.getCode(), feature2.getMessage());
}
float ret = calculSimilar(feature1.getData(), feature2.getData());
return R.ok(ret);
}
@Override
public R<Float> featureComparison(byte[] imageData1, byte[] imageData2) {
if(Objects.isNull(imageData1) || Objects.isNull(imageData2)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
BufferedImage bufferedImage1 = ImageIO.read(new ByteArrayInputStream(imageData1));
BufferedImage bufferedImage2 = ImageIO.read(new ByteArrayInputStream(imageData2));
return featureComparison(bufferedImage1, bufferedImage2);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
/**
* 获取默认人脸检测模型
* @return
*/
private FaceDetModel getDefaultDetModel() {
FaceDetConfig detectModelConfig = new FaceDetConfig();
detectModelConfig.setModelEnum(FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);
detectModelConfig.setConfidenceThreshold(0.98);
log.debug("创建默认人脸检测模型：ULTRA_LIGHT_FAST_GENERIC_FACE");
FaceDetModel detectModel = FaceDetModelFactory.getInstance().getModel(detectModelConfig);
return detectModel;
}
@Override
public R<DetectionResponse> extractFeatures(BufferedImage image) {
R<DetectionResponse> detectedResult = config.getDetectModel().detect(image);
if(!detectedResult.isSuccess()){
return detectedResult;
}
if(Objects.isNull(detectedResult.getData()) || Objects.isNull(detectedResult.getData().getDetectionInfoList()) || detectedResult.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
try (NDManager manager = model.getNDManager().newSubManager()) {
DJLImagePreprocessor djlImagePreprocessor = new DJLImagePreprocessor(djlImage, manager);
for (DetectionInfo detectionInfo : detectedResult.getData().getDetectionInfoList()){
DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
FaceInfo faceInfo = detectionInfo.getFaceInfo();
float[] features = null;
Image subImage = djlImage;
//人脸对齐
if(config.isAlign()){
//人脸对齐
double[][] pointsArray = FaceUtils.facePoints(faceInfo.getKeyPoints());
djlImagePreprocessor.enableCrop(rectangle).enableAffine(pointsArray, 96, 112);
subImage = djlImagePreprocessor.process();
}else{
//裁剪
djlImagePreprocessor.enableCrop(rectangle);
if(config.isCropFace()){
subImage = djlImagePreprocessor.process();
}
}
features = featureExtraction(subImage);
faceInfo.setFeature(features);
}
}
((Mat)djlImage.getWrappedImage()).release();
return detectedResult;
}
@Override
public R<DetectionResponse> extractFeatures(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return extractFeatures(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<DetectionResponse> extractFeatures(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return extractFeatures(image);
}
@Override
public R<float[]> extractTopFaceFeature(BufferedImage image) {
R<DetectionResponse> detectedResult = config.getDetectModel().detect(image);
if(!detectedResult.isSuccess()){
return R.fail(detectedResult.getCode(), detectedResult.getMessage());
}
if(Objects.isNull(detectedResult.getData()) || Objects.isNull(detectedResult.getData().getDetectionInfoList()) || detectedResult.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
float[] features = null;
try (NDManager manager = model.getNDManager().newSubManager()) {
DJLImagePreprocessor djlImagePreprocessor = new DJLImagePreprocessor(djlImage, manager);
//只取第一个人脸
DetectionInfo detectionInfo = detectedResult.getData().getDetectionInfoList().get(0);
DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
FaceInfo faceInfo = detectionInfo.getFaceInfo();
Image subImage = djlImage;
//人脸对齐
if(config.isAlign()){
//人脸对齐
double[][] pointsArray = FaceUtils.facePoints(faceInfo.getKeyPoints());
djlImagePreprocessor.enableCrop(rectangle).enableAffine(pointsArray, 96, 112);
subImage = djlImagePreprocessor.process();
}else{
//裁剪
djlImagePreprocessor.enableCrop(rectangle);
if(config.isCropFace()){
subImage = djlImagePreprocessor.process();
}
}
features = featureExtraction(subImage);
}
((Mat)djlImage.getWrappedImage()).release();
return Objects.isNull(features) ? R.fail(R.Status.Unknown) : R.ok(features);
}
@Override
public R<float[]> extractTopFaceFeature(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return extractTopFaceFeature(image);
}
@Override
public R<float[]> extractTopFaceFeature(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return extractTopFaceFeature(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return register(faceRegisterInfo, bufferedImage);
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, BufferedImage sourceImage) {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
}
//提取最大人脸特征
R<float[]> featureResponse = extractTopFaceFeature(sourceImage);
if(!featureResponse.isSuccess()){
return R.fail(featureResponse.getCode(), featureResponse.getMessage());
}
return register(faceRegisterInfo, featureResponse.getData());
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, InputStream inputStream) {
if(Objects.isNull(inputStream)){
throw new FaceException("图像输入流无效");
}
BufferedImage image = null;
try {
image = ImageIO.read(inputStream);
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
}
return register(faceRegisterInfo, image);
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return register(faceRegisterInfo, ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, float[] feature) {
if(vectorDBClient == null){
return R.fail(1000, "向量数据库未初始化成功");
}
if(Objects.isNull(feature)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "人脸特征为空");
}
FaceVector faceVector = new FaceVector();
if(faceRegisterInfo != null){
faceVector.setId(faceRegisterInfo.getId());
faceVector.setMetadata(faceRegisterInfo.getMetadata());
}
faceVector.setVector(feature);
return R.ok(vectorDBClient.insert(faceVector));
}
@Override
public void removeRegister(String... keys) {
vectorDBClient.deleteBatch(Arrays.asList(keys));
}
@Override
public void clearFace() {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
}
vectorDBClient.dropCollection(VectorDBConstants.Defaults.DEFAULT_COLLECTION_NAME);
}
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
upsertFace(faceRegisterInfo, bufferedImage);
}
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, BufferedImage sourceImage) {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
}
if(Objects.isNull(faceRegisterInfo)){
throw new FaceException("注册信息为空");
}
if(StringUtils.isBlank(faceRegisterInfo.getId())){
throw new FaceException("注册信息中ID为空");
}
//提取最大人脸特征
R<float[]> featureResponse = extractTopFaceFeature(sourceImage);
if(!featureResponse.isSuccess()){
throw new FaceException(featureResponse.getMessage());
}
upsertFace(faceRegisterInfo, featureResponse.getData());
}
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, float[] feature) {
if(Objects.isNull(feature)){
throw new FaceException("人脸特征为空");
}
if(Objects.isNull(vectorDBClient)){
throw new FaceException("未初始化人脸库");
}
FaceVector faceVector = new FaceVector();
if(faceRegisterInfo != null){
faceVector.setId(faceRegisterInfo.getId());
faceVector.setMetadata(faceRegisterInfo.getMetadata());
}
faceVector.setVector(feature);
vectorDBClient.upsert(faceVector);
}
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
upsertFace(faceRegisterInfo, ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<List<FaceSearchResult>> searchByTopFace(String imagePath, FaceSearchParams params) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return searchByTopFace(bufferedImage, params);
}
@Override
public R<List<FaceSearchResult>> searchByTopFace(byte[] imageData, FaceSearchParams params) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return searchByTopFace(ImageIO.read(new ByteArrayInputStream(imageData)), params);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<List<FaceSearchResult>> searchByTopFace(BufferedImage sourceImage, FaceSearchParams params) {
if(vectorDBClient == null){
return R.fail(1000, "向量数据库未初始化成功");
}
//提取最大人脸特征
R<float[]> featureResponse = extractTopFaceFeature(sourceImage);
if(!featureResponse.isSuccess()){
return R.fail(featureResponse.getCode(), featureResponse.getMessage());
}
return R.ok(search(featureResponse.getData(), params));
}
@Override
public List<FaceSearchResult> search(float[] feature, FaceSearchParams params) {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
}
if(Objects.isNull(feature)){
throw new FaceException("人脸特征为空");
}
if(Objects.isNull(params)){
throw new FaceException("人脸查询参数为空");
}
//设置默认值
float threshold = Objects.isNull(params.getThreshold()) ? FaceDetectConstant.FACENET_DEFAULT_SIMILARITY_THRESHOLD : params.getThreshold();
int topK = Objects.isNull(params.getTopK()) ? 1 : params.getTopK();
boolean normalize = Objects.isNull(params.getNormalizeSimilarity()) ? NORMALIZE_SIMILARITY : params.getNormalizeSimilarity();
FaceSearchParams searchParams = new FaceSearchParams(topK, threshold, normalize);
List<FaceSearchResult> searchResults = vectorDBClient.search(feature, searchParams);
return searchResults;
}
@Override
public R<DetectionResponse> search(String imagePath, FaceSearchParams params) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return search(bufferedImage, params);
}
@Override
public R<DetectionResponse> search(BufferedImage sourceImage, FaceSearchParams params) {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
}
//提取所有人脸特征
R<DetectionResponse> detectionResponse = extractFeatures(sourceImage);
if(!detectionResponse.isSuccess()){
return detectionResponse;
}
//设置默认值
float threshold = Objects.isNull(params.getThreshold()) ? FaceDetectConstant.FACENET_DEFAULT_SIMILARITY_THRESHOLD : params.getThreshold();
int topK = Objects.isNull(params.getTopK()) ? 1 : params.getTopK();
boolean normalize = Objects.isNull(params.getNormalizeSimilarity()) ? NORMALIZE_SIMILARITY : params.getNormalizeSimilarity();
FaceSearchParams searchParams = new FaceSearchParams(topK, threshold, normalize);
for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){
if(Objects.nonNull(detectionInfo.getFaceInfo()) && Objects.nonNull(detectionInfo.getFaceInfo().getFeature())){
List<FaceSearchResult> searchResults = vectorDBClient.search(detectionInfo.getFaceInfo().getFeature(), searchParams);
detectionInfo.getFaceInfo().setFaceSearchResults(searchResults);
}
}
return detectionResponse;
}
@Override
public R<DetectionResponse> search(byte[] imageData, FaceSearchParams params) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return search(ImageIO.read(new ByteArrayInputStream(imageData)), params);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceVector> getFaceInfoById(String id) {
if(vectorDBClient == null){
return R.fail(1000, "向量数据库未初始化成功");
}
return R.ok(vectorDBClient.getFaceInfoById(id));
}
@Override
public R<List<FaceVector>> listFaces(long pageNum, long pageSize) {
if(vectorDBClient == null){
return R.fail(1000, "向量数据库未初始化成功");
}
return R.ok(vectorDBClient.listFaces(pageNum, pageSize));
}
@Override
public void loadFaceFeatures() {
if(Objects.isNull(vectorDBClient)){
throw new FaceException("未初始化人脸库");
}
vectorDBClient.loadFaceFeatures();
}
@Override
public void releaseFaceFeatures() {
if(Objects.isNull(vectorDBClient)){
throw new FaceException("未初始化人脸库");
}
vectorDBClient.releaseFaceFeatures();
}
@Override
public void close() {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if(Objects.nonNull(vectorDBClient)){
vectorDBClient.close();
}
} catch (Exception e) {
log.warn("关闭 vectorDBClient 失败", e);
}
}
@Override
public boolean isLoadFaceCompleted() {
return isLoadCompleted;
}
@Override
public GenericObjectPool<Predictor<Image, float[]>> getPool() {
return predictorPool;
}
}

378
face/src/main/java/cn/smartjavaai/face/model/facerec/FaceRecModel.java Normal file
View File

@@ -0,0 +1,378 @@
package cn.smartjavaai.face.model.facerec;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.entity.FaceRegisterInfo;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.entity.FaceVector;
import org.apache.commons.pool2.impl.GenericObjectPool;
import java.awt.image.BufferedImage;
import java.io.InputStream;
import java.util.List;
/**
* 人脸识别模型
* @author dwj
*/
public interface FaceRecModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(FaceRecConfig config); // 加载模型
/**
* 计算相似度
* @param feature1 图1特征
* @param feature2 图2特征
* @return
*/
default float calculSimilar(float[] feature1, float[] feature2){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征比较
* @param imagePath1 图1路径
* @param imagePath2 图2路径
* @return
*/
default R<Float> featureComparison(String imagePath1, String imagePath2){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征比较
* @param sourceImage1 图1BufferedImage
* @param sourceImag2 图2BufferedImage
* @return
*/
default R<Float> featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征比较
* @param imageData1
* @param imageData2
* @return
*/
default R<Float> featureComparison(byte[] imageData1, byte[] imageData2){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 注册人脸
* 提取分数最高人脸进行注册
* @param faceRegisterInfo 注册人脸信息
* @param imagePath 图片路径
* @return
*/
default R<String> register(FaceRegisterInfo faceRegisterInfo, String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 注册人脸
* 提取分数最高人脸进行注册
* @param faceRegisterInfo 注册人脸信息
* @param inputStream
* @return
*/
default R<String> register(FaceRegisterInfo faceRegisterInfo, InputStream inputStream){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 注册人脸
* 提取分数最高人脸进行注册
* @param faceRegisterInfo 注册人脸信息
* @param sourceImage
* @return
*/
default R<String> register(FaceRegisterInfo faceRegisterInfo, BufferedImage sourceImage){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 注册人脸
* 提取分数最高人脸进行注册
* @param faceRegisterInfo 注册人脸信息
* @param imageData
* @return
*/
default R<String> register(FaceRegisterInfo faceRegisterInfo, byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 注册人脸
* 提取分数最高人脸进行注册
* @param faceRegisterInfo 注册人脸信息
* @param feature 人脸特征
* @return
*/
default R<String> register(FaceRegisterInfo faceRegisterInfo, float[] feature){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 更新或注册人脸
* 自动提取分数最高人脸进行更新
* @param faceRegisterInfo 注册人脸信息
* @param imagePath
* @return
*/
default void upsertFace(FaceRegisterInfo faceRegisterInfo, String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 更新或注册人脸
* 自动提取分数最高人脸进行更新
* @param faceRegisterInfo 注册人脸信息
* @param sourceImage
* @return
*/
default void upsertFace(FaceRegisterInfo faceRegisterInfo, BufferedImage sourceImage){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 更新或注册人脸
* 自动提取分数最高人脸进行更新
* @param faceRegisterInfo 注册人脸信息
* @param feature 人脸特征
* @return
*/
default void upsertFace(FaceRegisterInfo faceRegisterInfo, float[] feature){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 更新或注册人脸
* 自动提取分数最高人脸进行更新
* @param faceRegisterInfo 注册人脸信息
* @param imageData
* @return
*/
default void upsertFace(FaceRegisterInfo faceRegisterInfo, byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 查询人脸（查询图片中所有人脸）
* @param imagePath
* @param params 人脸查询参数
* @return
*/
default R<DetectionResponse> search(String imagePath, FaceSearchParams params){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 查询人脸（查询图片中所有人脸）
* 适用于多人脸场景
* @param sourceImage
* @return
*/
default R<DetectionResponse> search(BufferedImage sourceImage, FaceSearchParams params){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 查询人脸（查询图片中所有人脸）
* 适用于多人脸场景
* @param imageData
* @return
*/
default R<DetectionResponse> search(byte[] imageData, FaceSearchParams params){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 查询人脸
* 适用于多人脸场景
* @param feature 人脸特征
* @return
*/
default List<FaceSearchResult> search(float[] feature, FaceSearchParams params){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 查询人脸
* 从图像中提取分数最高的人脸特征，并在人脸库中进行 1:N 查询
* 适用于单人脸场景
* @param imagePath
* @param params 人脸查询参数
* @return
*/
default R<List<FaceSearchResult>> searchByTopFace(String imagePath, FaceSearchParams params){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 查询人脸
* 从图像中提取分数最高的人脸特征，并在人脸库中进行 1:N 查询
* 适用于单人脸场景
* @param sourceImage
* @return
*/
default R<List<FaceSearchResult>> searchByTopFace(BufferedImage sourceImage, FaceSearchParams params){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 查询人脸
* 从图像中提取分数最高的人脸特征，并在人脸库中进行 1:N 查询
* 适用于单人脸场景
* @param imageData
* @return
*/
default R<List<FaceSearchResult>> searchByTopFace(byte[] imageData, FaceSearchParams params){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 使用人脸ID获取人脸信息
* @param id
* @return
*/
default R<FaceVector> getFaceInfoById(String id){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 获取人脸列表
* @param pageNum
* @param pageSize
* @return
*/
default R<List<FaceVector>> listFaces(long pageNum, long pageSize){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 删除已注册人脸
* @param keys
* @return
*/
default void removeRegister(String... keys){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 清空人脸库数据
*/
default void clearFace(){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征提取（所有人脸）
* 适用于多人脸场景
* @param imagePath 图片路径
* @return
*/
default R<DetectionResponse> extractFeatures(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征提取（所有人脸）
* 适用于多人脸场景
* @param imageData 图片字节流
* @return
*/
default R<DetectionResponse> extractFeatures(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征提取（所有人脸）
* 适用于多人脸场景
* @param image BufferedImage
* @return
*/
default R<DetectionResponse> extractFeatures(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征提取（提取分数最高人脸特征）
* 适用于单人脸场景
* @param image BufferedImage
* @return
*/
default R<float[]> extractTopFaceFeature(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征提取（提取分数最高人脸特征）
* 适用于单人脸场景
* @param imagePath 图片路径
* @return
*/
default R<float[]> extractTopFaceFeature(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 特征提取（提取分数最高人脸特征）
* 适用于单人脸场景
* @param imageData 图片字节流
* @return
*/
default R<float[]> extractTopFaceFeature(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 加载人脸特征
*/
default void loadFaceFeatures(){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 释放人脸特征缓存
*/
default void releaseFaceFeatures(){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 是否加载人脸库完成
* @return
*/
default boolean isLoadFaceCompleted(){
throw new UnsupportedOperationException("默认不支持该功能");
}
default GenericObjectPool<Predictor<Image, float[]>> getPool() {
throw new UnsupportedOperationException("默认不支持该功能");
}
}

947
face/src/main/java/cn/smartjavaai/face/model/facerec/SeetaFace6FaceRecModel.java Normal file
View File

@@ -0,0 +1,947 @@
package cn.smartjavaai.face.model.facerec;
import ai.djl.engine.Engine;
import cn.smartjavaai.common.entity.DetectionInfo;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.entity.FaceRegisterInfo;
import cn.smartjavaai.face.entity.FaceResult;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.enums.SimilarityType;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.vector.config.MilvusConfig;
import cn.smartjavaai.face.vector.config.SQLiteConfig;
import cn.smartjavaai.face.vector.core.VectorDBClient;
import cn.smartjavaai.face.vector.core.VectorDBFactory;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import com.seeta.pool.*;
import com.seeta.sdk.*;
import cn.smartjavaai.face.seetaface.NativeLoader;
import io.milvus.param.MetricType;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
/**
* SeetaFace6 人脸模型
* @author dwj
*/
@SuppressWarnings("AliMissingOverrideAnnotation")
@Slf4j
public class SeetaFace6FaceRecModel implements FaceRecModel{
/**
* 特征维度
*/
private static final int DIMENSION = 1024;
private FaceRecConfig config;
private FaceDetectorPool faceDetectorPool;
private FaceRecognizerPool faceRecognizerPool;
private FaceLandmarkerPool faceLandmarkerPool;
private FaceDatabasePool faceDatabasePool;
private VectorDBClient vectorDBClient = null;
/**
* 是否加载人脸库完毕
*/
private static volatile boolean isLoadCompleted = false;
/**
* 是否归一化相似度
*/
public static final boolean NORMALIZE_SIMILARITY = false;
@Override
public void loadModel(FaceRecConfig config) {
this.config = config;
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
}
//加载依赖库
NativeLoader.loadNativeLibraries(config.getDevice());
log.debug("Loading seetaFace6 library successfully.");
String[] faceDetectorModelPath = {config.getModelPath() + File.separator + "face_detector.csta"};
String[] faceRecognizerModelPath = {config.getModelPath() + File.separator + "face_recognizer.csta"};
//轻量模型
if(config.getModelEnum() == FaceRecModelEnum.SEETA_FACE6_LIGHT_MODEL){
faceRecognizerModelPath = new String[] { config.getModelPath() + File.separator + "face_recognizer_light.csta" };
}
String[] faceLandmarkerModelPath = {config.getModelPath() + File.separator + "face_landmarker_pts5.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = config.getGpuId();
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
}
try {
SeetaModelSetting faceDetectorPoolSetting = new SeetaModelSetting(gpuId, faceDetectorModelPath, device);
SeetaConfSetting faceDetectorPoolConfSetting = new SeetaConfSetting(faceDetectorPoolSetting);
SeetaModelSetting faceRecognizerPoolSetting = new SeetaModelSetting(gpuId, faceRecognizerModelPath, device);
SeetaConfSetting faceRecognizerPoolConfSetting = new SeetaConfSetting(faceRecognizerPoolSetting);
SeetaModelSetting faceLandmarkerPoolSetting = new SeetaModelSetting(gpuId, faceLandmarkerModelPath, device);
SeetaConfSetting faceLandmarkerPoolConfSetting = new SeetaConfSetting(faceLandmarkerPoolSetting);
SeetaModelSetting faceDatabasePoolSetting = new SeetaModelSetting(gpuId, faceRecognizerModelPath, device);
SeetaConfSetting faceDatabasePoolConfSetting = new SeetaConfSetting(faceDatabasePoolSetting);
this.faceDetectorPool = new FaceDetectorPool(faceDetectorPoolConfSetting);
this.faceRecognizerPool = new FaceRecognizerPool(faceRecognizerPoolConfSetting);
this.faceLandmarkerPool = new FaceLandmarkerPool(faceLandmarkerPoolConfSetting);
this.faceDatabasePool = new FaceDatabasePool(faceDatabasePoolConfSetting);
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
faceDetectorPool.setMaxTotal(predictorPoolSize);
faceRecognizerPool.setMaxTotal(predictorPoolSize);
faceLandmarkerPool.setMaxTotal(predictorPoolSize);
faceDatabasePool.setMaxTotal(predictorPoolSize);
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
//初始化人脸库
if(config.getVectorDBConfig() != null && config.getVectorDBConfig().getType() != null){
if(config.getVectorDBConfig() instanceof MilvusConfig){
MilvusConfig milvusConfig = ((MilvusConfig) config.getVectorDBConfig());
//设置向量维度
milvusConfig.setDimension(DIMENSION);
//相似度计算方式 为空，设置默认值
if(Objects.isNull(milvusConfig.getMetricType())){
//seetaface6 默认使用余弦相似度
milvusConfig.setMetricType(MetricType.COSINE);
}
}else if (config.getVectorDBConfig() instanceof SQLiteConfig){
SQLiteConfig sqliteConfig = (SQLiteConfig) config.getVectorDBConfig();
if(Objects.isNull(sqliteConfig.getSimilarityType())){
//seetaface6 默认使用余弦相似度
sqliteConfig.setSimilarityType(SimilarityType.COSINE);
}
}
//创建向量数据库
vectorDBClient = VectorDBFactory.createClient(config.getVectorDBConfig());
// 加载人脸数据库
if(config.isAutoLoadFace()){
new Thread(new Runnable() {
@Override
public void run() {
try {
log.debug("start load face...");
vectorDBClient.initialize();
isLoadCompleted = true;
log.debug("Load face success!");
} catch (Exception e) {
e.printStackTrace();
}
}
}).start();
}
}
} catch (FileNotFoundException e) {
throw new FaceException(e);
}
}
@Override
public float calculSimilar(float[] feature1, float[] feature2) {
if(Objects.isNull(feature1) || Objects.isNull(feature2)){
throw new FaceException("特征向量无效");
}
FaceRecognizer faceRecognizer = null;
try {
faceRecognizer = faceRecognizerPool.borrowObject();
return faceRecognizer.CalculateSimilarity(feature1, feature2);
} catch (Exception e) {
throw new FaceException(e);
}finally {
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<Float> featureComparison(String imagePath1, String imagePath2) {
if(!FileUtils.isFileExists(imagePath1) || !FileUtils.isFileExists(imagePath2)){
throw new FaceException("图像文件不存在");
}
BufferedImage image1 = null;
BufferedImage image2 = null;
try {
image1 = ImageIO.read(new File(Paths.get(imagePath1).toAbsolutePath().toString()));
image2 = ImageIO.read(new File(Paths.get(imagePath2).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return featureComparison(image1, image2);
}
@Override
public R<Float> featureComparison(BufferedImage image1, BufferedImage image2) {
if(!ImageUtils.isImageValid(image1) || !ImageUtils.isImageValid(image2)){
return R.fail(R.Status.INVALID_IMAGE);
}
R<float[]> feature1 = extractTopFaceFeature(image1);
if(!feature1.isSuccess()){
return R.fail(feature1.getCode(), feature1.getMessage());
}
R<float[]> feature2 = extractTopFaceFeature(image2);
if(!feature2.isSuccess()){
return R.fail(feature2.getCode(), feature2.getMessage());
}
return R.ok(calculSimilar(feature1.getData(), feature2.getData()));
}
@Override
public R<Float> featureComparison(byte[] imageData1, byte[] imageData2) {
if(Objects.isNull(imageData1) || Objects.isNull(imageData2)){
throw new FaceException("图像无效");
}
BufferedImage image1 = null;
BufferedImage image2 = null;
try {
image1 = ImageIO.read(new ByteArrayInputStream(imageData1));
image2 = ImageIO.read(new ByteArrayInputStream(imageData2));
} catch (IOException e) {
throw new FaceException("无效图片", e);
}
return featureComparison(image1, image2);
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return register(faceRegisterInfo, bufferedImage);
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
//提取特征向量
R<float[]> featureResponse = extractTopFaceFeature(image);
if(!featureResponse.isSuccess()){
return R.fail(featureResponse.getCode(), featureResponse.getMessage());
}
return register(faceRegisterInfo, featureResponse.getData());
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new ByteArrayInputStream(imageData));
} catch (IOException e) {
throw new FaceException(e);
}
return register(faceRegisterInfo, bufferedImage);
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, InputStream inputStream) {
if(Objects.isNull(inputStream)){
throw new FaceException("图像输入流无效");
}
BufferedImage image = null;
try {
image = ImageIO.read(inputStream);
} catch (IOException e) {
throw new FaceException("无效的图片输入流", e);
}
return register(faceRegisterInfo, image);
}
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, float[] feature) {
if(Objects.isNull(feature)){
return R.fail(R.Status.Unknown.getCode(), "人脸注册失败：人脸特征为空");
}
FaceVector faceVector = new FaceVector();
if(Objects.nonNull(faceRegisterInfo)){
faceVector.setId(faceRegisterInfo.getId());
faceVector.setMetadata(faceRegisterInfo.getMetadata());
}
faceVector.setVector(feature);
return R.ok(vectorDBClient.insert(faceVector));
}
// /**
// * 注册已裁剪后人脸
// * @param key
// * @param faceData
// * @return
// */
// private boolean registerCroppedFace(String key, FaceData faceData) {
// FaceDatabase faceDatabase = null;
// try {
// faceDatabase = faceDatabasePool.borrowObject();
// SeetaImageData cropImageData = new SeetaImageData(faceData.getWidth(), faceData.getHeight(), faceData.getChannel());
// cropImageData.data = faceData.getImgData();
// long index = faceDatabase.RegisterByCroppedFace(cropImageData);
// if (index < 0) {
// log.debug("register face fail: key={}, index={}", key, index);
// return false;
// }
// int rows = 0;
// try {
// rows = new FaceDao(config.getFaceDbPath()).updateIndex(index, faceData);
// } catch (SQLException | ClassNotFoundException e) {
// throw new FaceException(e);
// }
// return rows > 0;
// } catch (FaceException e) {
// throw e;
// } catch (Exception e) {
// throw new FaceException(e);
// }finally {
// if (faceDatabase != null) {
// try {
// faceDatabasePool.returnObject(faceDatabase); //归还
// } catch (Exception e) {
// log.warn("归还Predictor失败", e);
// }
// }
// }
// }
@Override
public R<DetectionResponse> search(String imagePath, FaceSearchParams params) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return search(bufferedImage, params);
}
@Override
public R<DetectionResponse> search(BufferedImage image, FaceSearchParams params) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
//提取所有人脸特征
R<DetectionResponse> detectionResponse = extractFeatures(image);
if(!detectionResponse.isSuccess()){
return detectionResponse;
}
//设置默认值
float threshold = Objects.isNull(params.getThreshold()) ? FaceDetectConstant.SEETAFACE_DEFAULT_SIMILARITY_THRESHOLD : params.getThreshold();
int topK = Objects.isNull(params.getTopK()) ? 1 : params.getTopK();
boolean normalize = Objects.isNull(params.getNormalizeSimilarity()) ? NORMALIZE_SIMILARITY : params.getNormalizeSimilarity();
FaceSearchParams searchParams = new FaceSearchParams(topK, threshold, normalize);
for (DetectionInfo detectionInfo : detectionResponse.getData().getDetectionInfoList()){
if(Objects.nonNull(detectionInfo.getFaceInfo()) && Objects.nonNull(detectionInfo.getFaceInfo().getFeature())){
List<FaceSearchResult> searchResults = vectorDBClient.search(detectionInfo.getFaceInfo().getFeature(), searchParams);
detectionInfo.getFaceInfo().setFaceSearchResults(searchResults);
}
}
return detectionResponse;
}
@Override
public R<DetectionResponse> search(byte[] imageData, FaceSearchParams params) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new ByteArrayInputStream(imageData));
} catch (IOException e) {
throw new FaceException(e);
}
return search(bufferedImage, params);
}
@Override
public List<FaceSearchResult> search(float[] feature, FaceSearchParams params) {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
}
if(Objects.isNull(feature)){
throw new FaceException("人脸特征为空");
}
if(Objects.isNull(params)){
throw new FaceException("人脸查询参数为空");
}
//设置默认值
float threshold = Objects.isNull(params.getThreshold()) ? FaceDetectConstant.SEETAFACE_DEFAULT_SIMILARITY_THRESHOLD : params.getThreshold();
int topK = Objects.isNull(params.getTopK()) ? 1 : params.getTopK();
boolean normalize = Objects.isNull(params.getNormalizeSimilarity()) ? NORMALIZE_SIMILARITY : params.getNormalizeSimilarity();
FaceSearchParams searchParams = new FaceSearchParams(topK, threshold, normalize);
List<FaceSearchResult> searchResults = vectorDBClient.search(feature, searchParams);
return searchResults;
}
@Override
public R<List<FaceSearchResult>> searchByTopFace(String imagePath, FaceSearchParams params) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return searchByTopFace(bufferedImage, params);
}
@Override
public R<List<FaceSearchResult>> searchByTopFace(BufferedImage sourceImage, FaceSearchParams params) {
if(!ImageUtils.isImageValid(sourceImage)){
return R.fail(R.Status.INVALID_IMAGE);
}
//提取分数最高人脸特征
R<float[]> featureResponse = extractTopFaceFeature(sourceImage);
if(!featureResponse.isSuccess()){
return R.fail(featureResponse.getCode(), featureResponse.getMessage());
}
//设置默认值
float threshold = Objects.isNull(params.getThreshold()) ? FaceDetectConstant.SEETAFACE_DEFAULT_SIMILARITY_THRESHOLD : params.getThreshold();
int topK = Objects.isNull(params.getTopK()) ? 1 : params.getTopK();
boolean normalize = Objects.isNull(params.getNormalizeSimilarity()) ? NORMALIZE_SIMILARITY : params.getNormalizeSimilarity();
FaceSearchParams searchParams = new FaceSearchParams(topK, threshold, normalize);
List<FaceSearchResult> searchResults = vectorDBClient.search(featureResponse.getData(), searchParams);
return R.ok(searchResults);
}
@Override
public R<List<FaceSearchResult>> searchByTopFace(byte[] imageData, FaceSearchParams params) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new ByteArrayInputStream(imageData));
} catch (IOException e) {
throw new FaceException(e);
}
return searchByTopFace(bufferedImage, params);
}
@Override
public void removeRegister(String... keys) {
if(keys == null || keys.length == 0){
throw new FaceException("keys不允许为空");
}
vectorDBClient.deleteBatch(Arrays.asList(keys));
}
@Override
public void clearFace(){
vectorDBClient.dropCollection(null);
}
/**
* 检查是否存在人脸库
* @return
*/
// private boolean checkFaceDb(){
// if(Objects.nonNull(config) && StringUtils.isNotBlank(config.getFaceDbPath())){
// File file = new File(config.getFaceDbPath());
// return file.exists() && file.isFile();
// }
// return false;
// }
private FaceResult searchFaceDb(long index,float similar) {
if(index >= 0){
String key = null;
// try {
// key = new FaceDao(config.getFaceDbPath()).findKeyByIndex(index);
// } catch (SQLException | ClassNotFoundException e) {
// throw new FaceException("查询人脸库失败", e);
// }
return new FaceResult(key, similar);
}
return null;
}
/**
* 加载人脸库
* @throws SQLException
* @throws ClassNotFoundException
*/
// private void loadFaceDb() {
// if(!checkFaceDb()){
// log.debug("未配置人脸库");
// return;
// }
// //分页查询人脸库
// int pageNo = 0, pageSize = 100;
// while (true) {
// List<FaceData> list = null;
// try {
// list = new FaceDao(config.getFaceDbPath()).findFace(pageNo, pageSize);
// } catch (SQLException | ClassNotFoundException e) {
// throw new FaceException("查询人脸库失败", e);
// }
// if (list == null) {
// break;
// }
// list.forEach(face -> {
// try {
// registerCroppedFace(face.getKey(), face);
// } catch (Exception e) {
// e.printStackTrace();
// }
// });
// if (list.size() < pageSize) {
// break;
// }
// pageNo++;
// }
// }
@Override
public R<DetectionResponse> extractFeatures(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return extractFeatures(image);
}
@Override
public R<DetectionResponse> extractFeatures(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return extractFeatures(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("特征提取异常", e);
}
}
@Override
public R<DetectionResponse> extractFeatures(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
FaceDetector faceDetector = null;
FaceLandmarker faceLandmarker = null;
FaceRecognizer faceRecognizer = null;
try {
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
faceRecognizer = faceRecognizerPool.borrowObject();
//默认使用Seetaface6检测模型
if(Objects.isNull(config.getDetectModel())){
List<float[]> featureList = new ArrayList<float[]>();
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
//提取特征
float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
return R.fail(R.Status.Unknown.getCode(), "人脸特征提取失败");
}
featureList.add(features);
seetaPointFSList.add(pointFS);
}
return R.ok(FaceUtils.featuresConvertToResponse(seetaResult, seetaPointFSList, featureList));
}else{
R<DetectionResponse> detectResponse = config.getDetectModel().detect(image);
if(!detectResponse.isSuccess()){
return detectResponse;
}
if(Objects.isNull(detectResponse.getData()) || Objects.isNull(detectResponse.getData().getDetectionInfoList()) || detectResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(DetectionInfo detectionInfo : detectResponse.getData().getDetectionInfoList()){
//提取特征
float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(faceInfo.getKeyPoints());
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
return R.fail(R.Status.Unknown.getCode(), "人脸特征提取失败");
}
faceInfo.setFeature(features);
}
return detectResponse;
}
} catch (FaceException e) {
throw e;
} catch (Exception e) {
throw new FaceException("人脸特征提取异常", e);
}finally {
if (faceDetector != null) {
try {
faceDetectorPool.returnObject(faceDetector); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<float[]> extractTopFaceFeature(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
float[] features = null;
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector faceDetector = null;
FaceLandmarker faceLandmarker = null;
FaceRecognizer faceRecognizer = null;
try {
faceRecognizer = faceRecognizerPool.borrowObject();
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = null;
//默认使用Seetaface6检测模型
if(Objects.isNull(config.getDetectModel())){
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], pointFS);
}else{
R<DetectionResponse> detectResponse = config.getDetectModel().detect(image);
if(!detectResponse.isSuccess()){
return R.fail(detectResponse.getCode(), detectResponse.getMessage());
}
if(Objects.isNull(detectResponse.getData()) || Objects.isNull(detectResponse.getData().getDetectionInfoList()) || detectResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
DetectionInfo detectionInfo = detectResponse.getData().getDetectionInfoList().get(0);
pointFS = FaceUtils.convertToSeetaPointF(detectionInfo.getFaceInfo().getKeyPoints());
}
//提取特征
features = new float[faceRecognizer.GetExtractFeatureSize()];
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
return R.fail(R.Status.Unknown.getCode(), "人脸特征提取失败");
}
return R.ok(features);
} catch (FaceException e) {
throw e;
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (faceDetector != null) {
try {
faceDetectorPool.returnObject(faceDetector); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
/**
* 特征提取
* @param image
* @return
*/
public float[] featureExtraction(BufferedImage image){
FaceRecognizer faceRecognizer = null;
try {
faceRecognizer = faceRecognizerPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//提取特征
float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
faceRecognizer.ExtractCroppedFace(imageData, features);
return features;
} catch (FaceException e) {
throw e;
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<float[]> extractTopFaceFeature(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return extractTopFaceFeature(image);
}
@Override
public R<float[]> extractTopFaceFeature(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return extractTopFaceFeature(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public void loadFaceFeatures() {
if(Objects.isNull(vectorDBClient)){
throw new FaceException("未初始化人脸库");
}
vectorDBClient.loadFaceFeatures();
}
@Override
public void releaseFaceFeatures() {
if(Objects.isNull(vectorDBClient)){
throw new FaceException("未初始化人脸库");
}
vectorDBClient.releaseFaceFeatures();
}
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
upsertFace(faceRegisterInfo, bufferedImage);
}
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, BufferedImage sourceImage) {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
}
if(Objects.isNull(faceRegisterInfo)){
throw new FaceException("注册信息为空");
}
if(StringUtils.isBlank(faceRegisterInfo.getId())){
throw new FaceException("注册信息中ID为空");
}
//提取最大人脸特征
R<float[]> featureResponse = extractTopFaceFeature(sourceImage);
if(!featureResponse.isSuccess()){
throw new FaceException(featureResponse.getMessage());
}
upsertFace(faceRegisterInfo, featureResponse.getData());
}
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, float[] feature) {
if(Objects.isNull(feature)){
throw new FaceException("人脸特征为空");
}
if(Objects.isNull(vectorDBClient)){
throw new FaceException("未初始化人脸库");
}
FaceVector faceVector = new FaceVector();
if(faceRegisterInfo != null){
faceVector.setId(faceRegisterInfo.getId());
faceVector.setMetadata(faceRegisterInfo.getMetadata());
}
faceVector.setVector(feature);
vectorDBClient.upsert(faceVector);
}
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new ByteArrayInputStream(imageData));
} catch (IOException e) {
throw new FaceException(e);
}
upsertFace(faceRegisterInfo, bufferedImage);
}
@Override
public R<FaceVector> getFaceInfoById(String id) {
if(vectorDBClient == null){
return R.fail(1000, "向量数据库未初始化成功");
}
return R.ok(vectorDBClient.getFaceInfoById(id));
}
@Override
public R<List<FaceVector>> listFaces(long pageNum, long pageSize) {
if(vectorDBClient == null){
return R.fail(1000, "向量数据库未初始化成功");
}
return R.ok(vectorDBClient.listFaces(pageNum, pageSize));
}
@Override
public void close() throws Exception {
if(Objects.nonNull(faceDetectorPool)){
faceDetectorPool.close();
}
if(Objects.nonNull(faceRecognizerPool)){
faceRecognizerPool.close();
}
if(Objects.nonNull(faceLandmarkerPool)){
faceLandmarkerPool.close();
}
if(Objects.nonNull(faceDatabasePool)){
faceDatabasePool.close();
}
if(Objects.nonNull(vectorDBClient)){
vectorDBClient.close();
}
}
@Override
public boolean isLoadFaceCompleted() {
return isLoadCompleted;
}
public FaceDetectorPool getFaceDetectorPool() {
return faceDetectorPool;
}
public FaceRecognizerPool getFaceRecognizerPool() {
return faceRecognizerPool;
}
public FaceLandmarkerPool getFaceLandmarkerPool() {
return faceLandmarkerPool;
}
public FaceDatabasePool getFaceDatabasePool() {
return faceDatabasePool;
}
}

101
face/src/main/java/cn/smartjavaai/face/model/facerec/criteria/FaceRecCriteriaFactory.java Normal file
View File

@@ -0,0 +1,101 @@
package cn.smartjavaai.face.model.facerec.criteria;
import ai.djl.Device;
import ai.djl.modality.cv.Image;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceNetConstant;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.model.facerec.translator.FaceFeatureTranslator;
import cn.smartjavaai.face.model.facerec.translator.FaceNetRecTranslator;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* 人脸识别 Criteria构建工厂
* @author dwj
*/
public class FaceRecCriteriaFactory {
public static Criteria<Image, float[]> createCriteria(FaceRecConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId());
}
Criteria<Image, float[]> criteria = null;
if(config.getModelEnum() == FaceRecModelEnum.FACENET_MODEL){
criteria =
Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelName("face_feature") // specify model file prefix
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null :
FaceNetConstant.MODEL_URL)
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optTranslator(new FaceNetRecTranslator())
.optDevice(device)
.optEngine("PyTorch") // Use PyTorch engine
.optProgress(new ProgressBar())
.build();
}else if (config.getModelEnum() == FaceRecModelEnum.INSIGHT_FACE_MOBILE_FACENET_MODEL){
if(StringUtils.isBlank(config.getModelPath())){
throw new RuntimeException("请指定模型路径");
}
List<Float> mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f);
String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(","));
criteria = Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(config.getModelPath()))
// .optTranslatorFactory(new ImageFeatureExtractorFactory())
// .optArgument("normalize", normalize)
// .optArgument("resize", "112,112")
.optTranslator(new FaceFeatureTranslator())
.optEngine("PyTorch") // Use PyTorch engine
.optProgress(new ProgressBar())
.optDevice(device)
.build();
}else if (config.getModelEnum() == FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL){
if(StringUtils.isBlank(config.getModelPath())){
throw new RuntimeException("请指定模型路径");
}
List<Float> mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f);
String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(","));
criteria = Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(config.getModelPath()))
// .optTranslatorFactory(new ImageFeatureExtractorFactory())
// .optArgument("normalize", normalize)
// .optArgument("resize", "112,112")
.optTranslator(new FaceFeatureTranslator())
.optEngine("PyTorch") // Use PyTorch engine
.optDevice(device)
.optProgress(new ProgressBar())
.build();
}else if (config.getModelEnum() == FaceRecModelEnum.ELASTIC_FACE_MODEL){
if(StringUtils.isBlank(config.getModelPath())){
throw new RuntimeException("请指定模型路径");
}
List<Float> mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f);
String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(","));
criteria = Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(config.getModelPath()))
// .optTranslatorFactory(new ImageFeatureExtractorFactory())
// .optArgument("normalize", normalize)
// .optArgument("resize", "112,112")
.optTranslator(new FaceFeatureTranslator())
.optEngine("PyTorch") // Use PyTorch engine
.optDevice(device)
.optProgress(new ProgressBar())
.build();
}
return criteria;
}
}

59
face/src/main/java/cn/smartjavaai/face/model/facerec/translator/FaceFeatureTranslator.java Normal file
View File

@@ -0,0 +1,59 @@
package cn.smartjavaai.face.model.facerec.translator;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.transform.Normalize;
import ai.djl.modality.cv.transform.Resize;
import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Pipeline;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
/**
* facenet人脸特征提取Translator
* @author dwj
* @date 2025/3/31
*/
public final class FaceFeatureTranslator implements Translator<Image, float[]> {
public FaceFeatureTranslator() {
}
/**
* {@inheritDoc}
*/
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
Pipeline pipeline = new Pipeline();
if(input.getWidth() != 112 || input.getHeight() != 112){
pipeline.add(new Resize(112,112));
}
pipeline
.add(new ToTensor())
.add(new Normalize(
new float[]{0.5F, 0.5F, 0.5F},
new float[]{0.5F, 0.5F, 0.5F}));
return pipeline.transform(new NDList(array));
}
/**
* {@inheritDoc}
*/
@Override
public float[] processOutput(TranslatorContext ctx, NDList list) {
NDArray embedding = list.singletonOrThrow();
embedding = embedding.div(embedding.norm()); // L2归一化
return embedding.toFloatArray();
}
@Override
public Batchifier getBatchifier() {
return Batchifier.STACK;
}
}

57
face/src/main/java/cn/smartjavaai/face/model/facerec/translator/FaceNetRecTranslator.java Normal file
View File

@@ -0,0 +1,57 @@
package cn.smartjavaai.face.model.facerec.translator;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.transform.Normalize;
import ai.djl.modality.cv.transform.Resize;
import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Pipeline;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
/**
* facenet人脸特征提取Translator
* @author dwj
* @date 2025/3/31
*/
public final class FaceNetRecTranslator implements Translator<Image, float[]> {
public FaceNetRecTranslator() {
}
/**
* {@inheritDoc}
*/
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
Pipeline pipeline = new Pipeline();
pipeline
//.add(new Resize(112))
.add(new ToTensor())
.add(new Normalize(
new float[]{0.5F, 0.5F, 0.5F},
new float[]{0.5F, 0.5F, 0.5F}));
return pipeline.transform(new NDList(array));
}
/**
* {@inheritDoc}
*/
@Override
public float[] processOutput(TranslatorContext ctx, NDList list) {
NDArray embedding = list.singletonOrThrow();
embedding = embedding.div(embedding.norm()); // L2归一化
return embedding.toFloatArray();
}
@Override
public Batchifier getBatchifier() {
return Batchifier.STACK;
}
}

446
face/src/main/java/cn/smartjavaai/face/model/liveness/CommonLivenessModel.java Normal file
View File

@@ -0,0 +1,446 @@
package cn.smartjavaai.face.model.liveness;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import ai.djl.util.JsonUtils;
import cn.smartjavaai.common.entity.DetectionInfo;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.preprocess.BufferedImagePreprocessor;
import cn.smartjavaai.common.utils.*;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.constant.MiniVisionConstant;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.liveness.criterial.LivenessCriteriaFactory;
import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator;
import com.seeta.sdk.FaceAntiSpoofing;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameUtils;
import org.opencv.core.Mat;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.file.Paths;
import java.util.*;
/**
* 通用活体检测模型
* @author dwj
*/
@Slf4j
public class CommonLivenessModel implements LivenessDetModel{
protected GenericObjectPool<Predictor<Image, Float>> predictorPool;
protected LivenessConfig config;
protected ZooModel<Image, Float> model;
@Override
public void loadModel(LivenessConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath不能为空");
}
this.config = config;
//设置真人阈值
Float realityThreshold = Objects.isNull(config.getRealityThreshold()) ? MiniVisionConstant.REALITY_THRESHOLD : config.getRealityThreshold();
this.config.setRealityThreshold(realityThreshold);
Criteria<Image, Float> criteria = LivenessCriteriaFactory.createCriteria(config);
try {
model = criteria.loadModel();
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("阿里通义实验室活体检测模型加载失败", e);
}
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
predictorPool.setMaxTotal(predictorPoolSize);
log.debug("当前设备: " + model.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
}
@Override
public R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
Predictor<Image, Float> predictor = null;
Image djlImage = null;
try {
predictor = predictorPool.borrowObject();
//预处理图片
BufferedImage processedImage = image;
if(config.getModelEnum() == LivenessModelEnum.IIC_FL_MODEL){
processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle)
.setExtendRatio(96f / 112f)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(128)
.enableCenterCrop(true)
.setCenterCropSize(112)
.process();
}
djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage));
Float result = predictor.predict(djlImage);
if(result >= config.getRealityThreshold()){
return R.ok(new LivenessResult(LivenessStatus.LIVE, result));
}else{
float nonLiveScore = BigDecimal.ONE.subtract(new BigDecimal(result)).floatValue();
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore));
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
if (djlImage != null){
((Mat)djlImage.getWrappedImage()).release();
}
}
}
@Override
public R<LivenessResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionRectangle);
}
@Override
public R<LivenessResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<LivenessResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData, faceDetectionRectangle);
}
@Override
public R<List<LivenessResult>> detect(String imagePath, DetectionResponse faceDetectionResponse) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionResponse);
}
@Override
public R<List<LivenessResult>> detect(byte[] imageData, DetectionResponse faceDetectionResponse) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<List<LivenessResult>> detect(BufferedImage image, DetectionResponse faceDetectionResponse) {
if(!ImageUtils.isImageValid(image)){
R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){
R.fail(R.Status.NO_FACE_DETECTED);
}
List<LivenessResult> livenessStatusList = new ArrayList<LivenessResult>();
for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){
R<LivenessResult> result = detect(image, detectionInfo.getDetectionRectangle());
if(!result.isSuccess()){
return R.fail(result.getCode(), result.getMessage());
}
livenessStatusList.add(result.getData());
}
return R.ok(livenessStatusList);
}
@Override
public R<List<LivenessResult>> detectBase64(String base64Image, DetectionResponse faceDetectionResponse) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData, faceDetectionResponse);
}
@Override
public R<LivenessResult> detectTopFace(BufferedImage image) {
if(Objects.isNull(config.getDetectModel())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型");
}
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
return detect(image, faceDetectionResponse.getData().getDetectionInfoList().get(0).getDetectionRectangle());
}
@Override
public R<LivenessResult> detectTopFace(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detectTopFace(image);
}
@Override
public R<LivenessResult> detectTopFace(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<LivenessResult> detectTopFaceBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detectTopFace(imageData);
}
@Override
public R<DetectionResponse> detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(Objects.isNull(config.getDetectModel())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型");
}
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){
R<LivenessResult> result = detect(image, detectionInfo.getDetectionRectangle());
if(!result.isSuccess()){
return R.fail(result.getCode(), result.getMessage());
}
if(Objects.isNull(detectionInfo.getFaceInfo())){
detectionInfo.setFaceInfo(new FaceInfo());
}
detectionInfo.getFaceInfo().setLivenessStatus(result.getData());
}
return faceDetectionResponse;
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<LivenessResult> detectVideo(InputStream videoInputStream) {
if(Objects.isNull(videoInputStream)){
return R.fail(R.Status.INVALID_VIDEO);
}
return detectVideo(new FFmpegFrameGrabber(videoInputStream));
}
@Override
public R<LivenessResult> detectVideo(String videoPath) {
if(!FileUtils.isFileExists(videoPath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
return detectVideo(new FFmpegFrameGrabber(videoPath));
}
private R<LivenessResult> detectVideo(FFmpegFrameGrabber grabber) {
try {
//滑动窗口
Deque<Float> scoreWindow = new ArrayDeque<>();
grabber.start();
// 获取视频总帧数
int totalFrames = grabber.getLengthInFrames();
log.debug("视频总帧数：{}，检测帧数：{}", totalFrames, config.getFrameCount());
if(totalFrames < config.getFrameCount()){
return R.fail(10001, "视频帧数低于检测帧数");
}
// 逐帧处理视频
for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) {
if(frameIndex >= config.getMaxVideoDetectFrames()){
return R.fail(10002, "超出最大检测帧数：" + config.getMaxVideoDetectFrames());
}
// 获取当前帧
Frame frame = grabber.grabImage();
if (frame != null) {
BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame);
R<LivenessResult> livenessStatus = detectTopFace(bufferedImage);
if(!livenessStatus.isSuccess()){
log.debug("第" + frameIndex + "帧处理失败：" + livenessStatus.getMessage());
continue;
}else{
log.debug("第" + frameIndex + "帧活体检测结果：" + JsonUtils.toJson(livenessStatus));
float liveScore = 0;
if(livenessStatus.getData().getStatus() == LivenessStatus.LIVE){
liveScore = livenessStatus.getData().getScore();
}else{
liveScore = BigDecimal.ONE.subtract(BigDecimal.valueOf(livenessStatus.getData().getScore())).floatValue();
}
scoreWindow.add(liveScore);
}
// 如果累计检测帧数 >= 配置值，开始判断
if (scoreWindow.size() >= config.getFrameCount()) {
float avgScore = (float) scoreWindow.stream()
.mapToDouble(Float::doubleValue)
.average()
.orElse(0.0);
log.debug("滑动窗口平均得分: {}", avgScore);
if (avgScore >= config.getRealityThreshold()) {
grabber.stop();
return R.ok(new LivenessResult(LivenessStatus.LIVE, avgScore));
} else {
grabber.stop();
float nonLiveScore = BigDecimal.ONE.subtract(BigDecimal.valueOf(avgScore)).floatValue();
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore));
}
}
}
}
grabber.stop();
if(scoreWindow.size() < config.getFrameCount()){
return R.fail(1000, "有效帧数量不足，无法完成活体检测");
}
} catch (Exception e) {
throw new FaceException(e);
}
return R.fail(R.Status.Unknown);
}
@Override
public GenericObjectPool<Predictor<Image, Float>> getPool() {
return predictorPool;
}
@Override
public void close() throws Exception {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
}

304
face/src/main/java/cn/smartjavaai/face/model/liveness/LivenessDetModel.java Normal file
View File

@@ -0,0 +1,304 @@
package cn.smartjavaai.face.model.liveness;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.face.config.LivenessConfig;
import org.apache.commons.pool2.impl.GenericObjectPool;
import java.awt.image.BufferedImage;
import java.io.InputStream;
import java.util.List;
/**
* 活体检测模型
* @author dwj
*/
public interface LivenessDetModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(LivenessConfig config); // 加载模型
/**
* 活体检测(多人脸)
* @param imagePath 图片路径
* @return
*/
default R<DetectionResponse> detect(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param image BufferedImage
* @return
*/
default R<DetectionResponse> detect(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param imageData 图片字节流
* @return
*/
default R<DetectionResponse> detect(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param base64Image
* @return
*/
default R<DetectionResponse> detectBase64(String base64Image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param imagePath 图片路径
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<LivenessResult>> detect(String imagePath, DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param imageData 图片数据
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<LivenessResult>> detect(byte[] imageData,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param image BufferedImage
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<LivenessResult>> detect(BufferedImage image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param base64Image
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<LivenessResult>> detectBase64(String base64Image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imagePath 图片路径
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imageData
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param image BufferedImage
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param base64Image
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imagePath 图片路径
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imageData
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param image BufferedImage
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param base64Image
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(分数最高人脸)
* @param image
* @return
*/
default R<LivenessResult> detectTopFace(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(分数最高人脸)
* @param imagePath
* @return
*/
default R<LivenessResult> detectTopFace(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(分数最高人脸)
* @param imageData
* @return
*/
default R<LivenessResult> detectTopFace(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(分数最高人脸)
* @param base64Image
* @return
*/
default R<LivenessResult> detectTopFaceBase64(String base64Image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 视频活体检测(逐帧检测)
* @param frameImage
* @param faceDetectionRectangle
* @return
*/
// default R<LivenessResult> detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
// throw new UnsupportedOperationException("默认不支持该功能");
// }
/**
* 视频活体检测(逐帧检测)
* @param frameData
* @param faceDetectionRectangle
* @return
*/
// default R<LivenessResult> detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
// throw new UnsupportedOperationException("默认不支持该功能");
// }
/**
* 视频活体检测(逐帧检测)
* @param frameImageData
* @return
*/
// default R<LivenessResult> detectVideoByFrame(byte[] frameImageData){
// throw new UnsupportedOperationException("默认不支持该功能");
// }
/**
* 视频活体检测(逐帧检测)
* @param frameImageData
* @return
*/
// default R<LivenessResult> detectVideoByFrame(BufferedImage frameImageData){
// throw new UnsupportedOperationException("默认不支持该功能");
// }
/**
* 视频活体检测
* @param videoInputStream
* @return
*/
default R<LivenessResult> detectVideo(InputStream videoInputStream){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 视频活体检测
* @param videoPath
* @return
*/
default R<LivenessResult> detectVideo(String videoPath){
throw new UnsupportedOperationException("默认不支持该功能");
}
default GenericObjectPool<Predictor<Image, Float>> getPool() {
throw new UnsupportedOperationException("默认不支持该功能");
}
}

290
face/src/main/java/cn/smartjavaai/face/model/liveness/MiniVisionLivenessModel.java Normal file
View File

@@ -0,0 +1,290 @@
package cn.smartjavaai.face.model.liveness;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.preprocess.BufferedImagePreprocessor;
import cn.smartjavaai.common.utils.ArrayUtils;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.constant.MiniVisionConstant;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator;
import cn.smartjavaai.common.utils.OpenCVUtils;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameUtils;
import org.opencv.core.Mat;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.file.Paths;
import java.util.*;
/**
* 小视科技 活体检测模型
* @author dwj
* @date 2025/6/27
*/
@Slf4j
public class MiniVisionLivenessModel extends CommonLivenessModel{
/**
* 个性化参数：seModelPath
*/
private static final String SE_MODEL_PATH_KEY = "seModelPath";
private GenericObjectPool<Predictor<Image, float[]>> predictorPool;
private GenericObjectPool<Predictor<Image, float[]>> sePredictorPool;
/**
* 模型策略
*/
private ModelStrategy modelStrategy;
private ZooModel<Image, float[]> model;
private ZooModel<Image, float[]> seModel;
@Override
public void loadModel(LivenessConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
String seModelPath = config.getCustomParam(SE_MODEL_PATH_KEY, String.class);
if(StringUtils.isBlank(config.getModelPath()) && StringUtils.isBlank(seModelPath)){
throw new FaceException("modelPath 和 seModelPath 至少有一个不能为空");
}
this.config = config;
//设置真人阈值
Float realityThreshold = Objects.isNull(config.getRealityThreshold()) ? MiniVisionConstant.REALITY_THRESHOLD : config.getRealityThreshold();
this.config.setRealityThreshold(realityThreshold);
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId());
}
if(StringUtils.isNotBlank(config.getModelPath()) && StringUtils.isBlank(seModelPath)){
//2.7_80x80_MiniFASNetV2
modelStrategy = ModelStrategy.MINIFASNET_V2;
}else if (StringUtils.isBlank(config.getModelPath()) && StringUtils.isNotBlank(seModelPath)){
//4_0_0_80x80_MiniFASNetV1SE
modelStrategy = ModelStrategy.MINIFASNET_V1_SE;
}else{
//融合
modelStrategy = ModelStrategy.FUSION;
}
if(modelStrategy == ModelStrategy.MINIFASNET_V2 || modelStrategy == ModelStrategy.FUSION){
//初始化 检测Criteria
Criteria<Image, float[]> criteria =
Criteria.builder()
.optEngine("OnnxRuntime")
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(config.getModelPath()))
.optTranslator(new MiniVisionTranslator())
.optProgress(new ProgressBar())
.optDevice(device)
.build();
try {
model = criteria.loadModel();
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("MiniFASNetV2模型加载失败", e);
}
}
if(modelStrategy == ModelStrategy.MINIFASNET_V1_SE || modelStrategy == ModelStrategy.FUSION){
//初始化 检测Criteria
Criteria<Image, float[]> seCriteria =
Criteria.builder()
.optEngine("OnnxRuntime")
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(seModelPath))
.optTranslator(new MiniVisionTranslator())
.optProgress(new ProgressBar())
.optDevice(device)
.build();
try {
seModel = seCriteria.loadModel();
this.sePredictorPool = new GenericObjectPool<>(new PredictorFactory<>(seModel));
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("MiniFASNetV1SE模型加载失败", e);
}
}
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
predictorPool.setMaxTotal(predictorPoolSize);
sePredictorPool.setMaxTotal(predictorPoolSize);
log.debug("当前设备: " + model.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
}
@Override
public R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
Predictor<Image, float[]> predictor = null;
Predictor<Image, float[]> sePredictor = null;
try {
float[] result = null;
float[] seResult = null;
if(Objects.nonNull(predictorPool)){
//预处理图片
BufferedImage processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle)
.setExtendRatio(2.7f)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(80)
.process();
predictor = predictorPool.borrowObject();
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage));
result = predictor.predict(djlImage);
((Mat)djlImage.getWrappedImage()).release();
}
if(Objects.nonNull(sePredictorPool)){
//预处理图片
BufferedImage processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle)
.setExtendRatio(4)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(80)
.process();
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage));
sePredictor = sePredictorPool.borrowObject();
seResult = sePredictor.predict(djlImage);
((Mat)djlImage.getWrappedImage()).release();
}
if(Objects.isNull(result) && Objects.isNull(seResult)){
throw new FaceException("活体检测错误");
}
//计算结果
int maxIndex = ArrayUtils.sumAndFindMaxIndex(result, seResult, 3);
BigDecimal score = Objects.isNull(result) ? BigDecimal.ZERO : BigDecimal.valueOf(result[maxIndex]);
BigDecimal seScore = Objects.isNull(seResult) ? BigDecimal.ZERO : BigDecimal.valueOf(seResult[maxIndex]);
BigDecimal avgSocre = score.add(seScore).divide(BigDecimal.valueOf(2), 2, RoundingMode.HALF_UP);
if(maxIndex == 1){
if(avgSocre.floatValue() >= config.getRealityThreshold()){
return R.ok(new LivenessResult(LivenessStatus.LIVE, avgSocre.floatValue()));
}else{
float nonLiveScore = BigDecimal.ONE.subtract(avgSocre).floatValue();
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore));
}
}else{
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, avgSocre.floatValue()));
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
if (sePredictor != null) {
try {
sePredictorPool.returnObject(sePredictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
sePredictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
public GenericObjectPool<Predictor<Image, float[]>> getPredictorPool() {
return predictorPool;
}
public GenericObjectPool<Predictor<Image, float[]>> getSePredictorPool() {
return sePredictorPool;
}
@Override
public void close() throws Exception {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (sePredictorPool != null) {
sePredictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 sePredictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
try {
if (seModel != null) {
seModel.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
/**
* 模型策略
*/
protected enum ModelStrategy {
MINIFASNET_V2,
MINIFASNET_V1_SE,
FUSION // 融合模型
}
}

596
face/src/main/java/cn/smartjavaai/face/model/liveness/Seetaface6LivenessModel.java Normal file
View File

@@ -0,0 +1,596 @@
package cn.smartjavaai.face.model.liveness;
import ai.djl.engine.Engine;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.face.constant.LivenessConstant;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.pool.*;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* seetaface6 活体检测模型
* @author dwj
* @date 2025/4/30
*/
@Slf4j
public class Seetaface6LivenessModel implements LivenessDetModel{
private FaceDetectorPool faceDetectorPool;
private FaceAntiSpoofingPool faceAntiSpoofingPool;
private FaceLandmarkerPool faceLandmarkerPool;
private LivenessConfig config;
@Override
public void loadModel(LivenessConfig config) {
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
}
this.config = config;
//加载依赖库
NativeLoader.loadNativeLibraries(config.getDevice());
log.debug("Loading seetaFace6 library successfully.");
String[] faceDetectorModelPath = {config.getModelPath() + File.separator + "face_detector.csta"};
String[] faceAntiSpoofingModelPath = {config.getModelPath() + File.separator + "fas_first.csta",config.getModelPath() + File.separator + "fas_second.csta"};
String[] faceLandmarkerModelPath = {config.getModelPath() + File.separator + "face_landmarker_pts5.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = config.getGpuId();
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
}
try {
SeetaModelSetting faceDetectorPoolSetting = new SeetaModelSetting(gpuId, faceDetectorModelPath, device);
SeetaConfSetting faceDetectorPoolConfSetting = new SeetaConfSetting(faceDetectorPoolSetting);
SeetaModelSetting faceLandmarkerPoolSetting = new SeetaModelSetting(gpuId, faceLandmarkerModelPath, device);
SeetaConfSetting faceLandmarkerPoolConfSetting = new SeetaConfSetting(faceLandmarkerPoolSetting);
SeetaModelSetting faceAntiSpoofingSetting = new SeetaModelSetting(gpuId, faceAntiSpoofingModelPath, device);
SeetaConfSetting faceAntiSpoofingPoolConfSetting = new SeetaConfSetting(faceAntiSpoofingSetting);
this.faceDetectorPool = new FaceDetectorPool(faceDetectorPoolConfSetting);
this.faceAntiSpoofingPool = new FaceAntiSpoofingPool(faceAntiSpoofingPoolConfSetting);
this.faceLandmarkerPool = new FaceLandmarkerPool(faceLandmarkerPoolConfSetting);
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
faceDetectorPool.setMaxTotal(predictorPoolSize);
faceAntiSpoofingPool.setMaxTotal(predictorPoolSize);
faceLandmarkerPool.setMaxTotal(predictorPoolSize);
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
//初始化模型参数
initConfig();
} catch (FileNotFoundException e) {
throw new FaceException(e);
}
}
/**
* 初始化模型参数
*/
private void initConfig(){
FaceAntiSpoofing faceAntiSpoofing = null;
//设置参数
try {
//人脸清晰度阈值
float faceClarityThreshold = LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD;
//活体阈值
float realityThreshold = LivenessConstant.DEFAULT_REALITY_THRESHOLD;
Float faceClarityThresholdValue = config.getCustomParam("faceClarityThreshold", Float.class);
if(Objects.nonNull(faceClarityThresholdValue)){
faceClarityThreshold = faceClarityThresholdValue;
}
Float realityThresholdValue = config.getRealityThreshold();
if(Objects.nonNull(realityThresholdValue)){
realityThreshold = realityThresholdValue;
}
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
faceAntiSpoofing.SetThreshold(faceClarityThreshold, realityThreshold);
faceAntiSpoofing.SetVideoFrameCount(config.getFrameCount());
} catch (Exception e) {
throw new FaceException(e);
} finally {
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
private R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints, boolean isImage) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(faceDetectionRectangle)){
return R.fail(R.Status.NO_FACE_DETECTED);
}
if(keyPoints == null || keyPoints.isEmpty()){
return R.fail(1002,"人脸关键点keyPoints为空");
}
FaceAntiSpoofing.Status status = null;
FaceAntiSpoofing faceAntiSpoofing = null;
try {
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(faceDetectionRectangle);
SeetaPointF[] landmarks = FaceUtils.convertToSeetaPointF(keyPoints);
//检测图片
if(isImage){
status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks);
}else{
//检测视频
status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks);
}
return R.ok(new LivenessResult(FaceUtils.convertToLivenessStatus(status)));
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<DetectionResponse> detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
FaceAntiSpoofing faceAntiSpoofing = null;
FaceLandmarker faceLandmarker = null;
FaceDetector detectPredictor = null;
try {
//默认使用Seetaface6检测模型
if(Objects.isNull(config.getDetectModel())){
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
detectPredictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
List<LivenessStatus> livenessStatusList = new ArrayList<LivenessStatus>();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(SeetaRect seetaRect : seetaResult){
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, landmarks);
seetaPointFSList.add(landmarks);
//检测图片
FaceAntiSpoofing.Status status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks);
livenessStatusList.add(FaceUtils.convertToLivenessStatus(status));
}
return R.ok(FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList, livenessStatusList));
}else{
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){
R<LivenessResult> result = detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints());
if(!result.isSuccess()){
return R.fail(result.getCode(), result.getMessage());
}
if(Objects.isNull(detectionInfo.getFaceInfo())){
detectionInfo.setFaceInfo(new FaceInfo());
}
detectionInfo.getFaceInfo().setLivenessStatus(result.getData());
}
return faceDetectionResponse;
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
if (detectPredictor != null) {
try {
faceDetectorPool.returnObject(detectPredictor);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<List<LivenessResult>> detect(String imagePath, DetectionResponse faceDetectionResponse) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionResponse);
}
@Override
public R<List<LivenessResult>> detect(BufferedImage image, DetectionResponse faceDetectionResponse) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
List<LivenessResult> livenessStatusList = new ArrayList<LivenessResult>();
for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
R<LivenessResult> result = detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints());
if(!result.isSuccess()){
return R.fail(result.getCode(), result.getMessage());
}
livenessStatusList.add(result.getData());
}
return R.ok(livenessStatusList);
}
@Override
public R<List<LivenessResult>> detect(byte[] imageData, DetectionResponse faceDetectionResponse) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<LivenessResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionRectangle, keyPoints);
}
@Override
public R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
return detect(image, faceDetectionRectangle, keyPoints, true);
}
@Override
public R<LivenessResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<LivenessResult> detectTopFace(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detectTopFace(image);
}
private R<LivenessResult> detectTopFace(BufferedImage image, boolean isImage) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
FaceAntiSpoofing faceAntiSpoofing = null;
FaceLandmarker faceLandmarker = null;
FaceDetector detectPredictor = null;
try {
//默认使用Seetaface6检测模型
if(Objects.isNull(config.getDetectModel())){
detectPredictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
return R.fail(R.Status.NO_FACE_DETECTED);
}
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], landmarks);
FaceAntiSpoofing.Status status = null;
//检测图片
if(isImage){
status = faceAntiSpoofing.Predict(imageData, seetaResult[0], landmarks);
}else{
status = faceAntiSpoofing.PredictVideo(imageData, seetaResult[0], landmarks);
}
return R.ok(new LivenessResult(FaceUtils.convertToLivenessStatus(status)));
}else{
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
DetectionInfo detectionInfo = faceDetectionResponse.getData().getDetectionInfoList().get(0);
return detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints(), isImage);
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
if (detectPredictor != null) {
try {
faceDetectorPool.returnObject(detectPredictor);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<LivenessResult> detectTopFace(BufferedImage image) {
return detectTopFace(image, true);
}
@Override
public R<LivenessResult> detectTopFace(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
public R<LivenessResult> detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!ImageUtils.isImageValid(frameImage)){
return R.fail(R.Status.INVALID_IMAGE);
}
return detect(frameImage,faceDetectionRectangle, keyPoints,false);
}
public R<LivenessResult> detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(Objects.isNull(frameData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(frameData)), faceDetectionRectangle, keyPoints, false);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
public R<LivenessResult> detectVideoByFrame(byte[] frameImageData) {
if(Objects.isNull(frameImageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detectVideoByFrame(ImageIO.read(new ByteArrayInputStream(frameImageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
public R<LivenessResult> detectVideoByFrame(BufferedImage frameImageData) {
return detectTopFace(frameImageData, false);
}
@Override
public R<LivenessResult> detectVideo(InputStream videoInputStream) {
if(Objects.isNull(videoInputStream)){
throw new FaceException("视频无效");
}
return detectVideo(new FFmpegFrameGrabber(videoInputStream));
}
@Override
public R<LivenessResult> detectVideo(String videoPath) {
if(!FileUtils.isFileExists(videoPath)){
throw new FaceException("视频文件不存在");
}
return detectVideo(new FFmpegFrameGrabber(videoPath));
}
private R<LivenessResult> detectVideo(FFmpegFrameGrabber grabber) {
FaceAntiSpoofing faceAntiSpoofing = null;
try {
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
//重置视频
faceAntiSpoofing.ResetVideo();
grabber.start();
// 获取视频总帧数
int totalFrames = grabber.getLengthInFrames();
int videoFrameCountConfig = faceAntiSpoofing.GetVideoFrameCount();
log.debug("视频总帧数：{}，检测帧数：{}", totalFrames, videoFrameCountConfig);
if(totalFrames < videoFrameCountConfig){
return R.fail(1001, "视频帧数低于检测帧数");
}
// 逐帧处理视频
for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) {
if(frameIndex >= config.getMaxVideoDetectFrames()){
return R.fail(10002, "超出最大检测帧数：" + config.getMaxVideoDetectFrames());
}
// 获取当前帧
Frame frame = grabber.grabImage();
if (frame != null) {
BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame);
R<LivenessResult> livenessStatus = detectVideoByFrame(bufferedImage);
if(!livenessStatus.isSuccess()){
log.debug("第" + frameIndex + "帧处理失败：" + livenessStatus.getMessage());
continue;
}
//满足检测帧数之后停止检测
if(livenessStatus.getData().getStatus() != LivenessStatus.DETECTING){
return livenessStatus;
}
}
}
grabber.stop();
} catch (FFmpegFrameGrabber.Exception e) {
throw new FaceException(e);
} catch (Exception e) {
throw new FaceException(e);
} finally {
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
return R.fail(1000, "有效帧数量不足，无法完成活体检测");
}
public FaceDetectorPool getFaceDetectorPool() {
return faceDetectorPool;
}
public FaceAntiSpoofingPool getFaceAntiSpoofingPool() {
return faceAntiSpoofingPool;
}
public FaceLandmarkerPool getFaceLandmarkerPool() {
return faceLandmarkerPool;
}
@Override
public void close() throws Exception {
try {
if (faceDetectorPool != null) {
faceDetectorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (faceAntiSpoofingPool != null) {
faceAntiSpoofingPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (faceLandmarkerPool != null) {
faceLandmarkerPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
}
}

50
face/src/main/java/cn/smartjavaai/face/model/liveness/criterial/LivenessCriteriaFactory.java Normal file
View File

@@ -0,0 +1,50 @@
package cn.smartjavaai.face.model.liveness.criterial;
import ai.djl.Device;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.translator.ImageFeatureExtractorFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.constant.FaceNetConstant;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.model.facerec.translator.FaceNetRecTranslator;
import cn.smartjavaai.face.model.liveness.translator.IicFrTranslator;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* 活体检测 Criteria构建工厂
* @author dwj
*/
public class LivenessCriteriaFactory {
public static Criteria<Image, Float> createCriteria(LivenessConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId());
}
Criteria<Image, Float> criteria = null;
if(config.getModelEnum() == LivenessModelEnum.IIC_FL_MODEL){
criteria = Criteria.builder()
.optEngine("OnnxRuntime")
.setTypes(ai.djl.modality.cv.Image.class, Float.class)
.optModelPath(Paths.get(config.getModelPath()))
.optTranslator(new IicFrTranslator())
.optProgress(new ProgressBar())
.optDevice(device)
.build();
}
return criteria;
}
}

42
face/src/main/java/cn/smartjavaai/face/model/liveness/translator/IicFrTranslator.java Normal file
View File

@@ -0,0 +1,42 @@
package cn.smartjavaai.face.model.liveness.translator;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
import java.util.Arrays;
public class IicFrTranslator implements Translator<Image, Float> {
@Override
public Float processOutput(TranslatorContext ctx, NDList list) {
NDArray prob = list.singletonOrThrow();
return prob.toFloatArray()[1];
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDManager manager = ctx.getNDManager();
NDArray array = input.toNDArray(manager, Image.Flag.COLOR);
array = array.transpose(2, 0, 1);
array = array.expandDims(0);
// 归一化
array = array.toType(DataType.FLOAT32, false).div(255.0f);
return new NDList(array);
}
@Override
public Batchifier getBatchifier() {
return null;
}
}

45
face/src/main/java/cn/smartjavaai/face/model/liveness/translator/MiniVisionTranslator.java Normal file
View File

@@ -0,0 +1,45 @@
package cn.smartjavaai.face.model.liveness.translator;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.transform.Pad;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.translate.*;
import java.util.Arrays;
/**
* minivision translator
* @author dwj
* @date 2025/6/27
*/
public class MiniVisionTranslator implements Translator<Image, float[]> {
@Override
public float[] processOutput(TranslatorContext ctx, NDList list) {
NDArray prob = list.singletonOrThrow();
NDArray softmax = prob.softmax(-1);
return softmax.toFloatArray();
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
array = array.toType(ai.djl.ndarray.types.DataType.FLOAT32, false);
// 调整数据布局: HWC -> CHW
array = array.transpose(2, 0, 1);
// 添加batch维度 (NCHW)
array = array.expandDims(0);
return new NDList(array);
}
@Override
public Batchifier getBatchifier() {
return null;
}
}

229
face/src/main/java/cn/smartjavaai/face/model/quality/FaceQualityModel.java Normal file
View File

@@ -0,0 +1,229 @@
package cn.smartjavaai.face.model.quality;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.config.QualityConfig;
import cn.smartjavaai.face.entity.FaceQualitySummary;
import cn.smartjavaai.face.entity.FaceQualityResult;
import java.awt.image.BufferedImage;
import java.util.List;
/**
* 质量评估模型
* @author dwj
* @date 2025/6/23
*/
public interface FaceQualityModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(QualityConfig config);
/**
* 亮度评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateBrightness(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 亮度评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateBrightness(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 亮度评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateBrightness(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 清晰度评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateClarity(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 清晰度评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateClarity(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 清晰度评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateClarity(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 完整度评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateCompleteness(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 完整度评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateCompleteness(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 完整度评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateCompleteness(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸姿态评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluatePose(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸姿态评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluatePose(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸姿态评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluatePose(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸分辨率评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateResolution(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸分辨率评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateResolution(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸分辨率评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateResolution(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 评估所有
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualitySummary> evaluateAll(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 评估所有
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualitySummary> evaluateAll(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 评估所有
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualitySummary> evaluateAll(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

765
face/src/main/java/cn/smartjavaai/face/model/quality/Seetaface6QualityModel.java Normal file
View File

@@ -0,0 +1,765 @@
package cn.smartjavaai.face.model.quality;
import ai.djl.engine.Engine;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.PoolUtils;
import cn.smartjavaai.face.config.QualityConfig;
import cn.smartjavaai.face.entity.FaceQualityResult;
import cn.smartjavaai.face.entity.FaceQualitySummary;
import cn.smartjavaai.face.enums.QualityGrade;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.seetaface.ClarityDLResult;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.pool.*;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;
import java.util.Objects;
/**
* seetaface6 质量评估模型
* @author dwj
* @date 2025/4/30
*/
@Slf4j
public class Seetaface6QualityModel implements FaceQualityModel {
private QualityConfig config;
/**
* 人脸亮度评估器池
*/
private QualityOfBrightnessPool qualityOfBrightnessPool;
/**
* 人脸清晰度评估器池
*/
private QualityOfClarityPool qualityOfClarityPool;
/**
* 人脸清晰度评估器池(深度学习)
*/
private QualityOfLBNPool qualityOfLBNPool;
/**
* 人脸完整度评估器池
*/
private QualityOfIntegrityPool qualityOfIntegrityPool;
/**
* 人脸姿态评估器池
*/
private QualityOfPosePool qualityOfPosePool;
/**
* 人脸姿态评估器池(深度学习)
*/
private QualityOfPoseExPool qualityOfPoseExPool;
/**
* 人脸分辨率评估器池
*/
private QualityOfResolutionPool qualityOfResolutionPool;
int predictorPoolSize = 0;
@Override
public void loadModel(QualityConfig config) {
DeviceEnum device = DeviceEnum.CPU;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice();
}
//加载依赖库
NativeLoader.loadNativeLibraries(device);
this.config = config;
predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
log.debug("Loading seetaFace6 library successfully.");
}
@Override
public R<FaceQualityResult> evaluateBrightness(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfBrightness qualityOfBrightness = null;
try {
if(Objects.isNull(this.qualityOfBrightnessPool)){
this.qualityOfBrightnessPool = new QualityOfBrightnessPool(new SeetaConfSetting());
qualityOfBrightnessPool.setMaxTotal(predictorPoolSize);
}
qualityOfBrightness = qualityOfBrightnessPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfBrightness.QualityLevel level = qualityOfBrightness.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("亮度评估错误", e);
} finally {
if (qualityOfBrightness != null) {
try {
qualityOfBrightnessPool.returnObject(qualityOfBrightness);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluateBrightness(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateBrightness(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluateBrightness(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateBrightness(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceQualityResult> evaluateClarity(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfClarity qualityOfClarity = null;
try {
if(Objects.isNull(this.qualityOfClarityPool)){
this.qualityOfClarityPool = new QualityOfClarityPool(new SeetaConfSetting());
qualityOfClarityPool.setMaxTotal(predictorPoolSize);
}
qualityOfClarity = qualityOfClarityPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfClarity.QualityLevel level = qualityOfClarity.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("清晰度评估错误", e);
} finally {
if (qualityOfClarity != null) {
try {
qualityOfClarityPool.returnObject(qualityOfClarity);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluateClarity(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateClarity(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluateClarity(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateClarity(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceQualityResult> evaluateCompleteness(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfIntegrity qualityOfIntegrity = null;
try {
if(Objects.isNull(this.qualityOfIntegrityPool)){
this.qualityOfIntegrityPool = new QualityOfIntegrityPool(new SeetaConfSetting());
qualityOfIntegrityPool.setMaxTotal(predictorPoolSize);
}
qualityOfIntegrity = qualityOfIntegrityPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfIntegrity.QualityLevel level = qualityOfIntegrity.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("完整度评估错误", e);
} finally {
if (qualityOfIntegrity != null) {
try {
qualityOfIntegrityPool.returnObject(qualityOfIntegrity);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluateCompleteness(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateCompleteness(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluateCompleteness(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateCompleteness(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceQualityResult> evaluatePose(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfPose qualityOfPose = null;
try {
if(Objects.isNull(this.qualityOfPosePool)){
this.qualityOfPosePool = new QualityOfPosePool(new SeetaConfSetting());
qualityOfPosePool.setMaxTotal(predictorPoolSize);
}
qualityOfPose = qualityOfPosePool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfPose.QualityLevel level = qualityOfPose.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("姿态评估错误", e);
} finally {
if (qualityOfPose != null) {
try {
qualityOfPosePool.returnObject(qualityOfPose);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluatePose(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluatePose(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluatePose(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluatePose(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceQualityResult> evaluateResolution(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfResolution qualityOfResolution = null;
try {
if(Objects.isNull(this.qualityOfResolutionPool)){
this.qualityOfResolutionPool = new QualityOfResolutionPool(new SeetaConfSetting());
qualityOfResolutionPool.setMaxTotal(predictorPoolSize);
}
qualityOfResolution = qualityOfResolutionPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfResolution.QualityLevel level = qualityOfResolution.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("姿态评估错误", e);
} finally {
if (qualityOfResolution != null) {
try {
qualityOfResolutionPool.returnObject(qualityOfResolution);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluateResolution(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateResolution(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluateResolution(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateResolution(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
public R<ClarityDLResult> evaluateClarityWithDL(BufferedImage image, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfLBN qualityOfLBN = null;
try {
if(Objects.isNull(this.qualityOfLBNPool)){
if(Objects.isNull(config)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "缺少必要配置（QualityConfig），请在调用前初始化模型配置");
}
if(StringUtils.isBlank(config.getModelPath())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "QualityConfig中，modelPath为空");
}
SeetaConfSetting setting = getClarityMLSetting();
this.qualityOfLBNPool = new QualityOfLBNPool(setting);
qualityOfLBNPool.setMaxTotal(predictorPoolSize);
}
qualityOfLBN = qualityOfLBNPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
int[] light = new int[1];
int[] blur = new int[1];
int[] noise = new int[1];
qualityOfLBN.Detect(imageData, pointFS, light, blur, noise);
return R.ok(new ClarityDLResult(light, blur, noise));
} catch (Exception e) {
throw new FaceException("亮度评估错误", e);
} finally {
if (qualityOfLBN != null) {
try {
qualityOfLBNPool.returnObject(qualityOfLBN);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
public R<ClarityDLResult> evaluateClarityWithDL(String imagePath, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateClarityWithDL(image, keyPoints);
}
public R<ClarityDLResult> evaluateClarityWithDL(byte[] imageData, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateClarityWithDL(ImageIO.read(new ByteArrayInputStream(imageData)), keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
public R<FaceQualityResult> evaluatePoseWithDL(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfPoseEx qualityOfPoseEx = null;
try {
if(Objects.isNull(this.qualityOfPoseExPool)){
if(Objects.isNull(config)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "缺少必要配置（QualityConfig），请在调用前初始化模型配置");
}
if(StringUtils.isBlank(config.getModelPath())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "QualityConfig中，modelPath为空");
}
SeetaConfSetting setting = getPoseMLSetting();
this.qualityOfPoseExPool = new QualityOfPoseExPool(setting);
qualityOfPoseExPool.setMaxTotal(predictorPoolSize);
}
qualityOfPoseEx = qualityOfPoseExPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
float[] scores = new float[1];
QualityOfPoseEx.QualityLevel level = qualityOfPoseEx.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("亮度评估错误", e);
} finally {
if (qualityOfPoseEx != null) {
try {
qualityOfPoseExPool.returnObject(qualityOfPoseEx);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
public R<FaceQualityResult> evaluatePoseWithDL(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluatePoseWithDL(image, rectangle, keyPoints);
}
public R<FaceQualityResult> evaluatePoseWithDL(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluatePoseWithDL(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
/**
* 获取清晰度模型配置(深度学习)
* @return
*/
private SeetaConfSetting getClarityMLSetting() throws FileNotFoundException {
String[] modelPath = {config.getModelPath() + File.separator + "quality_lbn.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = 0;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = config.getGpuId();
}
}
SeetaConfSetting setting = new SeetaConfSetting(new SeetaModelSetting(gpuId, modelPath, device));
return setting;
}
/**
* 获取人脸姿态模型配置(深度学习)
* @return
*/
private SeetaConfSetting getPoseMLSetting() throws FileNotFoundException {
String[] modelPath = {config.getModelPath() + File.separator + "pose_estimation.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = 0;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = config.getGpuId();
}
}
SeetaConfSetting setting = new SeetaConfSetting(new SeetaModelSetting(gpuId, modelPath, device));
return setting;
}
@Override
public R<FaceQualitySummary> evaluateAll(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateAll(image, rectangle, keyPoints);
}
@Override
public R<FaceQualitySummary> evaluateAll(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfBrightness qualityOfBrightness = null;
QualityOfClarity qualityOfClarity = null;
QualityOfIntegrity qualityOfIntegrity = null;
QualityOfPose qualityOfPose = null;
QualityOfResolution qualityOfResolution = null;
try {
if(Objects.isNull(this.qualityOfBrightnessPool)){
this.qualityOfBrightnessPool = new QualityOfBrightnessPool(new SeetaConfSetting());
qualityOfBrightnessPool.setMaxTotal(predictorPoolSize);
}
if(Objects.isNull(this.qualityOfClarityPool)){
this.qualityOfClarityPool = new QualityOfClarityPool(new SeetaConfSetting());
qualityOfClarityPool.setMaxTotal(predictorPoolSize);
}
if(Objects.isNull(this.qualityOfIntegrityPool)){
this.qualityOfIntegrityPool = new QualityOfIntegrityPool(new SeetaConfSetting());
qualityOfIntegrityPool.setMaxTotal(predictorPoolSize);
}
if(Objects.isNull(this.qualityOfPosePool)){
this.qualityOfPosePool = new QualityOfPosePool(new SeetaConfSetting());
qualityOfPosePool.setMaxTotal(predictorPoolSize);
}
if(Objects.isNull(this.qualityOfResolutionPool)){
this.qualityOfResolutionPool = new QualityOfResolutionPool(new SeetaConfSetting());
qualityOfResolutionPool.setMaxTotal(predictorPoolSize);
}
FaceQualitySummary summary = new FaceQualitySummary();
qualityOfBrightness = qualityOfBrightnessPool.borrowObject();
qualityOfClarity = qualityOfClarityPool.borrowObject();
qualityOfIntegrity = qualityOfIntegrityPool.borrowObject();
qualityOfPose = qualityOfPosePool.borrowObject();
qualityOfResolution = qualityOfResolutionPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scoresBrightness = new float[1];
QualityOfBrightness.QualityLevel level = qualityOfBrightness.check(imageData, seetaRect, pointFS, scoresBrightness);
summary.setBrightness(new FaceQualityResult(scoresBrightness[0], QualityGrade.valueOf(level.name())));
float[] scoresClarity = new float[1];
QualityOfClarity.QualityLevel clarityLevel = qualityOfClarity.check(imageData, seetaRect, pointFS, scoresClarity);
summary.setClarity(new FaceQualityResult(scoresClarity[0], QualityGrade.valueOf(clarityLevel.name())));
float[] scoresIntegrity = new float[1];
QualityOfIntegrity.QualityLevel integrityLevel = qualityOfIntegrity.check(imageData, seetaRect, pointFS, scoresIntegrity);
summary.setCompleteness(new FaceQualityResult(scoresIntegrity[0], QualityGrade.valueOf(integrityLevel.name())));
float[] scoresPose = new float[1];
QualityOfPose.QualityLevel poseLevel = qualityOfPose.check(imageData, seetaRect, pointFS, scoresPose);
summary.setPose(new FaceQualityResult(scoresPose[0], QualityGrade.valueOf(poseLevel.name())));
float[] scoresResolution = new float[1];
QualityOfResolution.QualityLevel resolutionLevel = qualityOfResolution.check(imageData, seetaRect, pointFS, scoresResolution);
summary.setResolution(new FaceQualityResult(scoresResolution[0], QualityGrade.valueOf(resolutionLevel.name())));
return R.ok(summary);
} catch (Exception e) {
throw new FaceException("亮度评估错误", e);
} finally {
PoolUtils.returnToPool(qualityOfBrightnessPool, qualityOfBrightness);
PoolUtils.returnToPool(qualityOfClarityPool, qualityOfClarity);
PoolUtils.returnToPool(qualityOfIntegrityPool, qualityOfIntegrity);
PoolUtils.returnToPool(qualityOfPosePool, qualityOfPose);
PoolUtils.returnToPool(qualityOfResolutionPool, qualityOfResolution);
}
}
@Override
public R<FaceQualitySummary> evaluateAll(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateAll(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
public QualityOfBrightnessPool getQualityOfBrightnessPool() {
return qualityOfBrightnessPool;
}
public QualityOfClarityPool getQualityOfClarityPool() {
return qualityOfClarityPool;
}
public QualityOfLBNPool getQualityOfLBNPool() {
return qualityOfLBNPool;
}
public QualityOfIntegrityPool getQualityOfIntegrityPool() {
return qualityOfIntegrityPool;
}
public QualityOfPosePool getQualityOfPosePool() {
return qualityOfPosePool;
}
public QualityOfPoseExPool getQualityOfPoseExPool() {
return qualityOfPoseExPool;
}
public QualityOfResolutionPool getQualityOfResolutionPool() {
return qualityOfResolutionPool;
}
@Override
public void close() throws Exception {
if(Objects.nonNull(qualityOfBrightnessPool)){
qualityOfBrightnessPool.close();
}
if(Objects.nonNull(qualityOfClarityPool)){
qualityOfClarityPool.close();
}
if(Objects.nonNull(qualityOfLBNPool)){
qualityOfLBNPool.close();
}
if(Objects.nonNull(qualityOfIntegrityPool)){
qualityOfIntegrityPool.close();
}
if(Objects.nonNull(qualityOfPosePool)){
qualityOfPosePool.close();
}
if(Objects.nonNull(qualityOfPoseExPool)){
qualityOfPoseExPool.close();
}
if(Objects.nonNull(qualityOfResolutionPool)){
qualityOfResolutionPool.close();
}
}
}

92
face/src/main/java/cn/smartjavaai/face/preprocess/DJLImagePreprocessor.java Normal file
View File

@@ -0,0 +1,92 @@
package cn.smartjavaai.face.preprocess;
import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.opencv.OpenCVImageFactory;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.utils.FaceAlignUtils;
import cn.smartjavaai.face.utils.FaceUtils;
import org.opencv.core.Mat;
import java.util.Objects;
/**
* 图片预处理
* @author dwj
* @date 2025/6/27
*/
public class DJLImagePreprocessor {
private Image image;
private NDManager manager;
// 裁剪参数
private boolean enableCrop = false;
private DetectionRectangle cropRect;
// 仿射变换参数
private boolean enableAffine = false;
private double[][] keyPoints;
private int affineTargetWidth;
private int affineTargetHeight;
public DJLImagePreprocessor(Image image, NDManager manager) {
this.image = image;
this.manager = manager;
}
// 启用裁剪
public DJLImagePreprocessor enableCrop(DetectionRectangle rect) {
this.enableCrop = true;
this.cropRect = rect;
return this;
}
// 启用仿射变换
public DJLImagePreprocessor enableAffine(double[][] keyPoints, int targetWidth, int targetHeight) {
if(Objects.isNull(keyPoints)){
throw new IllegalArgumentException("keyPoints must be not null");
}
this.enableAffine = true;
this.affineTargetWidth = targetWidth;
this.affineTargetHeight = targetHeight;
this.keyPoints = keyPoints;
return this;
}
// 处理流程
public Image process() {
Image result = image;
if(enableAffine){
result = warpAffine(keyPoints, affineTargetWidth, affineTargetHeight);
}else {
if(enableCrop){
result = result.getSubImage(cropRect.x, cropRect.y, cropRect.width, cropRect.height);
}
}
return result;
}
// 仿射变换
private Image warpAffine(double[][] keyPoints, int width, int height) {
NDArray srcPoints = manager.create(keyPoints);
NDArray dstPoints = null;
if(width == 512 && height == 512){
dstPoints = FaceUtils.faceTemplate512x512(manager);
}else if(width == 112 && height == 112){
dstPoints = FaceUtils.faceTemplate112x112(manager);
}else if(width == 96 && height == 112){
dstPoints = FaceUtils.faceTemplate96x112(manager);
}
// 5点仿射变换
Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints);
Mat mat = FaceAlignUtils.warpAffine((Mat) image.getWrappedImage(), affine_matrix, width, height);
Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat);
return alignedImg;
}
}

24
face/src/main/java/cn/smartjavaai/face/seetaface/ClarityDLResult.java Normal file
View File

@@ -0,0 +1,24 @@
package cn.smartjavaai.face.seetaface;
import com.seeta.sdk.QualityOfLBN;
import lombok.Data;
/**
* Seetaface6 清晰度（深度学习）结果
* @author dwj
* @date 2025/6/25
*/
@Data
public class ClarityDLResult {
private QualityOfLBN.LIGHTSTATE lightstate;
private QualityOfLBN.BLURSTATE blurstate;
private QualityOfLBN.NOISESTATE noisestate;
public ClarityDLResult(int[] light, int[] blur, int[] noise) {
this.lightstate = QualityOfLBN.LIGHTSTATE.values()[light[0]];
this.blurstate = QualityOfLBN.BLURSTATE.values()[blur[0]];
this.noisestate = QualityOfLBN.NOISESTATE.values()[noise[0]];
}
}

238
face/src/main/java/cn/smartjavaai/face/seetaface/NativeLoader.java Normal file
View File

@@ -0,0 +1,238 @@
package cn.smartjavaai.face.seetaface;
import cn.hutool.core.io.FileUtil;
import cn.hutool.setting.dialect.Props;
import cn.hutool.system.OsInfo;
import cn.hutool.system.SystemUtil;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.exception.FaceException;
import com.seeta.sdk.util.DllItem;
import com.seeta.sdk.util.LoadNativeCore;
import lombok.extern.slf4j.Slf4j;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.*;
import java.util.stream.Collectors;
/**
* 依赖库加载器
* @author dwj
*/
@Slf4j
public class NativeLoader {
private static Path seetaface6NativePath;
private static final String SEETAFACE_LIB_DIR = "seetaface6";
/**
* 定义dll 路径和加载顺序的文件
*/
private static final String PROPERTIES_FILE_NAME = "dll.properties";
// 使用 volatile 保证内存可见性
private static volatile boolean isDllLoaded = false;
public static void loadNativeLibraries(DeviceEnum device) {
try {
if (!isDllLoaded) {
synchronized (NativeLoader.class) {
if (!isDllLoaded) { // 双重检查
OsInfo osInfo = SystemUtil.getOsInfo();
//检查当前系统是否支持
if(!osInfo.isWindows() && !osInfo.isLinux()){
throw new FaceException("当前系统不支持：" + osInfo.getName());
}
//判断硬件架构是否支持GPU
if(device != null && device.equals(DeviceEnum.GPU)){
//GPU仅支持amd64
if(!osInfo.getArch().contains("amd64") && !osInfo.getArch().contains("x86_64")){
throw new FaceException("seetaface6 GPU模型不支持当前arch：" + osInfo.getArch());
}
}
seetaface6NativePath = Paths.get(Config.getCachePath(), SEETAFACE_LIB_DIR);
//创建目录
FileUtil.mkdir(seetaface6NativePath);
log.debug("seetaface6依赖库路径: " + seetaface6NativePath.toAbsolutePath().toString());
//拷贝依赖库到缓存目录
List<File> fileList = getLibFiles(osInfo, device);
if(fileList != null && !fileList.isEmpty()){
// 加载依赖库文件
fileList.forEach(file -> {
System.load(file.getAbsolutePath());
//log.debug(String.format("load %s finish", file.getAbsolutePath()));
});
}
log.debug("seetaface6 依赖库加载完毕");
isDllLoaded = true;
}
}
} else {
log.debug("SeetaFace DLL is already loaded.");
}
} catch (Exception e) {
throw new RuntimeException("Native library loading failed", e);
}
}
/**
* 拷贝依赖库到缓存目录
* @param osInfo
* @return
*/
private static List<File> getLibFiles(OsInfo osInfo,DeviceEnum deviceEnum){
try {
String device = getDevice(deviceEnum);
log.debug("当前设备：{}", device);
//获取dll文件列表
List<DllItem> baseList = new ArrayList<>();
List<DllItem> jniList = new ArrayList<>();
InputStream propsInputStream = LoadNativeCore.class.getResourceAsStream(getPropertiesPath());
Props props = new Props();
props.load(propsInputStream);
String prefix = getPrefix();
props.forEach((keyObj, valuObj) -> {
String key = (String) keyObj;
String value = (String) valuObj;
DllItem dllItem = new DllItem();
dllItem.setKey(key);
if (key.contains("base")) {
if (value.contains("tennis")) {
dllItem.setValue(prefix + "base/" + device + "/" + value);
} else {
dllItem.setValue(prefix + "base/" + value);
}
baseList.add(dllItem);
} else {
dllItem.setValue(prefix + value);
jniList.add(dllItem);
}
});
//给dll文件排序
List<String> basePath = getSortedPath(baseList);
List<String> sdkPath = getSortedPath(jniList);
List<File> fileList = new ArrayList<>();
//拷贝文件到临时目录
for (String baseSo : basePath) {
fileList.add(extractLibrary(baseSo));
}
for (String sdkSo : sdkPath) {
fileList.add(extractLibrary(sdkSo));
}
return fileList;
} catch (Exception e) {
throw new FaceException("拷贝依赖库失败",e);
}
}
private static String getDevice(DeviceEnum deviceEnum) {
String device = "CPU";
if ("amd64".equals(getArch()) && deviceEnum != null) {
device = deviceEnum == DeviceEnum.GPU ? "GPU" : "CPU";
}
return device;
}
/**
* 返回路径文件前缀
*
* @return
*/
private static String getPrefix() {
String arch = getArch();
//aarch64
String os = SystemUtil.getOsInfo().getName();
//Windows操作系统
if (os != null && os.toLowerCase().startsWith("windows")) {
os = "/windows/";
} else if (os != null && os.toLowerCase().startsWith("linux")) {//Linux操作系统
os = "/linux/";
} else { //其它操作系统
//安卓 乌班图等等，先不写
return null;
}
// "/seetaface6/windows/amd64"
return "/" + SEETAFACE_LIB_DIR + os + arch + "/";
}
private static String getArch() {
String arch = SystemUtil.getOsInfo().getArch().toLowerCase();
if (arch.startsWith("amd64")
|| arch.startsWith("x86_64")
|| arch.startsWith("x86-64")
|| arch.startsWith("x64")) {
arch = "amd64";
} else if (arch.contains("aarch")) {
arch = "aarch64";
} else if (arch.contains("arm")) {
arch = "arm";
}
return arch;
}
/**
* 获取dll配置文件路径
*
* @return String
*/
private static String getPropertiesPath() {
return getPrefix() + PROPERTIES_FILE_NAME;
}
/**
* 拷贝依赖库到临时目录
* @param libPath
* @return
* @throws IOException
*/
private static File extractLibrary(String libPath) throws IOException {
String resourcePath = libPath;
try (InputStream in = NativeLoader.class.getResourceAsStream(resourcePath)) {
if (in == null) throw new FileNotFoundException(resourcePath);
Path path = Paths.get(resourcePath);
String fileName = path.getFileName().toString();
Path targetPath = seetaface6NativePath.resolve(fileName);
if (Files.exists(targetPath)) {
//log.debug("target file already exists, skip copy: {}", targetPath.toAbsolutePath());
} else {
Files.copy(in, targetPath, StandardCopyOption.REPLACE_EXISTING);
log.debug("copy target path success: {}", targetPath.toAbsolutePath());
// 设置可执行权限
if (!SystemUtil.getOsInfo().getName().toLowerCase().contains("win")) {
targetPath.toFile().setExecutable(true);
}
}
return targetPath.toFile();
}
}
/**
* 将获得的配置进行排序 并生成路径
*
* @param list
* @return List<String>
*/
private static List<String> getSortedPath(List<DllItem> list) {
return list.stream().sorted(Comparator.comparing(dllItem -> {
int i = dllItem.getKey().lastIndexOf(".") + 1;
String substring = dllItem.getKey().substring(i);
return Integer.valueOf(substring);
})).map(DllItem::getValue).collect(Collectors.toList());
}
}

15
face/src/main/java/cn/smartjavaai/face/sqllite/ResultSetExtractor.java Normal file
View File

@@ -0,0 +1,15 @@
package cn.smartjavaai.face.sqllite;
import java.sql.ResultSet;
/**
* ResultSetExtractor
* @author dwj
* @param <T>
*/
public interface ResultSetExtractor<T> {
public abstract T extractData(ResultSet rs);
}

14
face/src/main/java/cn/smartjavaai/face/sqllite/RowMapper.java Normal file
View File

@@ -0,0 +1,14 @@
package cn.smartjavaai.face.sqllite;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* RowMapper
* @author dwj
* @param <T>
*/
public interface RowMapper<T> {
public abstract T mapRow(ResultSet rs, int index) throws SQLException;
}

376
face/src/main/java/cn/smartjavaai/face/sqllite/SqliteHelper.java Normal file
View File

@@ -0,0 +1,376 @@
package cn.smartjavaai.face.sqllite;
import cn.hutool.core.io.resource.ResourceUtil;
import lombok.extern.slf4j.Slf4j;
import org.sqlite.SQLiteConfig;
import org.sqlite.SQLiteDataSource;
import javax.sql.DataSource;
import java.io.File;
import java.lang.reflect.Field;
import java.sql.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* sqlite帮助类
* @author dwj
*/
@Slf4j
public class SqliteHelper {
private static final ConcurrentHashMap<String, SqliteHelper> INSTANCES = new ConcurrentHashMap<>();
private static final int MAX_CONNECTIONS = 10; // 最大连接数
private final String dbFilePath;
private final DataSource dataSource;
/**
* 获取SqliteHelper实例（单例模式）
* @param dbFilePath sqlite db 文件路径
* @return SqliteHelper实例
* @throws SQLException SQL异常
* @throws ClassNotFoundException 类未找到异常
*/
public static SqliteHelper getInstance(String dbFilePath) throws SQLException, ClassNotFoundException {
return INSTANCES.computeIfAbsent(dbFilePath, path -> {
try {
return new SqliteHelper(path);
} catch (Exception e) {
log.error("创建SqliteHelper实例失败", e);
throw new RuntimeException("创建SqliteHelper实例失败", e);
}
});
}
/**
* 私有构造函数
* @param dbFilePath sqlite db 文件路径
* @throws ClassNotFoundException 类未找到异常
* @throws SQLException SQL异常
*/
private SqliteHelper(String dbFilePath) throws ClassNotFoundException, SQLException {
this.dbFilePath = dbFilePath;
createDatabaseIfNotExists();
// 初始化连接池
SQLiteDataSource sqLiteDataSource = new SQLiteDataSource();
sqLiteDataSource.setUrl("jdbc:sqlite:" + dbFilePath);
// 配置SQLite连接
SQLiteConfig sqLiteConfig = new SQLiteConfig();
sqLiteConfig.setSharedCache(true);
sqLiteConfig.enableLoadExtension(true);
sqLiteConfig.setBusyTimeout(5000); // 5秒超时
sqLiteDataSource.setConfig(sqLiteConfig);
this.dataSource = sqLiteDataSource;
}
/**
* 获取数据库连接
* @return 数据库连接
* @throws SQLException SQL异常
*/
public Connection getConnection() throws SQLException {
return dataSource.getConnection();
}
/**
* 执行sql查询
* @param sql sql select 语句
* @param rse 结果集处理类对象
* @return 查询结果
* @throws SQLException SQL异常
*/
public <T> T executeQuery(String sql, ResultSetExtractor<T> rse) throws SQLException {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
return rse.extractData(rs);
}
}
/**
* 执行select查询，返回结果列表
* @param sql sql select 语句
* @param rm 结果集的行数据处理类对象
* @return 查询结果列表
* @throws SQLException SQL异常
*/
public <T> List<T> executeQuery(String sql, RowMapper<T> rm) throws SQLException {
List<T> rsList = new ArrayList<>();
try (Connection conn = getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
while (rs.next()) {
rsList.add(rm.mapRow(rs, rs.getRow()));
}
return rsList;
}
}
/**
* 简单查询某个字段
* @param sql SQL查询语句
* @return 查询结果
* @throws SQLException SQL异常
*/
public String executeQuery(String sql) throws SQLException {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) {
return rs.getString(1);
}
return null;
}
}
/**
* 执行数据库更新sql语句
* @param sql SQL更新语句
* @return 更新行数
* @throws SQLException SQL异常
*/
public int executeUpdate(String sql) throws SQLException {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement()) {
return stmt.executeUpdate(sql);
}
}
/**
* 执行多个sql更新语句
* @param sqls SQL更新语句数组
* @throws SQLException SQL异常
*/
public void executeUpdate(String... sqls) throws SQLException {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement()) {
for (String sql : sqls) {
stmt.executeUpdate(sql);
}
}
}
/**
* 执行数据库更新 sql List
* @param sqls sql列表
* @throws SQLException SQL异常
*/
public void executeUpdate(List<String> sqls) throws SQLException {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement()) {
for (String sql : sqls) {
stmt.executeUpdate(sql);
}
}
}
/**
* 执行select查询，返回结果列表
* @param sql sql select 语句
* @param clazz 实体泛型
* @return 实体集合
* @throws SQLException 异常信息
* @throws IllegalAccessException 非法访问异常
* @throws InstantiationException 实例化异常
*/
public <T> List<T> executeQueryList(String sql, Class<T> clazz) throws SQLException, IllegalAccessException, InstantiationException {
List<T> rsList = new ArrayList<>();
try (Connection conn = getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
while (rs.next()) {
T t = clazz.newInstance();
for (Field field : t.getClass().getDeclaredFields()) {
field.setAccessible(true);
field.set(t, rs.getObject(field.getName()));
}
rsList.add(t);
}
return rsList;
}
}
/**
* 执行sql查询,适用单条结果集
* @param sql sql select 语句
* @param clazz 结果集处理类对象
* @return 查询结果
* @throws SQLException SQL异常
* @throws IllegalAccessException 非法访问异常
* @throws InstantiationException 实例化异常
*/
public <T> T executeQuery(String sql, Class<T> clazz) throws SQLException, IllegalAccessException, InstantiationException {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) {
T t = clazz.newInstance();
for (Field field : t.getClass().getDeclaredFields()) {
field.setAccessible(true);
field.set(t, rs.getObject(field.getName()));
}
return t;
}
return null;
}
}
/**
* 执行数据库更新sql语句
* @param tableName 表名
* @param param key-value键值对,key:表中字段名,value:值
* @return 更新行数
* @throws SQLException SQL异常
*/
public int executeInsertOrUpdate(String tableName, Map<String, Object> param) throws SQLException {
try (Connection conn = getConnection()) {
// 保证字段和值顺序一致
List<String> keys = new ArrayList<>(param.keySet());
StringBuilder sql = new StringBuilder();
sql.append("INSERT OR REPLACE INTO ");
sql.append(tableName);
sql.append(" (");
for (String key : keys) {
sql.append(key).append(",");
}
sql.deleteCharAt(sql.length() - 1);
sql.append(") VALUES (");
for (int i = 0; i < keys.size(); i++) {
sql.append("?,");
}
sql.deleteCharAt(sql.length() - 1);
sql.append(");");
log.debug("sql: {}", sql.toString());
try (PreparedStatement pstmt = conn.prepareStatement(sql.toString())) {
for (int i = 0; i < keys.size(); i++) {
Object value = param.get(keys.get(i));
if (value instanceof byte[]) {
pstmt.setBytes(i + 1, (byte[]) value);
} else {
pstmt.setObject(i + 1, value);
}
}
return pstmt.executeUpdate();
}
}
}
/**
* 使用预编译语句执行更新
* @param sql SQL语句
* @param args 参数
* @return 更新行数
* @throws SQLException SQL异常
*/
public int executeUpdate(String sql, Object[] args) throws SQLException {
try (Connection conn = getConnection()) {
if (args == null || args.length == 0) {
try (Statement stmt = conn.createStatement()) {
return stmt.executeUpdate(sql);
}
} else {
try (PreparedStatement stmt = conn.prepareStatement(sql)) {
for (int i = 0; i < args.length; i++) {
stmt.setObject(i + 1, args[i]);
}
return stmt.executeUpdate();
}
}
}
}
/**
* 创建数据库文件（如果不存在）
*/
private void createDatabaseIfNotExists() {
File dbFile = new File(dbFilePath);
if (!dbFile.exists()) {
try {
// 创建数据库文件
dbFile.getParentFile().mkdirs(); // 创建父目录
dbFile.createNewFile();
log.debug("Created new SQLite database file: {}", dbFilePath);
} catch (Exception e) {
log.error("Failed to create database file: {}", dbFilePath, e);
throw new RuntimeException("Database file creation failed", e);
}
}
}
/**
* 初始化数据库表结构
*/
public void initializeDatabase(String tableName, String schemaResourcePath) throws SQLException {
// 检查表是否存在
if (isTableExists(tableName)) {
log.debug("Database table already exists");
return;
}
log.debug("Creating database tables...");
// 使用 Hutool 读取 SQL 资源文件
List<String> sqlStatements = readSqlResource(schemaResourcePath);
// 执行所有 SQL 语句
for (String sql : sqlStatements) {
if (!sql.trim().isEmpty()) {
executeUpdate(sql);
}
}
log.debug("Database tables created successfully");
}
/**
* 检查表是否存在
*/
private boolean isTableExists(String tableName) throws SQLException {
try (Connection conn = getConnection()) {
ResultSet rs = conn.getMetaData().getTables(null, null, tableName, null);
return rs.next();
} catch (SQLException e) {
log.warn("Error checking table existence: {}", e.getMessage());
return false;
}
}
/**
* 使用 Hutool 读取 SQL 资源文件并分割为语句列表
*/
private List<String> readSqlResource(String resourcePath) {
try {
// 读取整个资源文件内容
String content = ResourceUtil.readUtf8Str(resourcePath);
// 分割 SQL 语句（按分号分割）
return Arrays.stream(content.split(";"))
.map(String::trim)
.filter(sql -> !sql.isEmpty())
.collect(Collectors.toList());
} catch (Exception e) {
log.error("Failed to read SQL resource: {}", resourcePath, e);
throw new RuntimeException("SQL resource read error", e);
}
}
/**
* 关闭所有连接池
*/
public static void closeAll() {
INSTANCES.clear();
log.debug("所有SqliteHelper实例已关闭");
}
}

212
face/src/main/java/cn/smartjavaai/face/translator/FaceDetectionTranslator.java Normal file
View File

@@ -0,0 +1,212 @@
/*
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
* with the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0/
*
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package cn.smartjavaai.face.translator;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.*;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDArrays;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Face Detection Translator
*/
public class FaceDetectionTranslator implements Translator<Image, DetectedObjects> {
private double confThresh;
private double nmsThresh;
private int topK;
private double[] variance;
private int[][] scales;
private int[] steps;
public FaceDetectionTranslator(
double confThresh,
double nmsThresh,
double[] variance,
int topK,
int[][] scales,
int[] steps) {
this.confThresh = confThresh;
this.nmsThresh = nmsThresh;
this.variance = variance;
this.topK = topK;
this.scales = scales;
this.steps = steps;
}
/** {@inheritDoc} */
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
ctx.setAttachment("width", input.getWidth());
ctx.setAttachment("height", input.getHeight());
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW RGB -> BGR
// The network by default takes float32
if (!array.getDataType().equals(DataType.FLOAT32)) {
array = array.toType(DataType.FLOAT32, false);
}
NDArray mean =
ctx.getNDManager().create(new float[] {104f, 117f, 123f}, new Shape(3, 1, 1));
array = array.sub(mean);
return new NDList(array);
}
/** {@inheritDoc} */
@Override
public DetectedObjects processOutput(TranslatorContext ctx, NDList list) {
int width = (int) ctx.getAttachment("width");
int height = (int) ctx.getAttachment("height");
NDManager manager = ctx.getNDManager();
double scaleXY = variance[0];
double scaleWH = variance[1];
NDArray prob = list.get(1).get(":, 1:");
prob =
NDArrays.stack(
new NDList(
prob.argMax(1).toType(DataType.FLOAT32, false),
prob.max(new int[] {1})));
NDArray boxRecover = boxRecover(manager, width, height, scales, steps);
NDArray boundingBoxes = list.get(0);
NDArray bbWH = boundingBoxes.get(":, 2:").mul(scaleWH).exp().mul(boxRecover.get(":, 2:"));
NDArray bbXY =
boundingBoxes
.get(":, :2")
.mul(scaleXY)
.mul(boxRecover.get(":, 2:"))
.add(boxRecover.get(":, :2"))
.sub(bbWH.mul(0.5f));
boundingBoxes = NDArrays.concat(new NDList(bbXY, bbWH), 1);
NDArray landms = list.get(2);
landms = decodeLandm(landms, boxRecover, scaleXY);
// filter the result below the threshold
NDArray cutOff = prob.get(1).gt(confThresh);
boundingBoxes = boundingBoxes.transpose().booleanMask(cutOff, 1).transpose();
landms = landms.transpose().booleanMask(cutOff, 1).transpose();
prob = prob.booleanMask(cutOff, 1);
// start categorical filtering
long[] order = prob.get(1).argSort().get(":" + topK).toLongArray();
prob = prob.transpose();
List<String> retNames = new ArrayList<>();
List<Double> retProbs = new ArrayList<>();
List<BoundingBox> retBB = new ArrayList<>();
Map<Integer, List<BoundingBox>> recorder = new ConcurrentHashMap<>();
for (int i = order.length - 1; i >= 0; i--) {
long currMaxLoc = order[i];
float[] classProb = prob.get(currMaxLoc).toFloatArray();
int classId = (int) classProb[0];
double probability = classProb[1];
double[] boxArr = boundingBoxes.get(currMaxLoc).toDoubleArray();
double[] landmsArr = landms.get(currMaxLoc).toDoubleArray();
Rectangle rect = new Rectangle(boxArr[0], boxArr[1], boxArr[2], boxArr[3]);
List<BoundingBox> boxes = recorder.getOrDefault(classId, new ArrayList<>());
boolean belowIoU = true;
for (BoundingBox box : boxes) {
if (box.getIoU(rect) > nmsThresh) {
belowIoU = false;
break;
}
}
if (belowIoU) {
List<Point> keyPoints = new ArrayList<>();
for (int j = 0; j < 5; j++) { // 5 face landmarks
double x = landmsArr[j * 2];
double y = landmsArr[j * 2 + 1];
keyPoints.add(new Point(x * width, y * height));
}
Landmark landmark =
new Landmark(boxArr[0], boxArr[1], boxArr[2], boxArr[3], keyPoints);
boxes.add(landmark);
recorder.put(classId, boxes);
String className = "Face"; // classes.get(classId)
retNames.add(className);
retProbs.add(probability);
retBB.add(landmark);
}
}
return new DetectedObjects(retNames, retProbs, retBB);
}
private NDArray boxRecover(
NDManager manager, int width, int height, int[][] scales, int[] steps) {
int[][] aspectRatio = new int[steps.length][2];
for (int i = 0; i < steps.length; i++) {
int wRatio = (int) Math.ceil((float) width / steps[i]);
int hRatio = (int) Math.ceil((float) height / steps[i]);
aspectRatio[i] = new int[] {hRatio, wRatio};
}
List<double[]> defaultBoxes = new ArrayList<>();
for (int idx = 0; idx < steps.length; idx++) {
int[] scale = scales[idx];
for (int h = 0; h < aspectRatio[idx][0]; h++) {
for (int w = 0; w < aspectRatio[idx][1]; w++) {
for (int i : scale) {
double skx = i * 1.0 / width;
double sky = i * 1.0 / height;
double cx = (w + 0.5) * steps[idx] / width;
double cy = (h + 0.5) * steps[idx] / height;
defaultBoxes.add(new double[] {cx, cy, skx, sky});
}
}
}
}
double[][] boxes = new double[defaultBoxes.size()][defaultBoxes.get(0).length];
for (int i = 0; i < defaultBoxes.size(); i++) {
boxes[i] = defaultBoxes.get(i);
}
return manager.create(boxes).clip(0.0, 1.0);
}
// decode face landmarks, 5 points per face
private NDArray decodeLandm(NDArray pre, NDArray priors, double scaleXY) {
NDArray point1 =
pre.get(":, :2").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2"));
NDArray point2 =
pre.get(":, 2:4").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2"));
NDArray point3 =
pre.get(":, 4:6").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2"));
NDArray point4 =
pre.get(":, 6:8").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2"));
NDArray point5 =
pre.get(":, 8:10").mul(scaleXY).mul(priors.get(":, 2:")).add(priors.get(":, :2"));
return NDArrays.concat(new NDList(point1, point2, point3, point4, point5), 1);
}
}

242
face/src/main/java/cn/smartjavaai/face/translator/SCRFDFaceTranslator.java Normal file
View File

@@ -0,0 +1,242 @@
package cn.smartjavaai.face.translator;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.BoundingBox;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.modality.cv.output.Landmark;
import ai.djl.modality.cv.output.Point;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDArrays;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
import cn.smartjavaai.common.utils.LetterBoxUtils;
import cn.smartjavaai.common.utils.NMSUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* SCRFD Translator
*/
public class SCRFDFaceTranslator implements Translator<Image, DetectedObjects> {
private double confThresh;
private double nmsThresh;
private int topK;
private int[] steps;
private int inputWidth = 640;
private int inputHeight = 640;
public SCRFDFaceTranslator(
double confThresh,
double nmsThresh,
int topK,
int[] steps) {
this.confThresh = confThresh;
this.nmsThresh = nmsThresh;
this.topK = topK;
this.steps = steps;
}
/** {@inheritDoc} */
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
ctx.setAttachment("width", input.getWidth());
ctx.setAttachment("height", input.getHeight());
NDManager manager = ctx.getNDManager();
NDArray array = input.toNDArray(manager, Image.Flag.COLOR);
//Letter box resize 640x640 with padding (保持比例，补边缘)
LetterBoxUtils.ResizeResult letterBoxResult = LetterBoxUtils.letterbox(manager, array, inputWidth, inputHeight, 0f, LetterBoxUtils.PaddingPosition.LEFT_TOP);
ctx.setAttachment("scale", letterBoxResult.r);
array = letterBoxResult.image;
array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW RGB -> BGR
// The network by default takes float32
if (!array.getDataType().equals(DataType.FLOAT32)) {
array = array.toType(DataType.FLOAT32, false);
}
// 归一化
array = array.sub(127.5).mul(0.0078125);
return new NDList(array);
}
/** {@inheritDoc} */
@Override
public DetectedObjects processOutput(TranslatorContext ctx, NDList list) {
NDManager manager =
NDManager.newBaseManager(ctx.getNDManager().getDevice(), "PyTorch");
int sourceWidth = (int) ctx.getAttachment("width");
int sourceHeight = (int) ctx.getAttachment("height");
float detScale = (float) ctx.getAttachment("scale");
Map<String, NDArray> centerCache = new HashMap<>();
List<NDArray> scores_list = new ArrayList<>();
List<NDArray> bboxes_list = new ArrayList<>();
List<NDArray> kpss_list = new ArrayList<>();
// 步长长度
int fmc = steps.length;
int numAnchors = 2;
// 多尺度后处理
for (int idx = 0; idx < steps.length; idx++) {
int stride = steps[idx];
NDArray scores, bboxPreds, kpsPreds = null;
scores = list.get(idx);
bboxPreds = list.get(idx + fmc).mul(stride);
kpsPreds = list.get(idx + fmc * 2).mul(stride);
int height = inputHeight / stride;
int width = inputWidth / stride;
String key = height + "_" + width + "_" + stride;
// anchor centers cache
NDArray anchorCenters;
if (centerCache.containsKey(key)) {
anchorCenters = centerCache.get(key);
} else {
NDArray yv = manager.arange((float) height).reshape(height, 1).repeat(1, width);
NDArray xv = manager.arange((float) width).reshape(1, width).repeat(0, height);
// stack x, y 到最后一维
anchorCenters = xv.stack(yv, -1); // shape [height, width, 2]
// 乘 stride
anchorCenters = anchorCenters.mul(stride);
// 拉平成 [-1, 2]，等价于 NumPy 的 reshape((-1, 2))
long total = anchorCenters.getShape().get(0) * anchorCenters.getShape().get(1);
// anchorCenters 现在是 [height*width, 2]
anchorCenters = anchorCenters.reshape(total, 2);
// 在第一维重复 numAnchors 次，直接拉平成最终形状
anchorCenters = anchorCenters.repeat(0, numAnchors); // shape [N*numAnchors, 2]
if (centerCache.size() < 100) {
centerCache.put(key, anchorCenters);
}
}
NDArray pos_mask = scores.gte(confThresh); // scores >= thresh
NDArray pos_inds = pos_mask.nonzero(); // shape: [N, 2]
pos_inds = pos_inds.get(":, 0"); // 取第一列的索引
// System.out.println(Arrays.toString(pos_inds.toLongArray()));
// 计算 bbox
NDArray bboxes = distance2bbox(anchorCenters, bboxPreds); // [num_anchors, 4]
// 取出符合阈值的
NDArray pos_scores = scores.get(pos_inds);
NDArray pos_bboxes = bboxes.get(pos_inds);
scores_list.add(pos_scores);
bboxes_list.add(pos_bboxes);
NDArray kpss = distance2kps(anchorCenters, kpsPreds); // [num_anchors, num_kps*2]
kpss = kpss.reshape(kpss.getShape().get(0), -1, 2); // reshape (N, -1, 2)
NDArray pos_kpss = kpss.get(pos_inds);
kpss_list.add(pos_kpss);
}
// 1. 合并 scores
NDArray scores = NDArrays.concat(new NDList(scores_list), 0);
NDArray scoresRavel = scores.reshape(-1);
// 2. 得到排序索引
long[] orderLong = scoresRavel.argSort().flip(0).get(":" + topK).toLongArray();
// 3. 合并 bboxes
NDArray bboxes = NDArrays.concat(new NDList(bboxes_list), 0).div(detScale);
NDArray kpss = NDArrays.concat(new NDList(kpss_list), 0).div(detScale);
// 4. 拼接 [x1,y1,x2,y2,score]
NDArray preDet = bboxes.concat(scores.reshape(-1,1), 1);
// 5. 按 order 排序
preDet = preDet.get(manager.create(orderLong));
// 6. NMS
int[] keep = NMSUtils.nms(preDet.get(":,0:4"), preDet.get(":,4"), (float)nmsThresh);
NDArray det = preDet.get(manager.create(keep));
// System.out.println(Arrays.toString(det.toFloatArray()));
if (kpss != null) {
kpss = kpss.get(manager.create(orderLong));
kpss = kpss.get(manager.create(keep));
}
List<String> retNames = new ArrayList<>();
List<Double> retProbs = new ArrayList<>();
List<BoundingBox> retBB = new ArrayList<>();
long numDet = det.getShape().get(0); // N
long numCols = det.getShape().get(1); // 应该是 5: x1,y1,x2,y2,score
float[] flat = det.toFloatArray(); // 一维
for (int i = 0; i < numDet; i++) {
int base = (int) (i * numCols);
float x1 = flat[base] / sourceWidth;
float y1 = flat[base + 1] / sourceHeight;
float x2 = flat[base + 2] / sourceWidth;
float y2 = flat[base + 3] / sourceHeight;
float score = flat[base + 4];
retNames.add("face"); // 类别
retProbs.add((double) score);
float width = x2 - x1;
float height = y2 - y1;
Landmark rect = new Landmark(x1, y1, width, height, decodeKps(kpss.get(i)));
retBB.add(rect);
}
return new DetectedObjects(retNames, retProbs, retBB);
}
public NDArray distance2bbox(NDArray points, NDArray distance) {
// points: [N, 2], distance: [N, 4]
NDArray x1 = points.get(":, 0").sub(distance.get(":, 0")); // x - left
NDArray y1 = points.get(":, 1").sub(distance.get(":, 1")); // y - top
NDArray x2 = points.get(":, 0").add(distance.get(":, 2")); // x + right
NDArray y2 = points.get(":, 1").add(distance.get(":, 3")); // y + bottom
// stack([x1, y1, x2, y2], axis=-1)
NDList list = new NDList(x1.expandDims(1), y1.expandDims(1), x2.expandDims(1), y2.expandDims(1));
return NDArrays.concat(list, 1); // axis=1 表示最后一维
}
public NDArray distance2kps(NDArray points, NDArray distance) {
// points: [N, 2], distance: [N, 2*num_kps]
int numKps = (int) distance.getShape().get(1) / 2;
List<NDArray> preds = new ArrayList<>();
for (int i = 0; i < numKps * 2; i += 2) {
NDArray px = points.get(":, " + (i % 2)).add(distance.get(":, " + i));
NDArray py = points.get(":, " + ((i % 2) + 1)).add(distance.get(":, " + (i + 1)));
preds.add(px);
preds.add(py);
}
// stack(preds, axis=-1)
NDList stackList = new NDList();
for (NDArray arr : preds) {
stackList.add(arr.expandDims(1));
}
return NDArrays.concat(stackList, 1); // shape [N, num_kps*2]
}
public List<Point> decodeKps(NDArray kpss) {
// 转成一维 float 数组
float[] flat = kpss.toFloatArray();
// reshape 成二维 [5][2]
int numPoints = (int) kpss.getShape().get(0); // 5
int dim = (int) kpss.getShape().get(1); // 2
float[][] kpsArray = new float[numPoints][dim];
for (int i = 0; i < numPoints; i++) {
for (int j = 0; j < dim; j++) {
kpsArray[i][j] = flat[i * dim + j];
}
}
// 转成 Point 数组
List<Point> points = new ArrayList<>();
for (int i = 0; i < numPoints; i++) {
points.add(new Point(Math.round(kpsArray[i][0]), Math.round(kpsArray[i][1])));
}
return points;
}
}

462
face/src/main/java/cn/smartjavaai/face/translator/YoloV5FaceTranslator.java Normal file
View File

@@ -0,0 +1,462 @@
package cn.smartjavaai.face.translator;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.*;
import ai.djl.modality.cv.transform.*;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.*;
import java.util.*;
/**
* YoloV5Translator
* @author dwj
*/
public class YoloV5FaceTranslator implements Translator<Image, DetectedObjects> {
private int maxBoxes;
private YoloOutputType yoloOutputLayerType;
private float nmsThreshold;
protected float threshold;
// private BaseImageTranslator.SynsetLoader synsetLoader;
protected List<String> classes;
protected boolean applyRatio;
protected boolean removePadding;
protected Pipeline pipeline;
private Image.Flag flag;
private Batchifier batchifier;
protected int width;
protected int height;
/**
* Constructs an ImageTranslator with the provided builder.
*
* @param builder the data to build with
*/
protected YoloV5FaceTranslator(Builder builder) {
this.yoloOutputLayerType = builder.outputType;
this.nmsThreshold = builder.nmsThreshold;
maxBoxes = builder.maxBox;
this.threshold = builder.threshold;
// this.synsetLoader = builder.synsetLoader;
this.applyRatio = builder.applyRatio;
this.removePadding = builder.removePadding;
this.flag = builder.flag;
this.pipeline = builder.pipeline;
this.batchifier = builder.batchifier;
this.width = builder.width;
this.height = builder.height;
classes = Arrays.asList("face");
}
/**
* Creates a builder to build a {@code YoloV8Translator} with specified arguments.
*
* @return a new builder
*/
public static Builder builder() {
return new Builder();
}
/**
* Creates a builder to build a {@code YoloV8Translator} with specified arguments.
*
* @param arguments arguments to specify builder options
* @return a new builder
*/
public static Builder builder(Map<String, ?> arguments) {
Builder builder = new Builder();
builder.configPreProcess(arguments);
builder.configPostProcess(arguments);
return builder;
}
/** {@inheritDoc} */
protected DetectedObjects processFromBoxOutput(int imageWidth, int imageHeight, NDList list) {
float[] flattened = list.get(0).toFloatArray();
int sizeClasses = classes.size();
int stride = 15 + sizeClasses;
int size = flattened.length / stride;
ArrayList<Landmark> boxes = new ArrayList<>();
ArrayList<Float> scores = new ArrayList<>();
ArrayList<Integer> classIds = new ArrayList<>();
for (int i = 0; i < size; i++) {
int indexBase = i * stride;
float maxClass = 0;
int maxIndex = 0;
// for (int c = 0; c < sizeClasses; c++) {
// if (flattened[indexBase + c + 5] > maxClass) {
// maxClass = flattened[indexBase + c + 5];
// maxIndex = c;
// }
// }
float score = flattened[indexBase + 4];
if (score > threshold) {
float xPos = flattened[indexBase];
float yPos = flattened[indexBase + 1];
float w = flattened[indexBase + 2];
float h = flattened[indexBase + 3];
List<Point> keypoints = new ArrayList<>();
keypoints.add(new Point(flattened[indexBase + 5], flattened[indexBase + 6]));
keypoints.add(new Point(flattened[indexBase + 7], flattened[indexBase + 8]));
keypoints.add(new Point(flattened[indexBase + 9], flattened[indexBase + 10]));
keypoints.add(new Point(flattened[indexBase + 11], flattened[indexBase + 12]));
keypoints.add(new Point(flattened[indexBase + 13], flattened[indexBase + 14]));
Landmark rect =
new Landmark(Math.max(0, xPos - w / 2), Math.max(0, yPos - h / 2), w, h,keypoints);
boxes.add(rect);
scores.add(score);
classIds.add(maxIndex);
}
}
return nms(imageWidth, imageHeight, boxes, classIds, scores);
}
private DetectedObjects processFromDetectOutput() {
throw new UnsupportedOperationException(
"detect layer output is not supported yet, check correct YoloV5 export format");
}
@Override
public DetectedObjects processOutput(TranslatorContext ctx, NDList list) throws Exception {
int imageWidth = (Integer) ctx.getAttachment("width");
int imageHeight = (Integer) ctx.getAttachment("height");
switch (yoloOutputLayerType) {
case DETECT:
return processFromDetectOutput();
case AUTO:
if (list.get(0).getShape().dimension() > 2) {
return processFromDetectOutput();
} else {
return processFromBoxOutput(imageWidth, imageHeight, list);
}
case BOX:
default:
return processFromBoxOutput(imageWidth, imageHeight, list);
}
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) throws Exception {
NDArray array = input.toNDArray(ctx.getNDManager(), flag);
NDList list = pipeline.transform(new NDList(array));
Shape shape = list.get(0).getShape();
int processedWidth;
int processedHeight;
long[] dim = shape.getShape();
if (NDImageUtils.isCHW(shape)) {
processedWidth = (int) dim[dim.length - 1];
processedHeight = (int) dim[dim.length - 2];
} else {
processedWidth = (int) dim[dim.length - 2];
processedHeight = (int) dim[dim.length - 3];
}
ctx.setAttachment("width", input.getWidth());
ctx.setAttachment("height", input.getHeight());
ctx.setAttachment("processedWidth", processedWidth);
ctx.setAttachment("processedHeight", processedHeight);
return list;
}
protected DetectedObjects nms(
int imageWidth,
int imageHeight,
List<Landmark> boxes,
List<Integer> classIds,
List<Float> scores) {
List<String> retClasses = new ArrayList<>();
List<Double> retProbs = new ArrayList<>();
List<BoundingBox> retBB = new ArrayList<>();
for (int classId = 0; classId < classes.size(); classId++) {
List<Rectangle> r = new ArrayList<>();
List<Double> s = new ArrayList<>();
List<Integer> map = new ArrayList<>();
for (int j = 0; j < classIds.size(); ++j) {
if (classIds.get(j) == classId) {
r.add(boxes.get(j));
s.add(scores.get(j).doubleValue());
map.add(j);
}
}
if (r.isEmpty()) {
continue;
}
List<Integer> nms = Rectangle.nms(r, s, nmsThreshold);
for (int index : nms) {
int pos = map.get(index);
int id = classIds.get(pos);
retClasses.add(classes.get(id));
retProbs.add(scores.get(pos).doubleValue());
// Rectangle rect = boxes.get(pos);
Landmark rect = boxes.get(pos);
List<Point> keypoints = new ArrayList<>();
if (removePadding) {
int padW = (width - imageWidth) / 2;
int padH = (height - imageHeight) / 2;
rect.getPath().forEach(point -> {
keypoints.add(new Point(point.getX() - padW, point.getY() - padH));
});
rect =
new Landmark(
(rect.getX() - padW) / imageWidth,
(rect.getY() - padH) / imageHeight,
rect.getWidth() / imageWidth,
rect.getHeight() / imageHeight,keypoints);
} else if (applyRatio) {
rect.getPath().forEach(point -> {
keypoints.add(new Point(point.getX() / width, point.getY() / height));
});
rect =
new Landmark(
rect.getX() / width,
rect.getY() / height,
rect.getWidth() / width,
rect.getHeight() / height,keypoints);
}
retBB.add(rect);
}
}
return new DetectedObjects(retClasses, retProbs, retBB);
}
public static class Builder {
private int maxBox = 8400;
YoloOutputType outputType;
float nmsThreshold;
protected float threshold = 0.2F;
protected boolean applyRatio;
protected boolean removePadding;
protected int width = 224;
protected int height = 224;
protected Image.Flag flag;
protected Pipeline pipeline;
protected Batchifier batchifier;
public Builder() {
this.outputType = YoloOutputType.AUTO;
this.nmsThreshold = 0.4F;
}
public Builder optOutputType(YoloOutputType outputType) {
this.outputType = outputType;
return this;
}
public Builder optNmsThreshold(float nmsThreshold) {
this.nmsThreshold = nmsThreshold;
return this;
}
/**
* Builds the translator.
*
* @return the new translator
*/
public YoloV5FaceTranslator build() {
if (pipeline == null) {
addTransform(
array -> array.transpose(2, 0, 1).toType(DataType.FLOAT32, false).div(255));
}
// validate();
return new YoloV5FaceTranslator(this);
}
protected Builder self() {
return this;
}
public Builder addTransform(Transform transform) {
if (this.pipeline == null) {
this.pipeline = new Pipeline();
}
this.pipeline.add(transform);
return this.self();
}
public Builder optApplyRatio(boolean value) {
this.applyRatio = value;
return this.self();
}
public Builder optFlag(Image.Flag flag) {
this.flag = flag;
return this.self();
}
public Builder setPipeline(Pipeline pipeline) {
this.pipeline = pipeline;
return this.self();
}
public Builder setImageSize(int width, int height) {
this.width = width;
this.height = height;
return this.self();
}
public Builder optBatchifier(Batchifier batchifier) {
this.batchifier = batchifier;
return this.self();
}
public Builder optThreshold(float threshold) {
this.threshold = threshold;
return this.self();
}
/** {@inheritDoc} */
protected void configPostProcess(Map<String, ?> arguments) {
if (ArgumentsUtil.booleanValue(arguments, "optApplyRatio") || ArgumentsUtil.booleanValue(arguments, "applyRatio")) {
this.optApplyRatio(true);
}
this.threshold = ArgumentsUtil.floatValue(arguments, "threshold", 0.2F);
String centerFit = ArgumentsUtil.stringValue(arguments, "centerFit", "false");
this.removePadding = "true".equals(centerFit);
String type = ArgumentsUtil.stringValue(arguments, "outputType", "AUTO");
this.outputType = YoloOutputType.valueOf(type.toUpperCase(Locale.ENGLISH));
this.nmsThreshold = ArgumentsUtil.floatValue(arguments, "nmsThreshold", 0.4F);
maxBox = ArgumentsUtil.intValue(arguments, "maxBox", 8400);
}
protected void configPreProcess(Map<String, ?> arguments) {
if (this.pipeline == null) {
this.pipeline = new Pipeline();
}
this.width = ArgumentsUtil.intValue(arguments, "width", 224);
this.height = ArgumentsUtil.intValue(arguments, "height", 224);
if (arguments.containsKey("flag")) {
this.flag = Image.Flag.valueOf(arguments.get("flag").toString());
}
String pad = ArgumentsUtil.stringValue(arguments, "pad", "false");
if ("true".equals(pad)) {
this.addTransform(new Pad(0.0));
} else if (!"false".equals(pad)) {
double padding = Double.parseDouble(pad);
this.addTransform(new Pad(padding));
}
String resize = ArgumentsUtil.stringValue(arguments, "resize", "false");
int w;
int shortEdge;
if ("true".equals(resize)) {
this.addTransform(new Resize(this.width, this.height));
} else if (!"false".equals(resize)) {
String[] tokens = resize.split("\\s*,\\s*");
w = (int)Double.parseDouble(tokens[0]);
if (tokens.length > 1) {
shortEdge = (int)Double.parseDouble(tokens[1]);
} else {
shortEdge = w;
}
Image.Interpolation interpolation;
if (tokens.length > 2) {
interpolation = Image.Interpolation.valueOf(tokens[2]);
} else {
interpolation = Image.Interpolation.BILINEAR;
}
this.addTransform(new Resize(w, shortEdge, interpolation));
}
String resizeShort = ArgumentsUtil.stringValue(arguments, "resizeShort", "false");
if ("true".equals(resizeShort)) {
w = Math.max(this.width, this.height);
this.addTransform(new ResizeShort(w));
} else if (!"false".equals(resizeShort)) {
String[] tokens = resizeShort.split("\\s*,\\s*");
shortEdge = (int)Double.parseDouble(tokens[0]);
int longEdge;
if (tokens.length > 1) {
longEdge = (int)Double.parseDouble(tokens[1]);
} else {
longEdge = -1;
}
Image.Interpolation interpolation;
if (tokens.length > 2) {
interpolation = Image.Interpolation.valueOf(tokens[2]);
} else {
interpolation = Image.Interpolation.BILINEAR;
}
this.addTransform(new ResizeShort(shortEdge, longEdge, interpolation));
}
if (ArgumentsUtil.booleanValue(arguments, "centerCrop", false)) {
this.addTransform(new CenterCrop(this.width, this.height));
}
if (ArgumentsUtil.booleanValue(arguments, "centerFit")) {
this.addTransform(new CenterFit(this.width, this.height));
}
if (ArgumentsUtil.booleanValue(arguments, "toTensor", true)) {
this.addTransform(new ToTensor());
}
String normalize = ArgumentsUtil.stringValue(arguments, "normalize", "false");
if ("true".equals(normalize)) {
float[] MEAN = new float[]{0.485F, 0.456F, 0.406F};
float[] STD = new float[]{0.229F, 0.224F, 0.225F};
this.addTransform(new Normalize(MEAN, STD));
} else if (!"false".equals(normalize)) {
String[] tokens = normalize.split("\\s*,\\s*");
if (tokens.length != 6) {
throw new IllegalArgumentException("Invalid normalize value: " + normalize);
}
float[] mean = new float[]{Float.parseFloat(tokens[0]), Float.parseFloat(tokens[1]), Float.parseFloat(tokens[2])};
float[] std = new float[]{Float.parseFloat(tokens[3]), Float.parseFloat(tokens[4]), Float.parseFloat(tokens[5])};
this.addTransform(new Normalize(mean, std));
}
String range = (String)arguments.get("range");
if ("0,1".equals(range)) {
this.addTransform((a) -> {
return a.div(255.0F);
});
} else if ("-1,1".equals(range)) {
this.addTransform((a) -> {
return a.div(128.0F).sub(1);
});
}
if (arguments.containsKey("batchifier")) {
this.batchifier = Batchifier.fromString((String)arguments.get("batchifier"));
}
}
}
public static enum YoloOutputType {
BOX,
DETECT,
AUTO;
private YoloOutputType() {
}
}
}

476
face/src/main/java/cn/smartjavaai/face/translator/YoloV8FaceTranslator.java Normal file
View File

@@ -0,0 +1,476 @@
package cn.smartjavaai.face.translator;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.*;
import ai.djl.modality.cv.transform.*;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.*;
import java.util.*;
/**
* YoloV8Translator
* @author dwj
*/
public class YoloV8FaceTranslator implements Translator<Image, DetectedObjects> {
private int maxBoxes;
private YoloOutputType yoloOutputLayerType;
private float nmsThreshold;
protected float threshold;
// private BaseImageTranslator.SynsetLoader synsetLoader;
protected List<String> classes;
protected boolean applyRatio;
protected boolean removePadding;
protected Pipeline pipeline;
private Image.Flag flag;
private Batchifier batchifier;
protected int width;
protected int height;
/**
* Constructs an ImageTranslator with the provided builder.
*
* @param builder the data to build with
*/
protected YoloV8FaceTranslator(Builder builder) {
this.yoloOutputLayerType = builder.outputType;
this.nmsThreshold = builder.nmsThreshold;
maxBoxes = builder.maxBox;
this.threshold = builder.threshold;
// this.synsetLoader = builder.synsetLoader;
this.applyRatio = builder.applyRatio;
this.removePadding = builder.removePadding;
this.flag = builder.flag;
this.pipeline = builder.pipeline;
this.batchifier = builder.batchifier;
this.width = builder.width;
this.height = builder.height;
classes = Arrays.asList("face");
}
/**
* Creates a builder to build a {@code YoloV8Translator} with specified arguments.
*
* @return a new builder
*/
public static Builder builder() {
return new Builder();
}
/**
* Creates a builder to build a {@code YoloV8Translator} with specified arguments.
*
* @param arguments arguments to specify builder options
* @return a new builder
*/
public static Builder builder(Map<String, ?> arguments) {
Builder builder = new Builder();
builder.configPreProcess(arguments);
builder.configPostProcess(arguments);
return builder;
}
/** {@inheritDoc} */
protected DetectedObjects processFromBoxOutput(int imageWidth, int imageHeight, NDList list) {
NDArray rawResult = list.get(0);
NDArray reshapedResult = rawResult.transpose();
Shape shape = reshapedResult.getShape();
float[] buf = reshapedResult.toFloatArray();
int numberRows = Math.toIntExact(shape.get(0));
int nClasses = Math.toIntExact(shape.get(1));
int padding = nClasses - classes.size();
System.out.println(Arrays.toString(reshapedResult.get(0).toFloatArray()));
// if (padding != 0 && padding != 4) {
// throw new IllegalStateException(
// "Expected classes: " + (nClasses - 4) + ", got " + classes.size());
// }
ArrayList<Landmark> boxes = new ArrayList<>();
ArrayList<Float> scores = new ArrayList<>();
ArrayList<Integer> classIds = new ArrayList<>();
// reverse order search in heap; searches through #maxBoxes for optimization when set
for (int i = numberRows - 1; i > numberRows - maxBoxes; --i) {
int index = i * nClasses;
float maxClassProb = buf[index + 4];
// int maxIndex = -1;
// for (int c = 4; c < nClasses; c++) {
// float classProb = buf[index + c];
// if (classProb > maxClassProb) {
// maxClassProb = classProb;
// maxIndex = c;
// }
// }
// maxIndex -= padding;
if (maxClassProb > threshold) {
float xPos = buf[index]; // center x
float yPos = buf[index + 1]; // center y
float w = buf[index + 2];
float h = buf[index + 3];
Rectangle rect =
new Rectangle(Math.max(0, xPos - w / 2), Math.max(0, yPos - h / 2), w, h);
// boxes.add(rect);
scores.add(maxClassProb);
classIds.add(0);
List<Point> keypoints = new ArrayList<>();
keypoints.add(new Point(buf[index + 5], buf[index + 6]));
keypoints.add(new Point(buf[index + 8], buf[index + 9]));
keypoints.add(new Point(buf[index + 11], buf[index + 12]));
keypoints.add(new Point(buf[index + 14], buf[index + 15]));
keypoints.add(new Point(buf[index + 17], buf[index + 18]));
Landmark kps = new Landmark(Math.max(0, xPos - w / 2), Math.max(0, yPos - h / 2), w, h, keypoints);
boxes.add(kps);
}
}
return nms(imageWidth, imageHeight, boxes, classIds, scores);
}
private DetectedObjects processFromDetectOutput() {
throw new UnsupportedOperationException(
"detect layer output is not supported yet, check correct YoloV5 export format");
}
@Override
public DetectedObjects processOutput(TranslatorContext ctx, NDList list) throws Exception {
int imageWidth = (Integer) ctx.getAttachment("width");
int imageHeight = (Integer) ctx.getAttachment("height");
switch (yoloOutputLayerType) {
case DETECT:
return processFromDetectOutput();
case AUTO:
if (list.get(0).getShape().dimension() > 2) {
return processFromDetectOutput();
} else {
return processFromBoxOutput(imageWidth, imageHeight, list);
}
case BOX:
default:
return processFromBoxOutput(imageWidth, imageHeight, list);
}
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) throws Exception {
NDArray array = input.toNDArray(ctx.getNDManager(), flag);
NDList list = pipeline.transform(new NDList(array));
Shape shape = list.get(0).getShape();
int processedWidth;
int processedHeight;
long[] dim = shape.getShape();
if (NDImageUtils.isCHW(shape)) {
processedWidth = (int) dim[dim.length - 1];
processedHeight = (int) dim[dim.length - 2];
} else {
processedWidth = (int) dim[dim.length - 2];
processedHeight = (int) dim[dim.length - 3];
}
ctx.setAttachment("width", input.getWidth());
ctx.setAttachment("height", input.getHeight());
ctx.setAttachment("processedWidth", processedWidth);
ctx.setAttachment("processedHeight", processedHeight);
return list;
}
protected DetectedObjects nms(
int imageWidth,
int imageHeight,
List<Landmark> boxes,
List<Integer> classIds,
List<Float> scores) {
List<String> retClasses = new ArrayList<>();
List<Double> retProbs = new ArrayList<>();
List<BoundingBox> retBB = new ArrayList<>();
for (int classId = 0; classId < classes.size(); classId++) {
List<Rectangle> r = new ArrayList<>();
List<Double> s = new ArrayList<>();
List<Integer> map = new ArrayList<>();
for (int j = 0; j < classIds.size(); ++j) {
if (classIds.get(j) == classId) {
r.add(boxes.get(j));
s.add(scores.get(j).doubleValue());
map.add(j);
}
}
if (r.isEmpty()) {
continue;
}
List<Integer> nms = Rectangle.nms(r, s, nmsThreshold);
for (int index : nms) {
int pos = map.get(index);
int id = classIds.get(pos);
retClasses.add(classes.get(id));
retProbs.add(scores.get(pos).doubleValue());
// Rectangle rect = boxes.get(pos);
Landmark rect = boxes.get(pos);
List<Point> keypoints = new ArrayList<>();
if (removePadding) {
int padW = (width - imageWidth) / 2;
int padH = (height - imageHeight) / 2;
rect.getPath().forEach(point -> {
keypoints.add(new Point(point.getX() - padW, point.getY() - padH));
});
rect =
new Landmark(
(rect.getX() - padW) / imageWidth,
(rect.getY() - padH) / imageHeight,
rect.getWidth() / imageWidth,
rect.getHeight() / imageHeight,keypoints);
} else if (applyRatio) {
rect.getPath().forEach(point -> {
keypoints.add(new Point(point.getX() / width, point.getY() / height));
});
rect =
new Landmark(
rect.getX() / width,
rect.getY() / height,
rect.getWidth() / width,
rect.getHeight() / height,keypoints);
}
retBB.add(rect);
}
}
return new DetectedObjects(retClasses, retProbs, retBB);
}
public static class Builder {
private int maxBox = 8400;
YoloOutputType outputType;
float nmsThreshold;
protected float threshold = 0.2F;
protected boolean applyRatio;
protected boolean removePadding;
protected int width = 224;
protected int height = 224;
protected Image.Flag flag;
protected Pipeline pipeline;
protected Batchifier batchifier;
public Builder() {
this.outputType = YoloOutputType.AUTO;
this.nmsThreshold = 0.4F;
}
public Builder optOutputType(YoloOutputType outputType) {
this.outputType = outputType;
return this;
}
public Builder optNmsThreshold(float nmsThreshold) {
this.nmsThreshold = nmsThreshold;
return this;
}
/**
* Builds the translator.
*
* @return the new translator
*/
public YoloV8FaceTranslator build() {
if (pipeline == null) {
addTransform(
array -> array.transpose(2, 0, 1).toType(DataType.FLOAT32, false).div(255));
}
// validate();
return new YoloV8FaceTranslator(this);
}
protected Builder self() {
return this;
}
public Builder addTransform(Transform transform) {
if (this.pipeline == null) {
this.pipeline = new Pipeline();
}
this.pipeline.add(transform);
return this.self();
}
public Builder optApplyRatio(boolean value) {
this.applyRatio = value;
return this.self();
}
public Builder optFlag(Image.Flag flag) {
this.flag = flag;
return this.self();
}
public Builder setPipeline(Pipeline pipeline) {
this.pipeline = pipeline;
return this.self();
}
public Builder setImageSize(int width, int height) {
this.width = width;
this.height = height;
return this.self();
}
public Builder optBatchifier(Batchifier batchifier) {
this.batchifier = batchifier;
return this.self();
}
public Builder optThreshold(float threshold) {
this.threshold = threshold;
return this.self();
}
/** {@inheritDoc} */
protected void configPostProcess(Map<String, ?> arguments) {
if (ArgumentsUtil.booleanValue(arguments, "optApplyRatio") || ArgumentsUtil.booleanValue(arguments, "applyRatio")) {
this.optApplyRatio(true);
}
this.threshold = ArgumentsUtil.floatValue(arguments, "threshold", 0.2F);
String centerFit = ArgumentsUtil.stringValue(arguments, "centerFit", "false");
this.removePadding = "true".equals(centerFit);
String type = ArgumentsUtil.stringValue(arguments, "outputType", "AUTO");
this.outputType = YoloOutputType.valueOf(type.toUpperCase(Locale.ENGLISH));
this.nmsThreshold = ArgumentsUtil.floatValue(arguments, "nmsThreshold", 0.4F);
maxBox = ArgumentsUtil.intValue(arguments, "maxBox", 8400);
}
protected void configPreProcess(Map<String, ?> arguments) {
if (this.pipeline == null) {
this.pipeline = new Pipeline();
}
this.width = ArgumentsUtil.intValue(arguments, "width", 224);
this.height = ArgumentsUtil.intValue(arguments, "height", 224);
if (arguments.containsKey("flag")) {
this.flag = Image.Flag.valueOf(arguments.get("flag").toString());
}
String pad = ArgumentsUtil.stringValue(arguments, "pad", "false");
if ("true".equals(pad)) {
this.addTransform(new Pad(0.0));
} else if (!"false".equals(pad)) {
double padding = Double.parseDouble(pad);
this.addTransform(new Pad(padding));
}
String resize = ArgumentsUtil.stringValue(arguments, "resize", "false");
int w;
int shortEdge;
if ("true".equals(resize)) {
this.addTransform(new Resize(this.width, this.height));
} else if (!"false".equals(resize)) {
String[] tokens = resize.split("\\s*,\\s*");
w = (int)Double.parseDouble(tokens[0]);
if (tokens.length > 1) {
shortEdge = (int)Double.parseDouble(tokens[1]);
} else {
shortEdge = w;
}
Image.Interpolation interpolation;
if (tokens.length > 2) {
interpolation = Image.Interpolation.valueOf(tokens[2]);
} else {
interpolation = Image.Interpolation.BILINEAR;
}
this.addTransform(new Resize(w, shortEdge, interpolation));
}
String resizeShort = ArgumentsUtil.stringValue(arguments, "resizeShort", "false");
if ("true".equals(resizeShort)) {
w = Math.max(this.width, this.height);
this.addTransform(new ResizeShort(w));
} else if (!"false".equals(resizeShort)) {
String[] tokens = resizeShort.split("\\s*,\\s*");
shortEdge = (int)Double.parseDouble(tokens[0]);
int longEdge;
if (tokens.length > 1) {
longEdge = (int)Double.parseDouble(tokens[1]);
} else {
longEdge = -1;
}
Image.Interpolation interpolation;
if (tokens.length > 2) {
interpolation = Image.Interpolation.valueOf(tokens[2]);
} else {
interpolation = Image.Interpolation.BILINEAR;
}
this.addTransform(new ResizeShort(shortEdge, longEdge, interpolation));
}
if (ArgumentsUtil.booleanValue(arguments, "centerCrop", false)) {
this.addTransform(new CenterCrop(this.width, this.height));
}
if (ArgumentsUtil.booleanValue(arguments, "centerFit")) {
this.addTransform(new CenterFit(this.width, this.height));
}
if (ArgumentsUtil.booleanValue(arguments, "toTensor", true)) {
this.addTransform(new ToTensor());
}
String normalize = ArgumentsUtil.stringValue(arguments, "normalize", "false");
if ("true".equals(normalize)) {
float[] MEAN = new float[]{0.485F, 0.456F, 0.406F};
float[] STD = new float[]{0.229F, 0.224F, 0.225F};
this.addTransform(new Normalize(MEAN, STD));
} else if (!"false".equals(normalize)) {
String[] tokens = normalize.split("\\s*,\\s*");
if (tokens.length != 6) {
throw new IllegalArgumentException("Invalid normalize value: " + normalize);
}
float[] mean = new float[]{Float.parseFloat(tokens[0]), Float.parseFloat(tokens[1]), Float.parseFloat(tokens[2])};
float[] std = new float[]{Float.parseFloat(tokens[3]), Float.parseFloat(tokens[4]), Float.parseFloat(tokens[5])};
this.addTransform(new Normalize(mean, std));
}
String range = (String)arguments.get("range");
if ("0,1".equals(range)) {
this.addTransform((a) -> {
return a.div(255.0F);
});
} else if ("-1,1".equals(range)) {
this.addTransform((a) -> {
return a.div(128.0F).sub(1);
});
}
if (arguments.containsKey("batchifier")) {
this.batchifier = Batchifier.fromString((String)arguments.get("batchifier"));
}
}
}
public static enum YoloOutputType {
BOX,
DETECT,
AUTO;
private YoloOutputType() {
}
}
}

70
face/src/main/java/cn/smartjavaai/face/utils/FaceAlignUtils.java Normal file
View File

@@ -0,0 +1,70 @@
package cn.smartjavaai.face.utils;
import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.opencv.OpenCVImageFactory;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.OpenCVUtils;
import com.seeta.sdk.SeetaImageData;
import com.seeta.sdk.SeetaPointF;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;
import java.awt.image.BufferedImage;
/**
* 人脸对齐
* @author Calvin
*/
public class FaceAlignUtils {
/**
* 根据目标点，进行旋转仿射变换
* Perform rotation and affine transformation based on the target 5 points
*
* @param src
* @param rot_mat
* @return
*/
public static Mat warpAffine(Mat src, Mat rot_mat) {
Mat rot = new Mat();
// 进行仿射变换，变换后大小为src的大小
// Perform affine transformation, the size after transformation is the same as the size of src
Scalar scalar = new Scalar(135, 133, 132);
Size size = new Size(512, 512);
Imgproc.warpAffine(src, rot, rot_mat, size, 0, 0, scalar);
return rot;
}
public static Mat warpAffine(Mat src, Mat rot_mat, int width, int height) {
Mat rot = new Mat();
Size size = new Size(width, height);
Scalar scalar = new Scalar(135, 133, 132);
Imgproc.warpAffine(src, rot, rot_mat, size,0, 0, scalar);
return rot;
}
public static Mat warpAffine(Mat src, Mat rot_mat, int width, int height, int flags) {
Mat rot = new Mat();
Size size = new Size(width, height);
Imgproc.warpAffine(src, rot, rot_mat, size, flags);
return rot;
}
public static SeetaImageData faceAlign(BufferedImage sourceImage, SeetaPointF[] pointFS) {
NDManager manager = NDManager.newBaseManager();
//获取子图中人脸关键点坐标
double[][] pointsArray = FaceUtils.facePoints(pointFS);
NDArray srcPoints = manager.create(pointsArray);
NDArray dstPoints = FaceUtils.faceTemplate512x512(manager);
// 5点仿射变换
Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints);
Mat mat = FaceAlignUtils.warpAffine(OpenCVUtils.image2Mat(sourceImage), affine_matrix);
BufferedImage alignImage = OpenCVUtils.mat2Image(mat);
SeetaImageData imageData = new SeetaImageData(alignImage.getWidth(), alignImage.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(alignImage);
return imageData;
}
}

691
face/src/main/java/cn/smartjavaai/face/utils/FaceUtils.java Normal file
View File

@@ -0,0 +1,691 @@
package cn.smartjavaai.face.utils;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.BoundingBox;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.face.FaceAttribute;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.HeadPose;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.face.EyeStatus;
import cn.smartjavaai.common.enums.face.GenderType;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.face.exception.FaceException;
import com.seeta.sdk.*;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.List;
import java.util.stream.Collectors;
/**
* 人脸检测相关工具类
* @author dwj
* @date 2025/4/9
*/
public class FaceUtils {
/**
* 转换为FaceDetectedResult
* @param detection
* @param img
* @return
*/
public static DetectionResponse convertToDetectionResponse(DetectedObjects detection, Image img){
if(Objects.isNull(detection) || Objects.isNull(detection.getProbabilities())
|| detection.getProbabilities().isEmpty() || Objects.isNull(detection.items()) || detection.items().isEmpty()){
return null;
}
DetectionResponse detectionResponse = new DetectionResponse();
List<DetectedObjects.DetectedObject> detectedObjectList = detection.items();
List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
Iterator iterator = detectedObjectList.iterator();
int index = 0;
while(iterator.hasNext()) {
DetectedObjects.DetectedObject result = (DetectedObjects.DetectedObject)iterator.next();
BoundingBox box = result.getBoundingBox();
List<Point> keyPoints = new ArrayList<Point>();
box.getBounds().getPath().forEach(point -> {
keyPoints.add(new Point(point.getX(), point.getY()));
});
int x = (int)(box.getBounds().getX() * img.getWidth());
int y = (int)(box.getBounds().getY() * img.getHeight());
int width = (int)(box.getBounds().getWidth() * img.getWidth());
int height = (int)(box.getBounds().getHeight() * img.getHeight());
// 修正边界，防止越界
if (x < 0) x = 0;
if (y < 0) y = 0;
if (x + width > img.getWidth()) width = img.getWidth() - x;
if (y + height > img.getHeight()) height = img.getHeight() - y;
DetectionRectangle rectangle = new DetectionRectangle(x, y, width, height);
FaceInfo faceInfo = new FaceInfo(keyPoints);
DetectionInfo detectionInfo = new DetectionInfo(rectangle, detection.getProbabilities().get(index).floatValue(),faceInfo);
detectionInfoList.add(detectionInfo);
index++;
}
detectionResponse.setDetectionInfoList(detectionInfoList);
return detectionResponse;
}
/**
* 转换为FaceDetectedResult
* @param seetaResult
* @return
*/
public static DetectionResponse convertToDetectionResponse(SeetaRect[] seetaResult, List<SeetaPointF[]> seetaPointFSList){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return null;
}
DetectionResponse detectionResponse = new DetectionResponse();
List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
for(int i = 0; i < seetaResult.length; i++){
SeetaRect rect = seetaResult[i];
SeetaPointF[] seetaPointFS = seetaPointFSList.get(i);
//过滤置信度
/*if(config.getConfidenceThreshold() > 0){
continue;
}*/
DetectionRectangle rectangle = new DetectionRectangle(rect.x, rect.y, rect.width, rect.height);
List<Point> keyPoints = Arrays.stream(seetaPointFS)
.map(p -> new Point(p.x, p.y))
.collect(Collectors.toList());
FaceInfo faceInfo = new FaceInfo(keyPoints);
DetectionInfo detectionInfo = new DetectionInfo(rectangle, 0, faceInfo);
detectionInfoList.add(detectionInfo);
}
detectionResponse.setDetectionInfoList(detectionInfoList);
return detectionResponse;
}
/**
* 转换为FaceDetectedResult(人脸特征提取)
* @param seetaResult
* @return
*/
public static DetectionResponse featuresConvertToResponse(SeetaRect[] seetaResult, List<SeetaPointF[]> seetaPointFSList, List<float[]> featureList){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return null;
}
List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
for(int i = 0; i < seetaResult.length; i++){
SeetaRect rect = seetaResult[i];
DetectionRectangle rectangle = new DetectionRectangle(rect.x, rect.y, rect.width, rect.height);
FaceInfo faceInfo = new FaceInfo();
if(seetaPointFSList != null && seetaPointFSList.size() > 0){
SeetaPointF[] seetaPointFS = seetaPointFSList.get(i);
List<Point> keyPoints = Arrays.stream(seetaPointFS)
.map(p -> new Point(p.x, p.y))
.collect(Collectors.toList());
faceInfo.setKeyPoints(keyPoints);
}
if(featureList != null && featureList.size() > 0){
faceInfo.setFeature(featureList.get(i));
}
detectionInfoList.add(new DetectionInfo(rectangle, 0, faceInfo));
}
return new DetectionResponse(detectionInfoList);
}
/**
* 转换为FaceDetectedResult(人脸特征提取)
* @param rect
* @param seetaPointFS
* @param feature
* @return
*/
public static DetectionResponse featuresConvertToResponse(SeetaRect rect, SeetaPointF[] seetaPointFS, float[] feature){
List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
DetectionRectangle rectangle = new DetectionRectangle(rect.x, rect.y, rect.width, rect.height);
FaceInfo faceInfo = new FaceInfo();
List<Point> keyPoints = Arrays.stream(seetaPointFS)
.map(p -> new Point(p.x, p.y))
.collect(Collectors.toList());
faceInfo.setKeyPoints(keyPoints);
faceInfo.setFeature(feature);
detectionInfoList.add(new DetectionInfo(rectangle, 0, faceInfo));
return new DetectionResponse(detectionInfoList);
}
/**
* 绘制人脸框
* @param sourceImage
* @param detectionResponse
* @param savePath
* @throws IOException
*/
public static void drawBoundingBoxes(BufferedImage sourceImage, DetectionResponse detectionResponse, String savePath) throws IOException {
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
}
if(Objects.isNull(detectionResponse) || Objects.isNull(detectionResponse.getDetectionInfoList()) || detectionResponse.getDetectionInfoList().isEmpty()){
throw new FaceException("无目标数据");
}
Graphics2D graphics = sourceImage.createGraphics();
graphics.setColor(Color.RED);// 边框颜色
graphics.setStroke(new BasicStroke(2)); // 线宽2像素
graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿
int stroke = 2;
for(DetectionInfo detectionInfo : detectionResponse.getDetectionInfoList()){
DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
graphics.setColor(Color.RED);// 边框颜色
graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight());
drawText(graphics, "face", rectangle.getX(), rectangle.getY(), stroke, 4);
//绘制人脸关键点
if(detectionInfo.getFaceInfo() != null && detectionInfo.getFaceInfo().getKeyPoints() != null &&
!detectionInfo.getFaceInfo().getKeyPoints().isEmpty()){
drawLandmarks(graphics, detectionInfo.getFaceInfo().getKeyPoints());
}
}
graphics.dispose();
ImageIO.write(sourceImage, "png", new File(savePath));
}
/**
* 绘制人脸框
* @param sourceImage
* @param detectionResponse
* @throws IOException
*/
public static BufferedImage drawBoundingBoxes(BufferedImage sourceImage, DetectionResponse detectionResponse) throws IOException {
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
}
if(Objects.isNull(detectionResponse) || Objects.isNull(detectionResponse.getDetectionInfoList()) || detectionResponse.getDetectionInfoList().isEmpty()){
throw new FaceException("无目标数据");
}
Graphics2D graphics = sourceImage.createGraphics();
graphics.setColor(Color.RED);// 边框颜色
graphics.setStroke(new BasicStroke(2)); // 线宽2像素
graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿
int stroke = 2;
for(DetectionInfo detectionInfo : detectionResponse.getDetectionInfoList()){
DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
graphics.setColor(Color.RED);// 边框颜色
graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight());
drawText(graphics, "face", rectangle.getX(), rectangle.getY(), stroke, 4);
//绘制人脸关键点
if(detectionInfo.getFaceInfo() != null && detectionInfo.getFaceInfo().getKeyPoints() != null &&
!detectionInfo.getFaceInfo().getKeyPoints().isEmpty()){
drawLandmarks(graphics, detectionInfo.getFaceInfo().getKeyPoints());
}
}
graphics.dispose();
return sourceImage;
}
/**
* 绘制文字
* @param g
* @param text
* @param x
* @param y
* @param stroke
* @param padding
*/
private static void drawText(Graphics2D g, String text, int x, int y, int stroke, int padding) {
FontMetrics metrics = g.getFontMetrics();
x += stroke / 2;
y += stroke / 2;
int width = metrics.stringWidth(text) + padding * 2 - stroke / 2;
int height = metrics.getHeight() + metrics.getDescent();
int ascent = metrics.getAscent();
java.awt.Rectangle background = new java.awt.Rectangle(x, y, width, height);
g.fill(background);
g.setPaint(Color.WHITE);
g.drawString(text, x + padding, y + ascent);
}
/**
* 修正检测框
* @param rectangle
* @param imageWidth
* @param imageHeight
* @return
*/
public static DetectionRectangle correctRect(DetectionRectangle rectangle, int imageWidth, int imageHeight) {
int x = rectangle.getX();
int y = rectangle.getY();
int width = rectangle.getWidth();
int height = rectangle.getHeight();
// 修正x, y防止越界
if (x < 0) x = 0;
if (y < 0) y = 0;
// 宽高不能超出图片范围
if (x + width > imageWidth) {
width = imageWidth - x;
}
if (y + height > imageHeight) {
height = imageHeight - y;
}
// 防止最终 width 或 height 为负或为 0
if (width <= 0 || height <= 0) {
return null; // 无效区域
}
return new DetectionRectangle(x, y, width, height);
}
/**
* 子图中人脸关键点坐标 - Coordinates of key points in the image
*
* @param points
* @return
*/
public static double[][] facePoints(List<Point> points) {
// 图中关键点坐标 - Coordinates of key points in the image
// 1. left_eye_x , left_eye_y
// 2. right_eye_x , right_eye_y
// 3. nose_x , nose_y
// 4. left_mouth_x , left_mouth_y
// 5. right_mouth_x , right_mouth_y
double[][] pointsArray = new double[5][2]; // 保存人脸关键点 - Save facial key points
int i = 0;
for (Point point : points) {
pointsArray[i][0] = point.getX();
pointsArray[i][1] = point.getY();
i++;
}
return pointsArray;
}
/**
* 子图中人脸关键点坐标 - Coordinates of key points in the image
*
* @param pointFS
* @return
*/
public static double[][] facePoints(SeetaPointF[] pointFS) {
// 图中关键点坐标 - Coordinates of key points in the image
// 1. left_eye_x , left_eye_y
// 2. right_eye_x , right_eye_y
// 3. nose_x , nose_y
// 4. left_mouth_x , left_mouth_y
// 5. right_mouth_x , right_mouth_y
double[][] pointsArray = new double[5][2]; // 保存人脸关键点 - Save facial key points
int i = 0;
for (SeetaPointF point : pointFS) {
pointsArray[i][0] = point.getX();
pointsArray[i][1] = point.getY();
i++;
}
return pointsArray;
}
/**
* 512x512的目标点 - Target point of 512x512
* standard 5 landmarks for FFHQ faces with 512 x 512
*
* @param manager
* @return
*/
public static NDArray faceTemplate512x512(NDManager manager) {
double[][] coord5point = {
{192.98138, 239.94708}, // 512x512的目标点 - Target point of 512x512
{318.90277, 240.1936},
{256.63416, 314.01935},
{201.26117, 371.41043},
{313.08905, 371.15118}
};
NDArray points = manager.create(coord5point);
return points;
}
/**
* 112x112的目标点 - Target point of 112x112
* standard 5 landmarks for FFHQ faces with 112x112
*
* @param manager
* @return
*/
public static NDArray faceTemplate112x112(NDManager manager) {
double[][] coord5point = {
{30.29459953, 51.69630051}, // 112x112的目标点 - Target point of 512x512
{65.53179932, 51.50139999},
{48.02519989, 71.73660278},
{33.54930115, 87},
{62.72990036, 87}
};
NDArray points = manager.create(coord5point);
return points;
}
/**
* 96x112的目标点 - Target point of 96x112
* standard 5 landmarks for FFHQ faces with 96x112
*
* @param manager
* @return
*/
public static NDArray faceTemplate96x112(NDManager manager) {
double[][] coord5point = {
{30.29459953, 51.69630051},
{65.53179932, 51.50139999},
{48.02519989, 71.73660278},
{33.54930115, 92.3655014},
{62.72990036, 92.20410156}
};
NDArray points = manager.create(coord5point);
return points;
}
/**
* bgr转图片
* @return 图片
*/
public static BufferedImage toBufferedImage(SeetaImageData seetaImageData) {
int type = BufferedImage.TYPE_3BYTE_BGR;
BufferedImage image = new BufferedImage(seetaImageData.width, seetaImageData.height, type);
image.getRaster().setDataElements(0, 0, seetaImageData.width, seetaImageData.height, seetaImageData.data);
return image;
}
/**
* 绘制人脸关键点
* @param g
* @param keyPoints
*/
private static void drawLandmarks(Graphics2D g, List<Point> keyPoints) {
g.setColor(new Color(246, 96, 0));
BasicStroke bStroke = new BasicStroke(4.0F, 0, 0);
g.setStroke(bStroke);
for (Point point : keyPoints){
g.drawRect((int)point.getX(), (int)point.getY(), 2, 2);
}
}
/**
* 将DetectionRectangle转换为SeetaRect
* @param detectionRectangle
* @return
*/
public static SeetaRect convertToSeetaRect(DetectionRectangle detectionRectangle){
SeetaRect seetaRect = new SeetaRect();
seetaRect.x = detectionRectangle.getX();
seetaRect.y = detectionRectangle.getY();
seetaRect.width = detectionRectangle.getWidth();
seetaRect.height = detectionRectangle.getHeight();
return seetaRect;
}
/**
* 将PointList转换为SeetaPointF[]
* @param pointList
* @return
*/
public static SeetaPointF[] convertToSeetaPointF(List<Point> pointList){
return pointList.stream()
.map(p -> {
SeetaPointF sp = new SeetaPointF();
sp.x = p.getX();
sp.y = p.getY();
return sp;
})
.toArray(SeetaPointF[]::new);
}
/**
* 将SeetaAntiSpoofing.Status转换为LivenessStatus
* @param status
* @return
*/
public static LivenessStatus convertToLivenessStatus(FaceAntiSpoofing.Status status){
if(status == null){
return LivenessStatus.UNKNOWN;
}
switch (status) {
case REAL:
return LivenessStatus.LIVE;
case SPOOF:
return LivenessStatus.NON_LIVE;
case FUZZY:
return LivenessStatus.UNKNOWN;
case DETECTING:
return LivenessStatus.DETECTING;
default:
return LivenessStatus.UNKNOWN; // 默认返回未知
}
}
/**
* 转为GenderType
* @param gender
* @return
*/
public static GenderType convertToGenderType(GenderPredictor.GENDER gender){
if(gender == null){
return GenderType.UNKNOWN;
}
switch (gender) {
case MALE:
return GenderType.MALE;
case FEMALE:
return GenderType.FEMALE;
default:
return GenderType.UNKNOWN; // 默认返回未知
}
}
/**
* 转为EyeStatus
* @param eyeState
* @return
*/
public static EyeStatus convertToEyeStatus(EyeStateDetector.EYE_STATE eyeState){
if(eyeState == null){
return EyeStatus.UNKNOWN;
}
switch (eyeState) {
case EYE_OPEN:
return EyeStatus.OPEN;
case EYE_CLOSE:
return EyeStatus.CLOSED;
case EYE_RANDOM:
return EyeStatus.NON_EYE_REGION;
default:
return EyeStatus.UNKNOWN; // 默认返回未知
}
}
public static DetectionResponse convertToFaceAttributeResponse(SeetaRect[] seetaResult, List<SeetaPointF[]> seetaPointFSList, List<FaceAttribute> faceAttributeList){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return null;
}
List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
for(int i = 0; i < seetaResult.length; i++){
SeetaRect rect = seetaResult[i];
DetectionRectangle rectangle = new DetectionRectangle(rect.x, rect.y, rect.width, rect.height);
FaceInfo faceInfo = new FaceInfo();
if(seetaPointFSList != null && seetaPointFSList.size() > 0){
SeetaPointF[] seetaPointFS = seetaPointFSList.get(i);
List<Point> keyPoints = Arrays.stream(seetaPointFS)
.map(p -> new Point(p.x, p.y))
.collect(Collectors.toList());
faceInfo.setKeyPoints(keyPoints);
}
if(faceAttributeList != null && faceAttributeList.size() > 0){
faceInfo.setFaceAttribute(faceAttributeList.get(i));
}
detectionInfoList.add(new DetectionInfo(rectangle, 0, faceInfo));
}
return new DetectionResponse(detectionInfoList);
}
/**
* 转换为FaceDetectedResult
* @param seetaResult
* @return
*/
public static DetectionResponse convertToDetectionResponse(SeetaRect[] seetaResult, List<SeetaPointF[]> seetaPointFSList, List<LivenessStatus> livenessStatusList){
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return null;
}
DetectionResponse detectionResponse = new DetectionResponse();
List<DetectionInfo> detectionInfoList = new ArrayList<DetectionInfo>();
for(int i = 0; i < seetaResult.length; i++){
SeetaRect rect = seetaResult[i];
SeetaPointF[] seetaPointFS = seetaPointFSList.get(i);
//过滤置信度
/*if(config.getConfidenceThreshold() > 0){
continue;
}*/
DetectionRectangle rectangle = new DetectionRectangle(rect.x, rect.y, rect.width, rect.height);
List<Point> keyPoints = Arrays.stream(seetaPointFS)
.map(p -> new Point(p.x, p.y))
.collect(Collectors.toList());
FaceInfo faceInfo = new FaceInfo(keyPoints);
faceInfo.setLivenessStatus(new LivenessResult(livenessStatusList.get(i)));
DetectionInfo detectionInfo = new DetectionInfo(rectangle, 0, faceInfo);
detectionInfoList.add(detectionInfo);
}
detectionResponse.setDetectionInfoList(detectionInfoList);
return detectionResponse;
}
/**
* 绘制人脸属性
* @param sourceImage
* @param detectionResponse
* @param savePath
* @throws IOException
*/
public static void drawBoxesWithFaceAttribute(BufferedImage sourceImage, DetectionResponse detectionResponse, String savePath) throws IOException {
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
}
if(Objects.isNull(detectionResponse) || Objects.isNull(detectionResponse.getDetectionInfoList()) || detectionResponse.getDetectionInfoList().isEmpty()){
throw new FaceException("无目标数据");
}
Graphics2D graphics = sourceImage.createGraphics();
graphics.setColor(Color.RED);// 边框颜色
graphics.setStroke(new BasicStroke(2)); // 线宽2像素
graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿
int stroke = 2;
for(DetectionInfo detectionInfo : detectionResponse.getDetectionInfoList()){
DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
graphics.setColor(Color.RED);// 边框颜色
graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight());
//drawText(graphics, "face", rectangle.getX(), rectangle.getY(), stroke, 4);
//绘制人脸关键点
if(detectionInfo.getFaceInfo() != null && detectionInfo.getFaceInfo().getKeyPoints() != null &&
!detectionInfo.getFaceInfo().getKeyPoints().isEmpty()){
drawLandmarks(graphics, detectionInfo.getFaceInfo().getKeyPoints());
}
// 判断人脸框是否足够大
if (rectangle.getHeight() > 60 && detectionInfo.getFaceInfo() != null && detectionInfo.getFaceInfo().getFaceAttribute() != null) {
StringBuilder attrText = new StringBuilder();
FaceAttribute faceAttribute = detectionInfo.getFaceInfo().getFaceAttribute();
if (faceAttribute.getGenderType() != null) {
attrText.append(faceAttribute.getGenderType().name()).append(" ");
}
if (faceAttribute.getAge() != null) {
attrText.append(faceAttribute.getAge()).append("岁").append(" ");
}
if (faceAttribute.getWearingMask() != null) {
attrText.append(faceAttribute.getWearingMask() ? "戴口罩" : "未戴口罩").append(" ");
}
if (faceAttribute.getLeftEyeStatus() != null && faceAttribute.getRightEyeStatus() != null) {
attrText.append("眼睛:")
.append(faceAttribute.getLeftEyeStatus().name())
.append("/")
.append(faceAttribute.getRightEyeStatus().name())
.append(" ");
}
List<String> lines = new ArrayList<>();
if (faceAttribute.getGenderType() != null) {
lines.add("性别: " + faceAttribute.getGenderType().name());
}
if (faceAttribute.getAge() != null) {
lines.add("年龄: " + faceAttribute.getAge());
}
if (faceAttribute.getWearingMask() != null) {
lines.add("口罩: " + (faceAttribute.getWearingMask() ? "是" : "否"));
}
if (faceAttribute.getLeftEyeStatus() != null && faceAttribute.getRightEyeStatus() != null) {
lines.add("眼睛: " + faceAttribute.getLeftEyeStatus().name() + "/" + faceAttribute.getRightEyeStatus().name());
}
if (faceAttribute.getHeadPose() != null) {
//attrText.append("姿态:").append(faceAttribute.getHeadPose().toString());
HeadPose pose = faceAttribute.getHeadPose();
String pitch = pose.getPitch() != null ? String.valueOf(pose.getPitch().intValue()) : "-";
String yaw = pose.getYaw() != null ? String.valueOf(pose.getYaw().intValue()) : "-";
String roll = pose.getRoll() != null ? String.valueOf(pose.getRoll().intValue()) : "-";
lines.add("姿态: P=" + pitch + " Y=" + yaw + " R=" + roll);
}
if (!lines.isEmpty()) {
drawMultilineTextWithBackground(graphics, lines, rectangle.getX(), rectangle.getY()); // 适当偏移
}
}
}
graphics.dispose();
ImageIO.write(sourceImage, "png", new File(savePath));
}
private static void drawMultilineTextWithBackground(Graphics2D g, List<String> lines, int x, int y) {
Font font = new Font("SansSerif", Font.PLAIN, 14);
g.setFont(font);
FontMetrics fm = g.getFontMetrics();
int lineHeight = fm.getHeight();
int maxWidth = lines.stream().mapToInt(fm::stringWidth).max().orElse(0);
int padding = 4;
int boxWidth = maxWidth + padding * 2;
int boxHeight = lineHeight * lines.size() + padding * 2;
// 背景矩形
g.setColor(new Color(0, 0, 0, 128));
g.fillRoundRect(x, y, boxWidth, boxHeight, 8, 8);
// 绘制每一行文字
g.setColor(Color.WHITE);
for (int i = 0; i < lines.size(); i++) {
g.drawString(lines.get(i), x + padding, y + padding + (i + 1) * lineHeight - 4);
}
}
/**
* 将 Milvus 查询返回的得分转换为 0~1 范围的相似度
* @param metricType 向量度量方式：IP 或 L2
* @param score 原始得分（L2 为距离，IP 为相似度）
* @return 映射后的相似度（0~1）
*/
public static float convertScoreToSimilarity(String metricType, float score) {
switch (metricType.toUpperCase()) {
case "IP":
// 内积 IP 的范围为 [-1, 1]，归一化为 [0, 1]
return (score + 1.0f) / 2.0f;
case "L2":
// 欧氏距离 L2：距离越小越相似，1 / (1 + 距离) 映射到 (0, 1]
return 1.0f / (1.0f + score);
case "COSINE":
// 余弦相似度 COSINE：本身范围为 [-1, 1]，也需要归一化到 [0, 1]
return (score + 1.0f) / 2.0f;
default:
throw new IllegalArgumentException("Unsupported metricType: " + metricType);
}
}
}

119
face/src/main/java/cn/smartjavaai/face/utils/SVDUtils.java Normal file
View File

@@ -0,0 +1,119 @@
package cn.smartjavaai.face.utils;
import Jama.Matrix;
import Jama.SingularValueDecomposition;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
/**
* 仿射变换处理工具
*/
public class SVDUtils {
/**
* 计算仿射变换矩阵
* Calculate affine transformation matrix
*
* @param manager
* @param points1
* @param points2
* @return
*/
public static NDArray transformationFromPoints(
NDManager manager, NDArray points1, NDArray points2) {
// 按列计算均值
// Calculate column-wise mean
NDArray c1 = points1.mean(new int[]{0}); // axis=0 列操作 - axis=0 column operation
NDArray c2 = points2.mean(new int[]{0}); // axis=0 列操作 - axis=0 column operation
// 按列减去均值
// Subtract column-wise mean
points1 = points1.sub(c1);
points2 = points2.sub(c2);
// 计算全局标准差
// Calculate global standard deviation
double s1 = std(points1);
double s2 = std(points2);
// 矩阵除以全局标准差
// Matrix divided by global standard deviation
NDArray djl_s1 = manager.create(s1);
NDArray djl_s2 = manager.create(s2);
points1 = points1.div(djl_s1);
points2 = points2.div(djl_s2);
double[] points1D = points1.toDoubleArray();
double[] points2D = points2.toDoubleArray();
// DJL 格式转换成Jamma格式
// Convert DJL format to Jama format
double[][] m1 = new double[5][2];
double[][] m2 = new double[5][2];
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 2; j++) {
m1[i][j] = points1D[i * 2 + j];
}
}
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 2; j++) {
m2[i][j] = points2D[i * 2 + j];
}
}
Matrix p1 = new Matrix(m1);
Matrix p2 = new Matrix(m2);
// 进行奇异值分解
// Perform singular value decomposition
Matrix p3 = p1.transpose().times(p2);
SingularValueDecomposition s = p3.svd();
Matrix U = s.getU();
Matrix S = s.getS();
Matrix V = s.getV();
// TODO 为什么第2列的符号是反的？
// Why is the sign of the second column opposite?
m1 = U.getArray();
m1[0][1] = -m1[0][1];
m1[1][1] = -m1[1][1];
m2 = V.getArray();
m2[0][1] = -m2[0][1];
m2[1][1] = -m2[1][1];
Matrix R = (U.times(V)).transpose();
double[][] rArray = R.getArray();
NDArray newR = manager.create(rArray);
// np.vstack([np.hstack(((s2 / s1) * R, c2.T - (s2 / s1) * R * c1.T)), np.matrix([0.,0., 1.])])
// (s2 / s1) * R
NDArray leftPart = djl_s2.div(djl_s1).mul(newR);
// c2.T - (s2 / s1) * R * c1.T)
NDArray rightPart = c2.reshape(2, 1).sub(leftPart.matMul(c1.reshape(2, 1)));
// numpy.hstack(((s2 / s1) * R, c2.T - (s2 / s1) * R * c1.T))
NDArray upPart = leftPart.concat(rightPart, 1);
// np.matrix([0.,0., 1.])
double[] downArray = {0d, 0d, 1d};
NDArray downPart = manager.create(downArray).reshape(1, 3);
NDArray all = upPart.concat(downPart, 0);
// System.out.println("all: " + all);
return upPart;
}
/**
* 计算全局标准差
* Calculate global standard deviation
*
* @param points
* @return
*/
public static double std(NDArray points) {
points = points.square();
double[] doubleResult = points.toDoubleArray();
double std = 0;
for (int i = 0; i < doubleResult.length; i++) {
std = std + doubleResult[i];
}
std = (float) Math.sqrt(std / doubleResult.length);
return std;
}
}

13
face/src/main/java/cn/smartjavaai/face/utils/Seetaface6Utils.java Normal file
View File

@@ -0,0 +1,13 @@
package cn.smartjavaai.face.utils;
import cn.smartjavaai.face.enums.QualityGrade;
/**
* Seetaface6工具类
* @author dwj
* @date 2025/6/24
*/
public class Seetaface6Utils {
}

129
face/src/main/java/cn/smartjavaai/face/utils/SimilarityUtil.java Normal file
View File

@@ -0,0 +1,129 @@
package cn.smartjavaai.face.utils;
import cn.smartjavaai.face.enums.SimilarityType;
/**
* 特征相似度计算工具类
* 支持三种计算方式：IP（内积）、L2（欧氏距离）、COSINE（余弦相似度）
* 所有计算结果归一化到[0,1]范围
*/
public class SimilarityUtil {
/**
* 计算特征相似度
* @param features1 特征向量1
* @param features2 特征向量2
* @param similarityType 计算类型 (IP, L2, COSINE)
* @param normalizeScore 是否归一化结果到 [0,1]
* @return 相似度
*/
public static float calculate(float[] features1, float[] features2,
SimilarityType similarityType,
boolean normalizeScore) {
validateInput(features1, features2);
switch (similarityType) {
case IP:
return innerProductSimilarity(features1, features2, normalizeScore);
case L2:
return euclideanSimilarity(features1, features2, normalizeScore);
case COSINE:
return cosineSimilarity(features1, features2, normalizeScore);
default:
throw new IllegalArgumentException("不支持的相似度计算类型: " + similarityType);
}
}
// ================ 私有计算方法 ================
/**
* 计算内积相似度（归一化到[0,1]）
* 适用于归一化向量（结果范围[-1,1] -> [0,1]）
*/
private static float innerProductSimilarity(float[] v1, float[] v2, boolean normalize) {
float dot = dotProduct(v1, v2);
return normalize ? (dot + 1.0f) / 2.0f : dot;
}
/**
* 计算欧氏距离相似度（归一化到[0,1]）
* 距离越小相似度越高，距离为0时相似度为1
*/
private static float euclideanSimilarity(float[] v1, float[] v2, boolean normalize) {
float dist = euclideanDistance(v1, v2);
return normalize ? 1.0f / (1.0f + dist) : dist;
}
/**
* 计算余弦相似度（归一化到[0,1]）
* 适用于非归一化向量（结果范围[-1,1] -> [0,1]）
*/
private static float cosineSimilarity(float[] v1, float[] v2, boolean normalize) {
float dot = dotProduct(v1, v2);
float norm1 = vectorNorm(v1);
float norm2 = vectorNorm(v2);
if (norm1 <= 0 || norm2 <= 0) {
return 0.0f;
}
float cosine = dot / (norm1 * norm2);
return normalize ? (cosine + 1.0f) / 2.0f : cosine;
}
// ================ 基础向量操作 ================
/**
* 计算点积（内积）
*/
public static float dotProduct(float[] v1, float[] v2) {
float sum = 0.0f;
for (int i = 0; i < v1.length; i++) {
sum += v1[i] * v2[i];
}
return sum;
}
/**
* 计算欧氏距离
*/
public static float euclideanDistance(float[] v1, float[] v2) {
float sumSquaredDiff = 0.0f;
for (int i = 0; i < v1.length; i++) {
float diff = v1[i] - v2[i];
sumSquaredDiff += diff * diff;
}
return (float) Math.sqrt(sumSquaredDiff);
}
/**
* 计算向量模长
*/
public static float vectorNorm(float[] vector) {
float sum = 0.0f;
for (float v : vector) {
sum += v * v;
}
return (float) Math.sqrt(sum);
}
// ================ 输入验证 ================
/**
* 验证输入向量
*/
private static void validateInput(float[] v1, float[] v2) {
if (v1 == null || v2 == null) {
throw new IllegalArgumentException("特征向量不能为null");
}
if (v1.length == 0 || v2.length == 0) {
throw new IllegalArgumentException("特征向量不能为空");
}
if (v1.length != v2.length) {
throw new IllegalArgumentException("特征向量长度不一致: " +
v1.length + " vs " + v2.length);
}
}
}

42
face/src/main/java/cn/smartjavaai/face/utils/VectorUtils.java Normal file
View File

@@ -0,0 +1,42 @@
package cn.smartjavaai.face.utils;
import java.nio.ByteBuffer;
import java.nio.FloatBuffer;
/**
* 人脸向量工具类
* @author dwj
* @date 2025/5/30
*/
public class VectorUtils {
/**
* 将 float 数组转换为 byte 数组
* @param floats 人脸特征向量
* @return 转换后的字节数组
*/
public static byte[] toByteArray(float[] floats) {
ByteBuffer buffer = ByteBuffer.allocate(floats.length * 4); // 每个 float 占 4 个字节
buffer.asFloatBuffer().put(floats);
return buffer.array();
}
/**
* 将 byte 数组转换回 float 数组
* @param bytes 从数据库读取的字节数组
* @return 原始的人脸特征向量
*/
public static float[] toFloatArray(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return new float[0];
}
ByteBuffer buffer = ByteBuffer.wrap(bytes);
FloatBuffer floatBuffer = buffer.asFloatBuffer();
float[] floats = new float[floatBuffer.remaining()];
floatBuffer.get(floats);
return floats;
}
}

92
face/src/main/java/cn/smartjavaai/face/vector/config/MilvusConfig.java Normal file
View File

@@ -0,0 +1,92 @@
package cn.smartjavaai.face.vector.config;
import cn.smartjavaai.face.enums.IdStrategy;
import cn.smartjavaai.face.enums.VectorDBType;
import io.milvus.param.IndexType;
import io.milvus.param.MetricType;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* Milvus配置类
* @author smartjavaai
*/
@Data
@EqualsAndHashCode(callSuper = true)
public class MilvusConfig extends VectorDBConfig {
/**
* 服务器地址
*/
private String host = "localhost";
/**
* 服务器端口
*/
private int port = 19530;
/**
* 索引类型
*/
private IndexType indexType = IndexType.IVF_FLAT;
/**
* 聚类数量，用于IVF索引
*/
private int nlist = 1024;
/**
* 向量维度
*/
private int dimension;
/**
* ID策略
*/
private IdStrategy idStrategy = IdStrategy.AUTO;
/**
* 相似度计算方式
*/
private MetricType metricType;
/**
* 集合名称
*/
private String collectionName;
/**
* 是否使用内存缓存
*/
private boolean useMemoryCache = true;
/**
* 用户名
*/
private String username;
/**
* 密码
*/
private String password;
/**
* 构造函数
*/
public MilvusConfig() {
setType(VectorDBType.MILVUS);
}
/**
* 构造函数
* @param host 服务器地址
* @param port 服务器端口
*/
public MilvusConfig(String host, int port) {
this();
this.host = host;
this.port = port;
}
}

30
face/src/main/java/cn/smartjavaai/face/vector/config/SQLiteConfig.java Normal file
View File

@@ -0,0 +1,30 @@
package cn.smartjavaai.face.vector.config;
import cn.smartjavaai.face.enums.SimilarityType;
import cn.smartjavaai.face.enums.VectorDBType;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* @author dwj
* @date 2025/5/31
*/
@Data
@EqualsAndHashCode(callSuper = true)
public class SQLiteConfig extends VectorDBConfig {
/**
* 数据库路径(包含文件名称)
*/
private String dbPath;
/**
* 相似度计算方式
*/
private SimilarityType similarityType;
public SQLiteConfig() {
setType(VectorDBType.SQLITE);
}
}

20
face/src/main/java/cn/smartjavaai/face/vector/config/VectorDBConfig.java Normal file
View File

@@ -0,0 +1,20 @@
package cn.smartjavaai.face.vector.config;
import cn.smartjavaai.face.enums.VectorDBType;
import lombok.Data;
/**
* 向量数据库基础配置
* @author dwj
*/
@Data
public abstract class VectorDBConfig {
/**
* 向量数据库类型
*/
private VectorDBType type;
}

48
face/src/main/java/cn/smartjavaai/face/vector/constant/VectorDBConstants.java Normal file
View File

@@ -0,0 +1,48 @@
package cn.smartjavaai.face.vector.constant;
/**
* 向量数据库常量类
* @author dwj
*/
public class VectorDBConstants {
/**
* 字段名称常量
*/
public static class FieldNames {
/** ID字段名 */
public static final String ID_FIELD = "id";
/** 向量字段名 */
public static final String VECTOR_FIELD = "vector";
/** 元数据字段名 */
public static final String METADATA_FIELD = "metadata";
}
/**
* 默认配置常量
*/
public static class Defaults {
/** 默认搜索探针数 */
public static final int DEFAULT_SEARCH_PARAM_NPROBE = 10;
/** 默认向量维度 */
public static final int DEFAULT_VECTOR_DIMENSION = 512;
/** 默认元数据最大长度 */
public static final int DEFAULT_METADATA_MAX_LENGTH = 32 * 1024;
/** 默认ID字段最大长度 */
public static final int DEFAULT_ID_MAX_LENGTH = 36;
/**
* 默认集合名称
*/
public static final String DEFAULT_COLLECTION_NAME = "face";
}
/**
* 搜索参数常量
*/
public static class SearchParams {
/** 默认相似度阈值 */
public static final float DEFAULT_SIMILARITY_THRESHOLD = 0.7f;
/** 默认返回TOP-K结果数 */
public static final int DEFAULT_TOP_K = 10;
}
}

691
face/src/main/java/cn/smartjavaai/face/vector/core/MilvusClient.java Normal file
View File

@@ -0,0 +1,691 @@
package cn.smartjavaai.face.vector.core;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.face.enums.IdStrategy;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.vector.config.MilvusConfig;
import cn.smartjavaai.face.vector.constant.VectorDBConstants;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import io.milvus.client.MilvusServiceClient;
import io.milvus.grpc.*;
import io.milvus.param.*;
import io.milvus.param.collection.*;
import io.milvus.param.dml.*;
import io.milvus.param.index.CreateIndexParam;
import io.milvus.response.DescCollResponseWrapper;
import io.milvus.response.QueryResultsWrapper;
import io.milvus.response.SearchResultsWrapper;
import io.milvus.v2.service.collection.request.DescribeCollectionReq;
import io.milvus.v2.service.collection.response.DescribeCollectionResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.*;
import java.util.stream.Collectors;
/**
* Milvus向量数据库客户端实现
* @author dwj
*/
@Slf4j
public class MilvusClient implements VectorDBClient {
private final MilvusConfig config;
private MilvusServiceClient serviceClient;
private String collectionName;
/**
* 是否初始化完毕
*/
private boolean isInit;
public MilvusClient(MilvusConfig config) {
this.config = config;
}
@Override
public void initialize() {
try {
ConnectParam.Builder builder = ConnectParam.newBuilder()
.withHost(config.getHost())
.withPort(config.getPort());
if (StringUtils.isNotBlank(config.getUsername()) && StringUtils.isNotBlank(config.getPassword())) {
builder.withAuthorization(config.getUsername(), config.getPassword());
}
ConnectParam connectParam = builder.build();
serviceClient = new MilvusServiceClient(connectParam);
collectionName = StringUtils.isNotBlank(config.getCollectionName()) ? config.getCollectionName() : VectorDBConstants.Defaults.DEFAULT_COLLECTION_NAME;
createCollection(collectionName, config.getDimension());
boolean isAutoID = isAutoID(collectionName);
if(isAutoID && config.getIdStrategy() != IdStrategy.AUTO){
throw new VectorDBException("ID策略与当前Collection不匹配");
}
if(!isAutoID && config.getIdStrategy() == IdStrategy.AUTO){
throw new VectorDBException("ID策略与当前Collection不匹配");
}
if(config.isUseMemoryCache()){
// 加载集合到内存
loadFaceFeatures();
}
isInit = true;
} catch (Exception e) {
throw new VectorDBException("初始化Milvus客户端失败", e);
}
}
@Override
public void createCollection(String collectionName, int dimension) {
try {
if (hasCollection(collectionName)) {
log.debug("集合已存在:{}", collectionName);
return;
}
// 创建集合字段
FieldType idField = null;
switch (config.getIdStrategy()){
case AUTO://自动生成ID
idField = FieldType.newBuilder()
.withName(VectorDBConstants.FieldNames.ID_FIELD)
.withDataType(DataType.Int64)
.withPrimaryKey(true)
.withAutoID(true)
.build();
break;
case CUSTOM://自定义ID
idField = FieldType.newBuilder()
.withName(VectorDBConstants.FieldNames.ID_FIELD)
.withDataType(DataType.VarChar)
.withMaxLength(VectorDBConstants.Defaults.DEFAULT_ID_MAX_LENGTH)
.withPrimaryKey(true)
.withAutoID(false)
.build();
break;
}
FieldType vectorField = FieldType.newBuilder()
.withName(VectorDBConstants.FieldNames.VECTOR_FIELD)
.withDataType(DataType.FloatVector)
.withDimension(dimension)
.build();
FieldType metadataField = FieldType.newBuilder()
.withName(VectorDBConstants.FieldNames.METADATA_FIELD)
.withDataType(DataType.VarChar)
.withMaxLength(VectorDBConstants.Defaults.DEFAULT_METADATA_MAX_LENGTH)
.withNullable(true)//允许为空值
.build();
// 创建集合参数
CreateCollectionParam createCollectionParam = CreateCollectionParam.newBuilder()
.withCollectionName(collectionName)
.withDescription("人脸特征向量集合")
.addFieldType(idField)
.addFieldType(vectorField)
.addFieldType(metadataField)
.build();
R<RpcStatus> response = serviceClient.createCollection(createCollectionParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Milvus 创建集合失败：" + response.getMessage());
}
log.debug("创建集合成功");
// 创建索引
IndexType indexType = IndexType.IVF_FLAT;
if (config.getIndexType() != null) {
indexType = config.getIndexType();
}
CreateIndexParam indexParam = CreateIndexParam.newBuilder()
.withCollectionName(collectionName)
.withFieldName(VectorDBConstants.FieldNames.VECTOR_FIELD)
.withIndexType(indexType)
.withMetricType(config.getMetricType())
.withExtraParam(String.format("{\"nlist\":%d}", config.getNlist()))
.withSyncMode(Boolean.TRUE)//调用方法后等待 Milvus 执行完成
.build();
R<RpcStatus> createIndexResponse = serviceClient.createIndex(indexParam);
if (createIndexResponse.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Milvus 创建索引失败：" + createIndexResponse.getMessage());
}
log.debug("创建索引成功");
} catch (Exception e) {
throw new VectorDBException("创建Milvus集合失败", e);
}
}
/**
* 判断是否为自增长ID
* @param collectionName
* @return
*/
private boolean isAutoID(String collectionName) {
R<DescribeCollectionResponse> response = serviceClient.describeCollection(
DescribeCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build()
);
DescCollResponseWrapper wrapper = new DescCollResponseWrapper(response.getData());
return wrapper.getPrimaryField().isAutoID();
}
@Override
public void dropCollection(String collectionName) {
try {
if (!isInit){
throw new VectorDBException("Milvus未初始化完毕");
}
if (hasCollection(collectionName)) {
R<RpcStatus> response = serviceClient.dropCollection(DropCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build());
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Milvus 删除集合失败：" + response.getMessage());
}
isInit = false;
}
} catch (Exception e) {
throw new VectorDBException("删除Milvus集合失败", e);
}
}
@Override
public boolean hasCollection(String collectionName) {
try {
R<Boolean> response = serviceClient.hasCollection(
HasCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build()
);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Milvus 查询失败：" + response.getMessage());
}
return Boolean.TRUE.equals(response.getData());
} catch (Exception e) {
throw new VectorDBException("检查Milvus集合是否保存失败", e);
}
}
@Override
public String insert(FaceVector faceVector) {
try {
if (!isInit){
throw new VectorDBException("Milvus未初始化完毕");
}
//验证
if(faceVector == null){
throw new VectorDBException("插入数据失败：faceVector不能为空");
}
if(faceVector.getVector() == null || faceVector.getVector().length == 0){
throw new VectorDBException("插入数据失败：vector不能为空");
}
//自定义ID
if(config.getIdStrategy() == IdStrategy.CUSTOM){
if(StringUtils.isBlank(faceVector.getId())){
throw new VectorDBException("插入数据失败：ID生成策略-自定义ID，id不能为空");
}
}
// 转 float[] 为 List<Float>
List<Float> vectorList = new ArrayList<>();
for (float v : faceVector.getVector()) {
vectorList.add(v);
}
List<List<Float>> vectors = Collections.singletonList(vectorList);
//List<String> metadataList = Collections.singletonList(faceVector.getMetadata());
List<String> metadataList = Optional.ofNullable(faceVector.getMetadata())
.map(Collections::singletonList)
.orElse(Collections.emptyList());
List<InsertParam.Field> fields = null;
switch (config.getIdStrategy()){
case AUTO://自动生成ID
fields = Arrays.asList(
InsertParam.Field.builder().name(VectorDBConstants.FieldNames.VECTOR_FIELD).values(vectors).build(),
InsertParam.Field.builder().name(VectorDBConstants.FieldNames.METADATA_FIELD).values(metadataList).build()
);
break;
case CUSTOM://自定义ID
List<String> ids = Collections.singletonList(faceVector.getId());
fields = Arrays.asList(
InsertParam.Field.builder().name(VectorDBConstants.FieldNames.ID_FIELD).values(ids).build(),
InsertParam.Field.builder().name(VectorDBConstants.FieldNames.VECTOR_FIELD).values(vectors).build(),
InsertParam.Field.builder().name(VectorDBConstants.FieldNames.METADATA_FIELD).values(metadataList).build()
);
break;
}
InsertParam insertParam = InsertParam.newBuilder()
.withCollectionName(collectionName)
.withFields(fields)
.build();
R<MutationResult> response = serviceClient.insert(insertParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("插入失败: " + response.getMessage());
}
List<Long> autoIds = response.getData().getIDs().getIntId().getDataList();
return config.getIdStrategy() == IdStrategy.AUTO ? String.valueOf(autoIds.get(0)) : faceVector.getId();
} catch (Exception e) {
throw new VectorDBException("插入Milvus向量失败", e);
}
}
@Override
public void upsert(FaceVector faceVector) {
try {
if (!isInit){
throw new VectorDBException("Milvus未初始化完毕");
}
if(config.getIdStrategy() == IdStrategy.AUTO){
throw new VectorDBException("idStrategy为AUTO时,不支持更新操作");
}
//验证
if(faceVector == null){
throw new VectorDBException("更新数据失败：faceVector不能为空");
}
if(faceVector.getVector() == null || faceVector.getVector().length == 0){
throw new VectorDBException("更新数据失败：vector不能为空");
}
if(StringUtils.isBlank(faceVector.getId())){
throw new VectorDBException("更新数据失败：id不能为空");
}
// 转换向量为 List<Float>
List<Float> vectorList = new ArrayList<>();
for (float v : faceVector.getVector()) {
vectorList.add(v);
}
List<List<Float>> vectors = Collections.singletonList(vectorList);
List<String> metadataList = Optional.ofNullable(faceVector.getMetadata())
.map(Collections::singletonList)
.orElse(Collections.emptyList());
// 准备字段列表
List<UpsertParam.Field> fields = new ArrayList<>();
fields.add(UpsertParam.Field.builder()
.name(VectorDBConstants.FieldNames.ID_FIELD)
.values(Collections.singletonList(faceVector.getId()))
.build());
// 添加向量和元数据字段
fields.add(UpsertParam.Field.builder()
.name(VectorDBConstants.FieldNames.VECTOR_FIELD)
.values(vectors)
.build());
fields.add(UpsertParam.Field.builder()
.name(VectorDBConstants.FieldNames.METADATA_FIELD)
.values(metadataList)
.build());
// 构建Upsert参数
UpsertParam upsertParam = UpsertParam.newBuilder()
.withCollectionName(collectionName)
.withFields(fields)
.build();
// 执行Upsert操作
R<MutationResult> response = serviceClient.upsert(upsertParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Upsert操作失败: " + response.getMessage());
}
} catch (Exception e) {
throw new VectorDBException("Milvus Upsert操作失败", e);
}
}
@Override
public List<String> insertBatch(List<FaceVector> faceVectors) {
try {
if (!isInit){
throw new VectorDBException("Milvus未初始化完毕");
}
List<String> ids = faceVectors.stream()
.map(FaceVector::getId)
.collect(Collectors.toList());
List<List<Float>> vectors = new ArrayList<>();
for (FaceVector fv : faceVectors) {
List<Float> list = new ArrayList<>();
for (float f : fv.getVector()) {
list.add(f);
}
vectors.add(list);
}
List<String> metadataList = faceVectors.stream()
.map(FaceVector::getMetadata)
.collect(Collectors.toList());
InsertParam insertParam = InsertParam.newBuilder()
.withCollectionName(collectionName)
.withFields(Arrays.asList(
InsertParam.Field.builder().name(VectorDBConstants.FieldNames.ID_FIELD).values(ids).build(),
InsertParam.Field.builder().name(VectorDBConstants.FieldNames.VECTOR_FIELD).values(vectors).build(),
InsertParam.Field.builder().name(VectorDBConstants.FieldNames.METADATA_FIELD).values(metadataList).build()
))
.build();
R<MutationResult> insertResult = serviceClient.insert(insertParam);
return ids;
} catch (Exception e) {
throw new VectorDBException("批量插入Milvus向量失败", e);
}
}
@Override
public void delete(String id) {
try {
if (!isInit){
throw new VectorDBException("Milvus未初始化完毕");
}
String expr = String.format("%s == \"%s\"", VectorDBConstants.FieldNames.ID_FIELD, id);
DeleteParam deleteParam = DeleteParam.newBuilder()
.withCollectionName(collectionName)
.withExpr(expr)
.build();
R<MutationResult> response = serviceClient.delete(deleteParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("删除操作失败: " + response.getMessage());
}
} catch (Exception e) {
throw new VectorDBException("删除Milvus向量失败", e);
}
}
@Override
public void deleteBatch(List<String> ids) {
try {
if (!isInit){
throw new VectorDBException("Milvus未初始化完毕");
}
// 构建IN表达式: id in ["id1", "id2", ...]
StringBuilder expr = new StringBuilder(VectorDBConstants.FieldNames.ID_FIELD + " in [");
for (int i = 0; i < ids.size(); i++) {
if(config.getIdStrategy() == IdStrategy.AUTO){
expr.append(ids.get(i)); // 不加引号
}else{
expr.append("'" + ids.get(i) + "'"); // 不加引号
}
if (i < ids.size() - 1) {
expr.append(", ");
}
}
expr.append("]");
DeleteParam deleteParam = DeleteParam.newBuilder()
.withCollectionName(collectionName)
.withExpr(expr.toString())
.build();
R<MutationResult> response = serviceClient.delete(deleteParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("删除操作失败: " + response.getMessage());
}
} catch (Exception e) {
throw new VectorDBException("批量删除Milvus向量失败", e);
}
}
@Override
public List<FaceSearchResult> search(float[] queryVector, FaceSearchParams faceSearchParams) {
try {
if (!isInit){
throw new VectorDBException("Milvus未初始化完毕");
}
// 1. 包装查询向量
List<List<Float>> vectors = new ArrayList<>();
List<Float> floatList = new ArrayList<>();
for (float f : queryVector) {
floatList.add(f);
}
vectors.add(floatList);
// 2. 构造搜索参数
SearchParam searchParam = SearchParam.newBuilder()
.withCollectionName(collectionName)
.withVectorFieldName(VectorDBConstants.FieldNames.VECTOR_FIELD)
.withTopK(faceSearchParams.getTopK())
.withMetricType(config.getMetricType())
.withOutFields(Arrays.asList(VectorDBConstants.FieldNames.ID_FIELD, VectorDBConstants.FieldNames.METADATA_FIELD))
.withVectors(vectors)
.withParams("{\"nprobe\": 10}")//和nlist有关
.build();
R<SearchResults> resp = serviceClient.search(searchParam);
if (resp.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Milvus 查询失败: " + resp.getMessage());
}
SearchResults results = resp.getData();
SearchResultsWrapper wrapper = new SearchResultsWrapper(results.getResults());
// 3. 获取字段数据（FieldData）
List<SearchResultsWrapper.IDScore> scores = wrapper.getIDScore(0); // 默认只有一条 query 向量
List<FaceSearchResult> finalResults = new ArrayList<>();
for (int i = 0; i < scores.size(); i++) {
SearchResultsWrapper.IDScore score = scores.get(i);
float similarity = score.getScore();
if (faceSearchParams.getNormalizeSimilarity()) {
// 将分数转换为相似度
similarity = FaceUtils.convertScoreToSimilarity(config.getMetricType().name(), score.getScore());
}
if (similarity >= faceSearchParams.getThreshold()) {
// 获取 Metadata
String metadata = wrapper.getFieldData(VectorDBConstants.FieldNames.METADATA_FIELD, 0).get(i).toString();
// 获取 ID
String id = wrapper.getFieldData(VectorDBConstants.FieldNames.ID_FIELD, 0).get(i).toString();
finalResults.add(new FaceSearchResult(id, similarity, metadata));
}
}
return finalResults;
} catch (Exception e) {
throw new VectorDBException("搜索 Milvus 向量失败", e);
}
}
@Override
public long count(String collectionName) {
try {
if (serviceClient == null){
throw new VectorDBException("Milvus未初始化完毕");
}
R<QueryResults> response = serviceClient.query(
QueryParam.newBuilder()
.withCollectionName(collectionName)
.withOutFields(Collections.singletonList("count(*)"))
.build()
);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Milvus 查询失败，msg: " + response.getMessage());
}
List<FieldData> fields = response.getData().getFieldsDataList();
if (fields.isEmpty()) {
throw new VectorDBException("Milvus 返回空字段数据");
}
FieldData countField = fields.get(0);
List<Long> countValues = countField.getScalars().getLongData().getDataList();
if (countValues.isEmpty()) {
throw new VectorDBException("Milvus count(*) 返回为空");
}
return countValues.get(0); // count(*) 查询的结果
} catch (Exception e) {
throw new VectorDBException("获取 Milvus 集合数量失败", e);
}
}
@Override
public void close() {
if (serviceClient != null) {
serviceClient.close();
isInit = false;
}
}
@Override
public FaceVector getFaceInfoById(String id) {
try {
if (!isInit) {
throw new VectorDBException("Milvus未初始化完毕");
}
String expr = VectorDBConstants.FieldNames.ID_FIELD + " == '" + id + "'";
if(config.getIdStrategy() == IdStrategy.AUTO){
expr = VectorDBConstants.FieldNames.ID_FIELD + " == " + id;
}
// 5. 执行查询
R<QueryResults> response = serviceClient.query(
QueryParam.newBuilder()
.withCollectionName(collectionName)
.withExpr(expr)
.withOutFields(Arrays.asList(VectorDBConstants.FieldNames.ID_FIELD, VectorDBConstants.FieldNames.VECTOR_FIELD, VectorDBConstants.FieldNames.METADATA_FIELD))
.build()
);
// 处理响应
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("查询失败: " + response.getMessage());
}
QueryResultsWrapper wrapper = new QueryResultsWrapper(response.getData());
List<QueryResultsWrapper.RowRecord> records = wrapper.getRowRecords();
if (records.isEmpty()) {
return null;
}
// 提取第一条记录
QueryResultsWrapper.RowRecord row = records.get(0);
Object vectorObj = row.get(VectorDBConstants.FieldNames.VECTOR_FIELD);
float[] vector = null;
if (vectorObj instanceof List<?>) {
// Milvus SDK通常返回List<Float>，转成float[]
List<Float> vectorList = (List<Float>) vectorObj;
vector = new float[vectorList.size()];
for (int i = 0; i < vectorList.size(); i++) {
vector[i] = vectorList.get(i);
}
}
return new FaceVector(id, vector, (String) row.get(VectorDBConstants.FieldNames.METADATA_FIELD));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public List<FaceVector> listFaces(long pageNum, long pageSize) {
try {
if (!isInit) {
throw new VectorDBException("Milvus未初始化完毕");
}
if (pageNum < 1 || pageSize < 1) {
throw new IllegalArgumentException("pageNum和pageSize必须大于0");
}
long offset = (pageNum - 1) * pageSize;
// 构造查询参数，使用offset和limit实现分页
QueryParam queryParam = QueryParam.newBuilder()
.withCollectionName(collectionName)
.withOutFields(Arrays.asList(
VectorDBConstants.FieldNames.ID_FIELD,
VectorDBConstants.FieldNames.VECTOR_FIELD,
VectorDBConstants.FieldNames.METADATA_FIELD))
.withOffset(offset)
.withLimit(pageSize)
.build();
R<QueryResults> response = serviceClient.query(queryParam);
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("分页查询失败: " + response.getMessage());
}
QueryResultsWrapper wrapper = new QueryResultsWrapper(response.getData());
List<QueryResultsWrapper.RowRecord> records = wrapper.getRowRecords();
if (records.isEmpty()) {
return Collections.emptyList();
}
List<FaceVector> result = new ArrayList<>();
for (QueryResultsWrapper.RowRecord row : records) {
String id = (String) row.get(VectorDBConstants.FieldNames.ID_FIELD);
Object vectorObj = row.get(VectorDBConstants.FieldNames.VECTOR_FIELD);
float[] vector = null;
if (vectorObj instanceof List<?>) {
List<Float> vectorList = (List<Float>) vectorObj;
vector = new float[vectorList.size()];
for (int i = 0; i < vectorList.size(); i++) {
vector[i] = vectorList.get(i);
}
}
String metadata = (String) row.get(VectorDBConstants.FieldNames.METADATA_FIELD);
result.add(new FaceVector(id, vector, metadata));
}
return result;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public void releaseCollection(String collectionName) {
if (!isInit){
throw new VectorDBException("Milvus未初始化完毕");
}
R<RpcStatus> response = serviceClient.releaseCollection(ReleaseCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build());
if (response.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Milvus releaseCollection失败，msg: " + response.getMessage());
}
}
@Override
public void loadFaceFeatures() {
// 加载集合到内存
R<RpcStatus> loadResponse = serviceClient.loadCollection(LoadCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build());
if (loadResponse.getStatus() != R.Status.Success.getCode()) {
throw new VectorDBException("Milvus 加载集合到内存失败：" + loadResponse.getMessage());
}
long count = count(collectionName);
log.debug("加载集合到内存成功,人脸数量：{}", count);
}
@Override
public void releaseFaceFeatures() {
releaseCollection(collectionName);
}
}

289
face/src/main/java/cn/smartjavaai/face/vector/core/SQLiteClient.java Normal file
View File

@@ -0,0 +1,289 @@
package cn.smartjavaai.face.vector.core;
import cn.hutool.core.util.IdUtil;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.dao.FaceDao;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.face.utils.SimilarityUtil;
import cn.smartjavaai.face.vector.config.SQLiteConfig;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Collectors;
@Slf4j
public class SQLiteClient implements VectorDBClient {
private final FaceDao faceDao;
//private final List<FaceVector> memoryIndex = new CopyOnWriteArrayList<>();
private final ConcurrentHashMap<String, FaceVector> memoryIndex = new ConcurrentHashMap<>();
private int featureDimension; // 维度
private final ExecutorService executor = Executors.newFixedThreadPool(4);
private SQLiteConfig config;
/**
* 是否初始化完毕
*/
private boolean isInit;
public SQLiteClient(SQLiteConfig config) {
this.config = config;
String dbPath = config.getDbPath();
//如果未指定db路径，则使用默认路径
if(StringUtils.isBlank(config.getDbPath())){
dbPath = Config.getCachePath() + File.separator + "face.db";
log.debug("使用默认SQLite人脸库路径: {}", dbPath);
}
this.faceDao = FaceDao.getInstance(dbPath);
}
@Override
public void initialize() {
try {
// 加载所有特征到内存
loadAllFeaturesToMemory();
isInit = true;
log.debug("SQLiteVectorDB initialized with {} faces", memoryIndex.size());
} catch (Exception e) {
throw new VectorDBException("初始化失败", e);
}
}
// 以下方法保持接口兼容但忽略collectionName参数
@Override
public void createCollection(String collectionName, int dimension) {
this.featureDimension = dimension;
log.debug("特征维度设置为: {}", dimension);
}
@Override
public void dropCollection(String collectionName) {
if (!isInit){
throw new VectorDBException("人脸库未加载完毕");
}
clearAllData();
log.warn("所有数据已被清空");
}
@Override
public boolean hasCollection(String collectionName) {
throw new UnsupportedOperationException("Sqlite 不支持此操作");
}
@Override
public String insert(FaceVector faceVector) {
if (!isInit){
throw new VectorDBException("人脸库未加载完毕");
}
return insertBatch(Collections.singletonList(faceVector)).get(0);
}
@Override
public void upsert(FaceVector faceVector) {
// if (faceVector.getId() != null) {
// delete(faceVector.getId());
// }
insert(faceVector);
}
@Override
public List<String> insertBatch(List<FaceVector> faceVectors) {
if (!isInit){
throw new VectorDBException("人脸库未加载完毕");
}
List<String> ids = new ArrayList<>();
try {
for (FaceVector faceVector : faceVectors) {
String id = faceVector.getId() != null ?
faceVector.getId() : IdUtil.simpleUUID();
faceVector.setId(id);
// 保存到数据库
faceDao.insertOrUpdate(faceVector);
// 添加到内存索引
addToMemoryIndex(faceVector);
ids.add(id);
}
log.debug("插入了 {} 个人脸向量", faceVectors.size());
return ids;
} catch (Exception e) {
throw new VectorDBException("批量插入失败", e);
}
}
@Override
public void delete(String id) {
if (!isInit){
throw new VectorDBException("人脸库未加载完毕");
}
deleteBatch(Collections.singletonList(id));
}
@Override
public void deleteBatch(List<String> ids) {
if (!isInit){
throw new VectorDBException("人脸库未加载完毕");
}
try {
// 从数据库中删除
boolean isSuccess = faceDao.deleteFace(ids.toArray(new String[0]));
// 从内存中删除
ids.forEach(memoryIndex::remove);
if(!isSuccess){
throw new VectorDBException("删除失败");
}
} catch (Exception e) {
throw new VectorDBException("批量删除失败", e);
}
}
@Override
public List<FaceSearchResult> search(float[] queryVector, FaceSearchParams faceSearchParams) {
if (!isInit){
throw new VectorDBException("人脸库未加载完毕");
}
if (memoryIndex.isEmpty()) {
return Collections.emptyList();
}
// 并行计算相似度
List<CompletableFuture<FaceSearchResult>> futures = memoryIndex.values().stream()
.map(vector -> CompletableFuture.supplyAsync(() -> {
float similarity = SimilarityUtil.calculate(queryVector, vector.getVector(), config.getSimilarityType(), faceSearchParams.getNormalizeSimilarity());
return similarity >= faceSearchParams.getThreshold() ?
new FaceSearchResult(vector.getId(), similarity, vector.getMetadata()) :
null;
}, executor))
.collect(Collectors.toList());
// 收集结果并过滤null
List<FaceSearchResult> allResults = futures.stream()
.map(CompletableFuture::join)
.filter(Objects::nonNull)
.collect(Collectors.toList());
// 获取TopK结果
return allResults.stream()
.sorted(Comparator.comparingDouble(FaceSearchResult::getSimilarity).reversed())
.limit(faceSearchParams.getTopK())
.collect(Collectors.toList());
}
@Override
public long count(String collectionName) {
return memoryIndex.size();
}
@Override
public void close() {
executor.shutdown();
try {
if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
executor.shutdownNow();
Thread.currentThread().interrupt();
}
}
@Override
public FaceVector getFaceInfoById(String id) {
if (!isInit) {
throw new VectorDBException("人脸库未加载完毕");
}
// 先从内存缓存中获取
FaceVector faceVector = memoryIndex.get(id);
if (faceVector == null) {
// 如果内存中没有，则从数据库查询
try {
faceVector = faceDao.findById(id);
} catch (SQLException | ClassNotFoundException e) {
throw new VectorDBException("SQLite查询异常", e);
}
}
return faceVector;
}
@Override
public List<FaceVector> listFaces(long pageNum, long pageSize) {
if (!isInit) {
throw new VectorDBException("人脸库未加载完毕");
}
if (pageNum < 1 || pageSize < 1) {
throw new IllegalArgumentException("pageNum和pageSize必须大于0");
}
// 从数据库中查询指定分页的数据
try {
return faceDao.findFace((int)pageNum, (int)pageSize);
} catch (Exception e) {
throw new VectorDBException("分页查询失败", e);
}
}
// ============= 私有辅助方法 =============
private void loadAllFeaturesToMemory() {
try {
int pageSize = 1000;
int page = 0;
while (true) {
List<FaceVector> batch = faceDao.findFace(page, pageSize);
if (CollectionUtils.isEmpty(batch)) {
break;
}
for (FaceVector vector : batch) {
addToMemoryIndex(vector);
}
page++;
}
log.debug("从数据库加载了 {} 个特征向量到内存", memoryIndex.size());
} catch (Exception e) {
throw new VectorDBException("加载特征到内存失败", e);
}
}
private void addToMemoryIndex(FaceVector faceVector) {
memoryIndex.put(faceVector.getId(), faceVector);
}
private void clearAllData() {
if (!isInit){
throw new VectorDBException("人脸库未加载完毕");
}
try {
faceDao.deleteAll();
memoryIndex.clear();
} catch (Exception e) {
log.error("清空数据库失败", e);
}
}
@Override
public void loadFaceFeatures() {
// 加载所有特征到内存
loadAllFeaturesToMemory();
isInit = true;
log.debug("SQLiteVectorDB load success {} faces", memoryIndex.size());
}
@Override
public void releaseFaceFeatures() {
if (!isInit){
throw new VectorDBException("人脸库未加载完毕");
}
memoryIndex.clear();
isInit = false;
}
}

124
face/src/main/java/cn/smartjavaai/face/vector/core/VectorDBClient.java Normal file
View File

@@ -0,0 +1,124 @@
package cn.smartjavaai.face.vector.core;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import java.util.List;
/**
* 向量数据库客户端接口
* 定义与向量数据库交互的通用操作
* @author dwj
*/
public interface VectorDBClient extends AutoCloseable {
/**
* 初始化连接和集合
* @throws VectorDBException 初始化异常
*/
void initialize();
/**
* 创建集合
* @param collectionName 集合名称
* @param dimension 向量维度
*/
void createCollection(String collectionName, int dimension);
/**
* 删除集合
* @param collectionName 集合名称
*/
void dropCollection(String collectionName);
/**
* 检查集合是否存在
* @param collectionName 集合名称
* @return 是否存在
*/
boolean hasCollection(String collectionName);
/**
* 插入人脸向量
* @param faceVector
* @return
*/
String insert(FaceVector faceVector);
/**
* 更新或新增人脸向量
* @param faceVector
*/
void upsert(FaceVector faceVector);
/**
* 批量插入人脸向量
* @param faceVectors
* @return
*/
List<String> insertBatch(List<FaceVector> faceVectors);
/**
* 根据ID删除向量
* @param id
*/
void delete(String id);
/**
* 批量删除向量
* @param ids
*/
void deleteBatch(List<String> ids);
/**
* 搜索相似人脸
* @param queryVector
* @param faceSearchParams
* @return
*/
List<FaceSearchResult> search(float[] queryVector, FaceSearchParams faceSearchParams);
/**
* 获取集合中的向量数量
* @param collectionName 集合名称
* @return 向量数量
*/
long count(String collectionName);
/**
* 关闭连接
*/
@Override
void close();
/**
* 使用人脸ID获取人脸信息
* @param id
* @return
*/
FaceVector getFaceInfoById(String id);
/**
* 获取人脸列表
* @param pageNum
* @param pageSize
* @return
*/
List<FaceVector> listFaces(long pageNum, long pageSize);
/**
* 加载人脸特征到内存
*/
void loadFaceFeatures();
/**
* 释放人脸特征缓存
*/
void releaseFaceFeatures();
}

52
face/src/main/java/cn/smartjavaai/face/vector/core/VectorDBFactory.java Normal file
View File

@@ -0,0 +1,52 @@
package cn.smartjavaai.face.vector.core;
import cn.smartjavaai.face.vector.config.MilvusConfig;
import cn.smartjavaai.face.vector.config.SQLiteConfig;
import cn.smartjavaai.face.vector.config.VectorDBConfig;
import cn.smartjavaai.face.vector.exception.VectorDBException;
/**
* 向量数据库工厂类
* 用于创建不同类型的向量数据库客户端
* @author dwj
*/
public class VectorDBFactory {
private VectorDBFactory() {
// 私有构造函数，防止实例化
}
/**
* 创建向量数据库客户端
* @param config 配置信息
* @return 向量数据库客户端
* @throws VectorDBException 创建异常
*/
public static VectorDBClient createClient(VectorDBConfig config) {
if (config == null) {
throw new VectorDBException("配置不能为空");
}
VectorDBClient client;
switch (config.getType()) {
case SQLITE:
if (!(config instanceof SQLiteConfig)) {
throw new VectorDBException("SQLite类型需要SQLiteConfig配置");
}
client = new SQLiteClient((SQLiteConfig) config);
break;
case MILVUS:
if (!(config instanceof MilvusConfig)) {
throw new VectorDBException("Milvus类型需要MilvusConfig配置");
}
client = new MilvusClient((MilvusConfig) config);
break;
// 未来可以在这里添加其他向量数据库的支持
default:
throw new VectorDBException("不支持的向量数据库类型: " + config.getType());
}
return client;
}
}

68
face/src/main/java/cn/smartjavaai/face/vector/entity/FaceVector.java Normal file
View File

@@ -0,0 +1,68 @@
package cn.smartjavaai.face.vector.entity;
import lombok.Data;
import java.util.UUID;
/**
* 人脸向量实体类
* @author smartjavaai
*/
@Data
public class FaceVector {
/**
* 向量ID
*/
private String id;
/**
* 人脸特征向量
*/
private float[] vector;
/**
* 元数据，可以存储人脸相关的其他信息（JSON格式）
*/
private String metadata;
/**
* 默认构造函数
*/
public FaceVector() {
this.id = UUID.randomUUID().toString();
}
/**
* 构造函数
* @param vector 人脸特征向量
*/
public FaceVector(float[] vector) {
this();
this.vector = vector;
}
/**
* 构造函数
* @param vector 人脸特征向量
* @param metadata 元数据
*/
public FaceVector(float[] vector, String metadata) {
this();
this.vector = vector;
this.metadata = metadata;
}
/**
* 构造函数
* @param id 向量ID
* @param vector 人脸特征向量
* @param metadata 元数据
*/
public FaceVector(String id, float[] vector, String metadata) {
this.id = id;
this.vector = vector;
this.metadata = metadata;
}
}

26
face/src/main/java/cn/smartjavaai/face/vector/exception/VectorDBException.java Normal file
View File

@@ -0,0 +1,26 @@
package cn.smartjavaai.face.vector.exception;
/**
* 向量数据库异常
* @author smartjavaai
*/
public class VectorDBException extends RuntimeException {
/**
* 构造函数
* @param message 异常信息
*/
public VectorDBException(String message) {
super(message);
}
/**
* 构造函数
* @param message 异常信息
* @param cause 原始异常
*/
public VectorDBException(String message, Throwable cause) {
super(message, cause);
}
}

14
face/src/main/resources/db/schema.sql Normal file
View File

@@ -0,0 +1,14 @@
PRAGMA foreign_keys = false;
-- ----------------------------
-- Table structure for face
-- ----------------------------
CREATE TABLE "face" (
"id" TEXT NOT NULL,
"vector" blob NOT NULL,
"metadata" TEXT,
PRIMARY KEY ("id"),
CONSTRAINT "id" UNIQUE ("id" ASC)
);
PRAGMA foreign_keys = true;

87
face/src/test/java/Test.java Normal file
View File

@@ -0,0 +1,87 @@
import ai.djl.Application;
import ai.djl.repository.Artifact;
import ai.djl.repository.MRL;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ModelZoo;
import ai.djl.util.JsonUtils;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.enums.SimilarityType;
import cn.smartjavaai.face.factory.FaceDetModelFactory;
import cn.smartjavaai.face.factory.FaceRecModelFactory;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.FaceRecModel;
import cn.smartjavaai.face.utils.SimilarityUtil;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* @author dwj
* @date 2025/7/25
*/
@Slf4j
public class Test {
/**
* 获取人脸检测模型
* @return
*/
public static FaceDetModel getFaceDetModel(){
FaceDetConfig config = new FaceDetConfig();
config.setModelEnum(FaceDetModelEnum.RETINA_FACE);//人脸检测模型
config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);//只返回相似度大于该值的人脸
config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);//用于去除重复的人脸框，当两个框的重叠度超过该值时，只保留一个
return FaceDetModelFactory.getInstance().getModel(config);
}
/**
* 获取人脸识别模型
* @return
*/
public static FaceRecModel getFaceRecModel(){
FaceRecConfig config = new FaceRecConfig();
config.setModelEnum(FaceRecModelEnum.ELASTIC_FACE_MODEL);
config.setModelPath("/Users/wenjie/Documents/develop/model/arcfaceresnet100-11-int8.onnx");
// config.setModelPath("/Users/xxx/Documents/develop/model/InsightFace/model_mobilefacenet.pt");
//裁剪人脸：如果图片已经是裁剪过的，则请将此参数设置为false
config.setCropFace(true);
//开启人脸对齐：适用于人脸不正的场景，开启将提升人脸特征准确度，关闭可以提升性能
config.setAlign(true);
//指定人脸检测模型
config.setDetectModel(getFaceDetModel());
return FaceRecModelFactory.getInstance().getModel(config);
}
public static void main(String[] args) throws ModelNotFoundException, IOException {
// boolean withArtifacts =
// args.length > 0 && ("--artifact".equals(args[0]) || "-a".equals(args[0]));
// if (!withArtifacts) {
// logger.info("============================================================");
// logger.info("user ./gradlew listModel --args='-a' to show artifact detail");
// logger.info("============================================================");
// }
// Map<Application, List<MRL>> models = ModelZoo.listModels();
// for (Map.Entry<Application, List<MRL>> entry : models.entrySet()) {
// String appName = entry.getKey().toString();
// for (MRL mrl : entry.getValue()) {
// if (withArtifacts) {
// for (Artifact artifact : mrl.listArtifacts()) {
// log.info("{} djl://{}", appName, artifact);
// }
// } else {
// log.info("{} {}", appName, mrl);
// }
// }
// }
}
}