1、人脸模块:新增小视科技(MiniVision)活体检测模型

2、人脸模块:新增阿里通义工作室活体检测模型
3、人脸模块:新增2个表情识别模型
4、人脸模块:新增InsightFace、ElasticFace人脸识别模型
5、人脸模块:新增Seetaface6质量评估模型
6、目标检测模块:开放更多自定义模型参数
7、人脸模块:支持base64图片
8、实现接口 AutoCloseable,支持资源的自动释放
9、OCR模块:解决加方向矫正后无法连续识别bug
10、人脸模块:解决人脸更新后缓存问题
11、优化部分功能
This commit is contained in:
dengwenjie
2025-07-07 08:45:08 +08:00
parent 3e631a060b
commit 07a8a18835
168 changed files with 8562 additions and 2850 deletions

View File

@@ -6,11 +6,11 @@
<parent>
<groupId>cn.smartjavaai</groupId>
<artifactId>smartjavaai-parent</artifactId>
<version>1.0.17</version>
<version>1.0.19</version>
</parent>
<artifactId>smartjavaai-face</artifactId>
<version>1.0.17</version>
<version>1.0.19</version>
<name>smartjavaai-face</name>
<description>SmartJavaAI</description>
<url>https://github.com/geekwenjie/SmartJavaAI</url>

View File

@@ -0,0 +1,77 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* 人脸检测模型配置
* @author dwj
*/
@Data
public class FaceDetConfig {
/**
* 人脸检测模型枚举
*/
private FaceDetModelEnum modelEnum;
/**
* 置信度阈值
*/
private double confidenceThreshold = FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD;
/**
* 非极大抑制阈值 作用:消除重叠检测框,保留最优结果
*/
private double nmsThresh = FaceDetectConstant.NMS_THRESHOLD;
/**
* 模型路径
*/
private String modelPath;
/**
* 设备类型
*/
private DeviceEnum device;
/**
* 个性化配置(按模型类型动态解析)
*/
private Map<String, Object> customParams = new HashMap<>();
public FaceDetConfig() {
}
public FaceDetConfig(FaceDetModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public FaceDetConfig(FaceDetModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
public <T> T getCustomParam(String key, Class<T> clazz) {
Object value = customParams.get(key);
if (value == null) return null;
return clazz.cast(value);
}
/**
* 添加个性化配置项
*/
public void putCustomParam(String key, Object value) {
if (customParams == null) {
customParams = new HashMap<>();
}
customParams.put(key, value);
}
}

View File

@@ -0,0 +1,51 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.enums.ExpressionModelEnum;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.FaceRecModel;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* @author dwj
* @date 2025/7/1
*/
@Data
public class FaceExpressionConfig {
/**
* 模型枚举
*/
private ExpressionModelEnum modelEnum = ExpressionModelEnum.DensNet121;
/**
* 模型路径
*/
private String modelPath;
/**
* 设备类型
*/
private DeviceEnum device;
/**
* 人脸检测模型
*/
private FaceDetModel detectModel;
/**
* 是否对齐人脸
*/
private boolean align = true;
/**
* 是否裁剪人脸
*/
private boolean cropFace = true;
}

View File

@@ -1,39 +0,0 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.face.model.facerec.FaceModel;
import lombok.Data;
/**
* 人脸特征提取配置
* @author dwj
* @date 2025/4/24
*/
@Data
public class FaceExtractConfig {
/**
* 是否裁剪人脸
*/
private boolean cropFace = true;
/**
* 是否对齐人脸
*/
private boolean align = false;
/**
* 人脸检测模型
*/
private FaceModel detectModel;
public FaceExtractConfig() {
}
public FaceExtractConfig(boolean cropFace, boolean align, FaceModel detectModel) {
this.cropFace = cropFace;
this.align = align;
this.detectModel = detectModel;
}
}

View File

@@ -2,22 +2,26 @@ package cn.smartjavaai.face.config;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceModelEnum;
import cn.smartjavaai.face.enums.VectorDBType;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.FaceRecModel;
import cn.smartjavaai.face.vector.config.VectorDBConfig;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* 人脸检测识别模型配置
* @author dwj
*/
@Data
public class FaceModelConfig {
public class FaceRecConfig {
/**
* 人脸模型枚举
*/
private FaceModelEnum modelEnum;
private FaceRecModelEnum modelEnum;
/**
* 置信度阈值
@@ -49,16 +53,6 @@ public class FaceModelConfig {
*/
private DeviceEnum device;
/**
* gpu设备ID 当device为GPU时生效
*/
private int gpuId = 0;
/**
* 人脸特征提取配置
*/
private FaceExtractConfig extractConfig;
/**
* 向量数据库配置
@@ -70,15 +64,52 @@ public class FaceModelConfig {
*/
private boolean isAutoLoadFace = true;
public FaceModelConfig() {
/**
* 是否裁剪人脸
*/
private boolean cropFace = true;
/**
* 是否对齐人脸
*/
private boolean align = false;
/**
* 人脸检测模型
*/
private FaceDetModel detectModel;
/**
* 个性化配置按模型类型动态解析
*/
private Map<String, Object> customParams = new HashMap<>();
public FaceRecConfig() {
}
public FaceModelConfig(FaceModelEnum modelEnum) {
public FaceRecConfig(FaceRecModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public FaceModelConfig(FaceModelEnum modelEnum, String modelPath) {
public FaceRecConfig(FaceRecModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
public <T> T getCustomParam(String key, Class<T> clazz) {
Object value = customParams.get(key);
if (value == null) return null;
return clazz.cast(value);
}
/**
* 添加个性化配置项
*/
public void putCustomParam(String key, Object value) {
if (customParams == null) {
customParams = new HashMap<>();
}
customParams.put(key, value);
}
}

View File

@@ -1,12 +1,15 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.constant.LivenessConstant;
import cn.smartjavaai.face.enums.FaceModelEnum;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.FaceRecModel;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* 活体检测模型配置
* @author dwj
@@ -30,25 +33,26 @@ public class LivenessConfig {
private DeviceEnum device;
/**
* gpu设备ID 当device为GPU时生效
* 人脸检测模型
*/
private int gpuId = 0;
private FaceDetModel detectModel;
/**
* 人脸清晰度阈值
* 个性化配置(按模型类型动态解析)
*/
private float faceClarityThreshold = LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD;
private Map<String, Object> customParams = new HashMap<>();
/**
* 活体阈值
*/
private float realityThreshold = LivenessConstant.DEFAULT_REALITY_THRESHOLD;
/**
* 视频检测帧数
*/
private int frameCount = LivenessConstant.DEFAULT_FRAME_COUNT;
/**
* 真人阈值
*/
private Float realityThreshold;
public LivenessConfig() {
}
@@ -64,4 +68,21 @@ public class LivenessConfig {
public LivenessConfig(String modelPath) {
this.modelPath = modelPath;
}
// 可选封装方法,便于类型转换和调用
public <T> T getCustomParam(String key, Class<T> clazz) {
Object value = customParams.get(key);
if (value == null) return null;
return clazz.cast(value);
}
/**
* 添加个性化配置项
*/
public void putCustomParam(String key, Object value) {
if (customParams == null) {
customParams = new HashMap<>();
}
customParams.put(key, value);
}
}

View File

@@ -0,0 +1,52 @@
package cn.smartjavaai.face.config;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.constant.LivenessConstant;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.enums.QualityModelEnum;
import lombok.Data;
/**
* 质量评估配置
* @author dwj
*/
@Data
public class QualityConfig {
/**
* 活体检测模型枚举
*/
private QualityModelEnum modelEnum = QualityModelEnum.SEETA_FACE6_MODEL;
/**
* 模型路径
*/
private String modelPath;
/**
* 设备类型
*/
private DeviceEnum device;
/**
* gpu设备ID 当device为GPU时生效
*/
private int gpuId = 0;
public QualityConfig() {
}
public QualityConfig(QualityModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
public QualityConfig(QualityModelEnum modelEnum, String modelPath) {
this.modelEnum = modelEnum;
this.modelPath = modelPath;
}
public QualityConfig(String modelPath) {
this.modelPath = modelPath;
}
}

View File

@@ -0,0 +1,14 @@
package cn.smartjavaai.face.constant;
/**
* FaceNet人脸模型常量
* @author dwj
*/
public class FaceNetConstant {
/**
* 模型下载地址
*/
public static final String MODEL_URL = "https://resources.djl.ai/test-models/pytorch/face_feature.zip";
}

View File

@@ -0,0 +1,14 @@
package cn.smartjavaai.face.constant;
/**
* MiniVision模型常量
* @author dwj
* @date 2025/7/3
*/
public class MiniVisionConstant {
/**
* 真人阈值
*/
public static final Float REALITY_THRESHOLD = 0.5f;
}

View File

@@ -0,0 +1,28 @@
package cn.smartjavaai.face.constant;
/**
* RetinaFace人脸检测模型常量
* @author dwj
* @date 2025/7/2
*/
public class RetinaFaceConstant {
/**
* 特征图层的基础缩放比例
*/
public static final int[][] scales = {{16, 32}, {64, 128}, {256, 512}};
/**
* 特征图相对于原图的采样步长
*/
public static final int[] steps = {8, 16, 32};
/**
* 缩放系数
*/
public static final double[] variance = {0.1f, 0.2f};
/**
* 模型下载地址
*/
public static final String MODEL_URL = "https://resources.djl.ai/test-models/pytorch/retinaface.zip";
}

View File

@@ -0,0 +1,27 @@
package cn.smartjavaai.face.constant;
/**
* UltraLightFastGenericFace人脸检测模型常量
* @author dwj
*/
public class UltraLightFastGenericFaceConstant {
/**
* 特征图层的基础缩放比例
*/
public static final int[][] scales = {{10, 16, 24}, {32, 48}, {64, 96}, {128, 192, 256}};
/**
* 特征图相对于原图的采样步长
*/
public static final int[] steps = {8, 16, 32, 64};
/**
* 缩放系数
*/
public static final double[] variance = {0.1f, 0.2f};
/**
* 模型下载地址
*/
public static final String MODEL_URL = "https://resources.djl.ai/test-models/pytorch/ultranet.zip";
}

View File

@@ -0,0 +1,29 @@
package cn.smartjavaai.face.entity;
import cn.smartjavaai.face.enums.QualityGrade;
import lombok.Data;
/**
* 质量评估结果
* @author dwj
* @date 2025/6/23
*/
@Data
public class FaceQualityResult {
/**
* 评估得分
*/
private float score;
private QualityGrade grade;
public FaceQualityResult() {
}
public FaceQualityResult(float score, QualityGrade grade) {
this.score = score;
this.grade = grade;
}
}

View File

@@ -0,0 +1,23 @@
package cn.smartjavaai.face.entity;
import lombok.Data;
import java.util.Map;
/**
* 人脸质量检测汇总结果
* @author dwj
* @date 2025/6/27
*/
@Data
public class FaceQualitySummary {
private FaceQualityResult brightness; // 亮度
private FaceQualityResult clarity; // 清晰度
private FaceQualityResult completeness; // 完整度
private FaceQualityResult pose; // 姿态
private FaceQualityResult resolution; // 分辨率
private Map<String, Object> extraResults; // 额外检测结果
}

View File

@@ -0,0 +1,38 @@
package cn.smartjavaai.face.enums;
/**
* 表情识别模型枚举
* @author dwj
*/
public enum ExpressionModelEnum {
DensNet121("DensNet121"),
FrEmotion("FrEmotion");
private final String modelClassName;
ExpressionModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
public String getModelClassName() {
return modelClassName;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static ExpressionModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (ExpressionModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

View File

@@ -1,20 +1,18 @@
package cn.smartjavaai.face.enums;
/**
* 人脸模型枚举
* 人脸检测模型枚举
* @author dwj
* @date 2025/4/10
*/
public enum FaceModelEnum {
public enum FaceDetModelEnum {
RETINA_FACE("RetinaFaceModel"),
ULTRA_LIGHT_FAST_GENERIC_FACE("UltraLightFastGenericFaceModel"),
FACENET_MODEL("FaceNetModel"),
SEETA_FACE6_MODEL("SeetaFace6Model");
private final String modelClassName;
FaceModelEnum(String modelClassName) {
FaceDetModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
@@ -25,9 +23,9 @@ public enum FaceModelEnum {
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static FaceModelEnum fromName(String name) {
public static FaceDetModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (FaceModelEnum model : values()) {
for (FaceDetModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}

View File

@@ -0,0 +1,39 @@
package cn.smartjavaai.face.enums;
/**
* 人脸识别模型枚举
* @author dwj
*/
public enum FaceRecModelEnum {
FACENET_MODEL("FaceNetModel"),
SEETA_FACE6_MODEL("SeetaFace6Model"),
INSIGHT_FACE_IRSE50_MODEL("InsightFaceIRSE50Model"),
INSIGHT_FACE_MOBILE_FACENET_MODEL("InsightFaceMobilefacenetModel"),
ELASTIC_FACE_MODEL("ElasticFaceModel");
private final String modelClassName;
FaceRecModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
public String getModelClassName() {
return modelClassName;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static FaceRecModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (FaceRecModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

View File

@@ -7,7 +7,14 @@ package cn.smartjavaai.face.enums;
*/
public enum LivenessModelEnum {
SEETA_FACE6_MODEL("SeetaFace6Model");
// SeetaFace6
SEETA_FACE6_MODEL("SeetaFace6Model"),
// MiniVision
MINI_VISION_MODEL("MiniVisionModel"),
//阿里通义实验室
IIC_FL_MODEL("IicFlModel");
private final String modelClassName;

View File

@@ -0,0 +1,14 @@
package cn.smartjavaai.face.enums;
/**
* 质量等级枚举
* @author dwj
* @date 2025/6/23
*/
public enum QualityGrade {
LOW,//Quality level is low
MEDIUM,//Quality level is medium
HIGH,//Quality level is high
}

View File

@@ -0,0 +1,36 @@
package cn.smartjavaai.face.enums;
/**
* 质量评估模型枚举
* @author dwj
* @date 2025/4/10
*/
public enum QualityModelEnum {
SEETA_FACE6_MODEL("SeetaFace6Model");
private final String modelClassName;
QualityModelEnum(String modelClassName) {
this.modelClassName = modelClassName;
}
public String getModelClassName() {
return modelClassName;
}
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static QualityModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (QualityModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
}

View File

@@ -0,0 +1,101 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import cn.smartjavaai.face.enums.ExpressionModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.expression.CommonEmotionModel;
import cn.smartjavaai.face.model.expression.ExpressionModel;
import cn.smartjavaai.face.model.liveness.MiniVisionLivenessModel;
import cn.smartjavaai.face.model.liveness.Seetaface6LivenessModel;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 表情识别模型工厂
* @author dwj
*/
@Slf4j
public class ExpressionModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile ExpressionModelFactory instance;
private static final ConcurrentHashMap<ExpressionModelEnum, ExpressionModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<ExpressionModelEnum, Class<? extends ExpressionModel>> registry =
new ConcurrentHashMap<>();
public static ExpressionModelFactory getInstance() {
if (instance == null) {
synchronized (ExpressionModelFactory.class) {
if (instance == null) {
instance = new ExpressionModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param expressionModelEnum
* @param clazz
*/
private static void registerModel(ExpressionModelEnum expressionModelEnum, Class<? extends ExpressionModel> clazz) {
registry.put(expressionModelEnum, clazz);
}
/**
* 获取模型(通过配置)
* @param config
* @return
*/
public ExpressionModel getModel(FaceExpressionConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置活体检测模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private ExpressionModel createFaceModel(FaceExpressionConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
ExpressionModel model = null;
try {
model = (ExpressionModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
model.loadModel(config);
return model;
}
// 初始化默认算法
static {
registerModel(ExpressionModelEnum.DensNet121, CommonEmotionModel.class);
registerModel(ExpressionModelEnum.FrEmotion, CommonEmotionModel.class);
log.debug("缓存目录:{}", Config.getCachePath());
}
}

View File

@@ -1,10 +1,13 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceModelEnum;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.facedect.CommonFaceDetModel;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facedect.SeetaFace6FaceDetModel;
import cn.smartjavaai.face.model.facerec.*;
import lombok.extern.slf4j.Slf4j;
@@ -13,29 +16,29 @@ import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 人脸检测识别模型工厂
* 人脸检测模型工厂
* @author dwj
*/
@Slf4j
public class FaceModelFactory {
public class FaceDetModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile FaceModelFactory instance;
private static volatile FaceDetModelFactory instance;
private static final ConcurrentHashMap<String, FaceModel> modelMap = new ConcurrentHashMap<>();
private static final ConcurrentHashMap<String, FaceDetModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<String, Class<? extends FaceModel>> registry =
private static final Map<String, Class<? extends FaceDetModel>> registry =
new ConcurrentHashMap<>();
public static FaceModelFactory getInstance() {
public static FaceDetModelFactory getInstance() {
if (instance == null) {
synchronized (FaceModelFactory.class) {
synchronized (FaceDetModelFactory.class) {
if (instance == null) {
instance = new FaceModelFactory();
instance = new FaceDetModelFactory();
}
}
}
@@ -49,7 +52,7 @@ public class FaceModelFactory {
* @param name
* @param clazz
*/
private static void registerAlgorithm(String name, Class<? extends FaceModel> clazz) {
private static void registerAlgorithm(String name, Class<? extends FaceDetModel> clazz) {
registry.put(name.toLowerCase(), clazz);
}
@@ -59,12 +62,12 @@ public class FaceModelFactory {
* @param config
* @return
*/
public FaceModel getModel(FaceModelConfig config) {
public FaceDetModel getModel(FaceDetConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置人脸模型");
}
return modelMap.computeIfAbsent(config.getModelEnum().name(), k -> {
return createFaceModel(config);
return createFaceDetModel(config);
});
}
@@ -72,10 +75,10 @@ public class FaceModelFactory {
* 获取默认模型
* @return
*/
public FaceModel getModel() {
public FaceDetModel getModel() {
// 初始化默认配置
FaceModelConfig config = new FaceModelConfig();
config.setModelEnum(FaceModelEnum.RETINA_FACE);
FaceDetConfig config = new FaceDetConfig();
config.setModelEnum(FaceDetModelEnum.RETINA_FACE);
config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);
config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);
return getModel(config);
@@ -86,14 +89,14 @@ public class FaceModelFactory {
* @param config
* @return
*/
private FaceModel createFaceModel(FaceModelConfig config) {
private FaceDetModel createFaceDetModel(FaceDetConfig config) {
Class<?> clazz = registry.get(config.getModelEnum().getModelClassName().toLowerCase());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
FaceModel algorithm = null;
FaceDetModel algorithm = null;
try {
algorithm = (FaceModel) clazz.newInstance();
algorithm = (FaceDetModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
@@ -106,10 +109,10 @@ public class FaceModelFactory {
* 获取轻量级人脸模型
* @return
*/
public FaceModel getLightFaceModel() {
public FaceDetModel getLightFaceDetModel() {
// 初始化默认配置
FaceModelConfig config = new FaceModelConfig();
config.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);
FaceDetConfig config = new FaceDetConfig();
config.setModelEnum(FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);
config.setConfidenceThreshold(FaceDetectConstant.DEFAULT_CONFIDENCE_THRESHOLD);
config.setNmsThresh(FaceDetectConstant.NMS_THRESHOLD);
return getModel(config);
@@ -118,11 +121,9 @@ public class FaceModelFactory {
// 初始化默认算法
static {
registerAlgorithm("retinafacemodel", RetinaFaceModel.class);
registerAlgorithm("ultralightfastgenericfacemodel", UltraLightFastGenericFaceModel.class);
//人脸特征提取
registerAlgorithm("facenetmodel", FaceNetModel.class);
registerAlgorithm("seetaface6model", SeetaFace6Model.class);
registerAlgorithm("retinafacemodel", CommonFaceDetModel.class);
registerAlgorithm("ultralightfastgenericfacemodel", CommonFaceDetModel.class);
registerAlgorithm("seetaface6model", SeetaFace6FaceDetModel.class);
log.debug("缓存目录:{}", Config.getCachePath());
}

View File

@@ -0,0 +1,97 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.QualityConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.quality.FaceQualityModel;
import cn.smartjavaai.face.model.quality.Seetaface6QualityModel;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 质量评估模型工厂
* @author dwj
*/
@Slf4j
public class FaceQualityModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile FaceQualityModelFactory instance;
private static final ConcurrentHashMap<String, FaceQualityModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<String, Class<? extends FaceQualityModel>> registry =
new ConcurrentHashMap<>();
public static FaceQualityModelFactory getInstance() {
if (instance == null) {
synchronized (FaceQualityModelFactory.class) {
if (instance == null) {
instance = new FaceQualityModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param name
* @param clazz
*/
private static void registerModel(String name, Class<? extends FaceQualityModel> clazz) {
registry.put(name.toLowerCase(), clazz);
}
/**
* 获取模型(通过配置)
* @param config
* @return
*/
public FaceQualityModel getModel(QualityConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置质量评估模型");
}
return modelMap.computeIfAbsent(config.getModelEnum().name(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private FaceQualityModel createFaceModel(QualityConfig config) {
Class<?> clazz = registry.get(config.getModelEnum().getModelClassName().toLowerCase());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
FaceQualityModel model = null;
try {
model = (FaceQualityModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
model.loadModel(config);
return model;
}
// 初始化默认算法
static {
registerModel("seetaface6model", Seetaface6QualityModel.class);
log.debug("缓存目录:{}", Config.getCachePath());
}
}

View File

@@ -0,0 +1,103 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.facerec.*;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 人脸检测识别模型工厂
* @author dwj
*/
@Slf4j
public class FaceRecModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile FaceRecModelFactory instance;
private static final ConcurrentHashMap<FaceRecModelEnum, FaceRecModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<FaceRecModelEnum, Class<? extends FaceRecModel>> registry =
new ConcurrentHashMap<>();
public static FaceRecModelFactory getInstance() {
if (instance == null) {
synchronized (FaceRecModelFactory.class) {
if (instance == null) {
instance = new FaceRecModelFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param recModelEnum
* @param clazz
*/
private static void registerAlgorithm(FaceRecModelEnum recModelEnum, Class<? extends FaceRecModel> clazz) {
registry.put(recModelEnum, clazz);
}
/**
* 获取模型(通过配置)
* @param config
* @return
*/
public FaceRecModel getModel(FaceRecConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置人脸模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private FaceRecModel createFaceModel(FaceRecConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new FaceException("Unsupported model");
}
FaceRecModel algorithm = null;
try {
algorithm = (FaceRecModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new FaceException(e);
}
algorithm.loadModel(config);
return algorithm;
}
// 初始化默认算法
static {
registerAlgorithm(FaceRecModelEnum.FACENET_MODEL, CommonFaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL, CommonFaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.INSIGHT_FACE_MOBILE_FACENET_MODEL, CommonFaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.ELASTIC_FACE_MODEL, CommonFaceRecModel.class);
registerAlgorithm(FaceRecModelEnum.SEETA_FACE6_MODEL, SeetaFace6FaceRecModel.class);
log.debug("缓存目录:{}", Config.getCachePath());
}
}

View File

@@ -1,13 +1,12 @@
package cn.smartjavaai.face.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.enums.FaceModelEnum;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.facerec.*;
import cn.smartjavaai.face.model.liveness.CommonLivenessModel;
import cn.smartjavaai.face.model.liveness.LivenessDetModel;
import cn.smartjavaai.face.model.liveness.MiniVisionLivenessModel;
import cn.smartjavaai.face.model.liveness.Seetaface6LivenessModel;
import lombok.extern.slf4j.Slf4j;
@@ -25,12 +24,12 @@ public class LivenessModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile LivenessModelFactory instance;
private static final ConcurrentHashMap<String, LivenessDetModel> modelMap = new ConcurrentHashMap<>();
private static final ConcurrentHashMap<LivenessModelEnum, LivenessDetModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<String, Class<? extends LivenessDetModel>> registry =
private static final Map<LivenessModelEnum, Class<? extends LivenessDetModel>> registry =
new ConcurrentHashMap<>();
@@ -49,11 +48,11 @@ public class LivenessModelFactory {
/**
* 注册模型
* @param name
* @param livenessModelEnum
* @param clazz
*/
private static void registerModel(String name, Class<? extends LivenessDetModel> clazz) {
registry.put(name.toLowerCase(), clazz);
private static void registerModel(LivenessModelEnum livenessModelEnum, Class<? extends LivenessDetModel> clazz) {
registry.put(livenessModelEnum, clazz);
}
@@ -66,7 +65,7 @@ public class LivenessModelFactory {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new FaceException("未配置活体检测模型");
}
return modelMap.computeIfAbsent(config.getModelEnum().name(), k -> {
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceModel(config);
});
}
@@ -77,7 +76,7 @@ public class LivenessModelFactory {
* @return
*/
private LivenessDetModel createFaceModel(LivenessConfig config) {
Class<?> clazz = registry.get(config.getModelEnum().getModelClassName().toLowerCase());
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new FaceException("Unsupported algorithm");
}
@@ -94,7 +93,9 @@ public class LivenessModelFactory {
// 初始化默认算法
static {
registerModel("seetaface6model", Seetaface6LivenessModel.class);
registerModel(LivenessModelEnum.SEETA_FACE6_MODEL, Seetaface6LivenessModel.class);
registerModel(LivenessModelEnum.MINI_VISION_MODEL, MiniVisionLivenessModel.class);
registerModel(LivenessModelEnum.IIC_FL_MODEL, CommonLivenessModel.class);
log.debug("缓存目录:{}", Config.getCachePath());
}

View File

@@ -2,10 +2,9 @@ package cn.smartjavaai.face.model.attribute;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.FaceAttribute;
import cn.smartjavaai.common.entity.face.FaceAttribute;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.face.config.FaceAttributeConfig;
import cn.smartjavaai.common.enums.GenderType;
import java.awt.image.BufferedImage;
import java.util.List;
@@ -14,7 +13,7 @@ import java.util.List;
* 人脸属性识别模型
* @author dwj
*/
public interface FaceAttributeModel {
public interface FaceAttributeModel extends AutoCloseable{
/**
* 加载模型

View File

@@ -1,13 +1,16 @@
package cn.smartjavaai.face.model.attribute;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceAttribute;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.HeadPose;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.enums.EyeStatus;
import cn.smartjavaai.common.enums.face.EyeStatus;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.PoolUtils;
import cn.smartjavaai.face.config.FaceAttributeConfig;
import cn.smartjavaai.common.enums.GenderType;
import cn.smartjavaai.common.enums.face.GenderType;
import cn.smartjavaai.face.context.PredictorContext;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.seetaface.NativeLoader;
@@ -468,6 +471,28 @@ public class Seetaface6FaceAttributeModel implements FaceAttributeModel {
}
}
@Override
public void close() throws Exception {
if(Objects.nonNull(faceDetectorPool)){
faceDetectorPool.close();
}
if(Objects.nonNull(genderPredictorPool)){
genderPredictorPool.close();
}
if(Objects.nonNull(faceLandmarkerPool)){
faceLandmarkerPool.close();
}
if(Objects.nonNull(agePredictorPool)){
agePredictorPool.close();
}
if(Objects.nonNull(eyeStateDetectorPool)){
eyeStateDetectorPool.close();
}
if(Objects.nonNull(maskDetectorPool)){
maskDetectorPool.close();
}
if(Objects.nonNull(poseEstimatorPool)){
poseEstimatorPool.close();
}
}
}

View File

@@ -0,0 +1,364 @@
package cn.smartjavaai.face.model.expression;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.inference.Predictor;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.ndarray.NDManager;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.ExpressionResult;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.enums.face.FacialExpression;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.expression.criterial.EmotionCriteriaFactory;
import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator;
import cn.smartjavaai.face.preprocess.DJLImagePreprocessor;
import cn.smartjavaai.face.utils.FaceUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.opencv.face.Face;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* 通用人脸表情识别模型
* @author dwj
*/
@Slf4j
public class CommonEmotionModel implements ExpressionModel{
private FaceExpressionConfig config;
private ZooModel<Image, Classifications> model;
private ObjectPool<Predictor<Image, Classifications>> predictorPool;
@Override
public void loadModel(FaceExpressionConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath为空");
}
this.config = config;
Criteria<Image, Classifications> criteria = EmotionCriteriaFactory.createCriteria(config);
try {
model = criteria.loadModel();
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("DenseNetEmotionModel模型加载失败", e);
}
}
public Classifications detectCore(Image image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
Predictor<Image, Classifications> predictor = null;
try (NDManager manager = model.getNDManager().newSubManager()){
predictor = predictorPool.borrowObject();
DJLImagePreprocessor imagePreprocessor = new DJLImagePreprocessor(image, manager);
Image faceImg = image;
if(config.isAlign()){
//仿射变换
faceImg = imagePreprocessor.enableAffine(FaceUtils.facePoints(keyPoints), 512, 512)
.process();
return predictor.predict(faceImg);
}else{
if(config.isCropFace()){
//裁剪
faceImg = imagePreprocessor.enableCrop(faceDetectionRectangle)
.process();
}
}
return predictor.predict(faceImg);
} catch (Exception e) {
throw new FaceException("表情识别异常", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
@Override
public R<DetectionResponse> detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(Objects.isNull(config.getDetectModel())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型");
}
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
Classifications classifications = detectCore(djlImage, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints());
Classifications.Classification bestClass = classifications.best();
FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName());
ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability());
result.setClassifications(classifications);
faceInfo.setExpressionResult(result);
}
return faceDetectionResponse;
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<List<ExpressionResult>> detect(String imagePath, DetectionResponse faceDetectionResponse) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionResponse);
}
@Override
public R<List<ExpressionResult>> detect(byte[] imageData, DetectionResponse faceDetectionResponse) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<List<ExpressionResult>> detect(BufferedImage image, DetectionResponse faceDetectionResponse) {
if(!ImageUtils.isImageValid(image)){
R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){
R.fail(R.Status.NO_FACE_DETECTED);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
List<ExpressionResult> expressionResults = new ArrayList<>();
for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
Classifications classifications = detectCore(djlImage, detectionInfo.getDetectionRectangle(), faceInfo.getKeyPoints());
Classifications.Classification bestClass = classifications.best();
FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName());
ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability());
result.setClassifications(classifications);
expressionResults.add(result);
}
return R.ok(expressionResults);
}
@Override
public R<List<ExpressionResult>> detectBase64(String base64Image, DetectionResponse faceDetectionResponse) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData, faceDetectionResponse);
}
@Override
public R<ExpressionResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionRectangle, keyPoints);
}
@Override
public R<ExpressionResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<ExpressionResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
Classifications classifications = detectCore(djlImage, faceDetectionRectangle, keyPoints);
Classifications.Classification bestClass = classifications.best();
FacialExpression expression = FacialExpression.fromLabel(bestClass.getClassName());
ExpressionResult result = new ExpressionResult(expression, (float)bestClass.getProbability());
result.setClassifications(classifications);
return R.ok(result);
}
@Override
public R<ExpressionResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData, faceDetectionRectangle, keyPoints);
}
@Override
public R<ExpressionResult> detectTopFace(BufferedImage image) {
if(Objects.isNull(config.getDetectModel())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型");
}
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
DetectionInfo detectionInfo = faceDetectionResponse.getData().getDetectionInfoList().get(0);
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
return detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints());
}
@Override
public R<ExpressionResult> detectTopFace(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detectTopFace(image);
}
@Override
public R<ExpressionResult> detectTopFace(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<ExpressionResult> detectTopFaceBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detectTopFace(imageData);
}
@Override
public void close() {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
}

View File

@@ -0,0 +1,195 @@
package cn.smartjavaai.face.model.expression;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.entity.face.ExpressionResult;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import java.awt.image.BufferedImage;
import java.util.List;
/**
* @author dwj
* @date 2025/7/1
*/
public interface ExpressionModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(FaceExpressionConfig config); // 加载模型
/**
* 表情识别(多人脸)
* @param imagePath 图片路径
* @return
*/
default R<DetectionResponse> detect(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param image BufferedImage
* @return
*/
default R<DetectionResponse> detect(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param imageData 图片字节流
* @return
*/
default R<DetectionResponse> detect(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param base64Image
* @return
*/
default R<DetectionResponse> detectBase64(String base64Image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param imagePath 图片路径
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<ExpressionResult>> detect(String imagePath, DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param imageData 图片数据
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<ExpressionResult>> detect(byte[] imageData,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param image BufferedImage
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<ExpressionResult>> detect(BufferedImage image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(多人脸)
* @param base64Image
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<ExpressionResult>> detectBase64(String base64Image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(单人脸)
* @param imagePath 图片路径
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<ExpressionResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(单人脸)
* @param imageData
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<ExpressionResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(单人脸)
* @param image BufferedImage
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<ExpressionResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(单人脸)
* @param base64Image
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<ExpressionResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(分数最高人脸)
* @param image
* @return
*/
default R<ExpressionResult> detectTopFace(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(分数最高人脸)
* @param imagePath
* @return
*/
default R<ExpressionResult> detectTopFace(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(分数最高人脸)
* @param imageData
* @return
*/
default R<ExpressionResult> detectTopFace(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 表情识别(分数最高人脸)
* @param base64Image
* @return
*/
default R<ExpressionResult> detectTopFaceBase64(String base64Image){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

View File

@@ -0,0 +1,58 @@
package cn.smartjavaai.face.model.expression.criterial;
import ai.djl.Device;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import cn.smartjavaai.face.enums.ExpressionModelEnum;
import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator;
import cn.smartjavaai.face.model.expression.translator.FrEmotionTranslator;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.Objects;
/**
* 人脸表情识别 Criteria构建工厂
* @author dwj
* @date 2025/5/14
*/
public class EmotionCriteriaFactory {
public static Criteria<Image, Classifications> createCriteria(FaceExpressionConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu();
}
Criteria<Image, Classifications> criteria = null;
if(config.getModelEnum() == ExpressionModelEnum.DensNet121){
//开源项目地址https://github.com/sajjjadayobi/FaceLib
//初始化 检测Criteria
criteria =
Criteria.builder()
.optEngine("PyTorch")
.setTypes(Image.class, Classifications.class)
.optModelPath(Paths.get(config.getModelPath()))
.optTranslator(new DenseNetEmotionTranslator(224))
.optProgress(new ProgressBar())
.optDevice(device)
.build();
}else if (config.getModelEnum() == ExpressionModelEnum.FrEmotion){
//初始化 检测Criteria
criteria =
Criteria.builder()
.optEngine("OnnxRuntime")
.setTypes(ai.djl.modality.cv.Image.class, Classifications.class)
.optModelPath(Paths.get(config.getModelPath()))
.optTranslator(new FrEmotionTranslator(224))
.optProgress(new ProgressBar())
.build();
}
return criteria;
}
}

View File

@@ -0,0 +1,63 @@
package cn.smartjavaai.face.model.expression.translator;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
import java.util.Arrays;
import java.util.List;
/**
* @author dwj
* @date 2025/6/30
*/
public class DenseNetEmotionTranslator implements Translator<Image, Classifications> {
private final List<String> labels = Arrays.asList("angry", "disgust", "fear", "happy", "sad", "surprise", "neutral");
private int imageSize = 224;
public DenseNetEmotionTranslator(int imageSize) {
this.imageSize = imageSize;
}
@Override
public Classifications processOutput(TranslatorContext ctx, NDList list) {
NDArray output = list.singletonOrThrow();
output = output.softmax(1);
return new Classifications(labels, output);
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
// 直接转换为灰度NDArray
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
// 调整大小
Shape shape = array.getShape();
long height = shape.get(0);
long width = shape.get(1);
if (height != imageSize || width != imageSize) {
array = NDImageUtils.resize(array, imageSize, imageSize);
}
array = NDImageUtils.resize(array, imageSize, imageSize);
array = array.transpose(2, 0, 1);
array = array.expandDims(0);
// 归一化
array = array.toType(DataType.FLOAT32, false).div(255.0f);
return new NDList(array);
}
@Override
public Batchifier getBatchifier() {
return null;
}
}

View File

@@ -0,0 +1,61 @@
package cn.smartjavaai.face.model.expression.translator;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.util.NDImageUtils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
import java.util.Arrays;
import java.util.List;
/**
* @author dwj
* @date 2025/6/30
*/
public class FrEmotionTranslator implements Translator<Image, Classifications> {
private final List<String> labels = Arrays.asList("angry", "disgust", "fear", "happy", "sad", "surprise", "neutral");
private int imageSize = 224;
public FrEmotionTranslator(int imageSize) {
this.imageSize = imageSize;
}
@Override
public Classifications processOutput(TranslatorContext ctx, NDList list) {
NDArray output = list.singletonOrThrow();
output = output.softmax(1);
return new Classifications(labels, output);
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
// 调整大小
Shape shape = array.getShape();
long height = shape.get(0);
long width = shape.get(1);
if (height != imageSize || width != imageSize) {
array = NDImageUtils.resize(array, imageSize, imageSize);
}
array = array.transpose(2, 0, 1); // 变成 (3, 224, 224)
array = array.expandDims(0);
// 归一化
array = array.toType(DataType.FLOAT32, false).div(255.0f);
return new NDList(array);
}
@Override
public Batchifier getBatchifier() {
return null;
}
}

View File

@@ -1,7 +1,7 @@
package cn.smartjavaai.face.model.facerec;
package cn.smartjavaai.face.model.facedect;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
@@ -9,89 +9,59 @@ import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.translator.FaceDetectionTranslator;
import cn.smartjavaai.face.model.facedect.criterial.FaceDetCriteriaFactory;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.utils.OpenCVUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
/**
* RetinaFace实现
* DJL通用人脸检测模型实现
* @author dwj
*/
@Slf4j
public class RetinaFaceModel implements FaceModel, AutoCloseable{
public class CommonFaceDetModel implements FaceDetModel{
private ObjectPool<Predictor<Image, DetectedObjects>> predictorPool;
private ZooModel<Image, DetectedObjects> model;
/**
* 特征图层的基础缩放比例
*/
public static final int[][] scales = {{16, 32}, {64, 128}, {256, 512}};
/**
* 特征图相对于原图的采样步长
*/
public static final int[] steps = {8, 16, 32};
/**
* 缩放系数
*/
public static final double[] variance = {0.1f, 0.2f};
/**
* 加载模型
* @param config
*/
@Override
public void loadModel(FaceModelConfig config){
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu();
}
FaceDetectionTranslator translator =
new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), variance, FaceDetectConstant.MAX_FACE_LIMIT, scales, steps);
Criteria<Image, DetectedObjects> criteria =
Criteria.builder()
.setTypes(Image.class, DetectedObjects.class)
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : "https://resources.djl.ai/test-models/pytorch/retinaface.zip")
// Load model from local file, e.g:
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optModelName("retinaface") // specify model file prefix
.optTranslator(translator)
.optDevice(device)
.optProgress(new ProgressBar())
.optEngine("PyTorch") // Use PyTorch engine
.build();
public void loadModel(FaceDetConfig config){
Criteria<Image, DetectedObjects> criteria = FaceDetCriteriaFactory.createCriteria(config);
try {
model = criteria.loadModel();
// 创建池子每个线程独享 Predictor
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
log.info("当前设备: " + model.getNDManager().getDevice());
log.debug("当前设备: " + model.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("模型加载失败", e);
throw new FaceException("人脸检测模型加载失败", e);
}
}
@@ -104,9 +74,9 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{
* @throws Exception
*/
@Override
public DetectionResponse detect(String imagePath){
public R<DetectionResponse> detect(String imagePath){
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
return R.fail(R.Status.FILE_NOT_FOUND);
}
Image img = null;
try {
@@ -115,7 +85,7 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{
throw new FaceException("无效的图片", e);
}
DetectedObjects detection = detect(img);
return FaceUtils.convertToDetectionResponse(detection,img);
return R.ok(FaceUtils.convertToDetectionResponse(detection,img));
}
/**
@@ -125,14 +95,14 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{
* @throws Exception
*/
@Override
public DetectionResponse detect(InputStream imageInputStream){
public R<DetectionResponse> detect(InputStream imageInputStream){
if(Objects.isNull(imageInputStream)){
throw new FaceException("图像输入流无效");
return R.fail(R.Status.INVALID_IMAGE);
}
try {
Image img = ImageFactory.getInstance().fromInputStream(imageInputStream);
DetectedObjects detection = detect(img);
return FaceUtils.convertToDetectionResponse(detection,img);
return R.ok(FaceUtils.convertToDetectionResponse(detection,img));
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
}
@@ -140,19 +110,19 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{
}
@Override
public DetectionResponse detect(BufferedImage image) {
public R<DetectionResponse> detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
DetectedObjects detection = detect(img);
return FaceUtils.convertToDetectionResponse(detection,img);
return R.ok(FaceUtils.convertToDetectionResponse(detection,img));
}
@Override
public DetectionResponse detect(byte[] imageData) {
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
@@ -162,34 +132,44 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{
}
@Override
public void detectAndDraw(String imagePath, String outputPath) {
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<Void> detectAndDraw(String imagePath, String outputPath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
return R.fail(R.Status.FILE_NOT_FOUND);
}
try {
Image img = ImageFactory.getInstance().fromFile(Paths.get(imagePath));
DetectedObjects detectedObjects = detect(img);
if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){
throw new FaceException("未识别到人脸");
return R.fail(R.Status.NO_FACE_DETECTED);
}
img.drawBoundingBoxes(detectedObjects);
Path output = Paths.get(outputPath);
log.debug("Saving to {}", output.toAbsolutePath().toString());
img.save(Files.newOutputStream(output), "png");
return R.ok();
} catch (IOException e) {
throw new FaceException(e);
}
}
@Override
public BufferedImage detectAndDraw(BufferedImage sourceImage) {
public R<BufferedImage> detectAndDraw(BufferedImage sourceImage) {
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(sourceImage));
DetectedObjects detectedObjects = detect(img);
if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){
throw new FaceException("未识别到人脸");
return R.fail(R.Status.NO_FACE_DETECTED);
}
img.drawBoundingBoxes(detectedObjects);
try {
@@ -198,7 +178,7 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{
img.save(outputStream, "png");
// 将字节流转换为 BufferedImage
byte[] imageBytes = outputStream.toByteArray();
return ImageIO.read(new ByteArrayInputStream(imageBytes));
return R.ok(ImageIO.read(new ByteArrayInputStream(imageBytes)));
} catch (IOException e) {
throw new FaceException("导出图片失败", e);
}
@@ -209,13 +189,13 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{
* @param image
* @return
*/
private DetectedObjects detect(Image image){
public DetectedObjects detect(Image image){
Predictor<Image, DetectedObjects> predictor = null;
try {
predictor = predictorPool.borrowObject();
return predictor.predict(image);
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
throw new FaceException("人脸检测错误", e);
}finally {
if (predictor != null) {
try {
@@ -235,8 +215,19 @@ public class RetinaFaceModel implements FaceModel, AutoCloseable{
@Override
public void close() {
if (predictorPool != null) {
predictorPool.close();
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
}

View File

@@ -0,0 +1,88 @@
package cn.smartjavaai.face.model.facedect;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.config.FaceDetConfig;
import java.awt.image.BufferedImage;
import java.io.InputStream;
/**
* 人脸检测模型
* @author dwj
*/
public interface FaceDetModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(FaceDetConfig config); // 加载模型
/**
* 人脸检测
* @param imagePath 图片路径
* @return
*/
default R<DetectionResponse> detect(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param imageInputStream 图片输入流
* @return
*/
default R<DetectionResponse> detect(InputStream imageInputStream){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param image BufferedImage
* @return
*/
default R<DetectionResponse> detect(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param imageData
* @return
*/
default R<DetectionResponse> detect(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param base64Image
* @return
*/
default R<DetectionResponse> detectBase64(String base64Image) {
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 检测并绘制人脸
* @param imagePath 图片输入路径(包含文件名称)
* @param outputPath 图片输出路径(包含文件名称)
*/
default R<Void> detectAndDraw(String imagePath, String outputPath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 检测并绘制人脸
* @param sourceImage
* @return
*/
default R<BufferedImage> detectAndDraw(BufferedImage sourceImage){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

View File

@@ -0,0 +1,236 @@
package cn.smartjavaai.face.model.facedect;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.pool.*;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* SeetaFace6 人脸检测模型
* @author dwj
*/
@Slf4j
public class SeetaFace6FaceDetModel implements FaceDetModel{
private FaceDetConfig config;
private FaceDetectorPool faceDetectorPool;
private FaceLandmarkerPool faceLandmarkerPool;
@Override
public void loadModel(FaceDetConfig config) {
this.config = config;
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
}
//加载依赖库
NativeLoader.loadNativeLibraries(config.getDevice());
log.debug("Loading seetaFace6 library successfully.");
String[] faceDetectorModelPath = {config.getModelPath() + File.separator + "face_detector.csta"};
String[] faceLandmarkerModelPath = {config.getModelPath() + File.separator + "face_landmarker_pts5.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = 0;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
Integer gpuIdValue = config.getCustomParam("gpuId", Integer.class);
if(Objects.nonNull(gpuIdValue) && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = gpuIdValue;
}
}
try {
SeetaModelSetting faceDetectorPoolSetting = new SeetaModelSetting(gpuId, faceDetectorModelPath, device);
SeetaConfSetting faceDetectorPoolConfSetting = new SeetaConfSetting(faceDetectorPoolSetting);
SeetaModelSetting faceLandmarkerPoolSetting = new SeetaModelSetting(gpuId, faceLandmarkerModelPath, device);
SeetaConfSetting faceLandmarkerPoolConfSetting = new SeetaConfSetting(faceLandmarkerPoolSetting);
this.faceDetectorPool = new FaceDetectorPool(faceDetectorPoolConfSetting);
this.faceLandmarkerPool = new FaceLandmarkerPool(faceLandmarkerPoolConfSetting);
} catch (FileNotFoundException e) {
throw new FaceException(e);
}
}
@Override
public R<DetectionResponse> detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(InputStream imageInputStream) {
if(Objects.isNull(imageInputStream)){
return R.fail(R.Status.INVALID_IMAGE);
}
BufferedImage image = null;
try {
image = ImageIO.read(imageInputStream);
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector predictor = null;
FaceLandmarker faceLandmarker = null;
try {
predictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaRect[] seetaResult = predictor.Detect(imageData);
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
seetaPointFSList.add(pointFS);
}
return R.ok(FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList));
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (predictor != null) {
try {
faceDetectorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<Void> detectAndDraw(String imagePath, String outputPath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
try {
//创建保存路径
Path imageOutputPath = Paths.get(outputPath);
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
R<DetectionResponse> result = detect(image);
if(result.getCode() != R.Status.SUCCESS.getCode()){
return R.fail(result.getCode(), result.getMessage());
}
if(Objects.isNull(result.getData()) || Objects.isNull(result.getData().getDetectionInfoList()) || result.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
//绘制人脸框
FaceUtils.drawBoundingBoxes(image, result.getData(), imageOutputPath.toAbsolutePath().toString());
return R.ok();
} catch (IOException e) {
throw new FaceException(e);
}
}
@Override
public R<BufferedImage> detectAndDraw(BufferedImage sourceImage) {
if(!ImageUtils.isImageValid(sourceImage)){
return R.fail(R.Status.INVALID_IMAGE);
}
R<DetectionResponse> result = detect(sourceImage);
if(result.getCode() != R.Status.SUCCESS.getCode()){
return R.fail(result.getCode(), result.getMessage());
}
if(Objects.isNull(result.getData()) || Objects.isNull(result.getData().getDetectionInfoList()) || result.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
//绘制人脸框
try {
return R.ok(FaceUtils.drawBoundingBoxes(sourceImage, result.getData()));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public void close() throws Exception {
try {
if (faceDetectorPool != null) {
faceDetectorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (faceLandmarkerPool != null) {
faceLandmarkerPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
}
}

View File

@@ -0,0 +1,69 @@
package cn.smartjavaai.face.model.facedect.criterial;
import ai.djl.Device;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.config.FaceExpressionConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.constant.RetinaFaceConstant;
import cn.smartjavaai.face.constant.UltraLightFastGenericFaceConstant;
import cn.smartjavaai.face.enums.ExpressionModelEnum;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import cn.smartjavaai.face.model.expression.translator.DenseNetEmotionTranslator;
import cn.smartjavaai.face.model.expression.translator.FrEmotionTranslator;
import cn.smartjavaai.face.translator.FaceDetectionTranslator;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.Objects;
/**
* 人脸检测 Criteria构建工厂
* @author dwj
*/
public class FaceDetCriteriaFactory {
public static Criteria<Image, DetectedObjects> createCriteria(FaceDetConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu();
}
Criteria<Image, DetectedObjects> criteria = null;
if(config.getModelEnum() == FaceDetModelEnum.RETINA_FACE){
FaceDetectionTranslator translator =
new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), RetinaFaceConstant.variance, FaceDetectConstant.MAX_FACE_LIMIT, RetinaFaceConstant.scales, RetinaFaceConstant.steps);
criteria =
Criteria.builder()
.setTypes(Image.class, DetectedObjects.class)
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : RetinaFaceConstant.MODEL_URL)
// Load model from local file, e.g:
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optModelName("retinaface") // specify model file prefix
.optTranslator(translator)
.optDevice(device)
.optProgress(new ProgressBar())
.optEngine("PyTorch") // Use PyTorch engine
.build();
}else if (config.getModelEnum() == FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE){
FaceDetectionTranslator translator =
new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), UltraLightFastGenericFaceConstant.variance, FaceDetectConstant.MAX_FACE_LIMIT, UltraLightFastGenericFaceConstant.scales, UltraLightFastGenericFaceConstant.steps);
criteria =
Criteria.builder()
.setTypes(Image.class, DetectedObjects.class)
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : UltraLightFastGenericFaceConstant.MODEL_URL)
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optTranslator(translator)
.optProgress(new ProgressBar())
.optDevice(device)
.optEngine("PyTorch") // Use PyTorch engine
.build();
}
return criteria;
}
}

View File

@@ -1,32 +1,35 @@
package cn.smartjavaai.face.model.facerec;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.opencv.OpenCVImageFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.hutool.core.lang.UUID;
import cn.hutool.core.lang.generator.UUIDGenerator;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceExtractConfig;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.entity.FaceRegisterInfo;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.face.enums.FaceModelEnum;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import cn.smartjavaai.face.enums.SimilarityType;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.factory.FaceModelFactory;
import cn.smartjavaai.face.translator.FaceFeatureTranslator;
import cn.smartjavaai.face.factory.FaceDetModelFactory;
import cn.smartjavaai.face.model.facedect.FaceDetModel;
import cn.smartjavaai.face.model.facerec.criterial.FaceRecCriteriaFactory;
import cn.smartjavaai.face.preprocess.DJLImagePreprocessor;
import cn.smartjavaai.face.utils.*;
import cn.smartjavaai.face.vector.config.MilvusConfig;
import cn.smartjavaai.face.vector.config.SQLiteConfig;
@@ -37,31 +40,24 @@ import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import io.milvus.param.MetricType;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.opencv.core.Mat;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.*;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* FaceNet 人脸特征提取模型
* @author dwj
*/
@Slf4j
public class FaceNetModel implements FaceModel, AutoCloseable{
public class CommonFaceRecModel implements FaceRecModel{
/**
* 特征维度
@@ -79,23 +75,13 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
private ZooModel<Image, float[]> model;
private FaceModelConfig config;
private FaceRecConfig config;
/**
* 是否归一化相似度
*/
public static final boolean NORMALIZE_SIMILARITY = true;
public static final List<Float> mean =
Arrays.asList(
127.5f / 255.0f,
127.5f / 255.0f,
127.5f / 255.0f,
128.0f / 255.0f,
128.0f / 255.0f,
128.0f / 255.0f);
private VectorDBClient vectorDBClient;
@@ -104,42 +90,21 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
* @param config
*/
@Override
public void loadModel(FaceModelConfig config) {
public void loadModel(FaceRecConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
if(Objects.isNull(config.getExtractConfig())){
config.setExtractConfig(getDefaultConfig());
}else{
if(Objects.isNull(config.getExtractConfig().getDetectModel())){
throw new FaceException("请设置人脸检测模型");
}
}
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu();
if(Objects.isNull(config.getDetectModel())){
config.setDetectModel(getDefaultDetModel());
}
this.config = config;
String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(","));
Criteria<Image, float[]> faceFeatureCriteria =
Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelName("face_feature") // specify model file prefix
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null :
"https://resources.djl.ai/test-models/pytorch/face_feature.zip")
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optTranslator(new FaceFeatureTranslator())
.optArgument("normalize", normalize)
.optDevice(device)
.optEngine("PyTorch") // Use PyTorch engine
.optProgress(new ProgressBar())
.build();
Criteria<Image, float[]> faceFeatureCriteria = FaceRecCriteriaFactory.createCriteria(config);
try {
model = faceFeatureCriteria.loadModel();
// 创建池子每个线程独享 Predictor
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
log.info("当前设备: " + model.getNDManager().getDevice());
log.debug("当前设备: " + model.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("模型加载失败", e);
}
@@ -184,14 +149,13 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
}
}
private float[] featureExtraction(Image image){
image.getWrappedImage();
public float[] featureExtraction(Image image){
Predictor<Image, float[]> predictor = null;
try {
predictor = predictorPool.borrowObject();
return predictor.predict(image);
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
throw new FaceException("人脸特征提取错误", e);
}finally {
if (predictor != null) {
try {
@@ -218,7 +182,7 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
@Override
public float calculSimilar(float[] feature1, float[] feature2) {
//默认返回归一化结果
return SimilarityUtil.calculate(feature1, feature2, SimilarityType.IP, NORMALIZE_SIMILARITY);
return SimilarityUtil.calculate(feature1, feature2, SimilarityType.IP, true);
}
/**
@@ -228,105 +192,103 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
* @return
*/
@Override
public float featureComparison(String imagePath1, String imagePath2) {
public R<Float> featureComparison(String imagePath1, String imagePath2) {
if(!FileUtils.isFileExists(imagePath1) || !FileUtils.isFileExists(imagePath2)){
throw new FaceException("图像文件不存在");
return R.fail(R.Status.FILE_NOT_FOUND);
}
R<float[]> feature1 = extractTopFaceFeature(imagePath1);
if (!feature1.isSuccess()){
throw new FaceException(feature1.getMessage());
// 将图片路径转换为 BufferedImage
BufferedImage image1 = null;
BufferedImage image2 = null;
try {
image1 = ImageIO.read(new File(Paths.get(imagePath1).toAbsolutePath().toString()));
image2 = ImageIO.read(new File(Paths.get(imagePath2).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
R<float[]> feature2 = extractTopFaceFeature(imagePath2);
if (!feature2.isSuccess()){
throw new FaceException(feature2.getMessage());
}
float ret = calculSimilar(feature1.getData(), feature2.getData());
return ret;
return featureComparison(image1, image2);
}
@Override
public float featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2) {
public R<Float> featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2) {
if(!ImageUtils.isImageValid(sourceImage1) || !ImageUtils.isImageValid(sourceImag2)){
throw new FaceException("图像无效");
}
R<float[]> feature1 = extractTopFaceFeature(sourceImage1);
if (!feature1.isSuccess()){
throw new FaceException(feature1.getMessage());
return R.fail(feature1.getCode(), feature1.getMessage());
}
R<float[]> feature2 = extractTopFaceFeature(sourceImag2);
if (!feature2.isSuccess()){
throw new FaceException(feature2.getMessage());
return R.fail(feature2.getCode(), feature2.getMessage());
}
float ret = calculSimilar(feature1.getData(), feature2.getData());
return ret;
return R.ok(ret);
}
@Override
public float featureComparison(byte[] imageData1, byte[] imageData2) {
public R<Float> featureComparison(byte[] imageData1, byte[] imageData2) {
if(Objects.isNull(imageData1) || Objects.isNull(imageData2)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
R<float[]> feature1 = extractTopFaceFeature(imageData1);
if (!feature1.isSuccess()){
throw new FaceException(feature1.getMessage());
try {
BufferedImage bufferedImage1 = ImageIO.read(new ByteArrayInputStream(imageData1));
BufferedImage bufferedImage2 = ImageIO.read(new ByteArrayInputStream(imageData2));
return featureComparison(bufferedImage1, bufferedImage2);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
R<float[]> feature2 = extractTopFaceFeature(imageData2);
if (!feature2.isSuccess()){
throw new FaceException(feature2.getMessage());
}
float ret = calculSimilar(feature1.getData(), feature2.getData());
return ret;
}
/**
* 获取默认特征提取配置
* 获取默认人脸检测模型
* @return
*/
private FaceExtractConfig getDefaultConfig() {
FaceExtractConfig config = new FaceExtractConfig();
FaceModelConfig detectModelConfig = new FaceModelConfig();
detectModelConfig.setModelEnum(FaceModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);
private FaceDetModel getDefaultDetModel() {
FaceDetConfig detectModelConfig = new FaceDetConfig();
detectModelConfig.setModelEnum(FaceDetModelEnum.ULTRA_LIGHT_FAST_GENERIC_FACE);
detectModelConfig.setConfidenceThreshold(0.98);
log.debug("创建默认检测模型ULTRA_LIGHT_FAST_GENERIC_FACE");
FaceModel detectModel = FaceModelFactory.getInstance().getModel(detectModelConfig);
log.debug("创建检测模型完毕");
config.setDetectModel(detectModel);
return config;
log.debug("创建默认人脸检测模型ULTRA_LIGHT_FAST_GENERIC_FACE");
FaceDetModel detectModel = FaceDetModelFactory.getInstance().getModel(detectModelConfig);
return detectModel;
}
@Override
public R<DetectionResponse> extractFeatures(BufferedImage image) {
DetectionResponse detectedResult = config.getExtractConfig().getDetectModel().detect(image);
if(Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getDetectionInfoList()) || detectedResult.getDetectionInfoList().isEmpty()){
R<DetectionResponse> detectedResult = config.getDetectModel().detect(image);
if(!detectedResult.isSuccess()){
return detectedResult;
}
if(Objects.isNull(detectedResult.getData()) || Objects.isNull(detectedResult.getData().getDetectionInfoList()) || detectedResult.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
NDManager manager = NDManager.newBaseManager();
for (DetectionInfo detectionInfo : detectedResult.getDetectionInfoList()){
DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
FaceInfo faceInfo = detectionInfo.getFaceInfo();
float[] features = null;
//裁剪人脸
Image subImage = djlImage.getSubImage(rectangle.getX(), rectangle.getY() , rectangle.getWidth() , rectangle.getHeight());
//人脸对齐
if(config.getExtractConfig().isAlign()){
//获取子图中人脸关键点坐标
double[][] pointsArray = FaceUtils.facePoints(detectionInfo.getFaceInfo().getKeyPoints());
NDArray srcPoints = manager.create(pointsArray);
NDArray dstPoints = FaceUtils.faceTemplate512x512(manager);
// 5点仿射变换
Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints);
Mat mat = FaceAlignUtils.warpAffine((Mat) djlImage.getWrappedImage(), affine_matrix);
Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat);
features = featureExtraction(alignedImg);
}else{
//不对齐人脸
try (NDManager manager = model.getNDManager().newSubManager()) {
DJLImagePreprocessor djlImagePreprocessor = new DJLImagePreprocessor(djlImage, manager);
for (DetectionInfo detectionInfo : detectedResult.getData().getDetectionInfoList()){
DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
FaceInfo faceInfo = detectionInfo.getFaceInfo();
float[] features = null;
Image subImage = djlImage;
//人脸对齐
if(config.isAlign()){
//人脸对齐
double[][] pointsArray = FaceUtils.facePoints(faceInfo.getKeyPoints());
djlImagePreprocessor.enableCrop(rectangle).enableAffine(pointsArray, 96, 112);
subImage = djlImagePreprocessor.process();
}else{
//裁剪
djlImagePreprocessor.enableCrop(rectangle);
if(config.isCropFace()){
subImage = djlImagePreprocessor.process();
}
}
features = featureExtraction(subImage);
faceInfo.setFeature(features);
}
faceInfo.setFeature(features);
}
return R.ok(detectedResult);
return detectedResult;
}
@@ -359,37 +321,36 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
@Override
public R<float[]> extractTopFaceFeature(BufferedImage image) {
R<DetectionResponse> detectedResult = config.getDetectModel().detect(image);
if(!detectedResult.isSuccess()){
return R.fail(detectedResult.getCode(), detectedResult.getMessage());
}
if(Objects.isNull(detectedResult.getData()) || Objects.isNull(detectedResult.getData().getDetectionInfoList()) || detectedResult.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
float[] features = null;
if(config.getExtractConfig().isCropFace()){
DetectionResponse detectedResult = config.getExtractConfig().getDetectModel().detect(image);
if(Objects.isNull(detectedResult) || Objects.isNull(detectedResult.getDetectionInfoList()) || detectedResult.getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
try (NDManager manager = model.getNDManager().newSubManager()) {
DJLImagePreprocessor djlImagePreprocessor = new DJLImagePreprocessor(djlImage, manager);
//只取第一个人脸
DetectionInfo detectionInfo = detectedResult.getDetectionInfoList().get(0);
DetectionInfo detectionInfo = detectedResult.getData().getDetectionInfoList().get(0);
DetectionRectangle rectangle = detectionInfo.getDetectionRectangle();
//裁剪人脸
Image subImage = djlImage.getSubImage(rectangle.getX(), rectangle.getY() , rectangle.getWidth() , rectangle.getHeight());
FaceInfo faceInfo = detectionInfo.getFaceInfo();
Image subImage = djlImage;
//人脸对齐
if(config.getExtractConfig().isAlign()){
NDManager manager = NDManager.newBaseManager();
//获取子图中人脸关键点坐标
double[][] pointsArray = FaceUtils.facePoints(detectionInfo.getFaceInfo().getKeyPoints());
NDArray srcPoints = manager.create(pointsArray);
NDArray dstPoints = FaceUtils.faceTemplate512x512(manager);
// 5点仿射变换
Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints);
Mat mat = FaceAlignUtils.warpAffine((Mat) djlImage.getWrappedImage(), affine_matrix);
Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat);
features = featureExtraction(alignedImg);
if(config.isAlign()){
//人脸对齐
double[][] pointsArray = FaceUtils.facePoints(faceInfo.getKeyPoints());
djlImagePreprocessor.enableCrop(rectangle).enableAffine(pointsArray, 96, 112);
subImage = djlImagePreprocessor.process();
}else{
//不对齐人脸
features = featureExtraction(subImage);
//裁剪
djlImagePreprocessor.enableCrop(rectangle);
if(config.isCropFace()){
subImage = djlImagePreprocessor.process();
}
}
}else{
//不裁剪人脸直接提取特征
features = featureExtraction(djlImage);
features = featureExtraction(subImage);
}
return Objects.isNull(features) ? R.fail(R.Status.Unknown) : R.ok(features);
}
@@ -481,10 +442,10 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
@Override
public R<String> register(FaceRegisterInfo faceRegisterInfo, float[] feature) {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
return R.fail(1000, "向量数据库未初始化成功");
}
if(Objects.isNull(feature)){
throw new FaceException("人脸特征为空");
return R.fail(R.Status.PARAM_ERROR.getCode(), "人脸特征为空");
}
FaceVector faceVector = new FaceVector();
if(faceRegisterInfo != null){
@@ -603,7 +564,7 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
@Override
public R<List<FaceSearchResult>> searchByTopFace(BufferedImage sourceImage, FaceSearchParams params) {
if(vectorDBClient == null){
throw new VectorDBException("向量数据库未初始化成功");
return R.fail(1000, "向量数据库未初始化成功");
}
//提取最大人脸特征
R<float[]> featureResponse = extractTopFaceFeature(sourceImage);
@@ -704,12 +665,22 @@ public class FaceNetModel implements FaceModel, AutoCloseable{
@Override
public void close() {
if (predictorPool != null) {
predictorPool.close();
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
if(Objects.nonNull(vectorDBClient)){
vectorDBClient.close();
try {
if(Objects.nonNull(vectorDBClient)){
vectorDBClient.close();
}
} catch (Exception e) {
log.warn("关闭 vectorDBClient 失败", e);
}
}
@Override

View File

@@ -2,10 +2,10 @@ package cn.smartjavaai.face.model.facerec;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.entity.FaceRegisterInfo;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.common.entity.FaceSearchResult;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import java.awt.image.BufferedImage;
import java.io.InputStream;
@@ -15,69 +15,15 @@ import java.util.List;
* 人脸识别模型
* @author dwj
*/
public interface FaceModel {
public interface FaceRecModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(FaceModelConfig config); // 加载模型
void loadModel(FaceRecConfig config); // 加载模型
/**
* 人脸检测
* @param imagePath 图片路径
* @return
*/
default DetectionResponse detect(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param imageInputStream 图片输入流
* @return
*/
default DetectionResponse detect(InputStream imageInputStream){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param image BufferedImage
* @return
*/
default DetectionResponse detect(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸检测
* @param imageData
* @return
*/
default DetectionResponse detect(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 检测并绘制人脸
* @param imagePath 图片输入路径包含文件名称
* @param outputPath 图片输出路径包含文件名称
*/
default void detectAndDraw(String imagePath, String outputPath){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 检测并绘制人脸
* @param sourceImage
* @return
*/
default BufferedImage detectAndDraw(BufferedImage sourceImage){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 计算相似度
* @param feature1 图1特征
@@ -94,7 +40,7 @@ public interface FaceModel {
* @param imagePath2 图2路径
* @return
*/
default float featureComparison(String imagePath1, String imagePath2){
default R<Float> featureComparison(String imagePath1, String imagePath2){
throw new UnsupportedOperationException("默认不支持该功能");
}
@@ -104,7 +50,7 @@ public interface FaceModel {
* @param sourceImag2 图2BufferedImage
* @return
*/
default float featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2){
default R<Float> featureComparison(BufferedImage sourceImage1, BufferedImage sourceImag2){
throw new UnsupportedOperationException("默认不支持该功能");
}
@@ -115,7 +61,7 @@ public interface FaceModel {
* @param imageData2
* @return
*/
default float featureComparison(byte[] imageData1, byte[] imageData2){
default R<Float> featureComparison(byte[] imageData1, byte[] imageData2){
throw new UnsupportedOperationException("默认不支持该功能");
}

View File

@@ -3,10 +3,11 @@ package cn.smartjavaai.face.model.facerec;
import cn.smartjavaai.common.entity.DetectionInfo;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.entity.FaceRegisterInfo;
import cn.smartjavaai.face.entity.FaceResult;
@@ -18,7 +19,7 @@ import cn.smartjavaai.face.vector.config.MilvusConfig;
import cn.smartjavaai.face.vector.config.SQLiteConfig;
import cn.smartjavaai.face.vector.core.VectorDBClient;
import cn.smartjavaai.face.vector.core.VectorDBFactory;
import cn.smartjavaai.common.entity.FaceSearchResult;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import com.seeta.pool.*;
@@ -26,7 +27,6 @@ import com.seeta.sdk.*;
import cn.smartjavaai.face.seetaface.NativeLoader;
import io.milvus.param.MetricType;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import javax.imageio.ImageIO;
@@ -46,7 +46,7 @@ import java.util.Objects;
*/
@SuppressWarnings("AliMissingOverrideAnnotation")
@Slf4j
public class SeetaFace6Model implements FaceModel , AutoCloseable{
public class SeetaFace6FaceRecModel implements FaceRecModel{
/**
* 特征维度
@@ -54,7 +54,7 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
private static final int DIMENSION = 1024;
private FaceModelConfig config;
private FaceRecConfig config;
private FaceDetectorPool faceDetectorPool;
private FaceRecognizerPool faceRecognizerPool;
@@ -77,7 +77,7 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
@Override
public void loadModel(FaceModelConfig config) {
public void loadModel(FaceRecConfig config) {
this.config = config;
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
@@ -92,8 +92,9 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
int gpuId = 0;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = config.getGpuId();
Integer gpuIdValue = config.getCustomParam("gpuId", Integer.class);
if(Objects.nonNull(gpuIdValue) && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = gpuIdValue;
}
}
try {
@@ -156,204 +157,8 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
} catch (FileNotFoundException e) {
throw new FaceException(e);
}
}
@Override
public DetectionResponse detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public DetectionResponse detect(InputStream imageInputStream) {
if(Objects.isNull(imageInputStream)){
throw new FaceException("图像输入流无效");
}
BufferedImage image = null;
try {
image = ImageIO.read(imageInputStream);
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
}
return detect(image);
}
@Override
public DetectionResponse detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector predictor = null;
FaceLandmarker faceLandmarker = null;
try {
predictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaRect[] seetaResult = predictor.Detect(imageData);
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
seetaPointFSList.add(pointFS);
}
return FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList);
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (predictor != null) {
try {
faceDetectorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public DetectionResponse detect(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public void detectAndDraw(String imagePath, String outputPath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
try {
//创建保存路径
Path imageOutputPath = Paths.get(outputPath);
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
DetectionResponse result = detect(image);
if(Objects.isNull(result) || Objects.isNull(result.getDetectionInfoList()) || result.getDetectionInfoList().isEmpty()){
throw new FaceException("未识别到人脸");
}
//绘制人脸框
FaceUtils.drawBoundingBoxes(image, result, imageOutputPath.toAbsolutePath().toString());
} catch (IOException e) {
throw new FaceException(e);
}
}
@Override
public BufferedImage detectAndDraw(BufferedImage sourceImage) {
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
}
DetectionResponse detectedObjects = detect(sourceImage);
if(Objects.isNull(detectedObjects) || Objects.isNull(detectedObjects.getDetectionInfoList()) || detectedObjects.getDetectionInfoList().isEmpty()){
throw new FaceException("未识别到人脸");
}
//绘制人脸框
try {
return FaceUtils.drawBoundingBoxes(sourceImage, detectedObjects);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* 获取5点坐标,循序依次为左眼中心右眼中心鼻尖左嘴角和右嘴角
* @param imageData
* @return
*/
private SeetaPointF[] getMaskPoint(SeetaImageData imageData) {
FaceDetector faceDetector = null;
FaceLandmarker faceLandmarker = null;
try {
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("未检测到人脸");
}
//提取第一个人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], pointFS);
return pointFS;
} catch (FaceException e) {
throw e;
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (faceDetector != null) {
try {
faceDetectorPool.returnObject(faceDetector); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
/**
* 裁剪人脸
* @param imageData
* @return
*/
private SeetaImageData getMaxCropFace(SeetaImageData imageData){
FaceRecognizer faceRecognizer = null;
try {
faceRecognizer = faceRecognizerPool.borrowObject();
//提取第一个人脸的5点人脸标识
SeetaPointF[] pointFS = getMaskPoint(imageData);
//裁剪人脸
SeetaImageData cropImageData = new SeetaImageData(faceRecognizer.GetCropFaceWidthV2(), faceRecognizer.GetCropFaceHeightV2(), faceRecognizer.GetCropFaceChannelsV2());
faceRecognizer.CropFaceV2(imageData, pointFS, cropImageData);
return cropImageData;
} catch (Exception e) {
throw new FaceException(e);
}finally {
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public float calculSimilar(float[] feature1, float[] feature2) {
if(Objects.isNull(feature1) || Objects.isNull(feature2)){
@@ -377,7 +182,7 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
}
@Override
public float featureComparison(String imagePath1, String imagePath2) {
public R<Float> featureComparison(String imagePath1, String imagePath2) {
if(!FileUtils.isFileExists(imagePath1) || !FileUtils.isFileExists(imagePath2)){
throw new FaceException("图像文件不存在");
}
@@ -395,96 +200,26 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
@Override
public float featureComparison(BufferedImage image1, BufferedImage image2) {
public R<Float> featureComparison(BufferedImage image1, BufferedImage image2) {
if(!ImageUtils.isImageValid(image1) || !ImageUtils.isImageValid(image2)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
SeetaImageData imageData1 = new SeetaImageData(image1.getWidth(), image1.getHeight(), 3);
imageData1.data = ImageUtils.getMatrixBGR(image1);
SeetaImageData imageData2 = new SeetaImageData(image2.getWidth(), image2.getHeight(), 3);
imageData2.data = ImageUtils.getMatrixBGR(image2);
FaceRecognizer faceRecognizer = null;
FaceDatabase faceDatabase = null;
FaceLandmarker faceLandmarker = null;
FaceDetector faceDetector = null;
try {
faceRecognizer = faceRecognizerPool.borrowObject();
faceDatabase = faceDatabasePool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
faceDetector = faceDetectorPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData1);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
throw new FaceException("未检测到人脸");
}
//提取第一个人脸的5点人脸标识
SeetaPointF[] pointFS1 = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData1, seetaResult[0], pointFS1);
//裁剪人脸
// SeetaImageData cropImageData1 = new SeetaImageData(faceRecognizer.GetCropFaceWidthV2(), faceRecognizer.GetCropFaceHeightV2(), faceRecognizer.GetCropFaceChannelsV2());
// faceRecognizer.CropFaceV2(imageData1, pointFS1, cropImageData1);
//图片2检测人脸
SeetaRect[] seetaResult2 = faceDetector.Detect(imageData2);
if(Objects.isNull(seetaResult2) || seetaResult2.length == 0){
throw new FaceException("未检测到人脸");
}
//图片2提取第一个人脸的5点人脸标识
SeetaPointF[] pointFS2 = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData2, seetaResult2[0], pointFS2);
//图片2裁剪人脸
// SeetaImageData cropImageData2 = new SeetaImageData(faceRecognizer.GetCropFaceWidthV2(), faceRecognizer.GetCropFaceHeightV2(), faceRecognizer.GetCropFaceChannelsV2());
// faceRecognizer.CropFaceV2(imageData2, pointFS2, cropImageData2);
// return faceDatabase.CompareByCroppedFace(cropImageData1, cropImageData2);
return faceDatabase.Compare(imageData1, pointFS1, imageData2, pointFS2);
} catch (FaceException e) {
throw e;
} catch (Exception e) {
throw new FaceException(e);
}finally {
if (faceDetector != null) {
try {
faceDetectorPool.returnObject(faceDetector); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceRecognizer != null) {
try {
faceRecognizerPool.returnObject(faceRecognizer); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceDatabase != null) {
try {
faceDatabasePool.returnObject(faceDatabase); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
R<float[]> feature1 = extractTopFaceFeature(image1);
if(!feature1.isSuccess()){
return R.fail(feature1.getCode(), feature1.getMessage());
}
R<float[]> feature2 = extractTopFaceFeature(image2);
if(!feature2.isSuccess()){
return R.fail(feature2.getCode(), feature2.getMessage());
}
return R.ok(calculSimilar(feature1.getData(), feature2.getData()));
}
@Override
public float featureComparison(byte[] imageData1, byte[] imageData2) {
public R<Float> featureComparison(byte[] imageData1, byte[] imageData2) {
if(Objects.isNull(imageData1) || Objects.isNull(imageData2)){
throw new FaceException("图像无效");
}
@@ -842,37 +577,64 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
List<float[]> featureList = new ArrayList<float[]>();
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
FaceDetector faceDetector = null;
FaceLandmarker faceLandmarker = null;
FaceRecognizer faceRecognizer = null;
try {
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
faceRecognizer = faceRecognizerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
//提取特征
float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
log.warn("人脸特征提取失败");
//默认使用Seetaface6检测模型
if(Objects.isNull(config.getDetectModel())){
List<float[]> featureList = new ArrayList<float[]>();
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
featureList.add(features);
seetaPointFSList.add(pointFS);
for(SeetaRect seetaRect : seetaResult){
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, pointFS);
//提取特征
float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
return R.fail(R.Status.Unknown.getCode(), "人脸特征提取失败");
}
featureList.add(features);
seetaPointFSList.add(pointFS);
}
return R.ok(FaceUtils.featuresConvertToResponse(seetaResult, seetaPointFSList, featureList));
}else{
R<DetectionResponse> detectResponse = config.getDetectModel().detect(image);
if(!detectResponse.isSuccess()){
return detectResponse;
}
if(Objects.isNull(detectResponse.getData()) || Objects.isNull(detectResponse.getData().getDetectionInfoList()) || detectResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(DetectionInfo detectionInfo : detectResponse.getData().getDetectionInfoList()){
//提取特征
float[] features = new float[faceRecognizer.GetExtractFeatureSize()];
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(faceInfo.getKeyPoints());
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
boolean isSuccess = faceRecognizer.Extract(imageData, pointFS, features);
if(!isSuccess){
return R.fail(R.Status.Unknown.getCode(), "人脸特征提取失败");
}
faceInfo.setFeature(features);
}
return detectResponse;
}
return R.ok(FaceUtils.featuresConvertToResponse(seetaResult, seetaPointFSList, featureList));
} catch (FaceException e) {
throw e;
} catch (Exception e) {
@@ -915,17 +677,31 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
FaceLandmarker faceLandmarker = null;
FaceRecognizer faceRecognizer = null;
try {
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
faceRecognizer = faceRecognizerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
//提取人脸的5点人脸标识
SeetaPointF[] pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], pointFS);
SeetaPointF[] pointFS = null;
//默认使用Seetaface6检测模型
if(Objects.isNull(config.getDetectModel())){
faceDetector = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
//检测人脸
SeetaRect[] seetaResult = faceDetector.Detect(imageData);
if(Objects.isNull(seetaResult) || seetaResult.length == 0){
return R.fail(R.Status.NO_FACE_DETECTED);
}
pointFS = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], pointFS);
}else{
R<DetectionResponse> detectResponse = config.getDetectModel().detect(image);
if(!detectResponse.isSuccess()){
return R.fail(detectResponse.getCode(), detectResponse.getMessage());
}
if(Objects.isNull(detectResponse.getData()) || Objects.isNull(detectResponse.getData().getDetectionInfoList()) || detectResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
DetectionInfo detectionInfo = detectResponse.getData().getDetectionInfoList().get(0);
pointFS = FaceUtils.convertToSeetaPointF(detectionInfo.getFaceInfo().getKeyPoints());
}
//提取特征
features = new float[faceRecognizer.GetExtractFeatureSize()];
//CropFaceV2 + ExtractCroppedFace 已包含裁剪+人脸对齐
@@ -1058,7 +834,7 @@ public class SeetaFace6Model implements FaceModel , AutoCloseable{
@Override
public void upsertFace(FaceRegisterInfo faceRegisterInfo, byte[] imageData) {
FaceModel.super.upsertFace(faceRegisterInfo, imageData);
FaceRecModel.super.upsertFace(faceRegisterInfo, imageData);
}
@Override

View File

@@ -1,234 +0,0 @@
package cn.smartjavaai.face.model.facerec;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.translator.FaceDetectionTranslator;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.utils.OpenCVUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.*;
import java.nio.file.Paths;
import java.util.Objects;
/**
* @author dwj
*/
@Slf4j
public class UltraLightFastGenericFaceModel implements FaceModel, AutoCloseable{
private ObjectPool<Predictor<Image, DetectedObjects>> predictorPool;
/**
* 特征图层的基础缩放比例
*/
private static final int[][] scales = {{10, 16, 24}, {32, 48}, {64, 96}, {128, 192, 256}};
/**
* 特征图相对于原图的采样步长
*/
private static final int[] steps = {8, 16, 32, 64};
/**
* 缩放系数
*/
private static final double[] variance = {0.1f, 0.2f};
private ZooModel<Image, DetectedObjects> model;
/**
* 加载模型
* @param config
*/
@Override
public void loadModel(FaceModelConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu();
}
FaceDetectionTranslator translator =
new FaceDetectionTranslator(config.getConfidenceThreshold(), config.getNmsThresh(), variance, FaceDetectConstant.MAX_FACE_LIMIT, scales, steps);
Criteria<Image, DetectedObjects> criteria =
Criteria.builder()
.setTypes(Image.class, DetectedObjects.class)
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null : "https://resources.djl.ai/test-models/pytorch/ultranet.zip")
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optTranslator(translator)
.optProgress(new ProgressBar())
.optDevice(device)
.optEngine("PyTorch") // Use PyTorch engine
.build();
try {
model = criteria.loadModel();
// 创建池子:每个线程独享 Predictor
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
log.info("当前设备: " + model.getNDManager().getDevice());
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("模型加载失败", e);
}
}
/**
* 检测人脸
* @param imagePath 图片路径
* @return
* @throws Exception
*/
@Override
public DetectionResponse detect(String imagePath){
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
Image img = null;
try {
img = ImageFactory.getInstance().fromFile(Paths.get(imagePath));
} catch (IOException e) {
throw new FaceException("无效的图片", e);
}
DetectedObjects detection = detect(img);
return FaceUtils.convertToDetectionResponse(detection,img);
}
/**
* 检测人脸
* @param imageInputStream 图片流
* @return
* @throws Exception
*/
@Override
public DetectionResponse detect(InputStream imageInputStream){
try {
Image img = ImageFactory.getInstance().fromInputStream(imageInputStream);
DetectedObjects detection = detect(img);
return FaceUtils.convertToDetectionResponse(detection,img);
} catch (IOException e) {
throw new FaceException("无效图片输入流", e);
}
}
@Override
public DetectionResponse detect(BufferedImage image) {
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(image));
DetectedObjects detection = detect(img);
return FaceUtils.convertToDetectionResponse(detection,img);
}
@Override
public DetectionResponse detect(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public void detectAndDraw(String imagePath, String outputPath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
}
try {
Image img = ImageFactory.getInstance().fromFile(Paths.get(imagePath));
DetectedObjects detectedObjects = detect(img);
if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){
throw new FaceException("未识别到人脸");
}
img.drawBoundingBoxes(detectedObjects);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
// 调用 save 方法将 Image 写入字节流
img.save(new FileOutputStream(Paths.get(outputPath).toAbsolutePath().toString()), "png");
} catch (IOException e) {
throw new FaceException(e);
}
}
@Override
public BufferedImage detectAndDraw(BufferedImage sourceImage) {
if(!ImageUtils.isImageValid(sourceImage)){
throw new FaceException("图像无效");
}
Image img = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(sourceImage));
DetectedObjects detectedObjects = detect(img);
if(Objects.isNull(detectedObjects) || detectedObjects.getNumberOfObjects() == 0){
throw new FaceException("未识别到人脸");
}
img.drawBoundingBoxes(detectedObjects);
try {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
// 调用 save 方法将 Image 写入字节流
img.save(outputStream, "png");
// 将字节流转换为 BufferedImage
byte[] imageBytes = outputStream.toByteArray();
return ImageIO.read(new ByteArrayInputStream(imageBytes));
} catch (IOException e) {
throw new FaceException("导出图片失败", e);
}
}
/**
* 人脸检测
* @param image
* @return
*/
private DetectedObjects detect(Image image){
Predictor<Image, DetectedObjects> predictor = null;
try {
predictor = predictorPool.borrowObject();
return predictor.predict(image);
} catch (Exception e) {
throw new FaceException("目标检测错误", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
@Override
public void close() {
if (predictorPool != null) {
predictorPool.close();
}
}
}

View File

@@ -0,0 +1,111 @@
package cn.smartjavaai.face.model.facerec.criterial;
import ai.djl.Device;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.modality.cv.transform.Normalize;
import ai.djl.modality.cv.transform.Resize;
import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.modality.cv.translator.ImageFeatureExtractor;
import ai.djl.modality.cv.translator.ImageFeatureExtractorFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import ai.djl.translate.Translator;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceDetConfig;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.constant.FaceDetectConstant;
import cn.smartjavaai.face.constant.FaceNetConstant;
import cn.smartjavaai.face.constant.RetinaFaceConstant;
import cn.smartjavaai.face.constant.UltraLightFastGenericFaceConstant;
import cn.smartjavaai.face.enums.FaceDetModelEnum;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.model.facerec.translator.FaceFeatureTranslator;
import cn.smartjavaai.face.model.facerec.translator.FaceNetRecTranslator;
import cn.smartjavaai.face.translator.FaceDetectionTranslator;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* 人脸识别 Criteria构建工厂
* @author dwj
*/
public class FaceRecCriteriaFactory {
public static Criteria<Image, float[]> createCriteria(FaceRecConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu();
}
Criteria<Image, float[]> criteria = null;
if(config.getModelEnum() == FaceRecModelEnum.FACENET_MODEL){
criteria =
Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelName("face_feature") // specify model file prefix
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null :
FaceNetConstant.MODEL_URL)
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optTranslator(new FaceNetRecTranslator())
.optDevice(device)
.optEngine("PyTorch") // Use PyTorch engine
.optProgress(new ProgressBar())
.build();
}else if (config.getModelEnum() == FaceRecModelEnum.INSIGHT_FACE_MOBILE_FACENET_MODEL){
if(StringUtils.isBlank(config.getModelPath())){
throw new RuntimeException("请指定模型路径");
}
List<Float> mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f);
String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(","));
criteria = Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(config.getModelPath()))
// .optTranslatorFactory(new ImageFeatureExtractorFactory())
// .optArgument("normalize", normalize)
// .optArgument("resize", "112,112")
.optTranslator(new FaceFeatureTranslator())
.optEngine("PyTorch") // Use PyTorch engine
.optProgress(new ProgressBar())
.build();
}else if (config.getModelEnum() == FaceRecModelEnum.INSIGHT_FACE_IRSE50_MODEL){
if(StringUtils.isBlank(config.getModelPath())){
throw new RuntimeException("请指定模型路径");
}
List<Float> mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f);
String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(","));
criteria = Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(config.getModelPath()))
// .optTranslatorFactory(new ImageFeatureExtractorFactory())
// .optArgument("normalize", normalize)
// .optArgument("resize", "112,112")
.optTranslator(new FaceFeatureTranslator())
.optEngine("PyTorch") // Use PyTorch engine
.optProgress(new ProgressBar())
.build();
}else if (config.getModelEnum() == FaceRecModelEnum.ELASTIC_FACE_MODEL){
if(StringUtils.isBlank(config.getModelPath())){
throw new RuntimeException("请指定模型路径");
}
List<Float> mean = Arrays.asList(0.5f,0.5f,0.5f,0.5f,0.5f,0.5f);
String normalize = mean.stream().map(Object::toString).collect(Collectors.joining(","));
criteria = Criteria.builder()
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(config.getModelPath()))
// .optTranslatorFactory(new ImageFeatureExtractorFactory())
// .optArgument("normalize", normalize)
// .optArgument("resize", "112,112")
.optTranslator(new FaceFeatureTranslator())
.optEngine("PyTorch") // Use PyTorch engine
.optProgress(new ProgressBar())
.build();
}
return criteria;
}
}

View File

@@ -1,4 +1,4 @@
package cn.smartjavaai.face.translator;
package cn.smartjavaai.face.model.facerec.translator;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.transform.Normalize;
@@ -18,6 +18,8 @@ import ai.djl.translate.TranslatorContext;
*/
public final class FaceFeatureTranslator implements Translator<Image, float[]> {
public FaceFeatureTranslator() {
}
@@ -28,12 +30,14 @@ public final class FaceFeatureTranslator implements Translator<Image, float[]> {
public NDList processInput(TranslatorContext ctx, Image input) {
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
Pipeline pipeline = new Pipeline();
if(input.getWidth() != 112 || input.getHeight() != 112){
pipeline.add(new Resize(112));
}
pipeline
.add(new Resize(180))
.add(new ToTensor())
.add(new Normalize(
new float[]{127.5f / 255.0f, 127.5f / 255.0f, 127.5f / 255.0f},
new float[]{128.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f}));
new float[]{0.5F, 0.5F, 0.5F},
new float[]{0.5F, 0.5F, 0.5F}));
return pipeline.transform(new NDList(array));
}

View File

@@ -0,0 +1,57 @@
package cn.smartjavaai.face.model.facerec.translator;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.transform.Normalize;
import ai.djl.modality.cv.transform.Resize;
import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Pipeline;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
/**
* facenet人脸特征提取Translator
* @author dwj
* @date 2025/3/31
*/
public final class FaceNetRecTranslator implements Translator<Image, float[]> {
public FaceNetRecTranslator() {
}
/**
* {@inheritDoc}
*/
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
Pipeline pipeline = new Pipeline();
pipeline
//.add(new Resize(112))
.add(new ToTensor())
.add(new Normalize(
new float[]{0.5F, 0.5F, 0.5F},
new float[]{0.5F, 0.5F, 0.5F}));
return pipeline.transform(new NDList(array));
}
/**
* {@inheritDoc}
*/
@Override
public float[] processOutput(TranslatorContext ctx, NDList list) {
NDArray embedding = list.singletonOrThrow();
embedding = embedding.div(embedding.norm()); // L2归一化
return embedding.toFloatArray();
}
@Override
public Batchifier getBatchifier() {
return Batchifier.STACK;
}
}

View File

@@ -0,0 +1,421 @@
package cn.smartjavaai.face.model.liveness;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import ai.djl.util.JsonUtils;
import cn.smartjavaai.common.entity.DetectionInfo;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.preprocess.BufferedImagePreprocessor;
import cn.smartjavaai.common.utils.*;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.constant.MiniVisionConstant;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.liveness.criterial.LivenessCriteriaFactory;
import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator;
import com.seeta.sdk.FaceAntiSpoofing;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.file.Paths;
import java.util.*;
/**
* 通用活体检测模型
* @author dwj
*/
@Slf4j
public class CommonLivenessModel implements LivenessDetModel{
protected ObjectPool<Predictor<Image, Float>> predictorPool;
protected LivenessConfig config;
protected ZooModel<Image, Float> model;
@Override
public void loadModel(LivenessConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath不能为空");
}
this.config = config;
//设置真人阈值
Float realityThreshold = Objects.isNull(config.getRealityThreshold()) ? MiniVisionConstant.REALITY_THRESHOLD : config.getRealityThreshold();
this.config.setRealityThreshold(realityThreshold);
Criteria<Image, Float> criteria = LivenessCriteriaFactory.createCriteria(config);
try {
model = criteria.loadModel();
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("阿里通义实验室活体检测模型加载失败", e);
}
}
@Override
public R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
Predictor<Image, Float> predictor = null;
try {
predictor = predictorPool.borrowObject();
//预处理图片
BufferedImage processedImage = image;
if(config.getModelEnum() == LivenessModelEnum.IIC_FL_MODEL){
processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle)
.setExtendRatio(96f / 112f)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(128)
.enableCenterCrop(true)
.setCenterCropSize(112)
.process();
}
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage));
Float result = predictor.predict(djlImage);
if(result >= config.getRealityThreshold()){
return R.ok(new LivenessResult(LivenessStatus.LIVE, result));
}else{
float nonLiveScore = BigDecimal.ONE.subtract(new BigDecimal(result)).floatValue();
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore));
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
@Override
public R<LivenessResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionRectangle);
}
@Override
public R<LivenessResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<LivenessResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData, faceDetectionRectangle);
}
@Override
public R<List<LivenessResult>> detect(String imagePath, DetectionResponse faceDetectionResponse) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image, faceDetectionResponse);
}
@Override
public R<List<LivenessResult>> detect(byte[] imageData, DetectionResponse faceDetectionResponse) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<List<LivenessResult>> detect(BufferedImage image, DetectionResponse faceDetectionResponse) {
if(!ImageUtils.isImageValid(image)){
R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){
R.fail(R.Status.NO_FACE_DETECTED);
}
List<LivenessResult> livenessStatusList = new ArrayList<LivenessResult>();
for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){
R<LivenessResult> result = detect(image, detectionInfo.getDetectionRectangle());
if(!result.isSuccess()){
return R.fail(result.getCode(), result.getMessage());
}
livenessStatusList.add(result.getData());
}
return R.ok(livenessStatusList);
}
@Override
public R<List<LivenessResult>> detectBase64(String base64Image, DetectionResponse faceDetectionResponse) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData, faceDetectionResponse);
}
@Override
public R<LivenessResult> detectTopFace(BufferedImage image) {
if(Objects.isNull(config.getDetectModel())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型");
}
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
return detect(image, faceDetectionResponse.getData().getDetectionInfoList().get(0).getDetectionRectangle());
}
@Override
public R<LivenessResult> detectTopFace(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detectTopFace(image);
}
@Override
public R<LivenessResult> detectTopFace(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<LivenessResult> detectTopFaceBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detectTopFace(imageData);
}
@Override
public R<DetectionResponse> detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return detect(image);
}
@Override
public R<DetectionResponse> detect(BufferedImage image) {
if(Objects.isNull(config.getDetectModel())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "未指定检测模型");
}
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){
R<LivenessResult> result = detect(image, detectionInfo.getDetectionRectangle());
if(!result.isSuccess()){
return R.fail(result.getCode(), result.getMessage());
}
if(Objects.isNull(detectionInfo.getFaceInfo())){
detectionInfo.setFaceInfo(new FaceInfo());
}
detectionInfo.getFaceInfo().setLivenessStatus(result.getData());
}
return faceDetectionResponse;
}
@Override
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<DetectionResponse> detectBase64(String base64Image) {
if(StringUtils.isBlank(base64Image)){
return R.fail(R.Status.INVALID_IMAGE);
}
byte[] imageData = Base64ImageUtils.base64ToImage(base64Image);
return detect(imageData);
}
@Override
public R<LivenessResult> detectVideo(InputStream videoInputStream) {
if(Objects.isNull(videoInputStream)){
return R.fail(R.Status.INVALID_VIDEO);
}
return detectVideo(new FFmpegFrameGrabber(videoInputStream));
}
@Override
public R<LivenessResult> detectVideo(String videoPath) {
if(!FileUtils.isFileExists(videoPath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
return detectVideo(new FFmpegFrameGrabber(videoPath));
}
private R<LivenessResult> detectVideo(FFmpegFrameGrabber grabber) {
try {
//滑动窗口
Deque<Float> scoreWindow = new ArrayDeque<>();
grabber.start();
// 获取视频总帧数
int totalFrames = grabber.getLengthInFrames();
log.debug("视频总帧数:{},检测帧数:{}", totalFrames, config.getFrameCount());
if(totalFrames < config.getFrameCount()){
return R.fail(10001, "视频帧数低于检测帧数");
}
// 逐帧处理视频
for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) {
// 获取当前帧
Frame frame = grabber.grabImage();
if (frame != null) {
BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame);
R<LivenessResult> livenessStatus = detectTopFace(bufferedImage);
if(!livenessStatus.isSuccess()){
log.debug("" + frameIndex + "帧处理失败:" + livenessStatus.getMessage());
continue;
}else{
log.debug("" + frameIndex + "帧活体检测结果:" + JsonUtils.toJson(livenessStatus));
float liveScore = 0;
if(livenessStatus.getData().getStatus() == LivenessStatus.LIVE){
liveScore = livenessStatus.getData().getScore();
}else{
liveScore = BigDecimal.ONE.subtract(BigDecimal.valueOf(livenessStatus.getData().getScore())).floatValue();
}
scoreWindow.add(liveScore);
}
// 如果累计检测帧数 >= 配置值,开始判断
if (scoreWindow.size() >= config.getFrameCount()) {
float avgScore = (float) scoreWindow.stream()
.mapToDouble(Float::doubleValue)
.average()
.orElse(0.0);
log.debug("滑动窗口平均得分: {}", avgScore);
if (avgScore >= config.getRealityThreshold()) {
grabber.stop();
return R.ok(new LivenessResult(LivenessStatus.LIVE, avgScore));
} else {
grabber.stop();
float nonLiveScore = BigDecimal.ONE.subtract(BigDecimal.valueOf(avgScore)).floatValue();
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore));
}
}
}
}
grabber.stop();
if(scoreWindow.size() < config.getFrameCount()){
return R.fail(1000, "有效帧数量不足,无法完成活体检测");
}
} catch (Exception e) {
throw new FaceException(e);
}
return R.fail(R.Status.Unknown);
}
@Override
public void close() throws Exception {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
}

View File

@@ -3,8 +3,9 @@ package cn.smartjavaai.face.model.liveness;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.common.enums.LivenessStatus;
import java.awt.image.BufferedImage;
import java.io.InputStream;
@@ -14,7 +15,7 @@ import java.util.List;
* 活体检测模型
* @author dwj
*/
public interface LivenessDetModel {
public interface LivenessDetModel extends AutoCloseable{
/**
* 加载模型
@@ -28,7 +29,7 @@ public interface LivenessDetModel {
* @param imagePath 图片路径
* @return
*/
default DetectionResponse detect(String imagePath){
default R<DetectionResponse> detect(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
@@ -37,7 +38,7 @@ public interface LivenessDetModel {
* @param image BufferedImage
* @return
*/
default DetectionResponse detect(BufferedImage image){
default R<DetectionResponse> detect(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
@@ -46,29 +47,31 @@ public interface LivenessDetModel {
* @param imageData 图片字节流
* @return
*/
default DetectionResponse detect(byte[] imageData){
default R<DetectionResponse> detect(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param base64Image
* @return
*/
default R<DetectionResponse> detectBase64(String base64Image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param imagePath 图片路径
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default List<LivenessStatus> detect(String imagePath, DetectionResponse faceDetectionResponse){
default R<List<LivenessResult>> detect(String imagePath, DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imagePath 图片路径
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default LivenessStatus detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
@@ -76,38 +79,116 @@ public interface LivenessDetModel {
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default List<LivenessStatus> detect(byte[] imageData,DetectionResponse faceDetectionResponse){
default R<List<LivenessResult>> detect(byte[] imageData,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imageData
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default LivenessStatus detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param image BufferedImage
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default List<LivenessStatus> detect(BufferedImage image,DetectionResponse faceDetectionResponse){
default R<List<LivenessResult>> detect(BufferedImage image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(多人脸)
* @param base64Image
* @param faceDetectionResponse 人脸检测结果
* @return
*/
default R<List<LivenessResult>> detectBase64(String base64Image,DetectionResponse faceDetectionResponse){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imagePath 图片路径
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imageData
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param image BufferedImage
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default LivenessStatus detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
default R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param base64Image
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imagePath 图片路径
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param imageData
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param image BufferedImage
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(单人脸)
* @param base64Image
* @param faceDetectionRectangle 人脸检测结果-人脸框
* @return
*/
default R<LivenessResult> detectBase64(String base64Image, DetectionRectangle faceDetectionRectangle){
throw new UnsupportedOperationException("默认不支持该功能");
}
@@ -117,7 +198,7 @@ public interface LivenessDetModel {
* @param image
* @return
*/
default LivenessStatus detectTopFace(BufferedImage image){
default R<LivenessResult> detectTopFace(BufferedImage image){
throw new UnsupportedOperationException("默认不支持该功能");
}
@@ -127,7 +208,7 @@ public interface LivenessDetModel {
* @param imagePath
* @return
*/
default LivenessStatus detectTopFace(String imagePath){
default R<LivenessResult> detectTopFace(String imagePath){
throw new UnsupportedOperationException("默认不支持该功能");
}
@@ -136,20 +217,31 @@ public interface LivenessDetModel {
* @param imageData
* @return
*/
default LivenessStatus detectTopFace(byte[] imageData){
default R<LivenessResult> detectTopFace(byte[] imageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 活体检测(分数最高人脸)
* @param base64Image
* @return
*/
default R<LivenessResult> detectTopFaceBase64(String base64Image){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 视频活体检测(逐帧检测)
* @param frameImage
* @param faceDetectionRectangle
* @return
*/
default LivenessStatus detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
// default R<LivenessResult> detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
// throw new UnsupportedOperationException("默认不支持该功能");
// }
/**
* 视频活体检测(逐帧检测)
@@ -157,34 +249,34 @@ public interface LivenessDetModel {
* @param faceDetectionRectangle
* @return
*/
default LivenessStatus detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
// default R<LivenessResult> detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints){
// throw new UnsupportedOperationException("默认不支持该功能");
// }
/**
* 视频活体检测(逐帧检测)
* @param frameImageData
* @return
*/
default LivenessStatus detectVideoByFrame(byte[] frameImageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
// default R<LivenessResult> detectVideoByFrame(byte[] frameImageData){
// throw new UnsupportedOperationException("默认不支持该功能");
// }
/**
* 视频活体检测(逐帧检测)
* @param frameImageData
* @return
*/
default LivenessStatus detectVideoByFrame(BufferedImage frameImageData){
throw new UnsupportedOperationException("默认不支持该功能");
}
// default R<LivenessResult> detectVideoByFrame(BufferedImage frameImageData){
// throw new UnsupportedOperationException("默认不支持该功能");
// }
/**
* 视频活体检测
* @param videoInputStream
* @return
*/
default LivenessStatus detectVideo(InputStream videoInputStream){
default R<LivenessResult> detectVideo(InputStream videoInputStream){
throw new UnsupportedOperationException("默认不支持该功能");
}
@@ -193,7 +285,7 @@ public interface LivenessDetModel {
* @param videoPath
* @return
*/
default LivenessStatus detectVideo(String videoPath){
default R<LivenessResult> detectVideo(String videoPath){
throw new UnsupportedOperationException("默认不支持该功能");
}

View File

@@ -0,0 +1,265 @@
package cn.smartjavaai.face.model.liveness;
import ai.djl.Device;
import ai.djl.MalformedModelException;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.common.preprocess.BufferedImagePreprocessor;
import cn.smartjavaai.common.utils.ArrayUtils;
import cn.smartjavaai.common.utils.Base64ImageUtils;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.constant.MiniVisionConstant;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.model.liveness.translator.MiniVisionTranslator;
import cn.smartjavaai.common.utils.OpenCVUtils;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.file.Paths;
import java.util.*;
/**
* 小视科技 活体检测模型
* @author dwj
* @date 2025/6/27
*/
@Slf4j
public class MiniVisionLivenessModel extends CommonLivenessModel{
/**
* 个性化参数seModelPath
*/
private static final String SE_MODEL_PATH_KEY = "seModelPath";
private ObjectPool<Predictor<Image, float[]>> predictorPool;
private ObjectPool<Predictor<Image, float[]>> sePredictorPool;
/**
* 模型策略
*/
private ModelStrategy modelStrategy;
private ZooModel<Image, float[]> model;
private ZooModel<Image, float[]> seModel;
@Override
public void loadModel(LivenessConfig config) {
if(Objects.isNull(config)){
throw new FaceException("config为null");
}
String seModelPath = config.getCustomParam(SE_MODEL_PATH_KEY, String.class);
if(StringUtils.isBlank(config.getModelPath()) && StringUtils.isBlank(seModelPath)){
throw new FaceException("modelPath 和 seModelPath 至少有一个不能为空");
}
this.config = config;
//设置真人阈值
Float realityThreshold = Objects.isNull(config.getRealityThreshold()) ? MiniVisionConstant.REALITY_THRESHOLD : config.getRealityThreshold();
this.config.setRealityThreshold(realityThreshold);
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu();
}
if(StringUtils.isNotBlank(config.getModelPath()) && StringUtils.isBlank(seModelPath)){
//2.7_80x80_MiniFASNetV2
modelStrategy = ModelStrategy.MINIFASNET_V2;
}else if (StringUtils.isBlank(config.getModelPath()) && StringUtils.isNotBlank(seModelPath)){
//4_0_0_80x80_MiniFASNetV1SE
modelStrategy = ModelStrategy.MINIFASNET_V1_SE;
}else{
//融合
modelStrategy = ModelStrategy.FUSION;
}
if(modelStrategy == ModelStrategy.MINIFASNET_V2 || modelStrategy == ModelStrategy.FUSION){
//初始化 检测Criteria
Criteria<Image, float[]> criteria =
Criteria.builder()
.optEngine("OnnxRuntime")
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(config.getModelPath()))
.optTranslator(new MiniVisionTranslator())
.optProgress(new ProgressBar())
.build();
try {
model = criteria.loadModel();
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("MiniFASNetV2模型加载失败", e);
}
}
if(modelStrategy == ModelStrategy.MINIFASNET_V1_SE || modelStrategy == ModelStrategy.FUSION){
//初始化 检测Criteria
Criteria<Image, float[]> seCriteria =
Criteria.builder()
.optEngine("OnnxRuntime")
.setTypes(Image.class, float[].class)
.optModelPath(Paths.get(seModelPath))
.optTranslator(new MiniVisionTranslator())
.optProgress(new ProgressBar())
.build();
try {
seModel = seCriteria.loadModel();
this.sePredictorPool = new GenericObjectPool<>(new PredictorFactory<>(seModel));
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new FaceException("MiniFASNetV1SE模型加载失败", e);
}
}
}
@Override
public R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
Predictor<Image, float[]> predictor = null;
Predictor<Image, float[]> sePredictor = null;
try {
float[] result = null;
float[] seResult = null;
if(Objects.nonNull(predictorPool)){
//预处理图片
BufferedImage processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle)
.setExtendRatio(2.7f)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(80)
.process();
predictor = predictorPool.borrowObject();
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage));
result = predictor.predict(djlImage);
}
if(Objects.nonNull(sePredictorPool)){
//预处理图片
BufferedImage processedImage = new BufferedImagePreprocessor(image, faceDetectionRectangle)
.setExtendRatio(4)
.enableSquarePadding(true)
.enableScaling(true)
.setTargetSize(80)
.process();
Image djlImage = ImageFactory.getInstance().fromImage(OpenCVUtils.image2Mat(processedImage));
sePredictor = sePredictorPool.borrowObject();
seResult = sePredictor.predict(djlImage);
}
if(Objects.isNull(result) && Objects.isNull(seResult)){
throw new FaceException("活体检测错误");
}
//计算结果
int maxIndex = ArrayUtils.sumAndFindMaxIndex(result, seResult, 3);
BigDecimal score = Objects.isNull(result) ? BigDecimal.ZERO : BigDecimal.valueOf(result[maxIndex]);
BigDecimal seScore = Objects.isNull(seResult) ? BigDecimal.ZERO : BigDecimal.valueOf(seResult[maxIndex]);
BigDecimal avgSocre = score.add(seScore).divide(BigDecimal.valueOf(2), 2, RoundingMode.HALF_UP);
if(maxIndex == 1){
if(avgSocre.floatValue() >= config.getRealityThreshold()){
return R.ok(new LivenessResult(LivenessStatus.LIVE, avgSocre.floatValue()));
}else{
float nonLiveScore = BigDecimal.ONE.subtract(avgSocre).floatValue();
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, nonLiveScore));
}
}else{
return R.ok(new LivenessResult(LivenessStatus.NON_LIVE, avgSocre.floatValue()));
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
if (sePredictor != null) {
try {
sePredictorPool.returnObject(sePredictor); //归还
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
sePredictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
@Override
public void close() throws Exception {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (sePredictorPool != null) {
sePredictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 sePredictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
try {
if (seModel != null) {
seModel.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
/**
* 模型策略
*/
protected enum ModelStrategy {
MINIFASNET_V2,
MINIFASNET_V1_SE,
FUSION // 融合模型
}
}

View File

@@ -1,11 +1,14 @@
package cn.smartjavaai.face.model.liveness;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.common.enums.LivenessStatus;
import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.face.constant.LivenessConstant;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.utils.FaceUtils;
@@ -38,12 +41,14 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
private FaceAntiSpoofingPool faceAntiSpoofingPool;
private FaceLandmarkerPool faceLandmarkerPool;
private LivenessConfig config;
@Override
public void loadModel(LivenessConfig config) {
if(StringUtils.isBlank(config.getModelPath())){
throw new FaceException("modelPath is null");
}
this.config = config;
//加载依赖库
NativeLoader.loadNativeLibraries(config.getDevice());
log.debug("Loading seetaFace6 library successfully.");
@@ -54,8 +59,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
int gpuId = 0;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = config.getGpuId();
Integer gpuIdValue = config.getCustomParam("gpuId", Integer.class);
if(Objects.nonNull(gpuIdValue) && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = gpuIdValue;
}
}
@@ -72,36 +78,93 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
this.faceDetectorPool = new FaceDetectorPool(faceDetectorPoolConfSetting);
this.faceAntiSpoofingPool = new FaceAntiSpoofingPool(faceAntiSpoofingPoolConfSetting);
this.faceLandmarkerPool = new FaceLandmarkerPool(faceLandmarkerPoolConfSetting);
FaceAntiSpoofing faceAntiSpoofing = null;
//设置参数
try {
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
if(config.getFaceClarityThreshold() > 0 && config.getRealityThreshold() > 0){
faceAntiSpoofing.SetThreshold(config.getFaceClarityThreshold(), config.getRealityThreshold());
}
if(config.getFrameCount() > 0){
faceAntiSpoofing.SetVideoFrameCount(config.getFrameCount());
}
} catch (Exception e) {
throw new FaceException(e);
} finally {
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
//初始化模型参数
initConfig();
} catch (FileNotFoundException e) {
throw new FaceException(e);
}
}
/**
* 初始化模型参数
*/
private void initConfig(){
FaceAntiSpoofing faceAntiSpoofing = null;
//设置参数
try {
//人脸清晰度阈值
float faceClarityThreshold = LivenessConstant.DEFAULT_FACE_CLARITY_THRESHOLD;
//活体阈值
float realityThreshold = LivenessConstant.DEFAULT_REALITY_THRESHOLD;
Float faceClarityThresholdValue = config.getCustomParam("faceClarityThreshold", Float.class);
if(Objects.nonNull(faceClarityThresholdValue)){
faceClarityThreshold = faceClarityThresholdValue;
}
Float realityThresholdValue = config.getRealityThreshold();
if(Objects.nonNull(realityThresholdValue)){
realityThreshold = realityThresholdValue;
}
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
faceAntiSpoofing.SetThreshold(faceClarityThreshold, realityThreshold);
faceAntiSpoofing.SetVideoFrameCount(config.getFrameCount());
} catch (Exception e) {
throw new FaceException(e);
} finally {
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
private R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints, boolean isImage) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(faceDetectionRectangle)){
return R.fail(R.Status.NO_FACE_DETECTED);
}
if(keyPoints == null || keyPoints.isEmpty()){
return R.fail(1002,"人脸关键点keyPoints为空");
}
FaceAntiSpoofing.Status status = null;
FaceAntiSpoofing faceAntiSpoofing = null;
try {
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(faceDetectionRectangle);
SeetaPointF[] landmarks = FaceUtils.convertToSeetaPointF(keyPoints);
//检测图片
if(isImage){
status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks);
}else{
//检测视频
status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks);
}
return R.ok(new LivenessResult(FaceUtils.convertToLivenessStatus(status)));
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public DetectionResponse detect(String imagePath) {
public R<DetectionResponse> detect(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
@@ -114,9 +177,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
@Override
public DetectionResponse detect(byte[] imageData) {
public R<DetectionResponse> detect(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)));
@@ -126,36 +189,54 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
@Override
public DetectionResponse detect(BufferedImage image) {
public R<DetectionResponse> detect(BufferedImage image) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
FaceAntiSpoofing.Status status = null;
FaceAntiSpoofing faceAntiSpoofing = null;
FaceLandmarker faceLandmarker = null;
FaceDetector detectPredictor = null;
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
List<LivenessStatus> livenessStatusList = new ArrayList<LivenessStatus>();
try {
detectPredictor = faceDetectorPool.borrowObject();
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
throw new FaceException("无人脸数据");
//默认使用Seetaface6检测模型
if(Objects.isNull(config.getDetectModel())){
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
detectPredictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
List<SeetaPointF[]> seetaPointFSList = new ArrayList<SeetaPointF[]>();
List<LivenessStatus> livenessStatusList = new ArrayList<LivenessStatus>();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(SeetaRect seetaRect : seetaResult){
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, landmarks);
seetaPointFSList.add(landmarks);
//检测图片
FaceAntiSpoofing.Status status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks);
livenessStatusList.add(FaceUtils.convertToLivenessStatus(status));
}
return R.ok(FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList, livenessStatusList));
}else{
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
for(DetectionInfo detectionInfo : faceDetectionResponse.getData().getDetectionInfoList()){
R<LivenessResult> result = detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints());
if(!result.isSuccess()){
return R.fail(result.getCode(), result.getMessage());
}
if(Objects.isNull(detectionInfo.getFaceInfo())){
detectionInfo.setFaceInfo(new FaceInfo());
}
detectionInfo.getFaceInfo().setLivenessStatus(result.getData());
}
return faceDetectionResponse;
}
for(SeetaRect seetaRect : seetaResult){
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, landmarks);
seetaPointFSList.add(landmarks);
//检测图片
status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks);
livenessStatusList.add(FaceUtils.convertToLivenessStatus(status));
}
return FaceUtils.convertToDetectionResponse(seetaResult, seetaPointFSList, livenessStatusList);
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
@@ -184,9 +265,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
@Override
public List<LivenessStatus> detect(String imagePath, DetectionResponse faceDetectionResponse) {
public R<List<LivenessResult>> detect(String imagePath, DetectionResponse faceDetectionResponse) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
@@ -198,10 +279,48 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
return detect(image, faceDetectionResponse);
}
@Override
public LivenessStatus detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
public R<List<LivenessResult>> detect(BufferedImage image, DetectionResponse faceDetectionResponse) {
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
List<LivenessResult> livenessStatusList = new ArrayList<LivenessResult>();
for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){
FaceInfo faceInfo = detectionInfo.getFaceInfo();
if(Objects.isNull(faceInfo) || Objects.isNull(faceInfo.getKeyPoints())){
return R.fail(R.Status.Unknown.getCode(), "未检测到人脸关键点");
}
R<LivenessResult> result = detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints());
if(!result.isSuccess()){
return R.fail(result.getCode(), result.getMessage());
}
livenessStatusList.add(result.getData());
}
return R.ok(livenessStatusList);
}
@Override
public R<List<LivenessResult>> detect(byte[] imageData, DetectionResponse faceDetectionResponse) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<LivenessResult> detect(String imagePath, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
@@ -213,144 +332,15 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
return detect(image, faceDetectionRectangle, keyPoints);
}
private List<LivenessStatus> detect(BufferedImage image, DetectionResponse faceDetectionResponse,boolean isImage) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
if(Objects.isNull(faceDetectionResponse) || Objects.isNull(faceDetectionResponse.getDetectionInfoList()) || faceDetectionResponse.getDetectionInfoList().isEmpty()){
throw new FaceException("无人脸数据");
}
FaceAntiSpoofing.Status status = null;
FaceAntiSpoofing faceAntiSpoofing = null;
FaceLandmarker faceLandmarker = null;
List<LivenessStatus> livenessStatusList = new ArrayList<LivenessStatus>();
try {
for(DetectionInfo detectionInfo : faceDetectionResponse.getDetectionInfoList()){
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(detectionInfo.getDetectionRectangle());
SeetaPointF[] landmarks = null;
FaceInfo faceInfo = detectionInfo.getFaceInfo();
//如果没有人脸标识,则提取人脸标识
if(faceInfo == null || faceInfo.getKeyPoints() == null || faceInfo.getKeyPoints().isEmpty()){
//提取人脸的5点人脸标识
landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaRect, landmarks);
}else{
landmarks = FaceUtils.convertToSeetaPointF(faceInfo.getKeyPoints());
}
//检测图片
if(isImage){
status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks);
}else{
//检测视频
status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks);
}
livenessStatusList.add(FaceUtils.convertToLivenessStatus(status));
}
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
return livenessStatusList;
}
@Override
public List<LivenessStatus> detect(BufferedImage image, DetectionResponse faceDetectionResponse) {
return detect(image, faceDetectionResponse, true);
}
private LivenessStatus detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints, boolean isImage) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
}
if(Objects.isNull(faceDetectionRectangle)){
throw new FaceException("无人脸数据");
}
FaceAntiSpoofing.Status status = null;
FaceAntiSpoofing faceAntiSpoofing = null;
FaceLandmarker faceLandmarker = null;
try {
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(faceDetectionRectangle);
SeetaPointF[] landmarks = null;
if(keyPoints == null || keyPoints.isEmpty()){
throw new FaceException("人脸关键点keyPoints为空");
}
landmarks = FaceUtils.convertToSeetaPointF(keyPoints);
//检测图片
if(isImage){
status = faceAntiSpoofing.Predict(imageData, seetaRect, landmarks);
}else{
//检测视频
status = faceAntiSpoofing.PredictVideo(imageData, seetaRect, landmarks);
}
return FaceUtils.convertToLivenessStatus(status);
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
if (faceAntiSpoofing != null) {
try {
faceAntiSpoofingPool.returnObject(faceAntiSpoofing);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
if (faceLandmarker != null) {
try {
faceLandmarkerPool.returnObject(faceLandmarker);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public LivenessStatus detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
public R<LivenessResult> detect(BufferedImage image, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
return detect(image, faceDetectionRectangle, keyPoints, true);
}
@Override
public List<LivenessStatus> detect(byte[] imageData, DetectionResponse faceDetectionResponse) {
public R<LivenessResult> detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionResponse);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public LivenessStatus detect(byte[] imageData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(imageData)), faceDetectionRectangle, keyPoints);
@@ -361,9 +351,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
@Override
public LivenessStatus detectTopFace(String imagePath) {
public R<LivenessResult> detectTopFace(String imagePath) {
if(!FileUtils.isFileExists(imagePath)){
throw new FaceException("图像文件不存在");
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
@@ -376,34 +366,44 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
private LivenessStatus detectTopFace(BufferedImage image, boolean isImage) {
private R<LivenessResult> detectTopFace(BufferedImage image, boolean isImage) {
if(!ImageUtils.isImageValid(image)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
FaceAntiSpoofing.Status status = null;
FaceAntiSpoofing faceAntiSpoofing = null;
FaceLandmarker faceLandmarker = null;
FaceDetector detectPredictor = null;
try {
detectPredictor = faceDetectorPool.borrowObject();
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
throw new FaceException("无人脸数据");
}
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], landmarks);
//检测图片
if(isImage){
status = faceAntiSpoofing.Predict(imageData, seetaResult[0], landmarks);
//默认使用Seetaface6检测模型
if(Objects.isNull(config.getDetectModel())){
detectPredictor = faceDetectorPool.borrowObject();
faceLandmarker = faceLandmarkerPool.borrowObject();
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
//检测人脸
SeetaRect[] seetaResult = detectPredictor.Detect(imageData);
if(Objects.isNull(seetaResult)){
return R.fail(R.Status.NO_FACE_DETECTED);
}
SeetaPointF[] landmarks = new SeetaPointF[faceLandmarker.number()];
faceLandmarker.mark(imageData, seetaResult[0], landmarks);
FaceAntiSpoofing.Status status = null;
//检测图片
if(isImage){
status = faceAntiSpoofing.Predict(imageData, seetaResult[0], landmarks);
}else{
status = faceAntiSpoofing.PredictVideo(imageData, seetaResult[0], landmarks);
}
return R.ok(new LivenessResult(FaceUtils.convertToLivenessStatus(status)));
}else{
status = faceAntiSpoofing.PredictVideo(imageData, seetaResult[0], landmarks);
R<DetectionResponse> faceDetectionResponse = config.getDetectModel().detect(image);
if(Objects.isNull(faceDetectionResponse.getData()) || Objects.isNull(faceDetectionResponse.getData().getDetectionInfoList()) || faceDetectionResponse.getData().getDetectionInfoList().isEmpty()){
return R.fail(R.Status.NO_FACE_DETECTED);
}
DetectionInfo detectionInfo = faceDetectionResponse.getData().getDetectionInfoList().get(0);
return detect(image, detectionInfo.getDetectionRectangle(), detectionInfo.getFaceInfo().getKeyPoints(), isImage);
}
return FaceUtils.convertToLivenessStatus(status);
} catch (Exception e) {
throw new FaceException("活体检测错误", e);
} finally {
@@ -433,14 +433,14 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
@Override
public LivenessStatus detectTopFace(BufferedImage image) {
public R<LivenessResult> detectTopFace(BufferedImage image) {
return detectTopFace(image, true);
}
@Override
public LivenessStatus detectTopFace(byte[] imageData) {
public R<LivenessResult> detectTopFace(byte[] imageData) {
if(Objects.isNull(imageData)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detectTopFace(ImageIO.read(new ByteArrayInputStream(imageData)));
@@ -450,18 +450,16 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
@Override
public LivenessStatus detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
public R<LivenessResult> detectVideoByFrame(BufferedImage frameImage, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(!ImageUtils.isImageValid(frameImage)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
return detect(frameImage,faceDetectionRectangle, keyPoints,false);
}
@Override
public LivenessStatus detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
public R<LivenessResult> detectVideoByFrame(byte[] frameData, DetectionRectangle faceDetectionRectangle, List<Point> keyPoints) {
if(Objects.isNull(frameData)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detect(ImageIO.read(new ByteArrayInputStream(frameData)), faceDetectionRectangle, keyPoints, false);
@@ -470,10 +468,9 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
}
@Override
public LivenessStatus detectVideoByFrame(byte[] frameImageData) {
public R<LivenessResult> detectVideoByFrame(byte[] frameImageData) {
if(Objects.isNull(frameImageData)){
throw new FaceException("图像无效");
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return detectVideoByFrame(ImageIO.read(new ByteArrayInputStream(frameImageData)));
@@ -482,13 +479,12 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
}
@Override
public LivenessStatus detectVideoByFrame(BufferedImage frameImageData) {
public R<LivenessResult> detectVideoByFrame(BufferedImage frameImageData) {
return detectTopFace(frameImageData, false);
}
@Override
public LivenessStatus detectVideo(InputStream videoInputStream) {
public R<LivenessResult> detectVideo(InputStream videoInputStream) {
if(Objects.isNull(videoInputStream)){
throw new FaceException("视频无效");
}
@@ -496,24 +492,26 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
@Override
public LivenessStatus detectVideo(String videoPath) {
public R<LivenessResult> detectVideo(String videoPath) {
if(!FileUtils.isFileExists(videoPath)){
throw new FaceException("视频文件不存在");
}
return detectVideo(new FFmpegFrameGrabber(videoPath));
}
private LivenessStatus detectVideo(FFmpegFrameGrabber grabber) {
private R<LivenessResult> detectVideo(FFmpegFrameGrabber grabber) {
FaceAntiSpoofing faceAntiSpoofing = null;
try {
faceAntiSpoofing = faceAntiSpoofingPool.borrowObject();
//重置视频
faceAntiSpoofing.ResetVideo();
grabber.start();
// 获取视频总帧数
int totalFrames = grabber.getLengthInFrames();
int videoFrameCountConfig = faceAntiSpoofing.GetVideoFrameCount();
log.debug("视频总帧数:{},检测帧数:{}", totalFrames, videoFrameCountConfig);
if(totalFrames < videoFrameCountConfig){
throw new FaceException("视频帧数低于检测帧数");
return R.fail(1001, "视频帧数低于检测帧数");
}
// 逐帧处理视频
for (int frameIndex = 0; frameIndex < totalFrames; frameIndex++) {
@@ -521,9 +519,13 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
Frame frame = grabber.grabImage();
if (frame != null) {
BufferedImage bufferedImage = Java2DFrameUtils.toBufferedImage(frame);
LivenessStatus livenessStatus = detectVideoByFrame(bufferedImage);
R<LivenessResult> livenessStatus = detectVideoByFrame(bufferedImage);
if(!livenessStatus.isSuccess()){
log.debug("" + frameIndex + "帧处理失败:" + livenessStatus.getMessage());
continue;
}
//满足检测帧数之后停止检测
if(livenessStatus != LivenessStatus.DETECTING){
if(livenessStatus.getData().getStatus() != LivenessStatus.DETECTING){
return livenessStatus;
}
}
@@ -542,6 +544,31 @@ public class Seetaface6LivenessModel implements LivenessDetModel{
}
}
}
return LivenessStatus.UNKNOWN;
return R.fail(1000, "有效帧数量不足,无法完成活体检测");
}
@Override
public void close() throws Exception {
try {
if (faceDetectorPool != null) {
faceDetectorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (faceAntiSpoofingPool != null) {
faceAntiSpoofingPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (faceLandmarkerPool != null) {
faceLandmarkerPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
}
}

View File

@@ -0,0 +1,50 @@
package cn.smartjavaai.face.model.liveness.criterial;
import ai.djl.Device;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.translator.ImageFeatureExtractorFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceRecConfig;
import cn.smartjavaai.face.config.LivenessConfig;
import cn.smartjavaai.face.constant.FaceNetConstant;
import cn.smartjavaai.face.enums.FaceRecModelEnum;
import cn.smartjavaai.face.enums.LivenessModelEnum;
import cn.smartjavaai.face.model.facerec.translator.FaceNetRecTranslator;
import cn.smartjavaai.face.model.liveness.translator.IicFrTranslator;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* 活体检测 Criteria构建工厂
* @author dwj
*/
public class LivenessCriteriaFactory {
public static Criteria<Image, Float> createCriteria(LivenessConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu();
}
Criteria<Image, Float> criteria = null;
if(config.getModelEnum() == LivenessModelEnum.IIC_FL_MODEL){
criteria = Criteria.builder()
.optEngine("OnnxRuntime")
.setTypes(ai.djl.modality.cv.Image.class, Float.class)
.optModelPath(Paths.get(config.getModelPath()))
.optTranslator(new IicFrTranslator())
.optProgress(new ProgressBar())
.optDevice(device)
.build();
}
return criteria;
}
}

View File

@@ -0,0 +1,42 @@
package cn.smartjavaai.face.model.liveness.translator;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.translate.Batchifier;
import ai.djl.translate.Translator;
import ai.djl.translate.TranslatorContext;
import java.util.Arrays;
public class IicFrTranslator implements Translator<Image, Float> {
@Override
public Float processOutput(TranslatorContext ctx, NDList list) {
NDArray prob = list.singletonOrThrow();
return prob.toFloatArray()[1];
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDManager manager = ctx.getNDManager();
NDArray array = input.toNDArray(manager, Image.Flag.COLOR);
array = array.transpose(2, 0, 1);
array = array.expandDims(0);
// 归一化
array = array.toType(DataType.FLOAT32, false).div(255.0f);
return new NDList(array);
}
@Override
public Batchifier getBatchifier() {
return null;
}
}

View File

@@ -0,0 +1,45 @@
package cn.smartjavaai.face.model.liveness.translator;
import ai.djl.modality.Classifications;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.transform.Pad;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.translate.*;
import java.util.Arrays;
/**
* minivision translator
* @author dwj
* @date 2025/6/27
*/
public class MiniVisionTranslator implements Translator<Image, float[]> {
@Override
public float[] processOutput(TranslatorContext ctx, NDList list) {
NDArray prob = list.singletonOrThrow();
NDArray softmax = prob.softmax(-1);
return softmax.toFloatArray();
}
@Override
public NDList processInput(TranslatorContext ctx, Image input) {
NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
array = array.toType(ai.djl.ndarray.types.DataType.FLOAT32, false);
// 调整数据布局: HWC -> CHW
array = array.transpose(2, 0, 1);
// 添加batch维度 (NCHW)
array = array.expandDims(0);
return new NDList(array);
}
@Override
public Batchifier getBatchifier() {
return null;
}
}

View File

@@ -0,0 +1,229 @@
package cn.smartjavaai.face.model.quality;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.face.config.QualityConfig;
import cn.smartjavaai.face.entity.FaceQualitySummary;
import cn.smartjavaai.face.entity.FaceQualityResult;
import java.awt.image.BufferedImage;
import java.util.List;
/**
* 质量评估模型
* @author dwj
* @date 2025/6/23
*/
public interface FaceQualityModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(QualityConfig config);
/**
* 亮度评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateBrightness(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 亮度评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateBrightness(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 亮度评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateBrightness(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 清晰度评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateClarity(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 清晰度评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateClarity(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 清晰度评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateClarity(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 完整度评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateCompleteness(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 完整度评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateCompleteness(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 完整度评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateCompleteness(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸姿态评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluatePose(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸姿态评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluatePose(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸姿态评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluatePose(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸分辨率评估
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateResolution(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸分辨率评估
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateResolution(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 人脸分辨率评估
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualityResult> evaluateResolution(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 评估所有
* @param imagePath
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualitySummary> evaluateAll(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 评估所有
* @param image
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualitySummary> evaluateAll(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 评估所有
* @param imageData
* @param rectangle
* @param keyPoints
* @return
*/
default R<FaceQualitySummary> evaluateAll(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

View File

@@ -0,0 +1,717 @@
package cn.smartjavaai.face.model.quality;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.common.utils.FileUtils;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.PoolUtils;
import cn.smartjavaai.face.config.QualityConfig;
import cn.smartjavaai.face.entity.FaceQualityResult;
import cn.smartjavaai.face.entity.FaceQualitySummary;
import cn.smartjavaai.face.enums.QualityGrade;
import cn.smartjavaai.face.exception.FaceException;
import cn.smartjavaai.face.seetaface.ClarityDLResult;
import cn.smartjavaai.face.seetaface.NativeLoader;
import cn.smartjavaai.face.utils.FaceUtils;
import com.seeta.pool.*;
import com.seeta.sdk.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;
import java.util.Objects;
/**
* seetaface6 质量评估模型
* @author dwj
* @date 2025/4/30
*/
@Slf4j
public class Seetaface6QualityModel implements FaceQualityModel {
private QualityConfig config;
/**
* 人脸亮度评估器池
*/
private QualityOfBrightnessPool qualityOfBrightnessPool;
/**
* 人脸清晰度评估器池
*/
private QualityOfClarityPool qualityOfClarityPool;
/**
* 人脸清晰度评估器池(深度学习)
*/
private QualityOfLBNPool qualityOfLBNPool;
/**
* 人脸完整度评估器池
*/
private QualityOfIntegrityPool qualityOfIntegrityPool;
/**
* 人脸姿态评估器池
*/
private QualityOfPosePool qualityOfPosePool;
/**
* 人脸姿态评估器池(深度学习)
*/
private QualityOfPoseExPool qualityOfPoseExPool;
/**
* 人脸分辨率评估器池
*/
private QualityOfResolutionPool qualityOfResolutionPool;
@Override
public void loadModel(QualityConfig config) {
DeviceEnum device = DeviceEnum.CPU;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice();
}
//加载依赖库
NativeLoader.loadNativeLibraries(device);
this.config = config;
log.debug("Loading seetaFace6 library successfully.");
}
@Override
public R<FaceQualityResult> evaluateBrightness(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfBrightness qualityOfBrightness = null;
try {
if(Objects.isNull(this.qualityOfBrightnessPool)){
this.qualityOfBrightnessPool = new QualityOfBrightnessPool(new SeetaConfSetting());
}
qualityOfBrightness = qualityOfBrightnessPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfBrightness.QualityLevel level = qualityOfBrightness.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("亮度评估错误", e);
} finally {
if (qualityOfBrightness != null) {
try {
qualityOfBrightnessPool.returnObject(qualityOfBrightness);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluateBrightness(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateBrightness(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluateBrightness(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateBrightness(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceQualityResult> evaluateClarity(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfClarity qualityOfClarity = null;
try {
if(Objects.isNull(this.qualityOfClarityPool)){
this.qualityOfClarityPool = new QualityOfClarityPool(new SeetaConfSetting());
}
qualityOfClarity = qualityOfClarityPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfClarity.QualityLevel level = qualityOfClarity.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("清晰度评估错误", e);
} finally {
if (qualityOfClarity != null) {
try {
qualityOfClarityPool.returnObject(qualityOfClarity);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluateClarity(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateClarity(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluateClarity(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateClarity(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceQualityResult> evaluateCompleteness(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfIntegrity qualityOfIntegrity = null;
try {
if(Objects.isNull(this.qualityOfIntegrityPool)){
this.qualityOfIntegrityPool = new QualityOfIntegrityPool(new SeetaConfSetting());
}
qualityOfIntegrity = qualityOfIntegrityPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfIntegrity.QualityLevel level = qualityOfIntegrity.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("完整度评估错误", e);
} finally {
if (qualityOfIntegrity != null) {
try {
qualityOfIntegrityPool.returnObject(qualityOfIntegrity);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluateCompleteness(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateCompleteness(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluateCompleteness(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateCompleteness(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceQualityResult> evaluatePose(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfPose qualityOfPose = null;
try {
if(Objects.isNull(this.qualityOfPosePool)){
this.qualityOfPosePool = new QualityOfPosePool(new SeetaConfSetting());
}
qualityOfPose = qualityOfPosePool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfPose.QualityLevel level = qualityOfPose.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("姿态评估错误", e);
} finally {
if (qualityOfPose != null) {
try {
qualityOfPosePool.returnObject(qualityOfPose);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluatePose(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluatePose(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluatePose(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluatePose(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public R<FaceQualityResult> evaluateResolution(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfResolution qualityOfResolution = null;
try {
if(Objects.isNull(this.qualityOfResolutionPool)){
this.qualityOfResolutionPool = new QualityOfResolutionPool(new SeetaConfSetting());
}
qualityOfResolution = qualityOfResolutionPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scores = new float[1];
QualityOfResolution.QualityLevel level = qualityOfResolution.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("姿态评估错误", e);
} finally {
if (qualityOfResolution != null) {
try {
qualityOfResolutionPool.returnObject(qualityOfResolution);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
@Override
public R<FaceQualityResult> evaluateResolution(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateResolution(image, rectangle, keyPoints);
}
@Override
public R<FaceQualityResult> evaluateResolution(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateResolution(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
public R<ClarityDLResult> evaluateClarityWithDL(BufferedImage image, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfLBN qualityOfLBN = null;
try {
if(Objects.isNull(this.qualityOfLBNPool)){
if(Objects.isNull(config)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "缺少必要配置QualityConfig请在调用前初始化模型配置");
}
if(StringUtils.isBlank(config.getModelPath())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "QualityConfig中modelPath为空");
}
SeetaConfSetting setting = getClarityMLSetting();
this.qualityOfLBNPool = new QualityOfLBNPool(setting);
}
qualityOfLBN = qualityOfLBNPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
int[] light = new int[1];
int[] blur = new int[1];
int[] noise = new int[1];
qualityOfLBN.Detect(imageData, pointFS, light, blur, noise);
return R.ok(new ClarityDLResult(light, blur, noise));
} catch (Exception e) {
throw new FaceException("亮度评估错误", e);
} finally {
if (qualityOfLBN != null) {
try {
qualityOfLBNPool.returnObject(qualityOfLBN);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
public R<ClarityDLResult> evaluateClarityWithDL(String imagePath, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateClarityWithDL(image, keyPoints);
}
public R<ClarityDLResult> evaluateClarityWithDL(byte[] imageData, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateClarityWithDL(ImageIO.read(new ByteArrayInputStream(imageData)), keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
public R<FaceQualityResult> evaluatePoseWithDL(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfPoseEx qualityOfPoseEx = null;
try {
if(Objects.isNull(this.qualityOfPoseExPool)){
if(Objects.isNull(config)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "缺少必要配置QualityConfig请在调用前初始化模型配置");
}
if(StringUtils.isBlank(config.getModelPath())){
return R.fail(R.Status.PARAM_ERROR.getCode(), "QualityConfig中modelPath为空");
}
SeetaConfSetting setting = getPoseMLSetting();
this.qualityOfPoseExPool = new QualityOfPoseExPool(setting);
}
qualityOfPoseEx = qualityOfPoseExPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
float[] scores = new float[1];
QualityOfPoseEx.QualityLevel level = qualityOfPoseEx.check(imageData, seetaRect, pointFS, scores);
FaceQualityResult result = new FaceQualityResult(scores[0], QualityGrade.valueOf(level.name()));
return R.ok(result);
} catch (Exception e) {
throw new FaceException("亮度评估错误", e);
} finally {
if (qualityOfPoseEx != null) {
try {
qualityOfPoseExPool.returnObject(qualityOfPoseEx);
} catch (Exception e) {
log.warn("归还Predictor失败", e);
}
}
}
}
public R<FaceQualityResult> evaluatePoseWithDL(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluatePoseWithDL(image, rectangle, keyPoints);
}
public R<FaceQualityResult> evaluatePoseWithDL(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluatePoseWithDL(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
/**
* 获取清晰度模型配置(深度学习)
* @return
*/
private SeetaConfSetting getClarityMLSetting() throws FileNotFoundException {
String[] modelPath = {config.getModelPath() + File.separator + "quality_lbn.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = 0;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = config.getGpuId();
}
}
SeetaConfSetting setting = new SeetaConfSetting(new SeetaModelSetting(gpuId, modelPath, device));
return setting;
}
/**
* 获取人脸姿态模型配置(深度学习)
* @return
*/
private SeetaConfSetting getPoseMLSetting() throws FileNotFoundException {
String[] modelPath = {config.getModelPath() + File.separator + "pose_estimation.csta"};
SeetaDevice device = SeetaDevice.SEETA_DEVICE_AUTO;
int gpuId = 0;
if(Objects.nonNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? SeetaDevice.SEETA_DEVICE_CPU : SeetaDevice.SEETA_DEVICE_GPU;
if(config.getGpuId() >= 0 && device == SeetaDevice.SEETA_DEVICE_GPU){
gpuId = config.getGpuId();
}
}
SeetaConfSetting setting = new SeetaConfSetting(new SeetaModelSetting(gpuId, modelPath, device));
return setting;
}
@Override
public R<FaceQualitySummary> evaluateAll(String imagePath, DetectionRectangle rectangle, List<Point> keyPoints) {
if(!FileUtils.isFileExists(imagePath)){
return R.fail(R.Status.FILE_NOT_FOUND);
}
// 将图片路径转换为 BufferedImage
BufferedImage image = null;
try {
image = ImageIO.read(new File(Paths.get(imagePath).toAbsolutePath().toString()));
} catch (IOException e) {
throw new FaceException("无效图片路径", e);
}
return evaluateAll(image, rectangle, keyPoints);
}
@Override
public R<FaceQualitySummary> evaluateAll(BufferedImage image, DetectionRectangle rectangle, List<Point> keyPoints) {
//参数检查
if(!ImageUtils.isImageValid(image)){
return R.fail(R.Status.INVALID_IMAGE);
}
if(Objects.isNull(rectangle)){
return R.fail(R.Status.PARAM_ERROR.getCode(), "rectangle为空");
}
if(Objects.isNull(keyPoints) || keyPoints.isEmpty()){
return R.fail(R.Status.PARAM_ERROR.getCode(), "keyPoints为空");
}
QualityOfBrightness qualityOfBrightness = null;
QualityOfClarity qualityOfClarity = null;
QualityOfIntegrity qualityOfIntegrity = null;
QualityOfPose qualityOfPose = null;
QualityOfResolution qualityOfResolution = null;
try {
if(Objects.isNull(this.qualityOfBrightnessPool)){
this.qualityOfBrightnessPool = new QualityOfBrightnessPool(new SeetaConfSetting());
}
if(Objects.isNull(this.qualityOfClarityPool)){
this.qualityOfClarityPool = new QualityOfClarityPool(new SeetaConfSetting());
}
if(Objects.isNull(this.qualityOfIntegrityPool)){
this.qualityOfIntegrityPool = new QualityOfIntegrityPool(new SeetaConfSetting());
}
if(Objects.isNull(this.qualityOfPosePool)){
this.qualityOfPosePool = new QualityOfPosePool(new SeetaConfSetting());
}
if(Objects.isNull(this.qualityOfResolutionPool)){
this.qualityOfResolutionPool = new QualityOfResolutionPool(new SeetaConfSetting());
}
FaceQualitySummary summary = new FaceQualitySummary();
qualityOfBrightness = qualityOfBrightnessPool.borrowObject();
qualityOfClarity = qualityOfClarityPool.borrowObject();
qualityOfIntegrity = qualityOfIntegrityPool.borrowObject();
qualityOfPose = qualityOfPosePool.borrowObject();
qualityOfResolution = qualityOfResolutionPool.borrowObject();
SeetaImageData imageData = new SeetaImageData(image.getWidth(), image.getHeight(), 3);
imageData.data = ImageUtils.getMatrixBGR(image);
SeetaRect seetaRect = FaceUtils.convertToSeetaRect(rectangle);
SeetaPointF[] pointFS = FaceUtils.convertToSeetaPointF(keyPoints);
float[] scoresBrightness = new float[1];
QualityOfBrightness.QualityLevel level = qualityOfBrightness.check(imageData, seetaRect, pointFS, scoresBrightness);
summary.setBrightness(new FaceQualityResult(scoresBrightness[0], QualityGrade.valueOf(level.name())));
float[] scoresClarity = new float[1];
QualityOfClarity.QualityLevel clarityLevel = qualityOfClarity.check(imageData, seetaRect, pointFS, scoresClarity);
summary.setClarity(new FaceQualityResult(scoresClarity[0], QualityGrade.valueOf(clarityLevel.name())));
float[] scoresIntegrity = new float[1];
QualityOfIntegrity.QualityLevel integrityLevel = qualityOfIntegrity.check(imageData, seetaRect, pointFS, scoresIntegrity);
summary.setCompleteness(new FaceQualityResult(scoresIntegrity[0], QualityGrade.valueOf(integrityLevel.name())));
float[] scoresPose = new float[1];
QualityOfPose.QualityLevel poseLevel = qualityOfPose.check(imageData, seetaRect, pointFS, scoresPose);
summary.setPose(new FaceQualityResult(scoresPose[0], QualityGrade.valueOf(poseLevel.name())));
float[] scoresResolution = new float[1];
QualityOfResolution.QualityLevel resolutionLevel = qualityOfResolution.check(imageData, seetaRect, pointFS, scoresResolution);
summary.setResolution(new FaceQualityResult(scoresResolution[0], QualityGrade.valueOf(resolutionLevel.name())));
return R.ok(summary);
} catch (Exception e) {
throw new FaceException("亮度评估错误", e);
} finally {
PoolUtils.returnToPool(qualityOfBrightnessPool, qualityOfBrightness);
PoolUtils.returnToPool(qualityOfClarityPool, qualityOfClarity);
PoolUtils.returnToPool(qualityOfIntegrityPool, qualityOfIntegrity);
PoolUtils.returnToPool(qualityOfPosePool, qualityOfPose);
PoolUtils.returnToPool(qualityOfResolutionPool, qualityOfResolution);
}
}
@Override
public R<FaceQualitySummary> evaluateAll(byte[] imageData, DetectionRectangle rectangle, List<Point> keyPoints) {
if(Objects.isNull(imageData)){
return R.fail(R.Status.INVALID_IMAGE);
}
try {
return evaluateAll(ImageIO.read(new ByteArrayInputStream(imageData)), rectangle, keyPoints);
} catch (IOException e) {
throw new FaceException("错误的图像", e);
}
}
@Override
public void close() throws Exception {
if(Objects.nonNull(qualityOfBrightnessPool)){
qualityOfBrightnessPool.close();
}
if(Objects.nonNull(qualityOfClarityPool)){
qualityOfClarityPool.close();
}
if(Objects.nonNull(qualityOfLBNPool)){
qualityOfLBNPool.close();
}
if(Objects.nonNull(qualityOfIntegrityPool)){
qualityOfIntegrityPool.close();
}
if(Objects.nonNull(qualityOfPosePool)){
qualityOfPosePool.close();
}
if(Objects.nonNull(qualityOfPoseExPool)){
qualityOfPoseExPool.close();
}
if(Objects.nonNull(qualityOfResolutionPool)){
qualityOfResolutionPool.close();
}
}
}

View File

@@ -0,0 +1,94 @@
package cn.smartjavaai.face.preprocess;
import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.opencv.OpenCVImageFactory;
import cn.smartjavaai.common.entity.DetectionRectangle;
import cn.smartjavaai.common.utils.OpenCVUtils;
import cn.smartjavaai.face.utils.FaceAlignUtils;
import cn.smartjavaai.face.utils.FaceUtils;
import org.opencv.core.Mat;
import java.util.Objects;
/**
* 图片预处理
* @author dwj
* @date 2025/6/27
*/
public class DJLImagePreprocessor {
private Image image;
private NDManager manager;
// 裁剪参数
private boolean enableCrop = false;
private DetectionRectangle cropRect;
// 仿射变换参数
private boolean enableAffine = false;
private double[][] keyPoints;
private int affineTargetWidth;
private int affineTargetHeight;
public DJLImagePreprocessor(Image image, NDManager manager) {
this.image = image;
this.manager = manager;
}
// 启用裁剪
public DJLImagePreprocessor enableCrop(DetectionRectangle rect) {
this.enableCrop = true;
this.cropRect = rect;
return this;
}
// 启用仿射变换
public DJLImagePreprocessor enableAffine(double[][] keyPoints, int targetWidth, int targetHeight) {
if(Objects.isNull(keyPoints)){
throw new IllegalArgumentException("keyPoints must be not null");
}
this.enableAffine = true;
this.affineTargetWidth = targetWidth;
this.affineTargetHeight = targetHeight;
this.keyPoints = keyPoints;
return this;
}
// 处理流程
public Image process() {
Image result = image;
if (enableCrop) {
result = result.getSubImage(cropRect.x, cropRect.y, cropRect.width, cropRect.height);
}
if (enableAffine) {
result = warpAffine(keyPoints, affineTargetWidth, affineTargetHeight);
}
return result;
}
// 仿射变换
private Image warpAffine(double[][] keyPoints, int width, int height) {
NDArray srcPoints = manager.create(keyPoints);
NDArray dstPoints = null;
if(width == 512 && height == 512){
dstPoints = FaceUtils.faceTemplate512x512(manager);
}else if(width == 112 && height == 112){
dstPoints = FaceUtils.faceTemplate112x112(manager);
}else if(width == 96 && height == 112){
dstPoints = FaceUtils.faceTemplate96x112(manager);
}
// 5点仿射变换
Mat affine_matrix = OpenCVUtils.toOpenCVMat(manager, srcPoints, dstPoints);
Mat mat = FaceAlignUtils.warpAffine((Mat) image.getWrappedImage(), affine_matrix, width, height);
Image alignedImg = OpenCVImageFactory.getInstance().fromImage(mat);
return alignedImg;
}
}

View File

@@ -0,0 +1,24 @@
package cn.smartjavaai.face.seetaface;
import com.seeta.sdk.QualityOfLBN;
import lombok.Data;
/**
* Seetaface6 清晰度(深度学习)结果
* @author dwj
* @date 2025/6/25
*/
@Data
public class ClarityDLResult {
private QualityOfLBN.LIGHTSTATE lightstate;
private QualityOfLBN.BLURSTATE blurstate;
private QualityOfLBN.NOISESTATE noisestate;
public ClarityDLResult(int[] light, int[] blur, int[] noise) {
this.lightstate = QualityOfLBN.LIGHTSTATE.values()[light[0]];
this.blurstate = QualityOfLBN.BLURSTATE.values()[blur[0]];
this.noisestate = QualityOfLBN.NOISESTATE.values()[noise[0]];
}
}

View File

@@ -7,7 +7,6 @@ import cn.hutool.system.OsInfo;
import cn.hutool.system.SystemUtil;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.face.exception.FaceException;
import com.seeta.sdk.util.DllItem;
import com.seeta.sdk.util.LoadNativeCore;

View File

@@ -5,6 +5,7 @@ import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.opencv.OpenCVImageFactory;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.common.utils.OpenCVUtils;
import com.seeta.sdk.SeetaImageData;
import com.seeta.sdk.SeetaPointF;
import org.opencv.core.Mat;
@@ -39,7 +40,8 @@ public class FaceAlignUtils {
public static Mat warpAffine(Mat src, Mat rot_mat, int width, int height) {
Mat rot = new Mat();
Size size = new Size(width, height);
Imgproc.warpAffine(src, rot, rot_mat, size);
Scalar scalar = new Scalar(135, 133, 132);
Imgproc.warpAffine(src, rot, rot_mat, size,0, 0, scalar);
return rot;
}

View File

@@ -7,11 +7,14 @@ import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import cn.smartjavaai.common.entity.*;
import cn.smartjavaai.common.entity.Point;
import cn.smartjavaai.common.enums.EyeStatus;
import cn.smartjavaai.common.enums.GenderType;
import cn.smartjavaai.common.entity.face.FaceAttribute;
import cn.smartjavaai.common.entity.face.FaceInfo;
import cn.smartjavaai.common.entity.face.HeadPose;
import cn.smartjavaai.common.entity.face.LivenessResult;
import cn.smartjavaai.common.enums.face.EyeStatus;
import cn.smartjavaai.common.enums.face.GenderType;
import cn.smartjavaai.common.utils.ImageUtils;
import cn.smartjavaai.face.config.FaceModelConfig;
import cn.smartjavaai.common.enums.LivenessStatus;
import cn.smartjavaai.common.enums.face.LivenessStatus;
import cn.smartjavaai.face.exception.FaceException;
import com.seeta.sdk.*;
@@ -340,6 +343,44 @@ public class FaceUtils {
return points;
}
/**
* 112x112的目标点 - Target point of 112x112
* standard 5 landmarks for FFHQ faces with 112x112
*
* @param manager
* @return
*/
public static NDArray faceTemplate112x112(NDManager manager) {
double[][] coord5point = {
{30.29459953, 51.69630051}, // 112x112的目标点 - Target point of 512x512
{65.53179932, 51.50139999},
{48.02519989, 71.73660278},
{33.54930115, 87},
{62.72990036, 87}
};
NDArray points = manager.create(coord5point);
return points;
}
/**
* 96x112的目标点 - Target point of 96x112
* standard 5 landmarks for FFHQ faces with 96x112
*
* @param manager
* @return
*/
public static NDArray faceTemplate96x112(NDManager manager) {
double[][] coord5point = {
{30.29459953, 51.69630051},
{65.53179932, 51.50139999},
{48.02519989, 71.73660278},
{33.54930115, 92.3655014},
{62.72990036, 92.20410156}
};
NDArray points = manager.create(coord5point);
return points;
}
/**
* bgr转图片
* @return 图片
@@ -507,7 +548,7 @@ public class FaceUtils {
.map(p -> new Point(p.x, p.y))
.collect(Collectors.toList());
FaceInfo faceInfo = new FaceInfo(keyPoints);
faceInfo.setLivenessStatus(livenessStatusList.get(i));
faceInfo.setLivenessStatus(new LivenessResult(livenessStatusList.get(i)));
DetectionInfo detectionInfo = new DetectionInfo(rectangle, 0, faceInfo);
detectionInfoList.add(detectionInfo);
}

View File

@@ -1,137 +0,0 @@
package cn.smartjavaai.face.utils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.imgproc.Imgproc;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
/**
* OpenCV 工具类
*/
public class OpenCVUtils {
/**
* canny算法边缘检测
*
* @param src
* @return
*/
public static Mat canny(Mat src) {
Mat mat = src.clone();
Imgproc.Canny(src, mat, 100, 200);
return mat;
}
/**
* 画线
*
* @param mat
* @param point1
* @param point2
*/
public static void line(Mat mat, Point point1, Point point2) {
Imgproc.line(mat, point1, point2, new Scalar(255, 255, 255), 1);
}
/**
* NDArray to opencv_core.Mat
*
* @param manager
* @param srcPoints
* @param dstPoints
* @return
*/
public static Mat toOpenCVMat(NDManager manager, NDArray srcPoints, NDArray dstPoints) {
NDArray svdMat = SVDUtils.transformationFromPoints(manager, srcPoints, dstPoints);
double[] doubleArray = svdMat.toDoubleArray();
Mat newSvdMat = new Mat(2, 3, CvType.CV_64F);
for (int i = 0; i < 2; i++) {
for (int j = 0; j < 3; j++) {
newSvdMat.put(i, j, doubleArray[i * 3 + j]);
}
}
return newSvdMat;
}
/**
* double[][] points array to Mat
* @param points
* @return
*/
public static Mat toOpenCVMat(double[][] points) {
Mat mat = new Mat(5, 2, CvType.CV_64F);
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 2; j++) {
mat.put(i, j, points[i * 5 + j]);
}
}
return mat;
}
/**
* 变换矩阵的逆矩阵
*
* @param src
* @return
*/
public static Mat invertAffineTransform(Mat src) {
Mat dst = src.clone();
Imgproc.invertAffineTransform(src, dst);
return dst;
}
/**
* Mat to BufferedImage
*
* @param mat
* @return
*/
public static BufferedImage mat2Image(Mat mat) {
int width = mat.width();
int height = mat.height();
byte[] data = new byte[width * height * (int) mat.elemSize()];
Imgproc.cvtColor(mat, mat, 4);
mat.get(0, 0, data);
BufferedImage ret = new BufferedImage(width, height, 5);
ret.getRaster().setDataElements(0, 0, width, height, data);
return ret;
}
/**
* BufferedImage to Mat
*
* @param img
* @return
*/
public static Mat image2Mat(BufferedImage img) {
int width = img.getWidth();
int height = img.getHeight();
int channels;
// 获取图像类型
int imageType = img.getType();
// 判断是3通道还是4通道
if (imageType == BufferedImage.TYPE_3BYTE_BGR) {
channels = 3;
} else if (imageType == BufferedImage.TYPE_4BYTE_ABGR || imageType == BufferedImage.TYPE_4BYTE_ABGR_PRE) {
channels = 4;
} else {
// 如果不是已知格式,强制转换为 3 通道 BGR
BufferedImage convertedImg = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
convertedImg.getGraphics().drawImage(img, 0, 0, null);
img = convertedImg;
channels = 3;
}
byte[] data = ((DataBufferByte) img.getRaster().getDataBuffer()).getData();
Mat mat = new Mat(height, width, CvType.CV_8UC(channels));
mat.put(0, 0, data);
return mat;
}
}

View File

@@ -0,0 +1,13 @@
package cn.smartjavaai.face.utils;
import cn.smartjavaai.face.enums.QualityGrade;
/**
* Seetaface6工具类
* @author dwj
* @date 2025/6/24
*/
public class Seetaface6Utils {
}

View File

@@ -6,7 +6,7 @@ import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.vector.config.MilvusConfig;
import cn.smartjavaai.face.vector.constant.VectorDBConstants;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.common.entity.FaceSearchResult;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import io.milvus.client.MilvusServiceClient;
import io.milvus.grpc.*;

View File

@@ -4,11 +4,10 @@ import cn.hutool.core.util.IdUtil;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.face.dao.FaceDao;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.face.utils.FaceUtils;
import cn.smartjavaai.face.utils.SimilarityUtil;
import cn.smartjavaai.face.vector.config.SQLiteConfig;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.common.entity.FaceSearchResult;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
@@ -24,7 +23,8 @@ import java.util.stream.Collectors;
public class SQLiteClient implements VectorDBClient {
private final FaceDao faceDao;
private final List<FaceVector> memoryIndex = new CopyOnWriteArrayList<>();
//private final List<FaceVector> memoryIndex = new CopyOnWriteArrayList<>();
private final ConcurrentHashMap<String, FaceVector> memoryIndex = new ConcurrentHashMap<>();
private int featureDimension; // 维度
private final ExecutorService executor = Executors.newFixedThreadPool(4);
@@ -138,7 +138,7 @@ public class SQLiteClient implements VectorDBClient {
// 从数据库中删除
boolean isSuccess = faceDao.deleteFace(ids.toArray(new String[0]));
// 从内存中删除
memoryIndex.removeIf(v -> ids.contains(v.getId()));
ids.forEach(memoryIndex::remove);
if(!isSuccess){
throw new VectorDBException("删除失败");
}
@@ -156,7 +156,7 @@ public class SQLiteClient implements VectorDBClient {
return Collections.emptyList();
}
// 并行计算相似度
List<CompletableFuture<FaceSearchResult>> futures = memoryIndex.stream()
List<CompletableFuture<FaceSearchResult>> futures = memoryIndex.values().stream()
.map(vector -> CompletableFuture.supplyAsync(() -> {
float similarity = SimilarityUtil.calculate(queryVector, vector.getVector(), config.getSimilarityType(), faceSearchParams.getNormalizeSimilarity());
return similarity >= faceSearchParams.getThreshold() ?
@@ -219,7 +219,7 @@ public class SQLiteClient implements VectorDBClient {
}
private void addToMemoryIndex(FaceVector faceVector) {
memoryIndex.add(faceVector);
memoryIndex.put(faceVector.getId(), faceVector);
}
private void clearAllData() {

View File

@@ -2,7 +2,7 @@ package cn.smartjavaai.face.vector.core;
import cn.smartjavaai.face.entity.FaceSearchParams;
import cn.smartjavaai.face.vector.entity.FaceVector;
import cn.smartjavaai.common.entity.FaceSearchResult;
import cn.smartjavaai.common.entity.face.FaceSearchResult;
import cn.smartjavaai.face.vector.exception.VectorDBException;
import java.util.List;