ZF/SmartJavaAI
1
0
Fork 0
mirror of https://github.com/geekwenjie/SmartJavaAI.git synced 2026-09-12 20:58:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

1、【核心升级】升级DJL版本到0.34.0

Browse Source 
2、【平台支持】新增对 Linux ARM64 架构的全面支持
3、【通用视觉】集成零样本目标检测模型
4、【活体检测】优化视频检测流程，实现 Predictor 视频会话级复用
5、【人脸识别】SQLite人脸查询改进线程池
6、【人脸识别】修复 Milvus 向量库下 listFaces 接口的调用异常
This commit is contained in:
dengwenjie
2025-11-26 18:51:59 +08:00
parent 06cb54bbf7
commit efce16834b
49 changed files with 1517 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vision/src/main/java/cn/smartjavaai/action/model/ActionRecModelFactory.java
View File

@@ -55,7 +55,7 @@ public class ActionRecModelFactory {
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
return createModel(config);
});
}
@@ -64,7 +64,7 @@ public class ActionRecModelFactory {
* @param config
* @return
*/
private ActionRecModel createFaceDetModel(ActionRecModelConfig config) {
private ActionRecModel createModel(ActionRecModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");

13
vision/src/main/java/cn/smartjavaai/clip/model/ClipModel.java
View File

@@ -66,8 +66,9 @@ public interface ClipModel extends AutoCloseable{
/**
* 图片特征比较
* @param image1 图1
* @param image2 图2
* @param image1
* @param image2
* @param scale
* @return
*/
default R<Float> compareImage(Image image1, Image image2, float scale){
@@ -115,10 +116,12 @@ public interface ClipModel extends AutoCloseable{
throw new UnsupportedOperationException("默认不支持该功能");
}
/**
* 文本特征比较
* @param feature1 文本1
* @param feature2 文本2
* 特征比较
* @param feature1
* @param feature2
* @param scale
* @return
*/
default R<Float> compareFeatures(float[] feature1, float[] feature2, float scale){

4
vision/src/main/java/cn/smartjavaai/clip/model/ClipModelFactory.java
View File

@@ -56,7 +56,7 @@ public class ClipModelFactory {
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
return createModel(config);
});
}
@@ -65,7 +65,7 @@ public class ClipModelFactory {
* @param config
* @return
*/
private ClipModel createFaceDetModel(ClipModelConfig config) {
private ClipModel createModel(ClipModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");

4
vision/src/main/java/cn/smartjavaai/cls/model/ClsModelFactory.java
View File

@@ -55,7 +55,7 @@ public class ClsModelFactory {
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
return createModel(config);
});
}
@@ -64,7 +64,7 @@ public class ClsModelFactory {
* @param config
* @return
*/
private ClsModel createFaceDetModel(ClsModelConfig config) {
private ClsModel createModel(ClsModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");

4
vision/src/main/java/cn/smartjavaai/instanceseg/model/InstanceSegModelFactory.java
View File

@@ -54,7 +54,7 @@ public class InstanceSegModelFactory {
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
return createModel(config);
});
}
@@ -63,7 +63,7 @@ public class InstanceSegModelFactory {
* @param config
* @return
*/
private InstanceSegModel createFaceDetModel(InstanceSegModelConfig config) {
private InstanceSegModel createModel(InstanceSegModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");

4
vision/src/main/java/cn/smartjavaai/obb/model/ObbDetModelFactory.java
View File

@@ -54,7 +54,7 @@ public class ObbDetModelFactory {
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
return createModel(config);
});
}
@@ -63,7 +63,7 @@ public class ObbDetModelFactory {
* @param config
* @return
*/
private ObbDetModel createFaceDetModel(ObbDetModelConfig config) {
private ObbDetModel createModel(ObbDetModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");

4
vision/src/main/java/cn/smartjavaai/objectdetection/model/person/PersonDetModelFactory.java
View File

@@ -56,7 +56,7 @@ public class PersonDetModelFactory {
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
return createModel(config);
});
}
@@ -65,7 +65,7 @@ public class PersonDetModelFactory {
* @param config
* @return
*/
private PersonDetModel createFaceDetModel(PersonDetModelConfig config) {
private PersonDetModel createModel(PersonDetModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");

4
vision/src/main/java/cn/smartjavaai/pose/model/PoseDetModelFactory.java
View File

@@ -55,7 +55,7 @@ public class PoseDetModelFactory {
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
return createModel(config);
});
}
@@ -64,7 +64,7 @@ public class PoseDetModelFactory {
* @param config
* @return
*/
private PoseModel createFaceDetModel(PoseModelConfig config) {
private PoseModel createModel(PoseModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");

4
vision/src/main/java/cn/smartjavaai/semseg/model/SemSegModelFactory.java
View File

@@ -54,7 +54,7 @@ public class SemSegModelFactory {
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceDetModel(config);
return createModel(config);
});
}
@@ -63,7 +63,7 @@ public class SemSegModelFactory {
* @param config
* @return
*/
private SemSegModel createFaceDetModel(SemSegModelConfig config) {
private SemSegModel createModel(SemSegModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");

48
vision/src/main/java/cn/smartjavaai/zeroshot/config/ZeroDetConfig.java Normal file
View File

@@ -0,0 +1,48 @@
package cn.smartjavaai.zeroshot.config;
import cn.smartjavaai.common.config.ModelConfig;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.zeroshot.enums.ZeroDetModelEnum;
import lombok.Data;
import java.util.List;
/**
* 零样本目标检测模型参数配置
*
* @author dwj
*/
@Data
public class ZeroDetConfig extends ModelConfig {
/**
* 模型
*/
private ZeroDetModelEnum modelEnum;
/**
* 模型路径
*/
private String modelPath;
/**
* 置信度阈值
*/
private float threshold = 0.3f;
public ZeroDetConfig() {
}
public ZeroDetConfig(ZeroDetModelEnum modelEnum, DeviceEnum device) {
this.modelEnum = modelEnum;
setDevice(device);
}
public ZeroDetConfig(ZeroDetModelEnum modelEnum) {
this.modelEnum = modelEnum;
}
}

53
vision/src/main/java/cn/smartjavaai/zeroshot/criteria/ZeroDetCriteriaFactory.java Normal file
View File

@@ -0,0 +1,53 @@
package cn.smartjavaai.zeroshot.criteria;
import ai.djl.Device;
import ai.djl.huggingface.translator.ZeroShotObjectDetectionTranslatorFactory;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.VisionLanguageInput;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.modality.cv.translator.YoloWorldTranslatorFactory;
import ai.djl.repository.zoo.Criteria;
import ai.djl.training.util.ProgressBar;
import ai.djl.translate.TranslatorFactory;
import cn.smartjavaai.common.enums.DeviceEnum;
import cn.smartjavaai.zeroshot.config.ZeroDetConfig;
import cn.smartjavaai.zeroshot.enums.ZeroDetModelEnum;
import org.apache.commons.lang3.StringUtils;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 零样本目标检测Criteria工厂
* @author dwj
*/
public class ZeroDetCriteriaFactory {
public static Criteria<VisionLanguageInput, DetectedObjects> createCriteria(ZeroDetConfig config) {
Device device = null;
if(!Objects.isNull(config.getDevice())){
device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId());
}
TranslatorFactory translatorFactory = null;
if(config.getModelEnum() == ZeroDetModelEnum.OWLV2_BASE_PATCH16){
translatorFactory = new ZeroShotObjectDetectionTranslatorFactory();
}else if(config.getModelEnum() == ZeroDetModelEnum.YOLOV8S_WORLDV2){
translatorFactory = new YoloWorldTranslatorFactory();
}
Criteria<VisionLanguageInput, DetectedObjects> criteria =
Criteria.builder()
.setTypes(VisionLanguageInput.class, DetectedObjects.class)
.optModelUrls(StringUtils.isNotBlank(config.getModelPath()) ? null :
config.getModelEnum().getModelUri())
.optModelPath(StringUtils.isNotBlank(config.getModelPath()) ? Paths.get(config.getModelPath()) : null)
.optDevice(device)
.optEngine(config.getModelEnum().getEngine())
.optTranslatorFactory(translatorFactory)
.optProgress(new ProgressBar())
.build();
return criteria;
}
}

17
vision/src/main/java/cn/smartjavaai/zeroshot/entity/DetectParams.java Normal file
View File

@@ -0,0 +1,17 @@
package cn.smartjavaai.zeroshot.entity;
import lombok.Data;
/**
* 检测参数
* @author dwj
*/
@Data
public class DetectParams {
/**
* 置信度阈值
*/
private float threshold = 0.3f;
}

45
vision/src/main/java/cn/smartjavaai/zeroshot/enums/ZeroDetModelEnum.java Normal file
View File

@@ -0,0 +1,45 @@
package cn.smartjavaai.zeroshot.enums;
/**
* 零样本目标检测模型枚举
* @author dwj
*/
public enum ZeroDetModelEnum {
YOLOV8S_WORLDV2("PyTorch", "djl://ai.djl.pytorch/yolov8s-worldv2"),
OWLV2_BASE_PATCH16("PyTorch", "djl://ai.djl.huggingface.pytorch/google/owlv2-base-patch16");
/**
* 根据名称获取枚举 (忽略大小写和下划线变体)
*/
public static ZeroDetModelEnum fromName(String name) {
String formatted = name.trim().toUpperCase().replaceAll("[-_]", "");
for (ZeroDetModelEnum model : values()) {
if (model.name().replaceAll("_", "").equals(formatted)) {
return model;
}
}
throw new IllegalArgumentException("未知模型名称: " + name);
}
private final String modelUri;
/**
* 模型引擎
*/
private final String engine;
ZeroDetModelEnum(String engine, String modelUri) {
this.modelUri = modelUri;
this.engine = engine;
}
public String getModelUri() {
return modelUri;
}
public String getEngine() {
return engine;
}
}

29
vision/src/main/java/cn/smartjavaai/zeroshot/exception/ZeroDetException.java Normal file
View File

@@ -0,0 +1,29 @@
package cn.smartjavaai.zeroshot.exception;
/**
* 零样本目标检测异常
* @author dwj
*/
public class ZeroDetException extends RuntimeException{
public ZeroDetException() {
super();
}
public ZeroDetException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public ZeroDetException(String message, Throwable cause) {
super(message, cause);
}
public ZeroDetException(String message) {
super(message);
}
public ZeroDetException(Throwable cause) {
super(cause);
}
}

162
vision/src/main/java/cn/smartjavaai/zeroshot/model/CommonZeroDetModel.java Normal file
View File

@@ -0,0 +1,162 @@
package cn.smartjavaai.zeroshot.model;
import ai.djl.MalformedModelException;
import ai.djl.engine.Engine;
import ai.djl.inference.Predictor;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.VisionLanguageInput;
import ai.djl.modality.cv.output.DetectedObjects;
import ai.djl.repository.zoo.Criteria;
import ai.djl.repository.zoo.ModelNotFoundException;
import ai.djl.repository.zoo.ZooModel;
import cn.smartjavaai.common.cv.SmartImageFactory;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.common.pool.PredictorFactory;
import cn.smartjavaai.objectdetection.exception.DetectionException;
import cn.smartjavaai.vision.utils.DetectedObjectsFilter;
import cn.smartjavaai.vision.utils.DetectorUtils;
import cn.smartjavaai.zeroshot.config.ZeroDetConfig;
import cn.smartjavaai.zeroshot.criteria.ZeroDetCriteriaFactory;
import cn.smartjavaai.zeroshot.exception.ZeroDetException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.pool2.impl.GenericObjectPool;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Objects;
/**
* 零样本目标检测模型
* @author dwj
*/
@Slf4j
public class CommonZeroDetModel implements ZeroDetModel {
private ZeroDetConfig config;
private ZooModel<VisionLanguageInput, DetectedObjects> model;
private GenericObjectPool<Predictor<VisionLanguageInput, DetectedObjects>> predictorPool;
@Override
public void loadModel(ZeroDetConfig config) {
if(Objects.isNull(config.getModelEnum())){
throw new DetectionException("未配置模型枚举");
}
Criteria<VisionLanguageInput, DetectedObjects> criteria = ZeroDetCriteriaFactory.createCriteria(config);
this.config = config;
try {
model = criteria.loadModel();
// 创建池子：每个线程独享 Predictor
this.predictorPool = new GenericObjectPool<>(new PredictorFactory<>(model));
int predictorPoolSize = config.getPredictorPoolSize();
if(config.getPredictorPoolSize() <= 0){
predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数
}
predictorPool.setMaxTotal(predictorPoolSize);
log.debug("当前设备: " + model.getNDManager().getDevice());
log.debug("当前引擎: " + Engine.getInstance().getEngineName());
log.debug("模型推理器线程池最大数量: " + predictorPoolSize);
} catch (IOException | ModelNotFoundException | MalformedModelException e) {
throw new DetectionException("模型加载失败", e);
}
}
@Override
public R<DetectionResponse> detect(Image image, String[] candidates) {
DetectedObjects detectedObjects = detectCore(new VisionLanguageInput(image, candidates));
DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, image);
return R.ok(detectionResponse);
}
/**
* 模型核心推理方法
* @param input
* @return
*/
@Override
public DetectedObjects detectCore(VisionLanguageInput input) {
Predictor<VisionLanguageInput, DetectedObjects> predictor = null;
try {
predictor = predictorPool.borrowObject();
DetectedObjects detectedObjects = predictor.predict(input);
//过滤
if(Objects.nonNull(detectedObjects) && detectedObjects.getNumberOfObjects() > 0){
DetectedObjectsFilter detectedObjectsFilter = new DetectedObjectsFilter(null, config.getThreshold());
detectedObjects = detectedObjectsFilter.filter(detectedObjects);
}
return detectedObjects;
} catch (Exception e) {
throw new DetectionException("零样本目标检测错误", e);
}finally {
if (predictor != null) {
try {
predictorPool.returnObject(predictor); //归还
log.debug("释放资源");
} catch (Exception e) {
log.warn("归还Predictor失败", e);
try {
predictor.close(); // 归还失败才销毁
} catch (Exception ex) {
log.error("关闭Predictor失败", ex);
}
}
}
}
}
@Override
public R<DetectionResponse> detectAndDraw(Image image, String[] candidates) {
DetectedObjects detectedObjects = detectCore(new VisionLanguageInput(image, candidates));
image.drawBoundingBoxes(detectedObjects);
DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, image);
detectionResponse.setDrawnImage(image);
return R.ok(detectionResponse);
}
@Override
public R<DetectionResponse> detectAndDraw(String[] candidates, String imagePath, String outputPath) {
try {
Image img = SmartImageFactory.getInstance().fromFile(Paths.get(imagePath));
DetectedObjects detectedObjects = detectCore(new VisionLanguageInput(img, candidates));
img.drawBoundingBoxes(detectedObjects);
img.save(Files.newOutputStream(Paths.get(outputPath)), "png");
DetectionResponse detectionResponse = DetectorUtils.convertToDetectionResponse(detectedObjects, img);
return R.ok(detectionResponse);
} catch (IOException e) {
throw new ZeroDetException(e);
}
}
private boolean fromFactory = false;
@Override
public void setFromFactory(boolean fromFactory) {
this.fromFactory = fromFactory;
}
public boolean isFromFactory() {
return fromFactory;
}
@Override
public void close() throws Exception {
try {
if (predictorPool != null) {
predictorPool.close();
}
} catch (Exception e) {
log.warn("关闭 predictorPool 失败", e);
}
try {
if (model != null) {
model.close();
}
} catch (Exception e) {
log.warn("关闭 model 失败", e);
}
}
}

50
vision/src/main/java/cn/smartjavaai/zeroshot/model/ZeroDetModel.java Normal file
View File

@@ -0,0 +1,50 @@
package cn.smartjavaai.zeroshot.model;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.VisionLanguageInput;
import ai.djl.modality.cv.output.DetectedObjects;
import cn.smartjavaai.common.entity.DetectionResponse;
import cn.smartjavaai.common.entity.R;
import cn.smartjavaai.zeroshot.config.ZeroDetConfig;
/**
* 零样本目标检测模型
* @author dwj
*/
public interface ZeroDetModel extends AutoCloseable{
/**
* 加载模型
* @param config
*/
void loadModel(ZeroDetConfig config);
/**
* 零样本目标检测
* @param image
* @return
*/
default R<DetectionResponse> detect(Image image, String[] candidates){
throw new UnsupportedOperationException("默认不支持该功能");
}
default DetectedObjects detectCore(VisionLanguageInput input){
throw new UnsupportedOperationException("默认不支持该功能");
}
default R<DetectionResponse> detectAndDraw(Image image, String[] candidates){
throw new UnsupportedOperationException("默认不支持该功能");
}
default R<DetectionResponse> detectAndDraw(String[] candidates, String imagePath, String outputPath){
throw new UnsupportedOperationException("默认不支持该功能");
}
default void setFromFactory(boolean fromFactory){
throw new UnsupportedOperationException("默认不支持该功能");
}
}

109
vision/src/main/java/cn/smartjavaai/zeroshot/model/ZeroDetModelFactory.java Normal file
View File

@@ -0,0 +1,109 @@
package cn.smartjavaai.zeroshot.model;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.objectdetection.exception.DetectionException;
import cn.smartjavaai.zeroshot.config.ZeroDetConfig;
import cn.smartjavaai.zeroshot.enums.ZeroDetModelEnum;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 零样本目标检测 模型工厂
* @author dwj
*/
@Slf4j
public class ZeroDetModelFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile ZeroDetModelFactory instance;
private static final ConcurrentHashMap<ZeroDetModelEnum, ZeroDetModel> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<ZeroDetModelEnum, Class<? extends ZeroDetModel>> registry =
new ConcurrentHashMap<>();
// 私有构造函数，防止外部创建实例
private ZeroDetModelFactory() {}
// 双重检查锁定的单例方法
public static ZeroDetModelFactory getInstance() {
if (instance == null) {
synchronized (ZeroDetModelFactory.class) {
if (instance == null) {
instance = new ZeroDetModelFactory();
}
}
}
return instance;
}
/**
* 获取模型（通过配置）
* @param config
* @return
*/
public ZeroDetModel getModel(ZeroDetConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new DetectionException("未配置模型");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private ZeroDetModel createModel(ZeroDetConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new DetectionException("Unsupported model");
}
ZeroDetModel model = null;
try {
model = (ZeroDetModel) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new DetectionException(e);
}
model.loadModel(config);
model.setFromFactory(true);
return model;
}
/**
* 注册模型
* @param modelEnum
* @param clazz
*/
private static void registerAlgorithm(ZeroDetModelEnum modelEnum, Class<? extends ZeroDetModel> clazz) {
registry.put(modelEnum, clazz);
}
/**
* 移除缓存的模型
* @param modelEnum
*/
public static void removeFromCache(ZeroDetModelEnum modelEnum) {
modelMap.remove(modelEnum);
}
// 初始化默认算法
static {
registerAlgorithm(ZeroDetModelEnum.YOLOV8S_WORLDV2, CommonZeroDetModel.class);
registerAlgorithm(ZeroDetModelEnum.OWLV2_BASE_PATCH16, CommonZeroDetModel.class);
log.debug("缓存目录：{}", Config.getCachePath());
}
}