【通用视觉】集成 OpenAI CLIP 模型，支持以图搜图、以文搜图、以图搜文等功能

【通用视觉】新增 YOLO 图像分类模型支持【ASR/TTS】集成 Sherpa TTS（语音合成）与 ASR（语音识别）模块，支持中文、粤语、方言、英文等多种语言【目标检测】优化视频目标检测功能
2026-09-18 16:39:21 +00:00 · 2025-10-24 11:22:39 +08:00
parent 248398a46d
commit 8502b4dc79
68 changed files with 4463 additions and 44 deletions
--- a/vision/src/main/java/cn/smartjavaai/cls/criteria/ClsCriteriaFactory.java
+++ b/vision/src/main/java/cn/smartjavaai/cls/criteria/ClsCriteriaFactory.java
@@ -0,0 +1,73 @@
+package cn.smartjavaai.cls.criteria;
+
+import ai.djl.Device;
+import ai.djl.modality.Classifications;
+import ai.djl.modality.cv.Image;
+import ai.djl.repository.zoo.Criteria;
+import ai.djl.training.util.ProgressBar;
+import ai.djl.translate.Translator;
+import cn.smartjavaai.action.config.ActionRecModelConfig;
+import cn.smartjavaai.action.enums.ActionRecModelEnum;
+import cn.smartjavaai.action.exception.ActionException;
+import cn.smartjavaai.action.model.CommonActionTranslator;
+import cn.smartjavaai.cls.config.ClsModelConfig;
+import cn.smartjavaai.cls.enums.ClsModelEnum;
+import cn.smartjavaai.cls.translator.YoloClsTranslator;
+import cn.smartjavaai.common.enums.DeviceEnum;
+import cn.smartjavaai.common.utils.DJLCommonUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.nio.file.Paths;
+import java.util.Objects;
+
+/**
+ * 分类模型Criteria工厂
+ * @author dwj
+ */
+public class ClsCriteriaFactory {
+
+
+    /**
+     * 创建动作识别Criteria
+     * @param config
+     * @return
+     */
+    public static Criteria<Image, Classifications> createCriteria(ClsModelConfig config) {
+        Device device = null;
+        if(!Objects.isNull(config.getDevice())){
+            device = config.getDevice() == DeviceEnum.CPU ? Device.cpu() : Device.gpu(config.getGpuId());
+        }
+        Translator<Image, Classifications> translator = getTranslator(config);
+        //检查模型路径
+        if (StringUtils.isBlank(config.getModelPath())){
+            throw new ActionException("请指定模型路径");
+        }
+        boolean isUrl = DJLCommonUtils.hasSupportedProtocol(config.getModelPath());
+        Criteria<Image, Classifications> criteria =
+                Criteria.builder()
+                        .setTypes(Image.class, Classifications.class)
+                        .optModelUrls(isUrl ? config.getModelPath() : null)
+                        .optModelPath(isUrl ? null : Paths.get(config.getModelPath()))
+                        .optTranslator(translator)
+                        .optDevice(device)
+                        .optProgress(new ProgressBar())
+                        .optEngine(config.getModelEnum().getEngine())
+                        .build();
+        return criteria;
+    }
+
+    /**
+     * 获取分类模型Translator
+     * @param config
+     * @return
+     */
+    public static Translator<Image, Classifications> getTranslator(ClsModelConfig config) {
+        Translator<Image, Classifications> translator = null;
+        if(config.getModelEnum() == ClsModelEnum.YOLOV11
+                || config.getModelEnum() == ClsModelEnum.YOLOV8){
+        YoloClsTranslator.Builder builder = YoloClsTranslator.builder().optSynsetArtifactName("synset.txt");
+            translator = builder.build();
+        }
+        return translator;
+    }
+}