【通用视觉】集成 OpenAI CLIP 模型，支持以图搜图、以文搜图、以图搜文等功能

【通用视觉】新增 YOLO 图像分类模型支持
【ASR/TTS】集成 Sherpa TTS（语音合成）与 ASR（语音识别）模块，支持中文、粤语、方言、英文等多种语言
【目标检测】优化视频目标检测功能

examples/face-example/pom.xml

@@ -12,7 +12,7 @@
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <smartjavaai.version>1.0.25</smartjavaai.version>
+        <smartjavaai.version>1.0.26</smartjavaai.version>
         <!--如果打包运行，需要替换成你的main-->
         <exec.mainClass>smartai.examples.face.facedet.FaceDetDemo</exec.mainClass>
 

examples/ocr-examples/pom.xml

@@ -12,7 +12,7 @@
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <smartjavaai.version>1.0.25</smartjavaai.version>
+        <smartjavaai.version>1.0.26</smartjavaai.version>
         <!--如果打包运行，需要替换成你的main-->
         <exec.mainClass>smartai.examples.ocr.common.OcrRecognizeDemo</exec.mainClass>
 

examples/speech-examples/pom.xml

@@ -12,7 +12,7 @@
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <smartjavaai.version>1.0.25</smartjavaai.version>
+        <smartjavaai.version>1.0.26</smartjavaai.version>
         <!--如果打包运行，需要替换成你的main-->
         <exec.mainClass>smartai.examples.speech.asr.common.OcrRecognizeDemo</exec.mainClass>
 

examples/speech-examples/src/main/java/smartai/examples/speech/asr/SherpaAsrDemo.java

@@ -0,0 +1,389 @@
+package smartai.examples.speech.asr;
+
+import ai.djl.modality.audio.Audio;
+import ai.djl.modality.audio.AudioFactory;
+import ai.djl.util.JsonUtils;
+import cn.smartjavaai.common.entity.Language;
+import cn.smartjavaai.common.entity.R;
+import cn.smartjavaai.speech.asr.config.AsrModelConfig;
+import cn.smartjavaai.speech.asr.entity.AsrResult;
+import cn.smartjavaai.speech.asr.entity.WhisperParams;
+import cn.smartjavaai.speech.asr.enums.AsrModelEnum;
+import cn.smartjavaai.speech.asr.factory.SpeechRecognizerFactory;
+import cn.smartjavaai.speech.asr.model.SpeechRecognizer;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+import java.nio.file.Paths;
+
+/**
+ * 语音识别ASR demo
+ * sherpa-onnx模型及依赖库下载链接:
+ * 1、（推荐）依赖库官网下载：https://github.com/k2-fsa/sherpa-onnx/releases
+ * 2、（推荐）ASR模型官网下载：https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models
+ * 3、百度网盘下载：https://pan.baidu.com/s/19p3WhVEM7dgdkvXFaeeAxg?pwd=1234 提取码: 1234
+ * @author dwj
+ * @date 2025/10/23
+ */
+@Slf4j
+public class SherpaAsrDemo {
+
+
+    /**
+     * 语音识别：Sherpa Paraformer（中文）
+     */
+    @Test
+    public void testSherpaParaformerAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_PARAFORMER);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-paraformer-zh-2023-09-14");
+            config.setModelName("model.int8.onnx");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-paraformer-zh-2023-09-14/test_wavs/0.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa Dolphin（中文）
+     */
+    @Test
+    public void testSherpaDolphinAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_DOLPHIN);
+            config.setModelName("model.int8.onnx");
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02/test_wavs/0.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa zipformer（中文）
+     */
+    @Test
+    public void testSherpaZipformerAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_ZIPFORMERCTC);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            config.setModelName("model.int8.onnx");
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03/test_wavs/0.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa FireRedAsr（中英）
+     */
+    @Test
+    public void testSherpaFireRedAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_FIREREDASR);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/3.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa SenseVoice （多语言模型）
+     */
+    @Test
+    public void testSherpaSenseVoiceAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_SENSEVOICE);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17");
+            config.setModelName("model.onnx");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/test_wavs/zh.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa WenetCtc（多语言模型：粤语）
+     */
+    @Test
+    public void testSherpaWenetCtcAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_WENETCTC);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10");
+            config.setModelName("model.int8.onnx");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10/test_wavs/yue-0.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa FireRedAsr（方言：四川、天津、河南）
+     */
+    @Test
+    public void testSherpaFireRedAsrSichuan() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_FIREREDASR);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/3-sichuan.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa Telespeech（方言-天津、河南、四川）
+     */
+    @Test
+    public void testSherpaTelespeechAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_TELESPEECH);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.setModelName("model.int8.onnx");
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04/test_wavs/4-tianjin.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa Nemo（英文）
+     */
+    @Test
+    public void testSherpaNemoAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_NEMO);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-ctc-en-citrinet-512");
+            config.setModelName("model.onnx");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/0.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa Moonshine（英文）
+     */
+    @Test
+    public void testSherpaMoonshineAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_MOONSHINE);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-moonshine-tiny-en-int8");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 2);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+
+
+    /**
+     * 语音识别：Sherpa Whisper（英文）
+     */
+    @Test
+    public void testSherpaWhisperAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_WHISPER);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-whisper-tiny");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-whisper-tiny/test_wavs/0.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa Transducer（英文）
+     */
+    @Test
+    public void testSherpaTransducerAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_TRANSDUCER);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-gigaspeech-2023-12-12");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-gigaspeech-2023-12-12/test_wavs/1221-135766-0001.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 语音识别：Sherpa Canary（德语-英文）
+     */
+    @Test
+    public void testSherpaCanaryAsr() {
+        try {
+            //获取模型
+            AsrModelConfig config = new AsrModelConfig();
+            config.setModelEnum(AsrModelEnum.SHERPA_CANARY);
+            config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8");
+            config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+            config.putCustomParam("debug", false);
+            config.putCustomParam("numThreads", 1);
+            SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
+            Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/de.wav"));
+            R<AsrResult> result = recognizer.recognize(audio);
+            if (result.isSuccess()){
+                log.info("识别成功:{}", result.getData());
+            }else{
+                log.error("识别失败:{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+
+
+
+
+
+
+
+
+
+
+}

examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java

@@ -33,7 +33,7 @@ import java.io.InputStream;
 import java.nio.file.Paths;
 
 /**
- * 语音识别demo
+ * 语音识别demo（Vosk、Whisper）
  * 模型下载网盘：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234
  * 文档地址：http://doc.smartjavaai.cn/
  * @author dwj
@@ -41,6 +41,8 @@ import java.nio.file.Paths;
 @Slf4j
 public class SpeechRecognizeDemo {
 
+
+
     /**
      * 获取Whisper模型
      * 模型下载网盘：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234

examples/speech-examples/src/main/java/smartai/examples/speech/tts/TtsDemo.java

@@ -0,0 +1,193 @@
+package smartai.examples.speech.tts;
+
+
+import ai.djl.modality.audio.Audio;
+import cn.smartjavaai.common.entity.R;
+import cn.smartjavaai.common.enums.DeviceEnum;
+import cn.smartjavaai.speech.tts.config.TtsModelConfig;
+import cn.smartjavaai.speech.tts.entity.SherpaTtsParams;
+import cn.smartjavaai.speech.tts.enums.TtsModelEnum;
+import cn.smartjavaai.speech.tts.factory.TtsModelFactory;
+import cn.smartjavaai.speech.tts.model.SherpaTtsModel;
+import cn.smartjavaai.speech.tts.model.TtsModel;
+import cn.smartjavaai.speech.utils.AudioUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+/**
+ * 语音合成demo（TTS）
+ * sherpa-onnx模型及依赖库下载链接:
+ * 1、（推荐）依赖库官网下载：https://github.com/k2-fsa/sherpa-onnx/releases
+ * 2、（推荐）TTS模型官网下载：https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
+ * 3、百度网盘下载：https://pan.baidu.com/s/186REUf7p1z0HH9AZNnwCUg?pwd=1234 提取码: 1234
+ * @author dwj
+ * @date 2025/10/22
+ */
+@Slf4j
+public class TtsDemo {
+
+    public static String enText = "Today as always, men fall into two groups: slaves and free men. Whoever does not have"
+            + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a"
+            + " businessman, an official, or a scholar.";
+
+    public static String zhText = "有问题，请拨打110或者手机18601239876。我们的价值观是真诚热爱！";
+
+    String znEnText =
+            "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki."
+                    + " 你觉得中英文说的如何呢？";
+
+    //设备类型
+    public static DeviceEnum device = DeviceEnum.CPU;
+
+
+    /**
+     * 获取Vits中文模型
+     * @return
+     */
+    public TtsModel getVitsZhModel(){
+        TtsModelConfig modelConfig = new TtsModelConfig();
+        modelConfig.setModelEnum(TtsModelEnum.SHERPA_VITS);
+        modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/vits-zh-hf-keqing");
+        modelConfig.setModelName("keqing.onnx");
+        modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+        modelConfig.setDevice(device);
+        modelConfig.putCustomParam("debug", false);
+        modelConfig.putCustomParam("numThreads", 1);
+        return TtsModelFactory.getInstance().getModel(modelConfig);
+    }
+
+    /**
+     * 获取Matcha中文模型
+     * @return
+     */
+    public TtsModel getMatchaZhModel(){
+        TtsModelConfig modelConfig = new TtsModelConfig();
+        modelConfig.setModelEnum(TtsModelEnum.SHERPA_MATCHA);
+        modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker");
+        modelConfig.setModelName("model-steps-3.onnx");
+        modelConfig.putCustomParam("vocoder", "/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker/vocos-22khz-univ.onnx");
+        modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+        modelConfig.setDevice(device);
+        modelConfig.putCustomParam("debug", false);
+        modelConfig.putCustomParam("numThreads", 1);
+        return TtsModelFactory.getInstance().getModel(modelConfig);
+    }
+
+    /**
+     * 获取Kokoro多语言模型
+     * @return
+     */
+    public TtsModel getKokoroZhEnModel(){
+        TtsModelConfig modelConfig = new TtsModelConfig();
+        modelConfig.setModelEnum(TtsModelEnum.SHERPA_KOKORO);
+        modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/kokoro-multi-lang-v1_0");
+        modelConfig.setModelName("model.onnx");
+        modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+        modelConfig.setDevice(device);
+        modelConfig.putCustomParam("debug", false);
+        modelConfig.putCustomParam("numThreads", 2);
+        return TtsModelFactory.getInstance().getModel(modelConfig);
+    }
+
+    /**
+     * 获取Matcha模型（英文）
+     * @return
+     */
+    public TtsModel getMatchaEnModel(){
+        TtsModelConfig modelConfig = new TtsModelConfig();
+        modelConfig.setModelEnum(TtsModelEnum.SHERPA_MATCHA);
+        modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-en_US-ljspeech");
+        modelConfig.setModelName("model-steps-3.onnx");
+        modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
+        modelConfig.putCustomParam("vocoder", "/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker/vocos-22khz-univ.onnx");
+        modelConfig.setDevice(device);
+        modelConfig.putCustomParam("debug", false);
+        modelConfig.putCustomParam("numThreads", 2);
+        return TtsModelFactory.getInstance().getModel(modelConfig);
+    }
+
+    /**
+     * 语音合成Vits（中文）
+     */
+    @Test
+    public void testVitsZhTts() throws IOException {
+        TtsModel ttsModel = getVitsZhModel();
+        SherpaTtsParams params = new SherpaTtsParams();
+        //语速
+        params.setSpeed(1f);
+        //说话人ID
+        params.setSpeakerId(100);
+        R<Audio> result = ttsModel.generate(zhText, params);
+        if (result.isSuccess()){
+            Audio audio = result.getData();
+            AudioUtils.saveToWav(audio.getData(),"/Users/wenjie/Downloads/tts-vits-zh.wav");
+        }else{
+            System.out.println(result.getMessage());
+        }
+    }
+
+    /**
+     * 语音合成Matcha（中文）
+     */
+    @Test
+    public void testMatchaZhTts() throws IOException {
+        TtsModel ttsModel = getMatchaZhModel();
+        SherpaTtsParams params = new SherpaTtsParams();
+        //语速
+        params.setSpeed(1.5f);
+        //说话人ID
+        params.setSpeakerId(0);
+        R<Audio> result = ttsModel.generate(zhText, params);
+        if (result.isSuccess()){
+            Audio audio = result.getData();
+            AudioUtils.saveToWav(audio.getData(),"/Users/wenjie/Downloads/tts-matcha-zh.wav");
+        }else{
+            System.out.println(result.getMessage());
+        }
+    }
+
+    /**
+     * 语音合成Kokoro（中英）
+     */
+    @Test
+    public void testKokoroZhEnTts() throws IOException {
+        TtsModel ttsModel = getKokoroZhEnModel();
+        SherpaTtsParams params = new SherpaTtsParams();
+        //语速
+        params.setSpeed(1f);
+        //说话人ID
+        params.setSpeakerId(3);
+        R<Audio> result = ttsModel.generate(znEnText, params);
+        if (result.isSuccess()){
+            Audio audio = result.getData();
+            AudioUtils.saveToWav(audio.getData(),"/Users/wenjie/Downloads/tts-kokoro-zh-en.wav");
+        }else{
+            System.out.println(result.getMessage());
+        }
+    }
+
+    /**
+     * 语音合成Matcha（英文）
+     */
+    @Test
+    public void testMatchaEnTts() throws IOException {
+        TtsModel ttsModel = getMatchaEnModel();
+        SherpaTtsParams params = new SherpaTtsParams();
+        //语速
+        params.setSpeed(1.5f);
+        //说话人ID
+        params.setSpeakerId(0);
+        R<Audio> result = ttsModel.generate(enText, params);
+        if (result.isSuccess()){
+            Audio audio = result.getData();
+            AudioUtils.saveToWav(audio.getData(),"/Users/wenjie/Downloads/tts-kitten-en.wav");
+        }else{
+            System.out.println(result.getMessage());
+        }
+    }
+
+
+}

examples/translation-example/pom.xml

@@ -12,7 +12,7 @@
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <smartjavaai.version>1.0.25</smartjavaai.version>
+        <smartjavaai.version>1.0.26</smartjavaai.version>
         <!--如果打包运行，需要替换成你的main-->
         <exec.mainClass>smartai.examples.nlp.translation.TranslationDemo</exec.mainClass>
 

examples/vision-example/pom.xml

@@ -12,7 +12,7 @@
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <smartjavaai.version>1.0.25</smartjavaai.version>
+        <smartjavaai.version>1.0.26</smartjavaai.version>
         <!--如果打包运行，需要替换成你的main-->
         <exec.mainClass>smartai.examples.vision.ObjectDetectionDemo</exec.mainClass>
 

examples/vision-example/src/main/java/smartai/examples/vision/ClipDemo.java

@@ -0,0 +1,356 @@
+package smartai.examples.vision;
+
+import ai.djl.modality.cv.Image;
+import cn.smartjavaai.clip.config.ClipModelConfig;
+import cn.smartjavaai.clip.enums.ClipModelEnum;
+import cn.smartjavaai.clip.model.ClipModel;
+import cn.smartjavaai.clip.model.ClipModelFactory;
+import cn.smartjavaai.common.cv.SmartImageFactory;
+import cn.smartjavaai.common.entity.R;
+import cn.smartjavaai.common.enums.DeviceEnum;
+import cn.smartjavaai.common.enums.SimilarityType;
+import cn.smartjavaai.common.utils.ImageUtils;
+import cn.smartjavaai.common.utils.SimilarityUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * clip模型demo
+ * @author dwj
+ */
+@Slf4j
+public class ClipDemo {
+
+    //设备类型
+    public static DeviceEnum device = DeviceEnum.CPU;
+
+    @BeforeClass
+    public static void beforeAll() throws IOException {
+        //将图片处理的底层引擎切换为 OpenCV
+        SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV);
+        //修改缓存路径
+//        Config.setCachePath("/Users/xxx/smartjavaai_cache");
+    }
+
+    public ClipModel getModel(){
+        ClipModelConfig config = new ClipModelConfig();
+        config.setModelEnum(ClipModelEnum.OPENAI);
+        config.setModelPath("/Users/wenjie/Documents/develop/model/vision/clip/openai/clip.pt");
+        //从jar包中加载模型
+//        config.setModelPath("jar://META-INF/models/clip/openai.zip");
+        config.setDevice(device);
+        return ClipModelFactory.getInstance().getModel(config);
+    }
+
+    /**
+     * 提取图片特征
+     */
+    @Test
+    public void extractImageFeatures(){
+        try {
+            ClipModel model = getModel();
+            //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档
+            Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/clip/dog.jpg"));
+            //获取图片特征
+            R<float[]> features = model.extractImageFeatures(image);
+            if(features.isSuccess()){
+                log.info("图片特征：{}", features.getData());
+            }else{
+                log.info("图片特征获取失败：{}", features.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 提取文本特征
+     */
+    @Test
+    public void extractTextFeatures() {
+        try {
+            ClipModel model = getModel();
+            // 提取单个文本特征
+            String text = "a photo of a dog";
+            R<float[]> features = model.extractTextFeatures(text);
+            if(features.isSuccess()){
+                log.info("文本特征：{}", features.getData());
+            }else{
+                log.info("文本特征获取失败：{}", features.getMessage());
+            }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 文本搜索图像（基于图像和文本直接比对）
+     */
+    @Test
+    public void searchImagesByText() {
+        try {
+            ClipModel model = getModel();
+            String text = "a photo of a dog";
+
+            // 读取图片列表
+            List<File> images = ImageUtils.listImageFiles("src/main/resources/clip");
+            List<Float> similarities = new ArrayList<>();
+
+            // 1. 计算每张图片的相似度
+            for (File imageFile : images) {
+                Image image = SmartImageFactory.getInstance().fromFile(imageFile.toPath());
+                R<Float> similarity = model.compareTextAndImage(image, text);
+                if (similarity.isSuccess()) {
+                    similarities.add(similarity.getData());
+                    log.info("图片：{}，相似度：{}", imageFile.getName(), similarity.getData());
+                } else {
+                    log.warn("图片：{}，相似度计算失败：{}", imageFile.getName(), similarity.getMessage());
+                }
+            }
+            if (similarities.isEmpty()) {
+                log.warn("没有计算到有效的相似度结果");
+                return;
+            }
+
+            // 2. 计算 Softmax 概率
+            double total = similarities.stream()
+                    .mapToDouble(Math::exp)
+                    .sum();
+
+            List<Double> probabilities = similarities.stream()
+                    .map(v -> Math.exp(v) / total)
+                    .collect(Collectors.toList());
+
+            // 3. 找出相似度最高的图片
+            int maxIndex = IntStream.range(0, similarities.size())
+                    .boxed()
+                    .max(Comparator.comparing(similarities::get))
+                    .orElse(-1);
+
+            // 4. 打印结果
+            log.info("---- 结果统计 ----");
+            for (int i = 0; i < images.size(); i++) {
+                log.info("图片：{}，相似度：{}，概率：{}",
+                        images.get(i).getName(),
+                        similarities.get(i),
+                        String.format("%.4f", probabilities.get(i)));
+            }
+
+            log.info("最匹配的图片：{}，相似度：{}，Softmax 概率：{}",
+                    images.get(maxIndex).getName(),
+                    similarities.get(maxIndex),
+                    String.format("%.4f", probabilities.get(maxIndex)));
+
+        } catch (Exception e) {
+            log.error("执行 searchImagesByText 异常", e);
+        }
+    }
+
+    /**
+     * 文本搜索图像（基于图像和文本的特征值比对）
+     */
+    @Test
+    public void searchImagesByText2() {
+        try {
+            ClipModel model = getModel();
+            String text = "a photo of a dog";
+            // 读取图片列表
+            List<File> images = ImageUtils.listImageFiles("src/main/resources/clip");
+            List<Float> similarities = new ArrayList<>();
+            R<float[]> textFeatures = model.extractTextFeatures(text);
+            float scale = 100f; // 缩放因子，越大 softmax 差异越明显
+            // 1. 计算每张图片的相似度
+            for (File imageFile : images) {
+                Image image = SmartImageFactory.getInstance().fromFile(imageFile.toPath());
+                R<float[]> imageFeatures = model.extractImageFeatures(image);
+                if (imageFeatures.isSuccess()) {
+                    float similarity = SimilarityUtil.calculate(
+                            imageFeatures.getData(),
+                            textFeatures.getData(),
+                            SimilarityType.COSINE,
+                            false
+                    );
+                    similarities.add(similarity * scale);
+                } else {
+                    log.warn("图片：{}，特征提取失败：{}", imageFile.getName(), imageFeatures.getMessage());
+                    similarities.add(Float.NEGATIVE_INFINITY); // 特征提取失败，赋极小值
+                }
+            }
+
+            if (similarities.isEmpty()) {
+                log.warn("没有计算到有效的相似度结果");
+                return;
+            }
+
+            // 2. 计算 Softmax 概率
+            double total = similarities.stream()
+                    .mapToDouble(Math::exp)
+                    .sum();
+
+            List<Double> probabilities = similarities.stream()
+                    .map(v -> Math.exp(v) / total)
+                    .collect(Collectors.toList());
+
+            // 3. 找出相似度最高的图片
+            int maxIndex = IntStream.range(0, similarities.size())
+                    .boxed()
+                    .max(Comparator.comparing(similarities::get))
+                    .orElse(-1);
+
+            // 4. 打印结果
+            log.info("---- 结果统计 ----");
+            for (int i = 0; i < images.size(); i++) {
+                log.info("图片：{}，相似度：{}，概率：{}",
+                        images.get(i).getName(),
+                        similarities.get(i),
+                        String.format("%.4f", probabilities.get(i)));
+            }
+
+            log.info("最匹配的图片：{}，相似度：{}，Softmax 概率：{}",
+                    images.get(maxIndex).getName(),
+                    similarities.get(maxIndex),
+                    String.format("%.4f", probabilities.get(maxIndex)));
+
+        } catch (Exception e) {
+            log.error("执行 searchImagesByText 异常", e);
+        }
+    }
+
+
+    /**
+     * 图像搜索文本
+     */
+    @Test
+    public void searchTextByImage() {
+        try {
+            ClipModel model = getModel();
+            String[] textArray = {"a diagram", "a dog", "a cat"};
+            //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档
+            Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/clip/dog.jpg"));
+            //获取图片特征
+            R<float[]> features = model.extractImageFeatures(image);
+            List<Float> similarities = new ArrayList<>();
+            // 1. 计算每张图片的相似度
+            for (String text : textArray) {
+                R<Float> similarity = model.compareTextAndImage(image, text);
+                if (similarity.isSuccess()) {
+                    similarities.add(similarity.getData());
+                    log.info("文本：{}，相似度：{}", text, similarity.getData());
+                } else {
+                    log.warn("文本：{}，相似度计算失败：{}", text, similarity.getMessage());
+                }
+            }
+            if (similarities.isEmpty()) {
+                log.warn("没有计算到有效的相似度结果");
+                return;
+            }
+
+            // 2. 计算 Softmax 概率
+            double total = similarities.stream()
+                    .mapToDouble(Math::exp)
+                    .sum();
+
+            List<Double> probabilities = similarities.stream()
+                    .map(v -> Math.exp(v) / total)
+                    .collect(Collectors.toList());
+
+            // 3. 找出相似度最高的图片
+            int maxIndex = IntStream.range(0, similarities.size())
+                    .boxed()
+                    .max(Comparator.comparing(similarities::get))
+                    .orElse(-1);
+
+            // 4. 打印结果
+            log.info("---- 结果统计 ----");
+            for (int i = 0; i < textArray.length; i++) {
+                log.info("文本：{}，相似度：{}，概率：{}",
+                        textArray[i],
+                        similarities.get(i),
+                        String.format("%.4f", probabilities.get(i)));
+            }
+
+            log.info("最匹配的文本：{}，相似度：{}，Softmax 概率：{}",
+                    textArray[maxIndex],
+                    similarities.get(maxIndex),
+                    String.format("%.4f", probabilities.get(maxIndex)));
+
+        } catch (Exception e) {
+            log.error("执行 searchImagesByText 异常", e);
+        }
+    }
+
+    /**
+     * 以图搜图
+     */
+    @Test
+    public void searchImagesByImage() {
+        try {
+            ClipModel model = getModel();
+            //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档
+            Image image1 = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/cat2.jpeg"));
+            List<Float> similarities = new ArrayList<>();
+            // 读取图片列表
+            List<File> images = ImageUtils.listImageFiles("src/main/resources/clip");
+            // 1. 计算每张图片的相似度
+            for (File imageFile : images) {
+                Image image = SmartImageFactory.getInstance().fromFile(imageFile.toPath());
+                R<Float> similarity = model.compareImage(image1, image, 100);
+                if (similarity.isSuccess()) {
+                    similarities.add(similarity.getData());
+                    log.info("图片：{}，相似度：{}", imageFile.getName(), similarity.getData());
+                } else {
+                    log.warn("图片：{}，相似度计算失败：{}", imageFile.getName(), similarity.getMessage());
+                }
+            }
+            if (similarities.isEmpty()) {
+                log.warn("没有计算到有效的相似度结果");
+                return;
+            }
+
+            // 2. 计算 Softmax 概率
+            double total = similarities.stream()
+                    .mapToDouble(Math::exp)
+                    .sum();
+
+            List<Double> probabilities = similarities.stream()
+                    .map(v -> Math.exp(v) / total)
+                    .collect(Collectors.toList());
+
+            // 3. 找出相似度最高的图片
+            int maxIndex = IntStream.range(0, similarities.size())
+                    .boxed()
+                    .max(Comparator.comparing(similarities::get))
+                    .orElse(-1);
+
+            // 4. 打印结果
+            log.info("---- 结果统计 ----");
+            for (int i = 0; i < images.size(); i++) {
+                log.info("图片：{}，相似度：{}，概率：{}",
+                        images.get(i).getName(),
+                        similarities.get(i),
+                        String.format("%.4f", probabilities.get(i)));
+            }
+
+            log.info("最匹配的图片：{}，相似度：{}，Softmax 概率：{}",
+                    images.get(maxIndex).getName(),
+                    similarities.get(maxIndex),
+                    String.format("%.4f", probabilities.get(maxIndex)));
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+
+}

examples/vision-example/src/main/java/smartai/examples/vision/ClsDemo.java

@@ -0,0 +1,90 @@
+package smartai.examples.vision;
+
+import ai.djl.modality.Classifications;
+import ai.djl.modality.cv.Image;
+import cn.smartjavaai.cls.config.ClsModelConfig;
+import cn.smartjavaai.cls.enums.ClsModelEnum;
+import cn.smartjavaai.cls.model.ClsModel;
+import cn.smartjavaai.cls.model.ClsModelFactory;
+import cn.smartjavaai.common.cv.SmartImageFactory;
+import cn.smartjavaai.common.entity.DetectionResponse;
+import cn.smartjavaai.common.entity.R;
+import cn.smartjavaai.common.enums.DeviceEnum;
+import cn.smartjavaai.instanceseg.config.InstanceSegModelConfig;
+import cn.smartjavaai.instanceseg.enums.InstanceSegModelEnum;
+import cn.smartjavaai.instanceseg.model.InstanceSegModel;
+import cn.smartjavaai.instanceseg.model.InstanceSegModelFactory;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+/**
+ * 图像分类模型demo
+ * @author dwj
+ */
+@Slf4j
+public class ClsDemo {
+
+
+    //设备类型
+    public static DeviceEnum device = DeviceEnum.CPU;
+
+    @BeforeClass
+    public static void beforeAll() throws IOException {
+        //将图片处理的底层引擎切换为 OpenCV
+        SmartImageFactory.setEngine(SmartImageFactory.Engine.OPENCV);
+        //修改缓存路径
+//        Config.setCachePath("/Users/xxx/smartjavaai_cache");
+    }
+
+
+    public ClsModel getModel(){
+        ClsModelConfig config = new ClsModelConfig();
+        //实例分割模型，切换模型需要同时修改modelEnum及modelPath
+        config.setModelEnum(ClsModelEnum.YOLOV8);
+        //模型所在路径，synset.txt也需要放在同目录下
+        config.setModelPath("/Users/wenjie/Documents/develop/model/vision/cls/yolo11m-cls.onnx");
+        // 指定允许的类别
+//        config.setAllowedClasses(Arrays.asList("dog","car"));
+        //指定返回检测数量
+        config.setDevice(device);
+        //置信度阈值
+        config.setThreshold(0.5f);
+        return ClsModelFactory.getInstance().getModel(config);
+    }
+
+    /**
+     * 实例分割
+     */
+    @Test
+    public void detect(){
+        try {
+            ClsModel detectorModel = getModel();
+            //创建Image对象，可以从文件、url、InputStream创建、BufferedImage、Base64创建，具体使用方法可以查看文档
+            Image image = SmartImageFactory.getInstance().fromFile(Paths.get("src/main/resources/clip/dog.jpg"));
+            R<Classifications> result = detectorModel.detect(image);
+            if(result.isSuccess()){
+                if(CollectionUtils.isNotEmpty(result.getData().getClassNames())){
+                    //分数最高分类
+                    log.info("分类识别结果：{}", result.getData().best().toString());
+                    //按分数排序前5个结果
+//                log.info("动作识别结果：{}", result.getData().topK(5).toString());
+                }else{
+                    log.info("未识别到分类");
+                }
+            }else{
+                log.info("分类识别失败：{}", result.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+
+}