diff --git a/README.md b/README.md index dc6fdf3..db069fd 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,18 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 + + +
+

图像分类
(Image Classification)

+
+ + +
+ +
+ +
@@ -316,6 +328,33 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的
+ + +
+

语音合成(ASR)

+
+ + +
+ +
+ + + + +
+

CLIP

+ - 文搜图
+ - 图搜图
+ - 图搜文
+
+ + +
+ +
+ + @@ -351,8 +390,11 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 - 静默活体检测：图片、视频活体检测 - 人脸表情识别：7种表情识别 - 人脸质量评估：亮度评估、清晰度评估、完整度评估、姿态评估、遮挡评估 +- **图像分类** + - 支持多种主流模型：集成 YOLOv8、YOLOv11 等分类模型 + - 支持自定义模型加载：可无缝加载并部署用户自行训练的分类模型 - **目标检测** - - 支持多种主流模型：集成 YOLOv5、YOLOv8、YOLOv11、YOLOv12、Tensorflow Object Detection 等目标检测算法 + - 支持多种主流模型：集成 YOLOv5、YOLOv8、YOLOv11、YOLOv12、Tensorflow Object Detection 等目标检测模型 - 支持自定义模型加载：可无缝加载并部署用户自行训练的目标检测模型 - 集成行人检测模型 - **语义分割** @@ -365,6 +407,9 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 - 支持KINETICS400数据集中400个人类动作识别 - **姿态估计** - 集成YOLOv8-pose、YOLOv11-pose等模型 +- **CLIP** + - 支持提取图片及文本特征 + - 支持文搜图、图搜文、图搜图 - **OCR文字识别** - 支持PaddleOCR 3.0模型：集成最新PP-OCRv5、PP-OCRv4、表格结构识别模型(SLANet_plus)、文本行方向分类模型 - 支持任意角度识别，方向校准 @@ -373,9 +418,12 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 - 支持中文车牌识别：单层/双层检测，颜色识别，支持12种中文车牌 - **机器翻译** - 集成NLLB-200模型：支持200+语言互相翻译 -- **语音识别** +- **语音识别(ASR)** - 集成openai的whisper模型：支持100种语言 - 集成vosk语音识别 + - 集成sherpa-onnx语音识别 +- **语音合成(TTS)** + - 集成sherpa-onnx：支持中文、方言、粤语、英文、德语等多种语言 ## 🌟 AI集成方式对比 @@ -404,7 +452,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 | translate | 机器翻译模块 | | speech | 语音功能模块，包含 ASR 和 TTS | -可以根据需求对每个模块单独引入，也可以通过引入`smartjavaai-all`方式引入所有模块。 +可以根据需求对每个模块单独引入，也可以通过引入`all`方式引入所有模块。 ------------------------------------------------------------------------------- @@ -435,7 +483,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 cn.smartjavaai all - 1.0.25 + 1.0.26 ``` @@ -534,6 +582,16 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 --- +#### 图像分类模型(CLS) + +**YOLO 系列** + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|----------------|------------|----------------------------------------------------------------------| +| YOLOV11-cls | OnnxRuntime | 最流行的图像分类模型 | [Github](https://docs.ultralytics.com/zh/models/) | +| YOLOV8-cls | OnnxRuntime | 最流行的图像分类模型 | [Github](https://docs.ultralytics.com/zh/models/) | + + #### 目标检测模型 **YOLO 系列** @@ -583,6 +641,14 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 | YOLOV11-SEG | OnnxRuntime | Ultralytics在COCO 数据集 上训练的模型 | [Github](https://docs.ultralytics.com/zh/tasks/segment/) | | Mask R-CNN | MXNet | Mask R-CNN 是一种在目标检测基础上，同时为每个物体生成像素级分割区域的深度学习模型 | 无 | + +#### CLIP模型(Connecting text and images) + + +| 模型名称 | 引擎 | 模型简介 | 模型开源网站 | +|-------------|-------------|--------------------|------------| +| clip-vit-base-patch32 | PyTorch | openai的图片及文本特征提取模型 | [Github](https://github.com/openai/CLIP) | + --- #### OBB旋转框目标检测模型 @@ -694,14 +760,23 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 --- -#### 语音识别模型 +#### 语音识别模型(ASR) 这里仅介绍模型的开源项目，每个开源项目通常包含多个具体模型，本文不逐一列出。 -| 模型名称 | 模型简介 | 模型官网 | -|---------| ------------------------ |-----------------------------------------------| -| Whisper | OpenAI 开源的通用语音识别（ASR）模型，支持多语言转写和翻译，具有较高的识别精度，尤其在嘈杂环境中表现良好，适合离线和批量音频处理。 | [Github](https://github.com/ggml-org/whisper.cpp) | -| Vosk | 一个轻量级离线语音识别工具包，支持多种语言和平台（包括移动端与嵌入式设备），可在低资源环境中运行，适合实时语音识别场景。 | [Github](https://github.com/alphacep/vosk-api) | +| 模型名称 | 模型简介 | 模型官网 | +|---------|-------------------------------------------------------------------------|-----------------------------------------------| +| Whisper | OpenAI 开源的通用语音识别（ASR）模型，支持多语言转写和翻译，具有较高的识别精度，尤其在嘈杂环境中表现良好，适合离线和批量音频处理。 | [Github](https://github.com/ggml-org/whisper.cpp) | +| Vosk | 一个轻量级离线语音识别工具包，支持多种语言和平台（包括移动端与嵌入式设备），可在低资源环境中运行，适合实时语音识别场景。 | [Github](https://github.com/alphacep/vosk-api) | +| sherpa-onnx | Sherpa-ONNX 是一个基于 ONNX Runtime 的ASR 及 TTS 推理框架 | [Github](https://github.com/k2-fsa/sherpa-onnx) | + +#### 语音合成模型(TTS) + +这里仅介绍模型的开源项目，每个开源项目通常包含多个具体模型，本文不逐一列出。 + +| 模型名称 | 模型简介 | 模型官网 | +|---------|-------------------------------------------------------------------------|-----------------------------------------------| +| sherpa-onnx | Sherpa-ONNX 是一个基于 ONNX Runtime 的ASR 及 TTS 推理框架 | [Github](https://github.com/k2-fsa/sherpa-onnx) | --- @@ -715,7 +790,7 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ## 联系方式 -如您在使用过程中有任何问题或建议，欢迎添加微信，与我们交流并加入用户交流群 +如您在使用过程中有任何问题、建议，或希望进行技术交流与合作，欢迎添加微信与我联系，并加入用户交流群。 - **微信**: deng775747758 （请备注：SmartJavaAI） - **Email**: 775747758@qq.com @@ -739,6 +814,12 @@ SmartJavaAI是专为JAVA 开发者打造的一个功能丰富、开箱即用的 ## 近期更新日志 +## [v1.0.26] - 2025-10-24 +- 【通用视觉】集成 OpenAI CLIP 模型，支持以图搜图、以文搜图、以图搜文等功能 +- 【通用视觉】新增 YOLO 图像分类模型支持 +- 【ASR/TTS】集成 Sherpa TTS（语音合成）与 ASR（语音识别）模块，支持中文、粤语、方言、英文等多种语言 +- 【目标检测】优化视频目标检测功能 + ## [v1.0.25] - 2025-10-02 - 【人脸识别】 新增多种人脸识别模型 - 【底层优化】 支持自由选择 OpenCV 或 BufferedImage 作为图像引擎 diff --git a/all/pom.xml b/all/pom.xml index 5bb04f3..261ac47 100644 --- a/all/pom.xml +++ b/all/pom.xml @@ -6,11 +6,11 @@ cn.smartjavaai smartjavaai-parent - 1.0.25 + 1.0.26 all - 1.0.25 + 1.0.26 ${project.artifactId} SmartJavaAI https://github.com/geekwenjie/SmartJavaAI diff --git a/bom/pom.xml b/bom/pom.xml index 697dc70..6467e59 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -6,10 +6,10 @@ cn.smartjavaai smartjavaai-parent - 1.0.25 + 1.0.26 - 1.0.25 + 1.0.26 bom bom 统一版本管理的 BOM 包，同时支持 import 和全量依赖 diff --git a/common/pom.xml b/common/pom.xml index 06eabfa..9874179 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -6,7 +6,7 @@ cn.smartjavaai smartjavaai-parent - 1.0.25 + 1.0.26 common diff --git a/common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java b/common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java index 14a1628..2528338 100644 --- a/common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java +++ b/common/src/main/java/cn/smartjavaai/common/config/ModelConfig.java @@ -45,6 +45,12 @@ public class ModelConfig { return clazz.cast(value); } + public T getCustomParam(String key, Class clazz, T defaultValue) { + Object value = customParams.getOrDefault(key, defaultValue); + return clazz.cast(value); + } + + /** * 添加个性化配置项 */ diff --git a/face/src/main/java/cn/smartjavaai/face/enums/SimilarityType.java b/common/src/main/java/cn/smartjavaai/common/enums/SimilarityType.java similarity index 85% rename from face/src/main/java/cn/smartjavaai/face/enums/SimilarityType.java rename to common/src/main/java/cn/smartjavaai/common/enums/SimilarityType.java index 805350a..264df3d 100644 --- a/face/src/main/java/cn/smartjavaai/face/enums/SimilarityType.java +++ b/common/src/main/java/cn/smartjavaai/common/enums/SimilarityType.java @@ -1,4 +1,4 @@ -package cn.smartjavaai.face.enums; +package cn.smartjavaai.common.enums; /** * @author dwj diff --git a/common/src/main/java/cn/smartjavaai/common/utils/BufferedImageUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/BufferedImageUtils.java index fd9cb8f..9b590bc 100644 --- a/common/src/main/java/cn/smartjavaai/common/utils/BufferedImageUtils.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/BufferedImageUtils.java @@ -573,4 +573,36 @@ public class BufferedImageUtils { } } + + /** + * 绘制检测框 + * @param sourceImage + * @param detectionResponse + * @throws IOException + */ + public static void drawFaceSearchResult(Graphics2D graphics, DetectionInfo detectionInfo, String text) { + graphics.setColor(Color.RED);// 边框颜色 + graphics.setStroke(new BasicStroke(2)); // 线宽2像素 + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); // 抗锯齿 + int stroke = 2; + DetectionRectangle rectangle = detectionInfo.getDetectionRectangle(); + graphics.setColor(Color.RED);// 边框颜色 + graphics.drawRect(rectangle.getX(), rectangle.getY(), rectangle.getWidth(), rectangle.getHeight()); + //绘制人脸关键点 + //人脸查询结果 + if(detectionInfo.getFaceInfo().getFaceSearchResults() != null){ + for (FaceSearchResult faceSearchResult : detectionInfo.getFaceInfo().getFaceSearchResults()){ + if(StringUtils.isNotBlank(faceSearchResult.getMetadata())){ + JsonObject metadata = GsonUtils.parseToJsonObject(faceSearchResult.getMetadata()); + JsonElement nameElement = metadata.get("name"); + if(metadata.has("name")){ + Graphics2DUtils.drawText(graphics, nameElement.getAsString(), rectangle.getX(), rectangle.getY(), stroke, 4); + } + } + } + } + graphics.dispose(); + } + } diff --git a/common/src/main/java/cn/smartjavaai/common/utils/DJLCommonUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/DJLCommonUtils.java index 98d1559..1bd73e4 100644 --- a/common/src/main/java/cn/smartjavaai/common/utils/DJLCommonUtils.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/DJLCommonUtils.java @@ -11,6 +11,7 @@ import org.opencv.core.Mat; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -19,6 +20,16 @@ import java.util.Objects; */ public class DJLCommonUtils { + private static final List SUPPORTED_PROTOCOLS = Arrays.asList( + "file://", + "http://", + "https://", + "jar://", + "djl://", + "s3://", + "hdfs://" + ); + /** * 检查模型目录中是否存在 "serving.properties" 文件 * @@ -158,5 +169,17 @@ public class DJLCommonUtils { return new DetectedObjects(classNames, probabilities, boxes); } + /** + * 判断路径是否以已知协议开头 + * @param path 模型路径 + * @return 是否以支持的协议开头 + */ + public static boolean hasSupportedProtocol(String path) { + if (path == null || path.isEmpty()) { + return false; + } + return SUPPORTED_PROTOCOLS.stream().anyMatch(path::startsWith); + } + } diff --git a/common/src/main/java/cn/smartjavaai/common/utils/FileUtils.java b/common/src/main/java/cn/smartjavaai/common/utils/FileUtils.java index a3cef1d..218bffc 100644 --- a/common/src/main/java/cn/smartjavaai/common/utils/FileUtils.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/FileUtils.java @@ -1,6 +1,9 @@ package cn.smartjavaai.common.utils; import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; /** * 文件操作工具类 @@ -28,4 +31,132 @@ public class FileUtils { File file = new File(path); return file.exists() && file.isDirectory(); } + + /** + * 查找指定目录下指定后缀的文件 + * + * @param dir 目录 + * @param suffix 文件后缀，例如 ".txt"、".wav" + * @param recursive 是否递归子目录 + * @return 文件列表 + */ + public static List findFilesWithSuffix(File dir, String suffix, boolean recursive) { + List result = new ArrayList<>(); + if (dir == null || !dir.exists() || !dir.isDirectory()) { + return result; + } + searchFiles(dir, suffix, recursive, result); + return result; + } + + // 搜索方法 + private static void searchFiles(File dir, String suffix, boolean recursive, List result) { + File[] files = dir.listFiles(); + if (files == null) return; + + for (File file : files) { + if (file.isDirectory()) { + if(file.getName().endsWith(suffix)){ + result.add(file); + continue; + } + if (recursive) { + searchFiles(file, suffix, true, result); + } + } else if (file.isFile() && file.getName().endsWith(suffix)) { + result.add(file); + } + } + } + + /** + * 查找指定目录下指定文件名的文件 + * + * @param dir 目录 + * @param fileName 文件名（精确匹配） + * @param recursive 是否递归子目录 + * @return 文件列表 + */ + public static List findFilesByName(File dir, String fileName, boolean recursive) { + List result = new ArrayList<>(); + if (dir == null || !dir.exists() || !dir.isDirectory() || fileName == null) { + return result; + } + searchByName(dir, fileName, recursive, result); + return result; + } + + // 递归搜索方法 + private static void searchByName(File dir, String fileName, boolean recursive, List result) { + File[] files = dir.listFiles(); + if (files == null) return; + + for (File file : files) { + if (file.isDirectory()) { + if (file.getName().equals(fileName)){ + result.add(file); + continue; + } + if (recursive) { + searchByName(file, fileName, true, result); + } + } else if (file.isFile() && file.getName().equals(fileName)) { + result.add(file); + } + } + } + + /** + * 将文件列表转换为绝对路径字符串 + * + * @param files 文件列表 + * @return 绝对路径字符串，用逗号分隔 + */ + public static String joinAbsolutePaths(List files) { + if (files == null || files.isEmpty()) { + return ""; + } + return files.stream() + .map(File::getAbsolutePath) + .collect(Collectors.joining(",")); + } + + /** + * 在指定目录中查找文件名包含指定关键字的文件，可选指定后缀。 + * + * @param dirPath 要搜索的目录路径 + * @param keyword 文件名包含的关键字（可为 null） + * @param extension 文件后缀名（例如 ".wav"，可为 null） + * @param recursive 是否递归搜索子目录 + * @return 匹配的文件列表 + */ + public static List searchFiles(String dirPath, String keyword, String extension, boolean recursive) { + List result = new ArrayList<>(); + File dir = new File(dirPath); + + if (!dir.exists() || !dir.isDirectory()) { + System.err.println("目录不存在或不是目录：" + dirPath); + return result; + } + + File[] files = dir.listFiles(); + if (files == null) return result; + + for (File file : files) { + if (file.isDirectory() && recursive) { + // 递归子目录 + result.addAll(searchFiles(file.getAbsolutePath(), keyword, extension, true)); + } else if (file.isFile()) { + String name = file.getName().toLowerCase(); + boolean matchKeyword = (keyword == null || name.contains(keyword.toLowerCase())); + boolean matchExt = (extension == null || name.endsWith(extension.toLowerCase())); + + if (matchKeyword && matchExt) { + result.add(file); + } + } + } + return result; + } + } diff --git a/face/src/main/java/cn/smartjavaai/face/utils/SimilarityUtil.java b/common/src/main/java/cn/smartjavaai/common/utils/SimilarityUtil.java similarity index 97% rename from face/src/main/java/cn/smartjavaai/face/utils/SimilarityUtil.java rename to common/src/main/java/cn/smartjavaai/common/utils/SimilarityUtil.java index f949fde..f693618 100644 --- a/face/src/main/java/cn/smartjavaai/face/utils/SimilarityUtil.java +++ b/common/src/main/java/cn/smartjavaai/common/utils/SimilarityUtil.java @@ -1,6 +1,7 @@ -package cn.smartjavaai.face.utils; +package cn.smartjavaai.common.utils; -import cn.smartjavaai.face.enums.SimilarityType; + +import cn.smartjavaai.common.enums.SimilarityType; /** * 特征相似度计算工具类 diff --git a/examples/face-example/pom.xml b/examples/face-example/pom.xml index 9191c33..b9ec72b 100644 --- a/examples/face-example/pom.xml +++ b/examples/face-example/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.25 + 1.0.26 smartai.examples.face.facedet.FaceDetDemo diff --git a/examples/ocr-examples/pom.xml b/examples/ocr-examples/pom.xml index 3266564..927c9a5 100644 --- a/examples/ocr-examples/pom.xml +++ b/examples/ocr-examples/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.25 + 1.0.26 smartai.examples.ocr.common.OcrRecognizeDemo diff --git a/examples/speech-examples/pom.xml b/examples/speech-examples/pom.xml index 6b5d89f..69082b6 100644 --- a/examples/speech-examples/pom.xml +++ b/examples/speech-examples/pom.xml @@ -12,7 +12,7 @@ 11 11 UTF-8 - 1.0.25 + 1.0.26 smartai.examples.speech.asr.common.OcrRecognizeDemo diff --git a/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SherpaAsrDemo.java b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SherpaAsrDemo.java new file mode 100644 index 0000000..ff814d7 --- /dev/null +++ b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SherpaAsrDemo.java @@ -0,0 +1,389 @@ +package smartai.examples.speech.asr; + +import ai.djl.modality.audio.Audio; +import ai.djl.modality.audio.AudioFactory; +import ai.djl.util.JsonUtils; +import cn.smartjavaai.common.entity.Language; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.speech.asr.config.AsrModelConfig; +import cn.smartjavaai.speech.asr.entity.AsrResult; +import cn.smartjavaai.speech.asr.entity.WhisperParams; +import cn.smartjavaai.speech.asr.enums.AsrModelEnum; +import cn.smartjavaai.speech.asr.factory.SpeechRecognizerFactory; +import cn.smartjavaai.speech.asr.model.SpeechRecognizer; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.nio.file.Paths; + +/** + * 语音识别ASR demo + * sherpa-onnx模型及依赖库下载链接: + * 1、（推荐）依赖库官网下载：https://github.com/k2-fsa/sherpa-onnx/releases + * 2、（推荐）ASR模型官网下载：https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + * 3、百度网盘下载：https://pan.baidu.com/s/19p3WhVEM7dgdkvXFaeeAxg?pwd=1234 提取码: 1234 + * @author dwj + * @date 2025/10/23 + */ +@Slf4j +public class SherpaAsrDemo { + + + /** + * 语音识别：Sherpa Paraformer（中文） + */ + @Test + public void testSherpaParaformerAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_PARAFORMER); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-paraformer-zh-2023-09-14"); + config.setModelName("model.int8.onnx"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-paraformer-zh-2023-09-14/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Dolphin（中文） + */ + @Test + public void testSherpaDolphinAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_DOLPHIN); + config.setModelName("model.int8.onnx"); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa zipformer（中文） + */ + @Test + public void testSherpaZipformerAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_ZIPFORMERCTC); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + config.setModelName("model.int8.onnx"); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa FireRedAsr（中英） + */ + @Test + public void testSherpaFireRedAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_FIREREDASR); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/3.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa SenseVoice （多语言模型） + */ + @Test + public void testSherpaSenseVoiceAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_SENSEVOICE); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17"); + config.setModelName("model.onnx"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/test_wavs/zh.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa WenetCtc（多语言模型：粤语） + */ + @Test + public void testSherpaWenetCtcAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_WENETCTC); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10"); + config.setModelName("model.int8.onnx"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10/test_wavs/yue-0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa FireRedAsr（方言：四川、天津、河南） + */ + @Test + public void testSherpaFireRedAsrSichuan() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_FIREREDASR); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/3-sichuan.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Telespeech（方言-天津、河南、四川） + */ + @Test + public void testSherpaTelespeechAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_TELESPEECH); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.setModelName("model.int8.onnx"); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04/test_wavs/4-tianjin.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Nemo（英文） + */ + @Test + public void testSherpaNemoAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_NEMO); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-ctc-en-citrinet-512"); + config.setModelName("model.onnx"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Moonshine（英文） + */ + @Test + public void testSherpaMoonshineAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_MOONSHINE); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-moonshine-tiny-en-int8"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 2); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + + /** + * 语音识别：Sherpa Whisper（英文） + */ + @Test + public void testSherpaWhisperAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_WHISPER); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-whisper-tiny"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-whisper-tiny/test_wavs/0.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Transducer（英文） + */ + @Test + public void testSherpaTransducerAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_TRANSDUCER); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-gigaspeech-2023-12-12"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-gigaspeech-2023-12-12/test_wavs/1221-135766-0001.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音识别：Sherpa Canary（德语-英文） + */ + @Test + public void testSherpaCanaryAsr() { + try { + //获取模型 + AsrModelConfig config = new AsrModelConfig(); + config.setModelEnum(AsrModelEnum.SHERPA_CANARY); + config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8"); + config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + config.putCustomParam("debug", false); + config.putCustomParam("numThreads", 1); + SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config); + Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/de.wav")); + R result = recognizer.recognize(audio); + if (result.isSuccess()){ + log.info("识别成功:{}", result.getData()); + }else{ + log.error("识别失败:{}", result.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + + + + + + + + + +} diff --git a/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java index 7992a39..23795d8 100644 --- a/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java +++ b/examples/speech-examples/src/main/java/smartai/examples/speech/asr/SpeechRecognizeDemo.java @@ -33,7 +33,7 @@ import java.io.InputStream; import java.nio.file.Paths; /** - * 语音识别demo + * 语音识别demo（Vosk、Whisper） * 模型下载网盘：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 * 文档地址：http://doc.smartjavaai.cn/ * @author dwj @@ -41,6 +41,8 @@ import java.nio.file.Paths; @Slf4j public class SpeechRecognizeDemo { + + /** * 获取Whisper模型 * 模型下载网盘：https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234 diff --git a/examples/speech-examples/src/main/java/smartai/examples/speech/tts/TtsDemo.java b/examples/speech-examples/src/main/java/smartai/examples/speech/tts/TtsDemo.java new file mode 100644 index 0000000..1152c69 --- /dev/null +++ b/examples/speech-examples/src/main/java/smartai/examples/speech/tts/TtsDemo.java @@ -0,0 +1,193 @@ +package smartai.examples.speech.tts; + + +import ai.djl.modality.audio.Audio; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.speech.tts.config.TtsModelConfig; +import cn.smartjavaai.speech.tts.entity.SherpaTtsParams; +import cn.smartjavaai.speech.tts.enums.TtsModelEnum; +import cn.smartjavaai.speech.tts.factory.TtsModelFactory; +import cn.smartjavaai.speech.tts.model.SherpaTtsModel; +import cn.smartjavaai.speech.tts.model.TtsModel; +import cn.smartjavaai.speech.utils.AudioUtils; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; + +/** + * 语音合成demo（TTS） + * sherpa-onnx模型及依赖库下载链接: + * 1、（推荐）依赖库官网下载：https://github.com/k2-fsa/sherpa-onnx/releases + * 2、（推荐）TTS模型官网下载：https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + * 3、百度网盘下载：https://pan.baidu.com/s/186REUf7p1z0HH9AZNnwCUg?pwd=1234 提取码: 1234 + * @author dwj + * @date 2025/10/22 + */ +@Slf4j +public class TtsDemo { + + public static String enText = "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" + + " businessman, an official, or a scholar."; + + public static String zhText = "有问题，请拨打110或者手机18601239876。我们的价值观是真诚热爱！"; + + String znEnText = + "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki." + + " 你觉得中英文说的如何呢？"; + + //设备类型 + public static DeviceEnum device = DeviceEnum.CPU; + + + /** + * 获取Vits中文模型 + * @return + */ + public TtsModel getVitsZhModel(){ + TtsModelConfig modelConfig = new TtsModelConfig(); + modelConfig.setModelEnum(TtsModelEnum.SHERPA_VITS); + modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/vits-zh-hf-keqing"); + modelConfig.setModelName("keqing.onnx"); + modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + modelConfig.setDevice(device); + modelConfig.putCustomParam("debug", false); + modelConfig.putCustomParam("numThreads", 1); + return TtsModelFactory.getInstance().getModel(modelConfig); + } + + /** + * 获取Matcha中文模型 + * @return + */ + public TtsModel getMatchaZhModel(){ + TtsModelConfig modelConfig = new TtsModelConfig(); + modelConfig.setModelEnum(TtsModelEnum.SHERPA_MATCHA); + modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker"); + modelConfig.setModelName("model-steps-3.onnx"); + modelConfig.putCustomParam("vocoder", "/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker/vocos-22khz-univ.onnx"); + modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + modelConfig.setDevice(device); + modelConfig.putCustomParam("debug", false); + modelConfig.putCustomParam("numThreads", 1); + return TtsModelFactory.getInstance().getModel(modelConfig); + } + + /** + * 获取Kokoro多语言模型 + * @return + */ + public TtsModel getKokoroZhEnModel(){ + TtsModelConfig modelConfig = new TtsModelConfig(); + modelConfig.setModelEnum(TtsModelEnum.SHERPA_KOKORO); + modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/kokoro-multi-lang-v1_0"); + modelConfig.setModelName("model.onnx"); + modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + modelConfig.setDevice(device); + modelConfig.putCustomParam("debug", false); + modelConfig.putCustomParam("numThreads", 2); + return TtsModelFactory.getInstance().getModel(modelConfig); + } + + /** + * 获取Matcha模型（英文） + * @return + */ + public TtsModel getMatchaEnModel(){ + TtsModelConfig modelConfig = new TtsModelConfig(); + modelConfig.setModelEnum(TtsModelEnum.SHERPA_MATCHA); + modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-en_US-ljspeech"); + modelConfig.setModelName("model-steps-3.onnx"); + modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib")); + modelConfig.putCustomParam("vocoder", "/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker/vocos-22khz-univ.onnx"); + modelConfig.setDevice(device); + modelConfig.putCustomParam("debug", false); + modelConfig.putCustomParam("numThreads", 2); + return TtsModelFactory.getInstance().getModel(modelConfig); + } + + /** + * 语音合成Vits（中文） + */ + @Test + public void testVitsZhTts() throws IOException { + TtsModel ttsModel = getVitsZhModel(); + SherpaTtsParams params = new SherpaTtsParams(); + //语速 + params.setSpeed(1f); + //说话人ID + params.setSpeakerId(100); + R