mirror of
https://github.com/geekwenjie/SmartJavaAI.git
synced 2026-09-12 12:48:57 +00:00
【通用视觉】集成 OpenAI CLIP 模型,支持以图搜图、以文搜图、以图搜文等功能
【通用视觉】新增 YOLO 图像分类模型支持 【ASR/TTS】集成 Sherpa TTS(语音合成)与 ASR(语音识别)模块,支持中文、粤语、方言、英文等多种语言 【目标检测】优化视频目标检测功能
This commit is contained in:
@@ -12,7 +12,7 @@
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<smartjavaai.version>1.0.25</smartjavaai.version>
|
||||
<smartjavaai.version>1.0.26</smartjavaai.version>
|
||||
<!--如果打包运行,需要替换成你的main-->
|
||||
<exec.mainClass>smartai.examples.speech.asr.common.OcrRecognizeDemo</exec.mainClass>
|
||||
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
package smartai.examples.speech.asr;
|
||||
|
||||
import ai.djl.modality.audio.Audio;
|
||||
import ai.djl.modality.audio.AudioFactory;
|
||||
import ai.djl.util.JsonUtils;
|
||||
import cn.smartjavaai.common.entity.Language;
|
||||
import cn.smartjavaai.common.entity.R;
|
||||
import cn.smartjavaai.speech.asr.config.AsrModelConfig;
|
||||
import cn.smartjavaai.speech.asr.entity.AsrResult;
|
||||
import cn.smartjavaai.speech.asr.entity.WhisperParams;
|
||||
import cn.smartjavaai.speech.asr.enums.AsrModelEnum;
|
||||
import cn.smartjavaai.speech.asr.factory.SpeechRecognizerFactory;
|
||||
import cn.smartjavaai.speech.asr.model.SpeechRecognizer;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.nio.file.Paths;
|
||||
|
||||
/**
|
||||
* 语音识别ASR demo
|
||||
* sherpa-onnx模型及依赖库下载链接:
|
||||
* 1、(推荐)依赖库官网下载:https://github.com/k2-fsa/sherpa-onnx/releases
|
||||
* 2、(推荐)ASR模型官网下载:https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models
|
||||
* 3、百度网盘下载:https://pan.baidu.com/s/19p3WhVEM7dgdkvXFaeeAxg?pwd=1234 提取码: 1234
|
||||
* @author dwj
|
||||
* @date 2025/10/23
|
||||
*/
|
||||
@Slf4j
|
||||
public class SherpaAsrDemo {
|
||||
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa Paraformer(中文)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaParaformerAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_PARAFORMER);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-paraformer-zh-2023-09-14");
|
||||
config.setModelName("model.int8.onnx");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-paraformer-zh-2023-09-14/test_wavs/0.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa Dolphin(中文)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaDolphinAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_DOLPHIN);
|
||||
config.setModelName("model.int8.onnx");
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02/test_wavs/0.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa zipformer(中文)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaZipformerAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_ZIPFORMERCTC);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
config.setModelName("model.int8.onnx");
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03/test_wavs/0.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa FireRedAsr(中英)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaFireRedAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_FIREREDASR);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/3.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa SenseVoice (多语言模型)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaSenseVoiceAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_SENSEVOICE);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17");
|
||||
config.setModelName("model.onnx");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/test_wavs/zh.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa WenetCtc(多语言模型:粤语)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaWenetCtcAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_WENETCTC);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10");
|
||||
config.setModelName("model.int8.onnx");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10/test_wavs/yue-0.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa FireRedAsr(方言:四川、天津、河南)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaFireRedAsrSichuan() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_FIREREDASR);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/3-sichuan.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa Telespeech(方言-天津、河南、四川)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaTelespeechAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_TELESPEECH);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.setModelName("model.int8.onnx");
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04/test_wavs/4-tianjin.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa Nemo(英文)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaNemoAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_NEMO);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-ctc-en-citrinet-512");
|
||||
config.setModelName("model.onnx");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/0.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa Moonshine(英文)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaMoonshineAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_MOONSHINE);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-moonshine-tiny-en-int8");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 2);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa Whisper(英文)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaWhisperAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_WHISPER);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-whisper-tiny");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-whisper-tiny/test_wavs/0.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa Transducer(英文)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaTransducerAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_TRANSDUCER);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-gigaspeech-2023-12-12");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-zipformer-gigaspeech-2023-12-12/test_wavs/1221-135766-0001.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音识别:Sherpa Canary(德语-英文)
|
||||
*/
|
||||
@Test
|
||||
public void testSherpaCanaryAsr() {
|
||||
try {
|
||||
//获取模型
|
||||
AsrModelConfig config = new AsrModelConfig();
|
||||
config.setModelEnum(AsrModelEnum.SHERPA_CANARY);
|
||||
config.setModelPath("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8");
|
||||
config.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
config.putCustomParam("debug", false);
|
||||
config.putCustomParam("numThreads", 1);
|
||||
SpeechRecognizer recognizer = SpeechRecognizerFactory.getInstance().getModel(config);
|
||||
Audio audio = AudioFactory.newInstance().fromFile(Paths.get("/Users/wenjie/Documents/develop/model/speech/sherpa-asr/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/de.wav"));
|
||||
R<AsrResult> result = recognizer.recognize(audio);
|
||||
if (result.isSuccess()){
|
||||
log.info("识别成功:{}", result.getData());
|
||||
}else{
|
||||
log.error("识别失败:{}", result.getMessage());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -33,7 +33,7 @@ import java.io.InputStream;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
/**
|
||||
* 语音识别demo
|
||||
* 语音识别demo(Vosk、Whisper)
|
||||
* 模型下载网盘:https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234
|
||||
* 文档地址:http://doc.smartjavaai.cn/
|
||||
* @author dwj
|
||||
@@ -41,6 +41,8 @@ import java.nio.file.Paths;
|
||||
@Slf4j
|
||||
public class SpeechRecognizeDemo {
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 获取Whisper模型
|
||||
* 模型下载网盘:https://pan.baidu.com/s/1kiMF5MF641R7LTn1GpB2lQ?pwd=1234 提取码: 1234
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package smartai.examples.speech.tts;
|
||||
|
||||
|
||||
import ai.djl.modality.audio.Audio;
|
||||
import cn.smartjavaai.common.entity.R;
|
||||
import cn.smartjavaai.common.enums.DeviceEnum;
|
||||
import cn.smartjavaai.speech.tts.config.TtsModelConfig;
|
||||
import cn.smartjavaai.speech.tts.entity.SherpaTtsParams;
|
||||
import cn.smartjavaai.speech.tts.enums.TtsModelEnum;
|
||||
import cn.smartjavaai.speech.tts.factory.TtsModelFactory;
|
||||
import cn.smartjavaai.speech.tts.model.SherpaTtsModel;
|
||||
import cn.smartjavaai.speech.tts.model.TtsModel;
|
||||
import cn.smartjavaai.speech.utils.AudioUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
/**
|
||||
* 语音合成demo(TTS)
|
||||
* sherpa-onnx模型及依赖库下载链接:
|
||||
* 1、(推荐)依赖库官网下载:https://github.com/k2-fsa/sherpa-onnx/releases
|
||||
* 2、(推荐)TTS模型官网下载:https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
|
||||
* 3、百度网盘下载:https://pan.baidu.com/s/186REUf7p1z0HH9AZNnwCUg?pwd=1234 提取码: 1234
|
||||
* @author dwj
|
||||
* @date 2025/10/22
|
||||
*/
|
||||
@Slf4j
|
||||
public class TtsDemo {
|
||||
|
||||
public static String enText = "Today as always, men fall into two groups: slaves and free men. Whoever does not have"
|
||||
+ " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a"
|
||||
+ " businessman, an official, or a scholar.";
|
||||
|
||||
public static String zhText = "有问题,请拨打110或者手机18601239876。我们的价值观是真诚热爱!";
|
||||
|
||||
String znEnText =
|
||||
"中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki."
|
||||
+ " 你觉得中英文说的如何呢?";
|
||||
|
||||
//设备类型
|
||||
public static DeviceEnum device = DeviceEnum.CPU;
|
||||
|
||||
|
||||
/**
|
||||
* 获取Vits中文模型
|
||||
* @return
|
||||
*/
|
||||
public TtsModel getVitsZhModel(){
|
||||
TtsModelConfig modelConfig = new TtsModelConfig();
|
||||
modelConfig.setModelEnum(TtsModelEnum.SHERPA_VITS);
|
||||
modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/vits-zh-hf-keqing");
|
||||
modelConfig.setModelName("keqing.onnx");
|
||||
modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
modelConfig.setDevice(device);
|
||||
modelConfig.putCustomParam("debug", false);
|
||||
modelConfig.putCustomParam("numThreads", 1);
|
||||
return TtsModelFactory.getInstance().getModel(modelConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取Matcha中文模型
|
||||
* @return
|
||||
*/
|
||||
public TtsModel getMatchaZhModel(){
|
||||
TtsModelConfig modelConfig = new TtsModelConfig();
|
||||
modelConfig.setModelEnum(TtsModelEnum.SHERPA_MATCHA);
|
||||
modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker");
|
||||
modelConfig.setModelName("model-steps-3.onnx");
|
||||
modelConfig.putCustomParam("vocoder", "/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker/vocos-22khz-univ.onnx");
|
||||
modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
modelConfig.setDevice(device);
|
||||
modelConfig.putCustomParam("debug", false);
|
||||
modelConfig.putCustomParam("numThreads", 1);
|
||||
return TtsModelFactory.getInstance().getModel(modelConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取Kokoro多语言模型
|
||||
* @return
|
||||
*/
|
||||
public TtsModel getKokoroZhEnModel(){
|
||||
TtsModelConfig modelConfig = new TtsModelConfig();
|
||||
modelConfig.setModelEnum(TtsModelEnum.SHERPA_KOKORO);
|
||||
modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/kokoro-multi-lang-v1_0");
|
||||
modelConfig.setModelName("model.onnx");
|
||||
modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
modelConfig.setDevice(device);
|
||||
modelConfig.putCustomParam("debug", false);
|
||||
modelConfig.putCustomParam("numThreads", 2);
|
||||
return TtsModelFactory.getInstance().getModel(modelConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取Matcha模型(英文)
|
||||
* @return
|
||||
*/
|
||||
public TtsModel getMatchaEnModel(){
|
||||
TtsModelConfig modelConfig = new TtsModelConfig();
|
||||
modelConfig.setModelEnum(TtsModelEnum.SHERPA_MATCHA);
|
||||
modelConfig.setModelPath("/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-en_US-ljspeech");
|
||||
modelConfig.setModelName("model-steps-3.onnx");
|
||||
modelConfig.setLibPath(Paths.get("/Users/wenjie/smartjavaai_cache/sherpa-onnx-v1.12.14-osx-arm64-jni/lib"));
|
||||
modelConfig.putCustomParam("vocoder", "/Users/wenjie/Documents/develop/model/speech/tts/matcha-icefall-zh-baker/vocos-22khz-univ.onnx");
|
||||
modelConfig.setDevice(device);
|
||||
modelConfig.putCustomParam("debug", false);
|
||||
modelConfig.putCustomParam("numThreads", 2);
|
||||
return TtsModelFactory.getInstance().getModel(modelConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音合成Vits(中文)
|
||||
*/
|
||||
@Test
|
||||
public void testVitsZhTts() throws IOException {
|
||||
TtsModel ttsModel = getVitsZhModel();
|
||||
SherpaTtsParams params = new SherpaTtsParams();
|
||||
//语速
|
||||
params.setSpeed(1f);
|
||||
//说话人ID
|
||||
params.setSpeakerId(100);
|
||||
R<Audio> result = ttsModel.generate(zhText, params);
|
||||
if (result.isSuccess()){
|
||||
Audio audio = result.getData();
|
||||
AudioUtils.saveToWav(audio.getData(),"/Users/wenjie/Downloads/tts-vits-zh.wav");
|
||||
}else{
|
||||
System.out.println(result.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音合成Matcha(中文)
|
||||
*/
|
||||
@Test
|
||||
public void testMatchaZhTts() throws IOException {
|
||||
TtsModel ttsModel = getMatchaZhModel();
|
||||
SherpaTtsParams params = new SherpaTtsParams();
|
||||
//语速
|
||||
params.setSpeed(1.5f);
|
||||
//说话人ID
|
||||
params.setSpeakerId(0);
|
||||
R<Audio> result = ttsModel.generate(zhText, params);
|
||||
if (result.isSuccess()){
|
||||
Audio audio = result.getData();
|
||||
AudioUtils.saveToWav(audio.getData(),"/Users/wenjie/Downloads/tts-matcha-zh.wav");
|
||||
}else{
|
||||
System.out.println(result.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音合成Kokoro(中英)
|
||||
*/
|
||||
@Test
|
||||
public void testKokoroZhEnTts() throws IOException {
|
||||
TtsModel ttsModel = getKokoroZhEnModel();
|
||||
SherpaTtsParams params = new SherpaTtsParams();
|
||||
//语速
|
||||
params.setSpeed(1f);
|
||||
//说话人ID
|
||||
params.setSpeakerId(3);
|
||||
R<Audio> result = ttsModel.generate(znEnText, params);
|
||||
if (result.isSuccess()){
|
||||
Audio audio = result.getData();
|
||||
AudioUtils.saveToWav(audio.getData(),"/Users/wenjie/Downloads/tts-kokoro-zh-en.wav");
|
||||
}else{
|
||||
System.out.println(result.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 语音合成Matcha(英文)
|
||||
*/
|
||||
@Test
|
||||
public void testMatchaEnTts() throws IOException {
|
||||
TtsModel ttsModel = getMatchaEnModel();
|
||||
SherpaTtsParams params = new SherpaTtsParams();
|
||||
//语速
|
||||
params.setSpeed(1.5f);
|
||||
//说话人ID
|
||||
params.setSpeakerId(0);
|
||||
R<Audio> result = ttsModel.generate(enText, params);
|
||||
if (result.isSuccess()){
|
||||
Audio audio = result.getData();
|
||||
AudioUtils.saveToWav(audio.getData(),"/Users/wenjie/Downloads/tts-kitten-en.wav");
|
||||
}else{
|
||||
System.out.println(result.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user