ZF/SmartJavaAI
1
0
Fork 0
mirror of https://github.com/geekwenjie/SmartJavaAI.git synced 2026-09-12 20:58:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

- 新增 语音识别模块，集成 OpenAI 开源的 Whisper 和 Vosk

Browse Source 
- 修复 质量评估模型的 Bug
- 修复 OCR 模块 recognizeAndDraw 方法的 Bug
- 修复 车牌识别在未检测到车牌时的报错问题
- 优化 OCR 表格识别功能，新增导出方式
This commit is contained in:
dengwenjie
2025-08-11 10:38:50 +08:00
parent 9b6f5ea9a1
commit 166f0f9d20
46 changed files with 1325 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/config/AsrModelConfig.java
View File

@@ -4,7 +4,10 @@ import cn.smartjavaai.common.config.ModelConfig;
import cn.smartjavaai.speech.asr.enums.AsrModelEnum;
import lombok.Data;
import java.nio.file.Path;
/**
* Asr模型配置
* @author dwj
* @date 2025/7/31
*/
@@ -14,4 +17,9 @@ public class AsrModelConfig extends ModelConfig {
private AsrModelEnum modelEnum;
private String modelPath;
/**
* 依赖库目录
*/
private Path libPath;
}

12
smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/VoskParams.java
View File

@@ -12,19 +12,9 @@ import lombok.Data;
public class VoskParams extends RecParams{
/**
* 最大候选结果数
*/
private int maxAlternatives;
/**
* 限定词汇表 例：["yes", "no", "hello"]
* 限定词汇表
*/
private String grammar;
/**
* 是否返回词级别的识别结果（包含每个词的开始/结束时间和置信度）。
*/
private boolean words = true;
}

103
smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/factory/SpeechRecognizerFactory.java Normal file
View File

@@ -0,0 +1,103 @@
package cn.smartjavaai.speech.asr.factory;
import cn.smartjavaai.common.config.Config;
import cn.smartjavaai.speech.asr.config.AsrModelConfig;
import cn.smartjavaai.speech.asr.enums.AsrModelEnum;
import cn.smartjavaai.speech.asr.exception.AsrException;
import cn.smartjavaai.speech.asr.model.SpeechRecognizer;
import cn.smartjavaai.speech.asr.model.VoskRecognizer;
import cn.smartjavaai.speech.asr.model.WhisperRecognizer;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 语音识别模型工厂
* @author dwj
*/
@Slf4j
public class SpeechRecognizerFactory {
// 使用 volatile 和双重检查锁定来确保线程安全的单例模式
private static volatile SpeechRecognizerFactory instance;
/**
* 模型缓存
*/
private static final ConcurrentHashMap<AsrModelEnum, SpeechRecognizer> modelMap = new ConcurrentHashMap<>();
/**
* 模型注册表
*/
private static final Map<AsrModelEnum, Class<? extends SpeechRecognizer>> registry =
new ConcurrentHashMap<>();
public static SpeechRecognizerFactory getInstance() {
if (instance == null) {
synchronized (SpeechRecognizerFactory.class) {
if (instance == null) {
instance = new SpeechRecognizerFactory();
}
}
}
return instance;
}
/**
* 注册模型
* @param expressionModelEnum
* @param clazz
*/
private static void registerModel(AsrModelEnum expressionModelEnum, Class<? extends SpeechRecognizer> clazz) {
registry.put(expressionModelEnum, clazz);
}
/**
* 获取模型（通过配置）
* @param config
* @return
*/
public SpeechRecognizer getModel(AsrModelConfig config) {
if(Objects.isNull(config) || Objects.isNull(config.getModelEnum())){
throw new AsrException("未配置语音识别模型枚举");
}
return modelMap.computeIfAbsent(config.getModelEnum(), k -> {
return createFaceModel(config);
});
}
/**
* 使用ModelConfig创建模型
* @param config
* @return
*/
private SpeechRecognizer createFaceModel(AsrModelConfig config) {
Class<?> clazz = registry.get(config.getModelEnum());
if(clazz == null){
throw new AsrException("Unsupported model");
}
SpeechRecognizer model = null;
try {
model = (SpeechRecognizer) clazz.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new AsrException(e);
}
model.loadModel(config);
return model;
}
// 初始化默认算法
static {
registerModel(AsrModelEnum.WHISPER, WhisperRecognizer.class);
registerModel(AsrModelEnum.VOSK, VoskRecognizer.class);
log.debug("缓存目录：{}", Config.getCachePath());
}
}

113
smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/VoskRecognizer.java
View File

@@ -13,9 +13,6 @@ import cn.smartjavaai.speech.asr.pool.WhisperStatePool;
import cn.smartjavaai.speech.utils.AudioUtils;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import io.github.givimad.whisperjni.WhisperContext;
import io.github.givimad.whisperjni.WhisperJNI;
import io.github.givimad.whisperjni.WhisperState;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.vosk.LibVosk;
@@ -39,6 +36,8 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import static ai.djl.util.JsonUtils.GSON;
@@ -61,6 +60,10 @@ public class VoskRecognizer implements SpeechRecognizer{
throw new AsrException("Missing model file: " + testModelPath.toAbsolutePath());
}
try {
//加载自定义依赖库
if(Objects.nonNull(config.getLibPath())){
System.load(config.getLibPath().toAbsolutePath().toString());
}
model = new Model(config.getModelPath());
LibVosk.setLogLevel(LogLevel.DEBUG);
log.debug("Vosk init success");
@@ -86,7 +89,7 @@ public class VoskRecognizer implements SpeechRecognizer{
return recognize(audioStream, new VoskParams());
}
private R<AsrResult> recognizeAudioStream(AudioInputStream ais,RecParams params) {
private R<AsrResult> recognizeAudioStream(AudioInputStream ais, RecParams params) {
try (Recognizer recognizer = buildRecognizer(params, ais.getFormat().getSampleRate())){
AudioFormat audioFormat = ais.getFormat();
log.debug("sampleRate:{}", audioFormat.getSampleRate());
@@ -95,14 +98,49 @@ public class VoskRecognizer implements SpeechRecognizer{
byte[] b = new byte[4096];
List<AsrSegment> segments = new ArrayList<AsrSegment>();
StringBuilder text = new StringBuilder();
String temp = "";
while ((nbytes = ais.read(b)) >= 0) {
if (recognizer.acceptWaveForm(b, nbytes)) {
String result = recognizer.getResult();
AsrSegment segment = parseSegment(result);
segments.add(segment);
// log.info("result:{}", result);
AsrSegment segment = parseSegment(result, params);
if(segment != null){
segments.add(segment);
text.append(segment.getText());
}
}else{
temp = recognizer.getPartialResult();
// log.info("temp:{}", temp);
}
}
return R.ok(new AsrResult(text.toString(), segments));
if(StringUtils.isNotBlank(temp)){
AsrSegment segment = parsePartialSegment(temp, params);
if(segment != null){
segments.add(segment);
text.append(segment.getText());
}
}
//补全结果
String finalText = recognizer.getFinalResult();
if(StringUtils.isNotBlank(text.toString()) && StringUtils.isNotBlank(finalText)){
AsrSegment finalSegment = parseSegment(finalText, params);
if(finalSegment != null){
//需要补全
if(!text.toString().endsWith(finalSegment.getText())){
AsrSegment alignSegment = VoskRecognizer.alignSegment(segments.get(segments.size() - 1), finalSegment);
//如果匹配失败，则直接使用最终片段
if(alignSegment != null){
segments.set(segments.size() - 1,alignSegment);
}else{
segments.set(segments.size() - 1,finalSegment);
}
}
}
}
String result = segments.stream()
.map(AsrSegment::getText)
.collect(Collectors.joining("\n"));
return R.ok(new AsrResult(result, segments));
} catch (IOException e) {
throw new AsrException(e);
}
@@ -113,18 +151,65 @@ public class VoskRecognizer implements SpeechRecognizer{
* @param segment
* @return
*/
private AsrSegment parseSegment(String segment) {
private AsrSegment parseSegment(String segment, RecParams params) {
JsonObject json = GSON.fromJson(segment, JsonObject.class);
JsonArray resultArray = json.getAsJsonArray("result");
if(Objects.isNull(resultArray) || resultArray.size() == 0){
return null;
}
double segmentStart = resultArray.get(0).getAsJsonObject().get("start").getAsDouble();
double segmentEnd = resultArray.get(resultArray.size() - 1).getAsJsonObject().get("end").getAsDouble();
long startMs = Math.round(segmentStart * 1000);
long endMs = Math.round(segmentEnd * 1000);
String text = json.get("text").getAsString();
String noSpaces = text.replace(" ", "");
return new AsrSegment(noSpaces, startMs, endMs);
if(Objects.nonNull(params.getLanguage()) && params.getLanguage() == Language.ZH){
text = text.replace(" ", "");
}
return new AsrSegment(text, startMs, endMs);
}
/**
* 解析结果
* @param segment
* @return
*/
private AsrSegment parsePartialSegment(String segment, RecParams params) {
JsonObject json = GSON.fromJson(segment, JsonObject.class);
JsonArray resultArray = json.getAsJsonArray("partial_result");
if(Objects.isNull(resultArray) || resultArray.size() == 0){
return null;
}
double segmentStart = resultArray.get(0).getAsJsonObject().get("start").getAsDouble();
double segmentEnd = resultArray.get(resultArray.size() - 1).getAsJsonObject().get("end").getAsDouble();
long startMs = Math.round(segmentStart * 1000);
long endMs = Math.round(segmentEnd * 1000);
String text = json.get("partial").getAsString();
if(Objects.nonNull(params.getLanguage()) && params.getLanguage() == Language.ZH){
text = text.replace(" ", "");
}
return new AsrSegment(text, startMs, endMs);
}
/**
* 补全
* @param shortSeg
* @param longSeg
* @return
*/
public static AsrSegment alignSegment(AsrSegment shortSeg, AsrSegment longSeg) {
String shortText = shortSeg.getText();
String longText = longSeg.getText();
int index = longText.indexOf(shortText);
if (index == -1) {
// log.debug("短文本不在长文本中");
return null;
}
String resultText = longText.substring(index);
return new AsrSegment(resultText, shortSeg.getStartTime(), longSeg.getEndTime());
}
/**
* 创建识别器
* @param params
@@ -146,8 +231,8 @@ public class VoskRecognizer implements SpeechRecognizer{
// }
//暂时只支持返回一个结果
// recognizer.setMaxAlternatives(1);
recognizer.setWords(voskParams.isWords());
//recognizer.setPartialWords(true);
recognizer.setWords(true);
recognizer.setPartialWords(true);
return recognizer;
}
@@ -187,6 +272,7 @@ public class VoskRecognizer implements SpeechRecognizer{
tryStream = new BufferedInputStream(new ByteArrayInputStream(allBytes));
conversionStream = new BufferedInputStream(new ByteArrayInputStream(allBytes));
ais = AudioSystem.getAudioInputStream(tryStream);
return recognizeAudioStream(ais, params);
} catch (UnsupportedAudioFileException e) {
log.debug("Unsupported Audio file, Conversion to WAV is required");
needConversion = true;
@@ -207,6 +293,7 @@ public class VoskRecognizer implements SpeechRecognizer{
if(needConversion){
try {
tempFile = AudioUtils.audioFormatConversion(conversionStream, "wav");
// log.info("tempFile:{}", tempFile.getAbsolutePath());
} catch (EncoderException | IOException e) {
throw new AsrException(e);
}
@@ -228,7 +315,7 @@ public class VoskRecognizer implements SpeechRecognizer{
}
}
}
return recognizeAudioStream(ais, params);
return R.fail(R.Status.Unknown);
}
@Override

9
smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/WhisperRecognizer.java
View File

@@ -51,6 +51,10 @@ public class WhisperRecognizer implements SpeechRecognizer{
throw new AsrException("Missing model file: " + testModelPath.toAbsolutePath());
}
try {
//加载自定义依赖库
if(Objects.nonNull(config.getLibPath())){
System.setProperty("io.github.givimad.whisperjni.libdir",config.getLibPath().toAbsolutePath().toString());
}
WhisperJNI.loadLibrary();
WhisperJNI.setLibraryLogger(null);
whisper = new WhisperJNI();
@@ -116,6 +120,9 @@ public class WhisperRecognizer implements SpeechRecognizer{
StringBuilder text = new StringBuilder();
try {
WhisperParams whisperParams = (WhisperParams) params;
if(Objects.isNull(whisperParams.getParams().language)){
return R.fail(1003, "请指定语言");
}
//不是英语，需要检查是否是多语言模型
if(!Language.EN.getCode().equals(whisperParams.getParams().language)){
if(!whisper.isMultilingual(ctx)){
@@ -190,7 +197,7 @@ public class WhisperRecognizer implements SpeechRecognizer{
* 获取一个WhisperState对象
* @return
*/
private WhisperState getWhisperState(){
public WhisperState getWhisperState(){
try {
return statePool.borrowObject();
} catch (Exception e) {