diff --git a/pom.xml b/pom.xml index 4142bfb..2a2b9cb 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ smartjavaai-all smartjavaai-ocr smartjavaai-bom + smartjavaai-speech @@ -25,7 +26,6 @@ 8 UTF-8 0.32.0 - diff --git a/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Language.java b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Language.java new file mode 100644 index 0000000..8ee3deb --- /dev/null +++ b/smartjavaai-common/src/main/java/cn/smartjavaai/common/entity/Language.java @@ -0,0 +1,138 @@ +package cn.smartjavaai.common.entity; + +/** + * 语言枚举 + * @author dwj + */ +public enum Language { + + + EN("en", "English", "英语"), + ZH("zh", "Chinese", "中文"), + DE("de", "German", "德语"), + ES("es", "Spanish", "西班牙语"), + RU("ru", "Russian", "俄语"), + KO("ko", "Korean", "韩语"), + FR("fr", "French", "法语"), + JA("ja", "Japanese", "日语"), + PT("pt", "Portuguese", "葡萄牙语"), + TR("tr", "Turkish", "土耳其语"), + PL("pl", "Polish", "波兰语"), + CA("ca", "Catalan", "加泰罗尼亚语"), + NL("nl", "Dutch", "荷兰语"), + AR("ar", "Arabic", "阿拉伯语"), + SV("sv", "Swedish", "瑞典语"), + IT("it", "Italian", "意大利语"), + ID("id", "Indonesian", "印尼语"), + HI("hi", "Hindi", "印地语"), + FI("fi", "Finnish", "芬兰语"), + VI("vi", "Vietnamese", "越南语"), + HE("he", "Hebrew", "希伯来语"), + UK("uk", "Ukrainian", "乌克兰语"), + EL("el", "Greek", "希腊语"), + MS("ms", "Malay", "马来语"), + CS("cs", "Czech", "捷克语"), + RO("ro", "Romanian", "罗马尼亚语"), + DA("da", "Danish", "丹麦语"), + HU("hu", "Hungarian", "匈牙利语"), + TA("ta", "Tamil", "泰米尔语"), + NO("no", "Norwegian", "挪威语"), + TH("th", "Thai", "泰语"), + UR("ur", "Urdu", "乌尔都语"), + HR("hr", "Croatian", "克罗地亚语"), + BG("bg", "Bulgarian", "保加利亚语"), + LT("lt", "Lithuanian", "立陶宛语"), + LA("la", "Latin", "拉丁语"), + MI("mi", "Maori", "毛利语"), + ML("ml", "Malayalam", "马拉雅拉姆语"), + CY("cy", "Welsh", "威尔士语"), + SK("sk", "Slovak", "斯洛伐克语"), + TE("te", "Telugu", "泰卢固语"), + FA("fa", "Persian", "波斯语"), + LV("lv", "Latvian", "拉脱维亚语"), + BN("bn", "Bengali", "孟加拉语"), + SR("sr", "Serbian", "塞尔维亚语"), + AZ("az", "Azerbaijani", "阿塞拜疆语"), + SL("sl", "Slovenian", "斯洛文尼亚语"), + KN("kn", "Kannada", "卡纳达语"), + ET("et", "Estonian", "爱沙尼亚语"), + MK("mk", "Macedonian", "马其顿语"), + BR("br", "Breton", "布列塔尼语"), + EU("eu", "Basque", "巴斯克语"), + IS("is", "Icelandic", "冰岛语"), + HY("hy", "Armenian", "亚美尼亚语"), + NE("ne", "Nepali", "尼泊尔语"), + MN("mn", "Mongolian", "蒙古语"), + BS("bs", "Bosnian", "波斯尼亚语"), + KK("kk", "Kazakh", "哈萨克语"), + SQ("sq", "Albanian", "阿尔巴尼亚语"), + SW("sw", "Swahili", "斯瓦希里语"), + GL("gl", "Galician", "加利西亚语"), + MR("mr", "Marathi", "马拉地语"), + PA("pa", "Punjabi", "旁遮普语"), + SI("si", "Sinhala", "僧伽罗语"), + KM("km", "Khmer", "高棉语"), + SN("sn", "Shona", "修纳语"), + YO("yo", "Yoruba", "约鲁巴语"), + SO("so", "Somali", "索马里语"), + AF("af", "Afrikaans", "南非荷兰语"), + OC("oc", "Occitan", "奥克语"), + KA("ka", "Georgian", "格鲁吉亚语"), + BE("be", "Belarusian", "白俄罗斯语"), + TG("tg", "Tajik", "塔吉克语"), + SD("sd", "Sindhi", "信德语"), + GU("gu", "Gujarati", "古吉拉特语"), + AM("am", "Amharic", "阿姆哈拉语"), + YI("yi", "Yiddish", "意第绪语"), + LO("lo", "Lao", "老挝语"), + UZ("uz", "Uzbek", "乌兹别克语"), + FO("fo", "Faroese", "法罗语"), + HT("ht", "Haitian Creole", "海地克里奥尔语"), + PS("ps", "Pashto", "普什图语"), + TK("tk", "Turkmen", "土库曼语"), + NN("nn", "Nynorsk", "新挪威语"), + MT("mt", "Maltese", "马耳他语"), + SA("sa", "Sanskrit", "梵语"), + LB("lb", "Luxembourgish", "卢森堡语"), + MY("my", "Myanmar", "缅甸语"), + BO("bo", "Tibetan", "藏语"), + TL("tl", "Tagalog", "他加禄语"), + MG("mg", "Malagasy", "马尔加什语"), + AS("as", "Assamese", "阿萨姆语"), + TT("tt", "Tatar", "鞑靼语"), + HAW("haw", "Hawaiian", "夏威夷语"), + LN("ln", "Lingala", "林加拉语"), + HA("ha", "Hausa", "豪萨语"), + BA("ba", "Bashkir", "巴什基尔语"), + JW("jw", "Javanese", "爪哇语"), + SU("su", "Sundanese", "巽他语"), + YUE("yue", "Cantonese", "粤语"); + + private final String code; // Whisper语言代码 + private final String englishName; // 英文名称 + private final String chineseName; // 中文名称 + + Language(String code, String englishName, String chineseName) { + this.code = code; + this.englishName = englishName; + this.chineseName = chineseName; + } + + public String getCode() { + return code; + } + + public String getEnglishName() { + return englishName; + } + + public String getChineseName() { + return chineseName; + } + + @Override + public String toString() { + return code; + } + +} diff --git a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/Seetaface6QualityModel.java b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/Seetaface6QualityModel.java index 7dd8469..976b20b 100644 --- a/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/Seetaface6QualityModel.java +++ b/smartjavaai-face/src/main/java/cn/smartjavaai/face/model/quality/Seetaface6QualityModel.java @@ -87,7 +87,7 @@ public class Seetaface6QualityModel implements FaceQualityModel { //加载依赖库 NativeLoader.loadNativeLibraries(device); this.config = config; - int predictorPoolSize = config.getPredictorPoolSize(); + predictorPoolSize = config.getPredictorPoolSize(); if(config.getPredictorPoolSize() <= 0){ predictorPoolSize = Runtime.getRuntime().availableProcessors(); // 默认等于CPU核心数 } diff --git a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/config/DetectorModelConfig.java b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/config/DetectorModelConfig.java index b0b1a24..1d89dcb 100644 --- a/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/config/DetectorModelConfig.java +++ b/smartjavaai-objectdetection/src/main/java/cn/smartjavaai/objectdetection/config/DetectorModelConfig.java @@ -43,7 +43,7 @@ public class DetectorModelConfig extends ModelConfig { private int maxBox; /** - * 允许的类列表 + * 允许的分类列表 */ private List allowedClasses; diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/config/PlateRecModelConfig.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/config/PlateRecModelConfig.java index a8af403..d5be142 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/config/PlateRecModelConfig.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/config/PlateRecModelConfig.java @@ -28,20 +28,6 @@ public class PlateRecModelConfig extends ModelConfig { */ private PlateDetModel plateDetModel; - /** - * 置信度阈值 - */ - private float confidenceThreshold; - - /** - * iou阈值 - */ - private float iouThreshold; - - /** - * 检测结果数量 - */ - private int topK; } diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/OcrCommonRecModelImpl.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/OcrCommonRecModelImpl.java index 7fd4806..9f6c6f7 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/OcrCommonRecModelImpl.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/common/recognize/OcrCommonRecModelImpl.java @@ -286,18 +286,8 @@ public class OcrCommonRecModelImpl implements OcrCommonRecModel { if(Objects.isNull(ocrInfo) || Objects.isNull(ocrInfo.getLineList()) || ocrInfo.getLineList().isEmpty()){ throw new OcrException("未检测到文字"); } - try { - OcrUtils.drawRectWithText(sourceImage, ocrInfo, fontSize); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 调用 save 方法将 Image 写入字节流 - img.save(outputStream, "png"); - // 将字节流转换为 BufferedImage - byte[] imageBytes = outputStream.toByteArray(); - ((Mat) img.getWrappedImage()).release(); - return ImageIO.read(new ByteArrayInputStream(imageBytes)); - } catch (IOException e) { - throw new OcrException("导出图片失败", e); - } + OcrUtils.drawRectWithText(sourceImage, ocrInfo, fontSize); + return sourceImage; } @Override diff --git a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/plate/translator/Yolo5PlateDetectTranslator.java b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/plate/translator/Yolo5PlateDetectTranslator.java index 899b964..480a0ae 100644 --- a/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/plate/translator/Yolo5PlateDetectTranslator.java +++ b/smartjavaai-ocr/src/main/java/cn/smartjavaai/ocr/model/plate/translator/Yolo5PlateDetectTranslator.java @@ -17,6 +17,7 @@ import cn.smartjavaai.common.utils.LetterBoxUtils; import cn.smartjavaai.common.utils.NMSUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -92,6 +93,11 @@ public class Yolo5PlateDetectTranslator implements Translator topK) { diff --git a/smartjavaai-speech/pom.xml b/smartjavaai-speech/pom.xml new file mode 100644 index 0000000..f56f5a1 --- /dev/null +++ b/smartjavaai-speech/pom.xml @@ -0,0 +1,166 @@ + + + 4.0.0 + + cn.smartjavaai + smartjavaai-parent + 1.0.22 + + + smartjavaai-speech + + + 11 + 11 + UTF-8 + + + + + cn.smartjavaai + smartjavaai-common + ${project.version} + + + + + net.java.dev.jna + jna + 5.13.0 + + + + io.github.givimad + whisper-jni + 1.7.1 + + + + com.alphacephei + vosk + 0.3.45 + + + + ws.schild + jave-core + 3.5.0 + + + + ws.schild + jave-nativebin-osxm1 + 3.5.0 + + + + + 1.0.22 + smartjavaai-speech + SmartJavaAI + https://github.com/geekwenjie/SmartJavaAI + + + MIT License + https://opensource.org/licenses/MIT + + + + + + + org.sonatype.central + central-publishing-maven-plugin + 0.4.0 + true + + dengwenjie + true + ${project.groupId}:${project.artifactId}:${project.version} + + + + + org.apache.maven.plugins + maven-source-plugin + 3.1.0 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.1.0 + + + none + + -Xdoclint:none + + + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.1.0 + + + sign-artifacts + verify + + sign + + + + + + + + + + scm:git:git://github.com/geekwenjie/SmartJavaAI.git + scm:git:ssh://github.com/geekwenjie/SmartJavaAI.git + http://github.com/geekwenjie/SmartJavaAI/tree/master + + + + + + dengwenjie + https://s01.oss.sonatype.org/content/repositories/snapshots + + + dengwenjie + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + + dengwenjie + 775747758@qq.com + + Project Manager + Architect + + + + + + diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/audio/SmartAudioFactory.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/audio/SmartAudioFactory.java new file mode 100644 index 0000000..2312b6e --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/audio/SmartAudioFactory.java @@ -0,0 +1,176 @@ +package cn.smartjavaai.speech.asr.audio; + +import ai.djl.modality.audio.Audio; +import ai.djl.modality.audio.AudioFactory; +import cn.smartjavaai.speech.asr.exception.AsrException; +import cn.smartjavaai.speech.utils.AudioUtils; +import lombok.extern.slf4j.Slf4j; +import ws.schild.jave.EncoderException; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; + +/** + * @author dwj + * @date 2025/8/1 + */ +@Slf4j +public class SmartAudioFactory { + + + private static final SmartAudioFactory INSTANCE = new SmartAudioFactory(); + + private SmartAudioFactory() { + } + + public static SmartAudioFactory getInstance() { + return INSTANCE; + } + + + + public Audio fromFile(Path path, AudioFormat targetFormat) throws IOException { + try (AudioInputStream ais = AudioSystem.getAudioInputStream(path.toFile())) { + if(targetFormat != null){ + try (AudioInputStream convertedAis = AudioSystem.getAudioInputStream(targetFormat, ais);) { + byte[] bytes = read(convertedAis); + float[] floats = bytesToFloats(bytes, targetFormat.isBigEndian()); + return new Audio(floats, targetFormat.getSampleRate(), targetFormat.getChannels()); + } + }else{ + AudioFormat format = ais.getFormat(); + byte[] bytes = read(ais); + float[] floats = bytesToFloats(bytes, format.isBigEndian()); + return new Audio(floats, format.getSampleRate(), format.getChannels()); + } + } catch (UnsupportedAudioFileException e) { + log.debug("Unsupported Audio file, Conversion to WAV is required"); + byte[] allBytes = Files.readAllBytes(path); + try(AudioInputStream audioInputStream = convertWav(allBytes)){ + return fromAudioInputStream(audioInputStream, targetFormat); + } + } + } + + public Audio fromAudioInputStream(AudioInputStream ais, AudioFormat targetFormat) throws IOException { + try (AudioInputStream audioInputStream = ais) { + if(targetFormat != null){ + try (AudioInputStream convertedAis = AudioSystem.getAudioInputStream(targetFormat, audioInputStream);) { + byte[] bytes = read(convertedAis); + float[] floats = bytesToFloats(bytes, targetFormat.isBigEndian()); + return new Audio(floats, targetFormat.getSampleRate(), targetFormat.getChannels()); + } + }else{ + AudioFormat format = audioInputStream.getFormat(); + byte[] bytes = read(audioInputStream); + float[] floats = bytesToFloats(bytes, format.isBigEndian()); + return new Audio(floats, format.getSampleRate(), format.getChannels()); + } + } + } + + public AudioInputStream convertWav(byte[] allBytes) { + InputStream conversionStream = new BufferedInputStream(new ByteArrayInputStream(allBytes)); + File tempFile = null; + try { + tempFile = AudioUtils.audioFormatConversion(conversionStream, "wav"); + InputStream fis = new BufferedInputStream(new FileInputStream(tempFile)); + AudioInputStream ais = AudioSystem.getAudioInputStream(fis); + return ais; + } catch (EncoderException | IOException | UnsupportedAudioFileException e) { + throw new AsrException(e); + } finally { + if(tempFile != null && tempFile.exists()){ + tempFile.delete(); + } + if(conversionStream != null){ + try { + conversionStream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + + public Audio fromInputStream(InputStream is, AudioFormat targetFormat) throws IOException { + byte[] allBytes = is.readAllBytes(); + try (BufferedInputStream tryStream = new BufferedInputStream(new ByteArrayInputStream(allBytes)); + AudioInputStream ais = AudioSystem.getAudioInputStream(tryStream)) { + if(targetFormat != null){ + try (AudioInputStream convertedAis = AudioSystem.getAudioInputStream(targetFormat, ais);) { + byte[] bytes = read(convertedAis); + float[] floats = bytesToFloats(bytes, targetFormat.isBigEndian()); + return new Audio(floats, targetFormat.getSampleRate(), targetFormat.getChannels()); + } + }else{ + AudioFormat format = ais.getFormat(); + byte[] bytes = read(ais); + float[] floats = bytesToFloats(bytes, format.isBigEndian()); + return new Audio(floats, format.getSampleRate(), format.getChannels()); + } + } catch (UnsupportedAudioFileException e) { + log.debug("Unsupported Audio file, Conversion to WAV is required"); + try(AudioInputStream audioInputStream = convertWav(allBytes)){ + return fromAudioInputStream(audioInputStream, targetFormat); + } + } + } + + private byte[] read(AudioInputStream ais) throws IOException { + AudioFormat format = ais.getFormat(); + int frameSize = format.getFrameSize(); + + // Some audio formats may have unspecified frame size + if (frameSize == AudioSystem.NOT_SPECIFIED) { + frameSize = 1; + } + + int size = (int) ais.getFrameLength() * frameSize; + + if (ais.getFrameLength() == AudioSystem.NOT_SPECIFIED || size <= 0){ + // unknown length, use ByteArrayOutputStream to read all data + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int read; + while ((read = ais.read(buf)) != -1) { + baos.write(buf, 0, read); + } + return baos.toByteArray(); + }else{ + byte[] ret = new byte[size]; + byte[] buf = new byte[1024]; + int offset = 0; + int read; + while ((read = ais.read(buf)) != -1) { + System.arraycopy(buf, 0, ret, offset, read); + offset += read; + } + return ret; + } + + } + + private float[] bytesToFloats(byte[] bytes, boolean isBigEndian) { + ByteOrder order = isBigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; + ShortBuffer buffer = ByteBuffer.wrap(bytes).order(order).asShortBuffer(); + short[] shorts = new short[buffer.capacity()]; + buffer.get(shorts); + + // Feed in float values between -1.0f and 1.0f. + float[] floats = new float[shorts.length]; + for (int i = 0; i < shorts.length; i++) { + floats[i] = ((float) shorts[i]) / (float) Short.MAX_VALUE; + } + return floats; + } + +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/config/AsrModelConfig.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/config/AsrModelConfig.java new file mode 100644 index 0000000..9ae36aa --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/config/AsrModelConfig.java @@ -0,0 +1,17 @@ +package cn.smartjavaai.speech.asr.config; + +import cn.smartjavaai.common.config.ModelConfig; +import cn.smartjavaai.speech.asr.enums.AsrModelEnum; +import lombok.Data; + +/** + * @author dwj + * @date 2025/7/31 + */ +@Data +public class AsrModelConfig extends ModelConfig { + + private AsrModelEnum modelEnum; + + private String modelPath; +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/AsrResult.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/AsrResult.java new file mode 100644 index 0000000..854319f --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/AsrResult.java @@ -0,0 +1,30 @@ +package cn.smartjavaai.speech.asr.entity; + +import lombok.Data; + +import java.util.List; + +/** + * 语音识别结果 + * @author dwj + */ +@Data +public class AsrResult { + + private String text; + private List segments; + + + public AsrResult() { + } + + public AsrResult(String text) { + this.text = text; + } + + public AsrResult(String text, List segments) { + this.text = text; + this.segments = segments; + } + +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/AsrSegment.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/AsrSegment.java new file mode 100644 index 0000000..05ea172 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/AsrSegment.java @@ -0,0 +1,27 @@ +package cn.smartjavaai.speech.asr.entity; + +import lombok.Data; + +/** + * @author dwj + */ +@Data +public class AsrSegment { + + private String text; + private long startTime; + private long endTime; + + public AsrSegment(String text, long startTime, long endTime) { + this.text = text; + this.startTime = startTime; + this.endTime = endTime; + } + + public AsrSegment(String text) { + this.text = text; + } + + public AsrSegment() { + } +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/RecParams.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/RecParams.java new file mode 100644 index 0000000..c5706b2 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/RecParams.java @@ -0,0 +1,27 @@ +package cn.smartjavaai.speech.asr.entity; + +import cn.smartjavaai.common.entity.Language; +import lombok.Data; + +/** + * 语音识别参数 + * @author dwj + */ +@Data +public abstract class RecParams { + + /** + * 语言 + */ + private Language language = Language.ZH; + + public Language getLanguage() { + return language; + } + + public void setLanguage(Language language) { + this.language = language; + } + + +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/VoskParams.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/VoskParams.java new file mode 100644 index 0000000..298ce16 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/VoskParams.java @@ -0,0 +1,30 @@ +package cn.smartjavaai.speech.asr.entity; + +import cn.smartjavaai.common.entity.Language; +import io.github.givimad.whisperjni.WhisperFullParams; +import io.github.givimad.whisperjni.WhisperSamplingStrategy; +import lombok.Data; + +/** + * @author dwj + */ +@Data +public class VoskParams extends RecParams{ + + /** + * 最大候选结果数 + */ + private int maxAlternatives; + + /** + * 限定词汇表 例：["yes", "no", "hello"] + */ + private String grammar; + + /** + * 是否返回词级别的识别结果（包含每个词的开始/结束时间和置信度）。 + */ + private boolean words = true; + + +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/WhisperParams.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/WhisperParams.java new file mode 100644 index 0000000..ffb5e22 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/entity/WhisperParams.java @@ -0,0 +1,32 @@ +package cn.smartjavaai.speech.asr.entity; + +import cn.smartjavaai.common.entity.Language; +import io.github.givimad.whisperjni.WhisperFullParams; +import io.github.givimad.whisperjni.WhisperSamplingStrategy; +import lombok.Data; + +/** + * @author dwj + */ +@Data +public class WhisperParams extends RecParams{ + + private WhisperFullParams params; + + public WhisperParams(WhisperFullParams params) { + this.params = params; + } + + public WhisperParams() { + this.params = new WhisperFullParams(WhisperSamplingStrategy.BEAN_SEARCH); + this.params.language = Language.ZH.getCode(); + } + + @Override + public void setLanguage(Language language) { + super.setLanguage(language); + if (language != null) { + this.params.language = language.getCode(); // 关键：设置给底层 whisper 参数 + } + } +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/enums/AsrModelEnum.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/enums/AsrModelEnum.java new file mode 100644 index 0000000..7ce905e --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/enums/AsrModelEnum.java @@ -0,0 +1,26 @@ +package cn.smartjavaai.speech.asr.enums; + +/** + * 语音识别模型枚举 + * @author dwj + */ +public enum AsrModelEnum { + + WHISPER, + + VOSK; + + /** + * 根据名称获取枚举 (忽略大小写和下划线变体) + */ + public static AsrModelEnum fromName(String name) { + String formatted = name.trim().toUpperCase().replaceAll("[-_]", ""); + for (AsrModelEnum model : values()) { + if (model.name().replaceAll("_", "").equals(formatted)) { + return model; + } + } + throw new IllegalArgumentException("未知模型名称: " + name); + } + +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/exception/AsrException.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/exception/AsrException.java new file mode 100644 index 0000000..00cf9c1 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/exception/AsrException.java @@ -0,0 +1,29 @@ +package cn.smartjavaai.speech.asr.exception; + +/** + * 语音识别异常 + * @author dwj + */ +public class AsrException extends RuntimeException{ + + public AsrException() { + super(); + } + + public AsrException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public AsrException(String message, Throwable cause) { + super(message, cause); + } + + public AsrException(String message) { + super(message); + } + + public AsrException(Throwable cause) { + super(cause); + } + +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/SpeechRecognizer.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/SpeechRecognizer.java new file mode 100644 index 0000000..d158610 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/SpeechRecognizer.java @@ -0,0 +1,48 @@ +package cn.smartjavaai.speech.asr.model; + +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.speech.asr.config.AsrModelConfig; +import cn.smartjavaai.speech.asr.entity.AsrResult; +import cn.smartjavaai.speech.asr.entity.RecParams; +import cn.smartjavaai.speech.asr.exception.AsrException; + +import java.io.InputStream; + +/** + * 语音识别 + * @author dwj + */ +public interface SpeechRecognizer extends AutoCloseable{ + + /** + * 加载模型 + * @param config + */ + void loadModel(AsrModelConfig config); // 加载模型 + + + default R recognize(String audioPath, RecParams params){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default R recognize(byte[] audioData, RecParams params){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default R recognize(InputStream audioStream, RecParams params){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default R recognize(String audioPath){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default R recognize(byte[] audioData){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + + default R recognize(InputStream audioStream){ + throw new UnsupportedOperationException("默认不支持该功能"); + } + +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/VoskRecognizer.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/VoskRecognizer.java new file mode 100644 index 0000000..2a94a0f --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/VoskRecognizer.java @@ -0,0 +1,255 @@ +package cn.smartjavaai.speech.asr.model; + +import ai.djl.modality.audio.Audio; +import ai.djl.modality.audio.AudioFactory; +import ai.djl.util.JsonUtils; +import cn.smartjavaai.common.entity.Language; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.speech.asr.audio.SmartAudioFactory; +import cn.smartjavaai.speech.asr.config.AsrModelConfig; +import cn.smartjavaai.speech.asr.entity.*; +import cn.smartjavaai.speech.asr.exception.AsrException; +import cn.smartjavaai.speech.asr.pool.WhisperStatePool; +import cn.smartjavaai.speech.utils.AudioUtils; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.github.givimad.whisperjni.WhisperContext; +import io.github.givimad.whisperjni.WhisperJNI; +import io.github.givimad.whisperjni.WhisperState; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.vosk.LibVosk; +import org.vosk.LogLevel; +import org.vosk.Model; +import org.vosk.Recognizer; +import ws.schild.jave.Encoder; +import ws.schild.jave.EncoderException; +import ws.schild.jave.info.MultimediaInfo; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import static ai.djl.util.JsonUtils.GSON; + +/** + * Vosk 识别器 + * @author dwj + */ +@Slf4j +public class VoskRecognizer implements SpeechRecognizer{ + + private Model model; + + @Override + public void loadModel(AsrModelConfig config) { + if(StringUtils.isBlank(config.getModelPath())){ + throw new AsrException("modelPath is null"); + } + Path testModelPath = Paths.get(config.getModelPath()); + if(!testModelPath.toFile().exists()){ + throw new AsrException("Missing model file: " + testModelPath.toAbsolutePath()); + } + try { + model = new Model(config.getModelPath()); + LibVosk.setLogLevel(LogLevel.DEBUG); + log.debug("Vosk init success"); + } catch (IOException e) { + throw new AsrException(e); + } + } + + + @Override + public R recognize(String audioPath) { + return recognize(audioPath, new VoskParams()); + } + + + @Override + public R recognize(byte[] audioData) { + return recognize(audioData, new VoskParams()); + } + + @Override + public R recognize(InputStream audioStream) { + return recognize(audioStream, new VoskParams()); + } + + private R recognizeAudioStream(AudioInputStream ais,RecParams params) { + try (Recognizer recognizer = buildRecognizer(params, ais.getFormat().getSampleRate())){ + AudioFormat audioFormat = ais.getFormat(); + log.debug("sampleRate:{}", audioFormat.getSampleRate()); + log.debug("channels:{}", audioFormat.getChannels()); + int nbytes; + byte[] b = new byte[4096]; + List segments = new ArrayList(); + StringBuilder text = new StringBuilder(); + while ((nbytes = ais.read(b)) >= 0) { + if (recognizer.acceptWaveForm(b, nbytes)) { + String result = recognizer.getResult(); + AsrSegment segment = parseSegment(result); + segments.add(segment); + } + } + return R.ok(new AsrResult(text.toString(), segments)); + } catch (IOException e) { + throw new AsrException(e); + } + } + + /** + * 解析结果 + * @param segment + * @return + */ + private AsrSegment parseSegment(String segment) { + JsonObject json = GSON.fromJson(segment, JsonObject.class); + JsonArray resultArray = json.getAsJsonArray("result"); + double segmentStart = resultArray.get(0).getAsJsonObject().get("start").getAsDouble(); + double segmentEnd = resultArray.get(resultArray.size() - 1).getAsJsonObject().get("end").getAsDouble(); + long startMs = Math.round(segmentStart * 1000); + long endMs = Math.round(segmentEnd * 1000); + String text = json.get("text").getAsString(); + String noSpaces = text.replace(" ", ""); + return new AsrSegment(noSpaces, startMs, endMs); + } + + /** + * 创建识别器 + * @param params + * @param sampleRate + * @return + * @throws IOException + */ + private Recognizer buildRecognizer(RecParams params,float sampleRate) throws IOException { + if(!(params instanceof VoskParams)){ + throw new AsrException("params is not VoskParams"); + } + VoskParams voskParams = (VoskParams) params; + Recognizer recognizer = new Recognizer(model, sampleRate); + if(StringUtils.isNotBlank(voskParams.getGrammar())){ + recognizer.setGrammar(voskParams.getGrammar()); + } +// if(voskParams.getMaxAlternatives() > 0){ +// recognizer.setMaxAlternatives(voskParams.getMaxAlternatives()); +// } + //暂时只支持返回一个结果 +// recognizer.setMaxAlternatives(1); + recognizer.setWords(voskParams.isWords()); + //recognizer.setPartialWords(true); + return recognizer; + } + + + @Override + public R recognize(String audioPath, RecParams params) { + Path audioFilePath = Paths.get(audioPath); + if(!audioFilePath.toFile().exists()){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + try (InputStream is = Files.newInputStream(audioFilePath)){ + return recognize(is, params); + } catch (IOException e) { + throw new AsrException(e); + } + } + + @Override + public R recognize(byte[] audioData, RecParams params) { + try (InputStream is = new ByteArrayInputStream(audioData)) { + return recognize(is, params); + } catch (IOException e) { + throw new AsrException(e); + } + } + + @Override + public R recognize(InputStream audioStream, RecParams params) { + AudioInputStream ais = null; + InputStream tryStream = null; + InputStream conversionStream = null; + boolean needConversion = false; + try { + // 缓存全部字节数据 + byte[] allBytes = audioStream.readAllBytes(); + // 创建两个独立流 + tryStream = new BufferedInputStream(new ByteArrayInputStream(allBytes)); + conversionStream = new BufferedInputStream(new ByteArrayInputStream(allBytes)); + ais = AudioSystem.getAudioInputStream(tryStream); + } catch (UnsupportedAudioFileException e) { + log.debug("Unsupported Audio file, Conversion to WAV is required"); + needConversion = true; + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + if(tryStream != null){ + try { + tryStream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + //转换wav格式 + File tempFile = null; + if(needConversion){ + try { + tempFile = AudioUtils.audioFormatConversion(conversionStream, "wav"); + } catch (EncoderException | IOException e) { + throw new AsrException(e); + } + try (InputStream fis = new BufferedInputStream(new FileInputStream(tempFile))){ + ais = AudioSystem.getAudioInputStream(fis); + return recognizeAudioStream(ais, params); + } catch (IOException | UnsupportedAudioFileException e) { + throw new AsrException("音频转换异常", e); + } finally { + if(tempFile != null && tempFile.exists()){ + tempFile.delete(); + } + if(conversionStream != null){ + try { + conversionStream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + return recognizeAudioStream(ais, params); + } + + @Override + public void close() throws Exception { + if(model != null){ + model.close(); + } + } + + /** + * 创建高级识别器 + * @param sampleRate 采样率 + * @return + */ + public Recognizer createAdvancedRecognizer(float sampleRate){ + try { + return new Recognizer(model, sampleRate); + } catch (IOException e) { + throw new AsrException(e); + } + } + + +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/WhisperRecognizer.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/WhisperRecognizer.java new file mode 100644 index 0000000..e6eb3e8 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/model/WhisperRecognizer.java @@ -0,0 +1,212 @@ +package cn.smartjavaai.speech.asr.model; + +import ai.djl.modality.audio.Audio; +import ai.djl.modality.audio.AudioFactory; +import cn.smartjavaai.common.entity.Language; +import cn.smartjavaai.common.entity.R; +import cn.smartjavaai.common.enums.DeviceEnum; +import cn.smartjavaai.speech.asr.audio.SmartAudioFactory; +import cn.smartjavaai.speech.asr.config.AsrModelConfig; +import cn.smartjavaai.speech.asr.entity.AsrResult; +import cn.smartjavaai.speech.asr.entity.AsrSegment; +import cn.smartjavaai.speech.asr.entity.RecParams; +import cn.smartjavaai.speech.asr.entity.WhisperParams; +import cn.smartjavaai.speech.asr.exception.AsrException; +import cn.smartjavaai.speech.asr.pool.WhisperStatePool; +import io.github.givimad.whisperjni.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.vosk.Model; +import org.vosk.Recognizer; + +import javax.sound.sampled.AudioFormat; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * @author dwj + */ +@Slf4j +public class WhisperRecognizer implements SpeechRecognizer{ + + private WhisperJNI whisper; + + private WhisperContext ctx; + + private WhisperStatePool statePool; + + @Override + public void loadModel(AsrModelConfig config) { + if(StringUtils.isBlank(config.getModelPath())){ + throw new AsrException("modelPath is null"); + } + Path testModelPath = Paths.get(config.getModelPath()); + if(!testModelPath.toFile().exists()){ + throw new AsrException("Missing model file: " + testModelPath.toAbsolutePath()); + } + try { + WhisperJNI.loadLibrary(); + WhisperJNI.setLibraryLogger(null); + whisper = new WhisperJNI(); + WhisperJNI.setLibraryLogger(new WhisperJNI.LibraryLogger() { + @Override + public void log(String s) { + log.debug("WhisperJNI {}", s); + } + }); + ctx = whisper.initNoState(testModelPath); + Boolean initOpenVINO = config.getCustomParam("initOpenVINO", Boolean.class); + if(Objects.nonNull(initOpenVINO) && initOpenVINO){ + String device = Objects.isNull(config.getDevice()) ? DeviceEnum.CPU.name() : config.getDevice().name(); + whisper.initOpenVINO(ctx, device); + log.debug("WhisperJNI initOpenVINO success"); + } + statePool = new WhisperStatePool(whisper, ctx); + log.debug("WhisperJNI init success"); + } catch (IOException e) { + throw new AsrException(e); + } + + } + + + @Override + public R recognize(String audioPath, RecParams params) { + Path audioFilePath = Paths.get(audioPath); + if(!audioFilePath.toFile().exists()){ + return R.fail(R.Status.FILE_NOT_FOUND); + } + Audio audio = null; + try { + audio = SmartAudioFactory.getInstance().fromFile(audioFilePath, getDefaultAudioFormat()); + } catch (IOException e) { + throw new AsrException("读取音频异常",e); + } + return recognize(audio, params); + } + + @Override + public R recognize(byte[] audioData, RecParams params) { + try (InputStream is = new ByteArrayInputStream(audioData)) { + return recognize(is, params); + } catch (IOException e) { + throw new AsrException(e); + } + } + + @Override + public R recognize(InputStream audioStream, RecParams params) { + Audio audio = null; + try { + audio = SmartAudioFactory.getInstance().fromInputStream(audioStream, getDefaultAudioFormat()); + } catch (IOException e) { + throw new AsrException("读取音频异常",e); + } + return recognize(audio, params); + } + + public R recognize(Audio audio, RecParams params) { + WhisperState state = null; + StringBuilder text = new StringBuilder(); + try { + WhisperParams whisperParams = (WhisperParams) params; + //不是英语，需要检查是否是多语言模型 + if(!Language.EN.getCode().equals(whisperParams.getParams().language)){ + if(!whisper.isMultilingual(ctx)){ + return R.fail(1002, "当前为非多语种模型，仅支持英语识别，暂不支持其他语言，请更换多语种模型"); + } + } + state = statePool.borrowObject(); + int result = whisper.fullWithState(ctx, state, whisperParams.getParams(), audio.getData(), audio.getData().length); + if(result != 0) { + return R.fail(1000, "Transcription failed with code " + result); + } + int numSegments = whisper.fullNSegmentsFromState(state); + if(numSegments <= 0){ + return R.fail(1001, "未识别出有效语音"); + } + List segments = new ArrayList(numSegments); + for(int i = 0; i < numSegments; i++){ + long startTime = whisper.fullGetSegmentTimestamp0FromState(state,i); + long endTime = whisper.fullGetSegmentTimestamp1FromState(state,i); + String content = whisper.fullGetSegmentTextFromState(state, i); + segments.add(new AsrSegment(content, startTime * 10, endTime * 10)); + text.append(content).append("\n"); + } + return R.ok(new AsrResult(text.toString(), segments)); + } catch (Exception e) { + throw new AsrException(e); + } finally { + if(state != null){ + try { + statePool.returnObject(state); + } catch (Exception e) { + log.warn("returnObject失败", e); + } + } + } + } + + @Override + public R recognize(String audioPath) { + return recognize(audioPath, new WhisperParams()); + } + + @Override + public R recognize(byte[] audioData) { + return recognize(audioData, new WhisperParams()); + } + + @Override + public R recognize(InputStream audioStream) { + return recognize(audioStream, new WhisperParams()); + } + + public WhisperGrammar parseGrammar(String grammarText) { + try { + return whisper.parseGrammar(grammarText); + } catch (IOException e) { + throw new AsrException(e); + } + } + + @Override + public void close() throws Exception { + if(statePool != null){ + statePool.close(); + } + if(ctx != null){ + ctx.close(); + } + } + + /** + * 获取一个WhisperState对象 + * @return + */ + private WhisperState getWhisperState(){ + try { + return statePool.borrowObject(); + } catch (Exception e) { + throw new AsrException(e); + } + } + + private AudioFormat getDefaultAudioFormat(){ + return new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + 16000, + 16, + 1, + 2, + 16000, + false + ); + } +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/pool/WhisperStatePool.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/pool/WhisperStatePool.java new file mode 100644 index 0000000..ebea0f4 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/asr/pool/WhisperStatePool.java @@ -0,0 +1,47 @@ +package cn.smartjavaai.speech.asr.pool; + +import io.github.givimad.whisperjni.WhisperContext; +import io.github.givimad.whisperjni.WhisperJNI; +import io.github.givimad.whisperjni.WhisperState; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.DefaultPooledObject; +import org.apache.commons.pool2.impl.GenericObjectPool; + +/** + * WhisperState 实例对象池 + * @author dwj + */ +public class WhisperStatePool extends GenericObjectPool { + + public WhisperStatePool(WhisperJNI whisperInstance, WhisperContext context) { + super(new PooledObjectFactory(){ + + @Override + public void activateObject(PooledObject pooledObject) throws Exception { + + } + + @Override + public void destroyObject(PooledObject pooledObject) throws Exception { + pooledObject.getObject().close(); + } + + @Override + public PooledObject makeObject() throws Exception { + WhisperState state = whisperInstance.initState(context); + return new DefaultPooledObject(state); + } + + @Override + public void passivateObject(PooledObject pooledObject) throws Exception { + + } + + @Override + public boolean validateObject(PooledObject pooledObject) { + return false; + } + }); + } +} diff --git a/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/utils/AudioUtils.java b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/utils/AudioUtils.java new file mode 100644 index 0000000..8881096 --- /dev/null +++ b/smartjavaai-speech/src/main/java/cn/smartjavaai/speech/utils/AudioUtils.java @@ -0,0 +1,212 @@ +package cn.smartjavaai.speech.utils; + +import cn.hutool.core.lang.UUID; +import cn.smartjavaai.speech.asr.exception.AsrException; +import ws.schild.jave.Encoder; +import ws.schild.jave.EncoderException; +import ws.schild.jave.MultimediaObject; +import ws.schild.jave.encode.AudioAttributes; +import ws.schild.jave.encode.EncodingAttributes; +import ws.schild.jave.info.MultimediaInfo; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import java.io.*; +import java.nio.file.Files; + +/** + * 音频工具类 + * @author dwj + */ +public class AudioUtils { + + + public static byte[] read(AudioInputStream ais) throws IOException { + AudioFormat format = ais.getFormat(); + int frameSize = format.getFrameSize(); + + // Some audio formats may have unspecified frame size + if (frameSize == AudioSystem.NOT_SPECIFIED) { + frameSize = 1; + } + int size = (int) ais.getFrameLength() * frameSize; + if (ais.getFrameLength() == AudioSystem.NOT_SPECIFIED || size <= 0){ + // unknown length, use ByteArrayOutputStream to read all data + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int read; + while ((read = ais.read(buf)) != -1) { + baos.write(buf, 0, read); + } + return baos.toByteArray(); + }else{ + byte[] ret = new byte[size]; + byte[] buf = new byte[1024]; + int offset = 0; + int read; + while ((read = ais.read(buf)) != -1) { + System.arraycopy(buf, 0, ret, offset, read); + offset += read; + } + return ret; + } + } + + /** + * 音频格式转换 + * + * @param sourceFilePath + * @param targetFilePath + * @param format wav/mp3/amr + * @return + */ + public static byte[] getAudioFormatConversionBytes(String sourceFilePath,String targetFilePath,String format) { + InputStream fis = null; + ByteArrayOutputStream bos = null; + byte[] bytes = null; + try { + File sourceFile = new File(sourceFilePath); + if (sourceFile.isFile()) { + File targetFile = new File(targetFilePath); + + // 音频格式转换 + audioFormatConversion(sourceFile, targetFile, format); + + fis = new FileInputStream(targetFile); + bos = new ByteArrayOutputStream(); + + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = fis.read(buffer)) != -1) { + bos.write(buffer, 0, bytesRead); + } + bytes = bos.toByteArray(); + } + } catch (Exception e) { + throw new AsrException("音频格式转换异常：" + e.getMessage(), e); + } finally { + try { + if (fis != null) { + fis.close(); + } + if (bos != null) { + bos.close(); + } + } catch (IOException e) { + throw new AsrException("音频格式转换资源关闭异常：" + e.getMessage(), e); + } + } + return bytes; + } + + /** + * 音频格式转换 + * + * @param sourceFilePath + * @param targetFilePath + * @param format wav/mp3/amr + * @return + */ + public static InputStream getAudioFormatConversionIns(String sourceFilePath, String targetFilePath, String format) throws EncoderException, FileNotFoundException { + File sourceFile = new File(sourceFilePath); + if (sourceFile.isFile()) { + File targetFile = new File(targetFilePath); + // 音频格式转换 + audioFormatConversion(sourceFile, targetFile, format); + return new FileInputStream(targetFile); + } + return null; + } + + /** + * 音频格式转换 + * @param source 源音频文件 + * @param target 输出的音频文件 + * @param format wav/mp3/amr + */ + public static void audioFormatConversion(File source,File target,String format) throws EncoderException { + //Audio Attributes + AudioAttributes audio = new AudioAttributes(); + switch (format) { + case "wav": + audio.setCodec("pcm_s16le"); + break; + case "mp3": + audio.setCodec("libmp3lame"); + break; + case "amr": + audio.setCodec("libvo_amrwbenc"); + break; + default: + throw new IllegalArgumentException("不支持的音频格式：" + format); + } + audio.setBitRate(16000); + audio.setChannels(1); + audio.setSamplingRate(16000); + //Encoding attributes + EncodingAttributes attrs = new EncodingAttributes(); + attrs.setOutputFormat(format); + attrs.setAudioAttributes(audio); + //Encode + Encoder encoder = new Encoder(); + encoder.encode(new MultimediaObject(source), target, attrs); + } + + + /** + * 音频格式转换 + * @param sourceStream 源音频流 + * @param format wav/mp3/amr + * @return + */ + public static File audioFormatConversion(InputStream sourceStream, String format) throws EncoderException, IOException { + // 1. 写入临时源文件 + File sourceFile = Files.createTempFile("source-", ".tmp").toFile(); + try (OutputStream os = new FileOutputStream(sourceFile)) { + byte[] buffer = new byte[8192]; + int len; + while ((len = sourceStream.read(buffer)) != -1) { + os.write(buffer, 0, len); + } + } + // 2. 创建临时目标文件 + File targetFile = Files.createTempFile("target-", "." + format).toFile(); + // 3. 调用已有方法进行格式转换 + audioFormatConversion(sourceFile, targetFile, format); + // 4. 删除源文件（可选） + sourceFile.delete(); + return targetFile; + } + + /** + * 获取音频信息 + * @param source + * @return + */ + public static MultimediaInfo getAudioInfo(File source) throws EncoderException { + MultimediaObject mo = new MultimediaObject(source); + return mo.getInfo(); + } + + /** + * 获取音频信息 + * @param ais + * @return + */ + public static MultimediaInfo getAudioInfo(AudioInputStream ais) throws EncoderException, IOException { + File tempFile = Files.createTempFile("audio_" + UUID.fastUUID().toString(), null).toFile(); + tempFile.deleteOnExit(); // JVM退出时自动删除 + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = ais.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + return getAudioInfo(tempFile); + } + + + +}