From e178ae1846994db69be2c8f7e735582f4e8f0d32 Mon Sep 17 00:00:00 2001 From: jack ning Date: Tue, 23 Sep 2025 21:30:57 +0800 Subject: [PATCH] update --- .../springai/service/BaseSpringAIService.java | 54 ++-- .../com/bytedesk/ai/utils/AIFileUtils.java | 85 ++++++- .../com/bytedesk/ai/utils/FileContent.java | 28 -- .../java/com/bytedesk/ai/utils/mcp/readme.md | 19 -- .../ai/zhipuai/ZhipuMultiModelService.java | 240 ++++++++---------- .../core/message/MessagePersistService.java | 59 ++++- .../message/content/RobotStreamContent.java | 3 + 7 files changed, 266 insertions(+), 222 deletions(-) delete mode 100644 modules/ai/src/main/java/com/bytedesk/ai/utils/FileContent.java delete mode 100644 modules/ai/src/main/java/com/bytedesk/ai/utils/mcp/readme.md diff --git a/modules/ai/src/main/java/com/bytedesk/ai/springai/service/BaseSpringAIService.java b/modules/ai/src/main/java/com/bytedesk/ai/springai/service/BaseSpringAIService.java index 454292858b..c044a78e70 100644 --- a/modules/ai/src/main/java/com/bytedesk/ai/springai/service/BaseSpringAIService.java +++ b/modules/ai/src/main/java/com/bytedesk/ai/springai/service/BaseSpringAIService.java @@ -35,10 +35,6 @@ import com.bytedesk.core.message.MessageProtobuf; import com.bytedesk.core.message.MessageRestService; import com.bytedesk.core.message.MessageTypeEnum; import com.bytedesk.core.message.content.RobotStreamContent; -import com.bytedesk.core.message.content.ImageContent; -import com.bytedesk.core.message.content.VideoContent; -import com.bytedesk.core.message.content.FileContent; -import com.bytedesk.core.message.content.AudioContent; import com.bytedesk.core.thread.ThreadRestService; import com.bytedesk.core.uid.UidUtils; import com.bytedesk.kbase.llm_chunk.elastic.ChunkElastic; @@ -68,7 +64,6 @@ import com.bytedesk.kbase.llm_webpage.vector.WebpageVectorSearchResult; import com.bytedesk.kbase.llm_webpage.vector.WebpageVectorService; import lombok.extern.slf4j.Slf4j; -import com.bytedesk.ai.robot.RobotConsts; @Slf4j public abstract class BaseSpringAIService implements SpringAIService { @@ -1357,27 +1352,15 @@ public abstract class BaseSpringAIService implements SpringAIService { } String type = messageEntity.getType(); try { - String url = null; if (MessageTypeEnum.IMAGE.name().equals(type)) { - ImageContent ic = ImageContent.fromJson(content); - if (ic != null) - url = ic.getUrl(); - return buildBdMediaPayload(MessageTypeEnum.IMAGE, url); + // 直接返回标准 JSON（ImageContent），由下游 fromJson 解析 + return content; } else if (MessageTypeEnum.VIDEO.name().equals(type)) { - VideoContent vc = VideoContent.fromJson(content); - if (vc != null) - url = vc.getUrl(); - return buildBdMediaPayload(MessageTypeEnum.VIDEO, url); + return content; } else if (MessageTypeEnum.FILE.name().equals(type)) { - FileContent fc = FileContent.fromJson(content); - if (fc != null) - url = fc.getUrl(); - return buildBdMediaPayload(MessageTypeEnum.FILE, url); + return content; } else if (MessageTypeEnum.AUDIO.name().equals(type)) { - AudioContent ac = AudioContent.fromJson(content); - if (ac != null) - url = ac.getUrl(); - return buildBdMediaPayload(MessageTypeEnum.AUDIO, url); + return content; } } catch (Exception e) { log.debug("toBdMediaMarkerIfMedia parse failed, fallback to text: {}", e.getMessage()); @@ -1385,15 +1368,7 @@ public abstract class BaseSpringAIService implements SpringAIService { return content; } - private String buildBdMediaPayload(MessageTypeEnum type, String url) { - if (type == null || !StringUtils.hasText(url)) { - return ""; // 空 URL 则返回空字符串，避免污染 Prompt - } - // 简单拼接 JSON，避免额外依赖；ZhipuMultiModelService 会解析 - return RobotConsts.BD_MEDIA_PREFIX - + "{\"" + RobotConsts.BD_MEDIA_FIELD_TYPE + "\":\"" + type.name() + "\"," - + "\"" + RobotConsts.BD_MEDIA_FIELD_URL + "\":\"" + url + "\"}"; - } + // 不再构造自定义 BD_MEDIA 前缀；媒体消息直接使用标准 JSON（BaseContent.toJson）在下游解析 private void processDirectResponse(String query, List searchContentList, RobotProtobuf robot, MessageProtobuf messageProtobufQuery, @@ -1529,13 +1504,24 @@ public abstract class BaseSpringAIService implements SpringAIService { protected void sendStreamMessage(MessageProtobuf messageProtobufQuery, MessageProtobuf messageProtobufReply, SseEmitter emitter, String content) { + sendStreamMessage(messageProtobufQuery, messageProtobufReply, emitter, content, null); + } + + /** + * 发送流式片段（可携带推理内容） + */ + protected void sendStreamMessage(MessageProtobuf messageProtobufQuery, MessageProtobuf messageProtobufReply, + SseEmitter emitter, String content, String reasonContent) { log.info("BaseSpringAIService sendStreamMessage content {}", content); try { if (StringUtils.hasLength(content) && !isEmitterCompleted(emitter)) { // 使用 RobotStreamContent 包装流式片段，类型改为 ROBOT_STREAM - RobotStreamContent robotStream = RobotStreamContent.builder() - .answer(content) - .build(); + RobotStreamContent.RobotStreamContentBuilder builder = RobotStreamContent.builder() + .answer(content); + if (StringUtils.hasLength(reasonContent)) { + builder.reasonContent(reasonContent); + } + RobotStreamContent robotStream = builder.build(); messageProtobufReply.setContent(robotStream.toJson()); messageProtobufReply.setType(MessageTypeEnum.ROBOT_STREAM); // 保存消息到数据库 diff --git a/modules/ai/src/main/java/com/bytedesk/ai/utils/AIFileUtils.java b/modules/ai/src/main/java/com/bytedesk/ai/utils/AIFileUtils.java index ed412ec301..92f7247b6d 100644 --- a/modules/ai/src/main/java/com/bytedesk/ai/utils/AIFileUtils.java +++ b/modules/ai/src/main/java/com/bytedesk/ai/utils/AIFileUtils.java @@ -14,9 +14,14 @@ package com.bytedesk.ai.utils; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.net.HttpURLConnection; +import java.net.URL; +import java.io.ByteArrayOutputStream; +import java.util.Base64; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; @@ -24,14 +29,26 @@ import org.springframework.core.io.support.ResourcePatternResolver; import org.springframework.util.FileCopyUtils; import lombok.experimental.UtilityClass; +import lombok.Data; +import lombok.Builder; +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j @UtilityClass public class AIFileUtils { + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + public static class LocalFileContent { + private String filename; + private String content; + } - public static List getAllFiles() { - List fileContents = new ArrayList<>(); + public static List getAllFiles() { + List fileContents = new ArrayList<>(); try { ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); // 获取 aidemo/bytedesk 目录下所有的 .md 文件 @@ -47,7 +64,7 @@ public class AIFileUtils { String filename = resource.getFilename(); log.info("Reading file: {}", filename); - fileContents.add(new FileContent(filename, content)); + fileContents.add(LocalFileContent.builder().filename(filename).content(content).build()); } catch (IOException e) { log.error("Error reading file: " + resource.getFilename(), e); } @@ -78,4 +95,66 @@ public class AIFileUtils { return folders; } + /** + * 判断是否为本地回环 HTTP(S) 地址（127.* 或 localhost） + */ + public static boolean isLocalLoopbackHttpUrl(String url) { + try { + URL u = new URL(url); + String host = u.getHost(); + String protocol = u.getProtocol(); + if (!"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) { + return false; + } + if (host == null) return false; + String h = host.trim().toLowerCase(); + return h.equals("localhost") || h.startsWith("127."); + } catch (Exception e) { + return false; + } + } + + /** + * 读取 HTTP(S) 资源并转为 Base64，带大小上限 + */ + public static String fetchHttpAsBase64(String url, int maxBytes) { + HttpURLConnection conn = null; + try { + URL u = new URL(url); + conn = (HttpURLConnection) u.openConnection(); + conn.setConnectTimeout(3000); + conn.setReadTimeout(5000); + conn.setInstanceFollowRedirects(true); + conn.setRequestMethod("GET"); + int code = conn.getResponseCode(); + if (code != HttpURLConnection.HTTP_OK) { + log.warn("fetchHttpAsBase64 non-200: {} for url {}", code, url); + return null; + } + try (InputStream in = conn.getInputStream(); ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + byte[] buf = new byte[8192]; + int total = 0; + int n; + while ((n = in.read(buf)) > 0) { + bos.write(buf, 0, n); + total += n; + if (total > maxBytes) { + log.warn("fetchHttpAsBase64 exceeded maxBytes ({}), url {}", maxBytes, url); + return null; + } + } + byte[] bytes = bos.toByteArray(); + if (bytes.length == 0) return null; + return Base64.getEncoder().encodeToString(bytes); + } + } catch (Exception e) { + log.warn("fetchHttpAsBase64 error: {} for url {}", e.getMessage(), url); + return null; + } finally { + if (conn != null) { + try { conn.disconnect(); } catch (Exception ignore) {} + } + } + } + } diff --git a/modules/ai/src/main/java/com/bytedesk/ai/utils/FileContent.java b/modules/ai/src/main/java/com/bytedesk/ai/utils/FileContent.java deleted file mode 100644 index ed56d5684d..0000000000 --- a/modules/ai/src/main/java/com/bytedesk/ai/utils/FileContent.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * @Author: jackning 270580156@qq.com - * @Date: 2025-02-26 13:26:57 - * @LastEditors: jackning 270580156@qq.com - * @LastEditTime: 2025-07-21 07:51:50 - * @Description: bytedesk.com https://github.com/Bytedesk/bytedesk - * Please be aware of the BSL license restrictions before installing Bytedesk IM – - * selling, reselling, or hosting Bytedesk IM as a service is a breach of the terms and automatically terminates your rights under the license. - * Business Source License 1.1: https://github.com/Bytedesk/bytedesk/blob/main/LICENSE - * contact: 270580156@qq.com - * - * Copyright (c) 2025 by bytedesk.com, All Rights Reserved. - */ -package com.bytedesk.ai.utils; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -public class FileContent { - private String filename; - private String content; -} \ No newline at end of file diff --git a/modules/ai/src/main/java/com/bytedesk/ai/utils/mcp/readme.md b/modules/ai/src/main/java/com/bytedesk/ai/utils/mcp/readme.md deleted file mode 100644 index 5924eadab2..0000000000 --- a/modules/ai/src/main/java/com/bytedesk/ai/utils/mcp/readme.md +++ /dev/null @@ -1,19 +0,0 @@ - -# spring ai mcp - -- [overview](https://docs.spring.io/spring-ai/reference/api/mcp/mcp-overview.html) -- [mcp servers](https://github.com/modelcontextprotocol/servers) -- [mcp examples](https://github.com/spring-projects/spring-ai-examples) -- [building mcp with llms](https://modelcontextprotocol.io/tutorials/building-mcp-with-llms) diff --git a/modules/ai/src/main/java/com/bytedesk/ai/zhipuai/ZhipuMultiModelService.java b/modules/ai/src/main/java/com/bytedesk/ai/zhipuai/ZhipuMultiModelService.java index d936d3a209..655a2df2e5 100644 --- a/modules/ai/src/main/java/com/bytedesk/ai/zhipuai/ZhipuMultiModelService.java +++ b/modules/ai/src/main/java/com/bytedesk/ai/zhipuai/ZhipuMultiModelService.java @@ -15,11 +15,7 @@ package com.bytedesk.ai.zhipuai; import java.util.ArrayList; import java.util.List; -import java.util.Base64; -import java.net.HttpURLConnection; -import java.net.URL; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; +import com.bytedesk.ai.utils.AIFileUtils; import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.messages.Message; @@ -54,7 +50,6 @@ import ai.z.openapi.service.model.MessageContent; import ai.z.openapi.service.model.VideoUrl; import lombok.extern.slf4j.Slf4j; -import com.bytedesk.ai.robot.RobotConsts; import com.bytedesk.core.message.content.ImageContent; import com.bytedesk.core.message.content.VideoContent; import com.bytedesk.core.message.content.FileContent; @@ -166,9 +161,9 @@ public class ZhipuMultiModelService extends BaseSpringAIService { String url = ic != null ? ic.getUrl() : null; if (url != null && !url.isEmpty()) { String toSend = url; - if (isLocalLoopbackHttpUrl(url)) { + if (AIFileUtils.isLocalLoopbackHttpUrl(url)) { try { - String b64 = fetchHttpAsBase64(url, 8 * 1024 * 1024); + String b64 = AIFileUtils.fetchHttpAsBase64(url, 8 * 1024 * 1024); if (b64 != null && !b64.isEmpty()) { toSend = b64; log.debug("Converted local image url to base64 (MessageProtobuf) for zai image_url: {} -> (base64)", url); @@ -179,6 +174,10 @@ public class ZhipuMultiModelService extends BaseSpringAIService { } contents.add(MessageContent.builder().type(ZAI_IMAGE_URL) .imageUrl(ImageUrl.builder().url(toSend).build()).build()); + // 若有图片说明（label），追加一条文本消息 + if (ic != null && ic.getLabel() != null && !ic.getLabel().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_TEXT).text(ic.getLabel()).build()); + } } } else if (type == MessageTypeEnum.VIDEO) { VideoContent vc = VideoContent.fromJson(raw); @@ -186,20 +185,33 @@ public class ZhipuMultiModelService extends BaseSpringAIService { if (url != null && !url.isEmpty()) { contents.add(MessageContent.builder().type(ZAI_VIDEO_URL) .videoUrl(VideoUrl.builder().url(url).build()).build()); + // 若有视频说明（label），追加一条文本消息 + if (vc != null && vc.getLabel() != null && !vc.getLabel().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_TEXT).text(vc.getLabel()).build()); + } } } else if (type == MessageTypeEnum.FILE || type == MessageTypeEnum.AUDIO) { // AUDIO 暂按 file_url 处理 String url = null; + FileContent fc = null; + AudioContent ac = null; if (type == MessageTypeEnum.FILE) { - FileContent fc = FileContent.fromJson(raw); + fc = FileContent.fromJson(raw); url = fc != null ? fc.getUrl() : null; } else { - AudioContent ac = AudioContent.fromJson(raw); + ac = AudioContent.fromJson(raw); url = ac != null ? ac.getUrl() : null; } if (url != null && !url.isEmpty()) { contents.add(MessageContent.builder().type(ZAI_FILE_URL) .fileUrl(FileUrl.builder().url(url).build()).build()); + // 若有文件/音频说明（label），追加一条文本消息 + if (fc != null && fc.getLabel() != null && !fc.getLabel().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_TEXT).text(fc.getLabel()).build()); + } + if (ac != null && ac.getLabel() != null && !ac.getLabel().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_TEXT).text(ac.getLabel()).build()); + } } } else { // 其他类型按文本 @@ -212,7 +224,7 @@ public class ZhipuMultiModelService extends BaseSpringAIService { return contents; } - // 从传入文本中识别 BaseSpringAIService 注入的媒体标记并构建 zai 多模态内容；否则退化为纯文本。 + // 直接根据文本尝试解析媒体 JSON（ImageContent/VideoContent/FileContent/AudioContent），否则退化为纯文本。 private List buildUserContents(String text) { List contents = new ArrayList<>(); if (text == null) { @@ -220,123 +232,78 @@ public class ZhipuMultiModelService extends BaseSpringAIService { return contents; } String trimmed = text.trim(); - if (trimmed.startsWith(RobotConsts.BD_MEDIA_PREFIX)) { - // 形如 __BD_MEDIA__:{"type":"IMAGE","url":"https://..."} - String json = trimmed.substring(RobotConsts.BD_MEDIA_PREFIX.length()); - try { - // 简单解析避免引入依赖 - String type = extractJsonField(json, RobotConsts.BD_MEDIA_FIELD_TYPE); - String url = extractJsonField(json, RobotConsts.BD_MEDIA_FIELD_URL); - if (url != null && !url.isEmpty()) { - MessageTypeEnum mt = null; + // 尝试逐类解析标准 JSON + try { + ImageContent ic = ImageContent.fromJson(trimmed); + if (ic != null && ic.getUrl() != null && !ic.getUrl().isEmpty()) { + String url = ic.getUrl(); + String toSend = url; + if (AIFileUtils.isLocalLoopbackHttpUrl(url)) { try { - mt = MessageTypeEnum.valueOf(type); - } catch (Exception ignore) { - } - if (mt == MessageTypeEnum.IMAGE) { - // 若为本地回环地址（127.* 或 localhost），尝试抓取并转 Base64 以适配 SDK 要求 - String toSend = url; - if (isLocalLoopbackHttpUrl(url)) { - try { - String b64 = fetchHttpAsBase64(url, 8 * 1024 * 1024); // 上限 8MB - if (b64 != null && !b64.isEmpty()) { - toSend = b64; - log.debug("Converted local image url to base64 for zai image_url: {} -> (base64)", url); - } else { - log.warn("Failed to convert local image to base64, fallback to original url: {}", url); - } - } catch (Exception ce) { - log.warn("Convert local image to base64 error, fallback to original url: {} - {}", url, - ce.getMessage()); - } + String b64 = AIFileUtils.fetchHttpAsBase64(url, 8 * 1024 * 1024); + if (b64 != null && !b64.isEmpty()) { + toSend = b64; + log.debug("Converted local image url to base64 for zai image_url: {} -> (base64)", url); } - contents.add(MessageContent.builder().type(ZAI_IMAGE_URL) - .imageUrl(ImageUrl.builder().url(toSend).build()).build()); - } else if (mt == MessageTypeEnum.VIDEO) { - contents.add(MessageContent.builder().type(ZAI_VIDEO_URL) - .videoUrl(VideoUrl.builder().url(url).build()).build()); - } else if (mt == MessageTypeEnum.FILE || mt == MessageTypeEnum.AUDIO) { - // AUDIO 暂按 file_url 处理 - contents.add(MessageContent.builder().type(ZAI_FILE_URL) - .fileUrl(FileUrl.builder().url(url).build()).build()); - } else { - // 未知类型按文本 - contents.add(MessageContent.builder().type(ZAI_TEXT).text(text).build()); + } catch (Exception ce) { + log.warn("Convert local image to base64 error, fallback to original url: {} - {}", url, ce.getMessage()); } } - } catch (Exception e) { - log.debug("parse BD_MEDIA payload failed, fallback to text: {}", e.getMessage()); - contents.add(MessageContent.builder().type(ZAI_TEXT).text(text).build()); + contents.add(MessageContent.builder().type(ZAI_IMAGE_URL) + .imageUrl(ImageUrl.builder().url(toSend).build()).build()); + // 若有图片说明（label），追加一条文本消息 + if (ic.getLabel() != null && !ic.getLabel().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_TEXT).text(ic.getLabel()).build()); + } + return contents; } - } else { - // 无标记，按纯文本 - contents.add(MessageContent.builder().type(ZAI_TEXT).text(text).build()); - } + } catch (Exception ignore) {} + + try { + VideoContent vc = VideoContent.fromJson(trimmed); + if (vc != null && vc.getUrl() != null && !vc.getUrl().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_VIDEO_URL) + .videoUrl(VideoUrl.builder().url(vc.getUrl()).build()).build()); + // 若有视频说明（label），追加一条文本消息 + if (vc.getLabel() != null && !vc.getLabel().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_TEXT).text(vc.getLabel()).build()); + } + return contents; + } + } catch (Exception ignore) {} + + try { + FileContent fc = FileContent.fromJson(trimmed); + if (fc != null && fc.getUrl() != null && !fc.getUrl().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_FILE_URL) + .fileUrl(FileUrl.builder().url(fc.getUrl()).build()).build()); + // 若有文件说明（label），追加一条文本消息 + if (fc.getLabel() != null && !fc.getLabel().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_TEXT).text(fc.getLabel()).build()); + } + return contents; + } + } catch (Exception ignore) {} + + try { + AudioContent ac = AudioContent.fromJson(trimmed); + if (ac != null && ac.getUrl() != null && !ac.getUrl().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_FILE_URL) + .fileUrl(FileUrl.builder().url(ac.getUrl()).build()).build()); + // 若有音频说明（label），追加一条文本消息 + if (ac.getLabel() != null && !ac.getLabel().isEmpty()) { + contents.add(MessageContent.builder().type(ZAI_TEXT).text(ac.getLabel()).build()); + } + return contents; + } + } catch (Exception ignore) {} + + // 以上均无法解析为媒体 JSON，则按纯文本 + contents.add(MessageContent.builder().type(ZAI_TEXT).text(text).build()); return contents; } - // 判断是否为本地回环 HTTP(S) 地址（127.* 或 localhost） - private boolean isLocalLoopbackHttpUrl(String url) { - try { - URL u = new URL(url); - String host = u.getHost(); - String protocol = u.getProtocol(); - if (!"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) { - return false; - } - if (host == null) - return false; - String h = host.trim().toLowerCase(); - return h.equals("localhost") || h.startsWith("127."); - } catch (Exception e) { - return false; - } - } - - // 读取 HTTP(S) 资源并转为 Base64，带大小上限 - private String fetchHttpAsBase64(String url, int maxBytes) { - HttpURLConnection conn = null; - try { - URL u = new URL(url); - conn = (HttpURLConnection) u.openConnection(); - conn.setConnectTimeout(3000); - conn.setReadTimeout(5000); - conn.setInstanceFollowRedirects(true); - conn.setRequestMethod("GET"); - int code = conn.getResponseCode(); - if (code != HttpURLConnection.HTTP_OK) { - log.warn("fetchHttpAsBase64 non-200: {} for url {}", code, url); - return null; - } - try (InputStream in = conn.getInputStream(); ByteArrayOutputStream bos = new ByteArrayOutputStream()) { - byte[] buf = new byte[8192]; - int total = 0; - int n; - while ((n = in.read(buf)) > 0) { - bos.write(buf, 0, n); - total += n; - if (total > maxBytes) { - log.warn("fetchHttpAsBase64 exceeded maxBytes ({}), url {}", maxBytes, url); - return null; - } - } - byte[] bytes = bos.toByteArray(); - if (bytes.length == 0) - return null; - return Base64.getEncoder().encodeToString(bytes); - } - } catch (Exception e) { - log.warn("fetchHttpAsBase64 error: {} for url {}", e.getMessage(), url); - return null; - } finally { - if (conn != null) { - try { - conn.disconnect(); - } catch (Exception ignore) { - } - } - } - } + // 工具函数迁移至 AIFileUtils // zai-sdk 消息内容类型常量 private static final String ZAI_TEXT = "text"; @@ -344,22 +311,7 @@ public class ZhipuMultiModelService extends BaseSpringAIService { private static final String ZAI_VIDEO_URL = "video_url"; private static final String ZAI_FILE_URL = "file_url"; - // 提取极简 JSON 字段值，只支持一层字符串字段。 - private String extractJsonField(String json, String field) { - if (json == null) - return null; - String key = "\"" + field + "\":"; - int i = json.indexOf(key); - if (i < 0) - return null; - int start = json.indexOf('"', i + key.length()); - if (start < 0) - return null; - int end = json.indexOf('"', start + 1); - if (end < 0) - return null; - return json.substring(start + 1, end); - } + // 保留：如需自定义解析辅助，可在此处添加 private String extractFinalTextFromResponse(ChatCompletionResponse response) { try { @@ -631,6 +583,19 @@ public class ZhipuMultiModelService extends BaseSpringAIService { if (data.getChoices() != null && !data.getChoices().isEmpty()) { Delta delta = data.getChoices().get(0).getDelta(); String piece = extractDeltaText(delta); + // 提取模型推理内容（reasoningContent） + String reasoning = null; + try { + java.lang.reflect.Method getReasoning = delta.getClass() + .getMethod("getReasoningContent"); + Object rv = getReasoning.invoke(delta); + if (rv instanceof String rs && !rs.isEmpty()) { + reasoning = rs; + } + } catch (NoSuchMethodException ignore) { + // 某些SDK版本没有该字段 + } catch (Exception ignore) { + } if (piece != null) { String pieceTrim = piece.trim(); if (pieceTrim.equalsIgnoreCase("null")) { @@ -639,7 +604,8 @@ public class ZhipuMultiModelService extends BaseSpringAIService { } if (!pieceTrim.isEmpty()) { finalAnswer.append(pieceTrim); - sendStreamMessage(messageProtobufQuery, messageProtobufReply, emitter, pieceTrim); + // 将推理内容附带在 RobotStreamContent.reasonContent 字段 + sendStreamMessage(messageProtobufQuery, messageProtobufReply, emitter, pieceTrim, reasoning); } else { log.debug("SSE piece is empty after trim, delta={}", delta); } diff --git a/modules/core/src/main/java/com/bytedesk/core/message/MessagePersistService.java b/modules/core/src/main/java/com/bytedesk/core/message/MessagePersistService.java index 1128fe214c..a7f00999cc 100644 --- a/modules/core/src/main/java/com/bytedesk/core/message/MessagePersistService.java +++ b/modules/core/src/main/java/com/bytedesk/core/message/MessagePersistService.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Service; import com.bytedesk.core.thread.ThreadEntity; import com.bytedesk.core.thread.ThreadRestService; +import com.bytedesk.core.message.content.RobotStreamContent; import jakarta.annotation.Nonnull; import lombok.AllArgsConstructor; @@ -56,7 +57,57 @@ public class MessagePersistService { Optional messageOpt = messageRestService.findByUid(uid); if (messageOpt.isPresent()) { MessageEntity message = messageOpt.get(); - message.setContent(message.getContent() + messageProtobuf.getContent()); + try { + // 解析已存与本次分片为 RobotStreamContent，按字段拼接 + String existingJson = message.getContent(); + String incomingJson = messageProtobuf.getContent(); + + RobotStreamContent existing = null; + RobotStreamContent incoming = null; + try { + if (existingJson != null && !existingJson.isEmpty()) { + existing = RobotStreamContent.fromJson(existingJson, RobotStreamContent.class); + } + } catch (Exception ignore) { + // 旧数据或非JSON，忽略 + } + try { + if (incomingJson != null && !incomingJson.isEmpty()) { + incoming = RobotStreamContent.fromJson(incomingJson, RobotStreamContent.class); + } + } catch (Exception ignore) { + } + + if (existing == null || incoming == null) { + // 兜底：任一无法解析时，仍旧进行字符串拼接以不丢数据 + message.setContent((existingJson == null ? "" : existingJson) + + (incomingJson == null ? "" : incomingJson)); + } else { + String mergedAnswer = concatSafe(existing.getAnswer(), incoming.getAnswer()); + String mergedReason = concatSafe(existing.getReasonContent(), incoming.getReasonContent()); + + // 沿用已有的其它字段（question、sources、kbUid、robotUid、regenerationContext） + RobotStreamContent merged = RobotStreamContent.builder() + .question(existing.getQuestion() != null ? existing.getQuestion() + : incoming.getQuestion()) + .answer(mergedAnswer) + .reasonContent(mergedReason) + .sources(existing.getSources() != null ? existing.getSources() + : incoming.getSources()) + .regenerationContext(existing.getRegenerationContext() != null + ? existing.getRegenerationContext() + : incoming.getRegenerationContext()) + .kbUid(existing.getKbUid() != null ? existing.getKbUid() : incoming.getKbUid()) + .robotUid(existing.getRobotUid() != null ? existing.getRobotUid() + : incoming.getRobotUid()) + .build(); + message.setContent(merged.toJson()); + } + } catch (Exception ex) { + log.warn("Failed to merge ROBOT_STREAM content using JSON, fallback to raw append: {}", + ex.getMessage()); + message.setContent(message.getContent() + messageProtobuf.getContent()); + } messageRestService.save(message); } return; @@ -94,6 +145,12 @@ public class MessagePersistService { messageRestService.save(message); } + private String concatSafe(String a, String b) { + if (a == null || a.isEmpty()) return b == null ? "" : b; + if (b == null || b.isEmpty()) return a; + return a + b; + } + // 处理消息通知，已处理的消息返回true，未处理的消息返回false public Boolean dealWithMessageNotification(@NonNull MessageTypeEnum type, MessageProtobuf messageProtobuf) { // String content = messageProtobuf.getContent(); diff --git a/modules/core/src/main/java/com/bytedesk/core/message/content/RobotStreamContent.java b/modules/core/src/main/java/com/bytedesk/core/message/content/RobotStreamContent.java index 0f1a372574..5d87678d5b 100644 --- a/modules/core/src/main/java/com/bytedesk/core/message/content/RobotStreamContent.java +++ b/modules/core/src/main/java/com/bytedesk/core/message/content/RobotStreamContent.java @@ -28,6 +28,9 @@ public class RobotStreamContent extends BaseContent { private String answer; + // 推理内容（例如部分模型返回的 reasoningContent） + private String reasonContent; + // 答案来源信息列表 private List sources;