diff --git a/modules/ai/SpringAI_RAG_Refactor.md b/modules/ai/SpringAI_RAG_Refactor.md new file mode 100644 index 0000000000..6cc27f80bb --- /dev/null +++ b/modules/ai/SpringAI_RAG_Refactor.md @@ -0,0 +1,47 @@ +# SpringAI RAG Controller 重构说明 + +## 问题描述 +原有的 `SpringAIRagController` 使用了特定的 `OllamaChatModel`，当 Ollama 没有启用时会导致以下错误： +``` +Parameter 2 of constructor in com.bytedesk.ai.springai.rag.SpringAIRagController required a bean of type 'org.springframework.ai.ollama.OllamaChatModel' that could not be found. +``` + +## 解决方案 +将特定的 `OllamaChatModel` 替换为通用的 `ChatModel` 接口，使控制器能够适配任何 ChatModel 实现。 + +## 主要更改 + +### 1. SpringAIRagController.java +- **字段更改**: 将 `private final OllamaChatModel bytedeskOllamaChatModel` 改为 `private final ChatModel chatModel` +- **导入更改**: 移除 `OllamaChatModel` 导入，添加 `ChatModel` 导入 +- **注解更改**: 添加 `@ConditionalOnBean(ChatModel.class)` 确保只有在有 ChatModel 可用时才创建控制器 +- **方法更新**: 所有使用 `bytedeskOllamaChatModel` 的地方都替换为 `chatModel` + +### 2. 新增配置类 + +#### ChatModelConfig.java +- 提供备用的 ChatModel 配置 +- 只有在没有其他 ChatModel 实现时才创建 FallbackChatModel +- 受 `bytedesk.features.java-ai=true` 配置控制 + +#### FallbackChatModel.java +- 简单的 ChatModel 实现作为备用 +- 当没有其他 ChatModel 可用时返回友好的错误消息 +- 提醒用户配置正确的 ChatModel 实现 + +## 兼容性 +此更改向后兼容： +- 当 Ollama 启用时，会优先使用 OllamaChatModel +- 当其他 ChatModel 实现（如 OpenAI）可用时，会使用相应的实现 +- 当没有任何 ChatModel 可用时，会使用 FallbackChatModel 避免启动错误 + +## 配置要求 +确保在 `application.properties` 中设置： +```properties +bytedesk.features.java-ai=true +``` + +## 测试建议 +1. 测试 Ollama 启用的情况 +2. 测试 Ollama 禁用但有其他 ChatModel 的情况 +3. 测试完全没有 ChatModel 的情况（应该使用 FallbackChatModel） diff --git a/modules/ai/src/main/java/com/bytedesk/ai/springai/config/ChatModelConfig.java b/modules/ai/src/main/java/com/bytedesk/ai/springai/config/ChatModelConfig.java new file mode 100644 index 0000000000..830a486220 --- /dev/null +++ b/modules/ai/src/main/java/com/bytedesk/ai/springai/config/ChatModelConfig.java @@ -0,0 +1,47 @@ +/* + * @Author: jackning 270580156@qq.com + * @Date: 2025-05-31 10:00:00 + * @LastEditors: jackning 270580156@qq.com + * @LastEditTime: 2025-05-31 13:49:47 + * @Description: bytedesk.com https://github.com/Bytedesk/bytedesk + * Please be aware of the BSL license restrictions before installing Bytedesk IM – + * selling, reselling, or hosting Bytedesk IM as a service is a breach of the terms and automatically terminates your rights under the license. + * Business Source License 1.1: https://github.com/Bytedesk/bytedesk/blob/main/LICENSE + * contact: 270580156@qq.com + * + * Copyright (c) 2025 by bytedesk.com, All Rights Reserved. + */ +package com.bytedesk.ai.springai.config; + +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import lombok.extern.slf4j.Slf4j; + +/** + * ChatModel配置类 + * 提供ChatModel的默认实现，当没有其他ChatModel可用时作为备用 + */ +@Slf4j +@Configuration +public class ChatModelConfig { + + /** + * 当没有其他ChatModel bean时，提供一个默认的ChatModel实现 + * 这个bean只有在启用Java AI功能时才会创建 + */ + @Bean + @Primary + @ConditionalOnMissingBean(ChatModel.class) + @ConditionalOnProperty(name = "bytedesk.features.java-ai", havingValue = "true", matchIfMissing = false) + public ChatModel defaultChatModel() { + log.warn("No ChatModel implementation found, using fallback implementation"); + + // 返回一个简单的ChatModel实现作为备用 + return new FallbackChatModel(); + } +} diff --git a/modules/ai/src/main/java/com/bytedesk/ai/springai/config/FallbackChatModel.java b/modules/ai/src/main/java/com/bytedesk/ai/springai/config/FallbackChatModel.java new file mode 100644 index 0000000000..a0648cac11 --- /dev/null +++ b/modules/ai/src/main/java/com/bytedesk/ai/springai/config/FallbackChatModel.java @@ -0,0 +1,50 @@ +/* + * @Author: jackning 270580156@qq.com + * @Date: 2025-05-31 10:00:00 + * @LastEditors: jackning 270580156@qq.com + * @LastEditTime: 2025-05-31 10:00:00 + * @Description: bytedesk.com https://github.com/Bytedesk/bytedesk + * Please be aware of the BSL license restrictions before installing Bytedesk IM – + * selling, reselling, or hosting Bytedesk IM as a service is a breach of the terms and automatically terminates your rights under the license. + * Business Source License 1.1: https://github.com/Bytedesk/bytedesk/blob/main/LICENSE + * contact: 270580156@qq.com + * + * Copyright (c) 2025 by bytedesk.com, All Rights Reserved. + */ +package com.bytedesk.ai.springai.config; + +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.chat.model.Generation; + +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +/** + * 备用ChatModel实现 + * 当没有其他ChatModel可用时使用此实现 + */ +@Slf4j +public class FallbackChatModel implements ChatModel { + + @Override + public ChatResponse call(Prompt prompt) { + log.warn("FallbackChatModel is being used - please configure a proper ChatModel implementation"); + + // 创建一个简单的响应 + String responseText = "AI服务暂时不可用，请稍后再试。请确保已正确配置ChatModel实现（如Ollama、OpenAI等）。"; + + AssistantMessage assistantMessage = new AssistantMessage(responseText); + Generation generation = new Generation(assistantMessage); + return new ChatResponse(List.of(generation)); + } + + @Override + public String call(String message) { + return call(new Prompt(new UserMessage(message))).getResult().getOutput().getText(); + } +} diff --git a/modules/ai/src/main/java/com/bytedesk/ai/springai/rag/SpringAIRagController.java b/modules/ai/src/main/java/com/bytedesk/ai/springai/rag/SpringAIRagController.java index ddffca3e5f..ebc585037c 100644 --- a/modules/ai/src/main/java/com/bytedesk/ai/springai/rag/SpringAIRagController.java +++ b/modules/ai/src/main/java/com/bytedesk/ai/springai/rag/SpringAIRagController.java @@ -14,6 +14,8 @@ package com.bytedesk.ai.springai.rag; import java.util.List; +import java.util.Map; +import java.util.HashMap; import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.chat.client.advisor.api.Advisor; @@ -22,7 +24,7 @@ import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.document.Document; -import org.springframework.ai.ollama.OllamaChatModel; +import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.rag.Query; import org.springframework.ai.rag.advisor.RetrievalAugmentationAdvisor; import org.springframework.ai.rag.generation.augmentation.ContextualQueryAugmenter; @@ -36,8 +38,10 @@ import org.springframework.ai.rag.retrieval.search.VectorStoreDocumentRetriever; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.elasticsearch.ElasticsearchVectorStore; import io.micrometer.observation.ObservationRegistry; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -59,14 +63,14 @@ import lombok.extern.slf4j.Slf4j; @RestController @RequestMapping("/spring/ai/rag") @RequiredArgsConstructor -@ConditionalOnProperty(name = "spring.ai.ollama.chat.enabled", havingValue = "true", matchIfMissing = false) +@ConditionalOnBean(ChatModel.class) public class SpringAIRagController { private final VectorStore vectorStore; - private final OllamaChatModel bytedeskOllamaChatModel; + private final EmbeddingModel embeddingModel; -// private final SpringAIVectorStoreService springAIVectorService; + private final ChatModel chatModel; private final ObservationRegistry observationRegistry; @@ -87,7 +91,7 @@ public class SpringAIRagController { .build()) .build(); // 使用chatClient，添加ObservationRegistry - ChatResponse response = ChatClient.builder(bytedeskOllamaChatModel, observationRegistry, null) + ChatResponse response = ChatClient.builder(chatModel, observationRegistry, null) .build() .prompt() .advisors(qaAdvisor) @@ -107,7 +111,7 @@ public class SpringAIRagController { @RequestParam(value = "message", defaultValue = "什么时间考试？") String message, @RequestParam(value = "kbUid", defaultValue = "") String kbUid) { - ChatClient chatClient = ChatClient.builder(bytedeskOllamaChatModel) + ChatClient chatClient = ChatClient.builder(chatModel) .defaultAdvisors(QuestionAnswerAdvisor.builder(vectorStore) .searchRequest(SearchRequest.builder().build()) .build()) @@ -136,13 +140,13 @@ public class SpringAIRagController { .similarityThreshold(0.50) .vectorStore(vectorStore) .build()) - // 允许为空 - // .queryAugmenter(ContextualQueryAugmenter.builder() - // .allowEmptyContext(true) - // .build()) + // 允许为空 + // .queryAugmenter(ContextualQueryAugmenter.builder() + // .allowEmptyContext(true) + // .build()) .build(); - String answer = ChatClient.builder(bytedeskOllamaChatModel) + String answer = ChatClient.builder(chatModel) .defaultAdvisors(retrievalAugmentationAdvisor) .build() .prompt() @@ -163,7 +167,7 @@ public class SpringAIRagController { Advisor retrievalAugmentationAdvisor = RetrievalAugmentationAdvisor.builder() .queryTransformers(RewriteQueryTransformer.builder() - .chatClientBuilder(ChatClient.builder(bytedeskOllamaChatModel).build().mutate()) + .chatClientBuilder(ChatClient.builder(chatModel).build().mutate()) .build()) .documentRetriever(VectorStoreDocumentRetriever.builder() .similarityThreshold(0.50) @@ -171,7 +175,7 @@ public class SpringAIRagController { .build()) .build(); - String answer = ChatClient.builder(bytedeskOllamaChatModel) + String answer = ChatClient.builder(chatModel) .defaultAdvisors(retrievalAugmentationAdvisor) .build() .prompt() @@ -206,13 +210,13 @@ public class SpringAIRagController { // conversation history and a follow-up query into a standalone query that // captures the essence of the conversation. CompressionQueryTransformer queryTransformer = CompressionQueryTransformer.builder() - .chatClientBuilder(ChatClient.builder(bytedeskOllamaChatModel).build().mutate()) + .chatClientBuilder(ChatClient.builder(chatModel).build().mutate()) .build(); Query transformedQuery = queryTransformer.transform(query); // 使用chatClient - String answer = ChatClient.builder(bytedeskOllamaChatModel) + String answer = ChatClient.builder(chatModel) // .defaultAdvisors(retrievalAugmentationAdvisor) .build() .prompt() @@ -241,13 +245,13 @@ public class SpringAIRagController { // to provide better results when querying a target system, such as a vector // store or a web search engine. QueryTransformer queryTransformer = RewriteQueryTransformer.builder() - .chatClientBuilder(ChatClient.builder(bytedeskOllamaChatModel).build().mutate()) + .chatClientBuilder(ChatClient.builder(chatModel).build().mutate()) .build(); Query transformedQuery = queryTransformer.transform(query); // 使用chatClient - String answer = ChatClient.builder(bytedeskOllamaChatModel) + String answer = ChatClient.builder(chatModel) // .defaultAdvisors(retrievalAugmentationAdvisor) .build() .prompt() @@ -272,14 +276,14 @@ public class SpringAIRagController { Query query = new Query("Hvad er Danmarks hovedstad?"); QueryTransformer queryTransformer = TranslationQueryTransformer.builder() - .chatClientBuilder(ChatClient.builder(bytedeskOllamaChatModel).build().mutate()) + .chatClientBuilder(ChatClient.builder(chatModel).build().mutate()) .targetLanguage("english") .build(); Query transformedQuery = queryTransformer.transform(query); // 使用chatClient - String answer = ChatClient.builder(bytedeskOllamaChatModel) + String answer = ChatClient.builder(chatModel) // .defaultAdvisors(retrievalAugmentationAdvisor) .build() .prompt() @@ -302,14 +306,14 @@ public class SpringAIRagController { @RequestParam(value = "kbUid", defaultValue = "") String kbUid) { MultiQueryExpander queryExpander = MultiQueryExpander.builder() - .chatClientBuilder(ChatClient.builder(bytedeskOllamaChatModel).build().mutate()) + .chatClientBuilder(ChatClient.builder(chatModel).build().mutate()) .numberOfQueries(3) // .includeOriginal(false) .build(); List queries = queryExpander.expand(new Query("How to run a Spring Boot app?")); // 使用chatClient - String answer = ChatClient.builder(bytedeskOllamaChatModel) + String answer = ChatClient.builder(chatModel) // .defaultAdvisors(retrievalAugmentationAdvisor) .build() .prompt() @@ -393,7 +397,7 @@ public class SpringAIRagController { ResponseEntity> observedChat( @RequestParam(value = "message", defaultValue = "什么时间考试？") String message) { - ChatClient chatClient = ChatClient.builder(bytedeskOllamaChatModel, observationRegistry, null) + ChatClient chatClient = ChatClient.builder(chatModel, observationRegistry, null) .build(); ChatResponse response = chatClient.prompt() @@ -406,4 +410,86 @@ public class SpringAIRagController { return ResponseEntity.ok(JsonResult.success(response)); } + // 查看当前ElasticsearchVectorStore详情 + // http://127.0.0.1:9003/spring/ai/rag/vector-store-info + @GetMapping("/vector-store-info") + ResponseEntity> getVectorStoreInfo() { + log.info("Getting VectorStore info"); + + Map vectorStoreInfo = new HashMap<>(); + + try { + vectorStoreInfo.put("type", vectorStore.getClass().getSimpleName()); + vectorStoreInfo.put("className", vectorStore.getClass().getName()); + + // 如果是ElasticsearchVectorStore，获取更详细的信息 + if (vectorStore instanceof ElasticsearchVectorStore) { + vectorStoreInfo.put("vectorStoreType", "ElasticsearchVectorStore"); + vectorStoreInfo.put("status", "Active"); + // 注意：由于ElasticsearchVectorStore的内部属性可能是私有的， + // 这里只能获取到公开的信息 + vectorStoreInfo.put("description", "Elasticsearch-based vector store for semantic search"); + } else { + vectorStoreInfo.put("vectorStoreType", "Unknown"); + } + + vectorStoreInfo.put("timestamp", System.currentTimeMillis()); + + } catch (Exception e) { + log.error("Error getting vector store info: {}", e.getMessage()); + vectorStoreInfo.put("error", e.getMessage()); + vectorStoreInfo.put("status", "Error"); + } + + return ResponseEntity.ok(JsonResult.success(vectorStoreInfo)); + } + + // 查看当前EmbeddingModel详情 + // http://127.0.0.1:9003/spring/ai/rag/embedding-model-info + @GetMapping("/embedding-model-info") + ResponseEntity> getEmbeddingModelInfo() { + log.info("Getting EmbeddingModel info"); + + Map embeddingModelInfo = new HashMap<>(); + + try { + embeddingModelInfo.put("type", embeddingModel.getClass().getSimpleName()); + embeddingModelInfo.put("className", embeddingModel.getClass().getName()); + + // 获取embedding维度信息 + try { + // 使用一个简单的测试文本来获取embedding维度 + var testEmbedding = embeddingModel.embed("test"); + embeddingModelInfo.put("dimensions", testEmbedding.length); + embeddingModelInfo.put("status", "Active"); + embeddingModelInfo.put("testEmbeddingSize", testEmbedding.length); + } catch (Exception e) { + log.warn("Could not get embedding dimensions: {}", e.getMessage()); + embeddingModelInfo.put("dimensionsError", e.getMessage()); + embeddingModelInfo.put("status", "Error"); + } + + // 根据类名判断embedding模型类型 + String className = embeddingModel.getClass().getSimpleName(); + if (className.contains("Ollama")) { + embeddingModelInfo.put("modelType", "Ollama Embedding Model"); + embeddingModelInfo.put("description", "Local Ollama embedding model (e.g., bgm-m3)"); + } else if (className.contains("Zhipuai") || className.contains("ZhiPu")) { + embeddingModelInfo.put("modelType", "Zhipuai Embedding Model"); + embeddingModelInfo.put("description", "Zhipuai cloud embedding model"); + } else { + embeddingModelInfo.put("modelType", "Unknown Embedding Model"); + } + + embeddingModelInfo.put("timestamp", System.currentTimeMillis()); + + } catch (Exception e) { + log.error("Error getting embedding model info: {}", e.getMessage()); + embeddingModelInfo.put("error", e.getMessage()); + embeddingModelInfo.put("status", "Error"); + } + + return ResponseEntity.ok(JsonResult.success(embeddingModelInfo)); + } + } diff --git a/modules/ai/src/main/java/com/bytedesk/ai/springai/rag/readme.md b/modules/ai/src/main/java/com/bytedesk/ai/springai/rag/readme.md deleted file mode 100644 index 5ccd3d3d40..0000000000 --- a/modules/ai/src/main/java/com/bytedesk/ai/springai/rag/readme.md +++ /dev/null @@ -1,4 +0,0 @@ -# RAG - -- [spring ai alibaba](https://java2ai.com/blog/spring-ai-alibaba-module-rag) -- [spring ai rag](https://docs.spring.io/spring-ai/reference/api/retrieval-augmented-generation.html?spm=0.29160081.0.0.75c73b5blqQmqQ) diff --git a/modules/core/src/main/java/com/bytedesk/core/rbac/role/RoleEntity.java b/modules/core/src/main/java/com/bytedesk/core/rbac/role/RoleEntity.java index cab6d5fe02..17e807a750 100755 --- a/modules/core/src/main/java/com/bytedesk/core/rbac/role/RoleEntity.java +++ b/modules/core/src/main/java/com/bytedesk/core/rbac/role/RoleEntity.java @@ -2,7 +2,7 @@ * @Author: jackning 270580156@qq.com * @Date: 2024-01-29 16:21:24 * @LastEditors: jackning 270580156@qq.com - * @LastEditTime: 2025-03-20 12:50:24 + * @LastEditTime: 2025-05-31 13:17:49 * @Description: bytedesk.com https://github.com/Bytedesk/bytedesk * Please be aware of the BSL license restrictions before installing Bytedesk IM – * selling, reselling, or hosting Bytedesk IM as a service is a breach of the terms and automatically terminates your rights under the license. @@ -37,12 +37,7 @@ import com.bytedesk.core.rbac.authority.AuthorityEntity; @AllArgsConstructor @NoArgsConstructor @EntityListeners({ RoleEntityListener.class }) -@Table(name = "bytedesk_core_role" -// , uniqueConstraints = { -// @UniqueConstraint(columnNames = { "name", "orgUid" }), // is_deleted=false -// 时不加限定 -// } -) +@Table(name = "bytedesk_core_role") public class RoleEntity extends BaseEntity { private static final long serialVersionUID = 1L; @@ -53,10 +48,6 @@ public class RoleEntity extends BaseEntity { private String description; - // 超级管理员、管理员等是平台角色，用户自定义角色是Level.ORGANIZATION - // @Builder.Default - // private String level = LevelEnum.PLATFORM.name(); - // 是否是系统角色 @Builder.Default @Column(name = "is_system", nullable = false) @@ -67,12 +58,6 @@ public class RoleEntity extends BaseEntity { @JoinTable(name = "bytedesk_core_role_authority", joinColumns = @JoinColumn(name = "role_id"), inverseJoinColumns = @JoinColumn(name = "authority_id")) private Set authorities = new HashSet<>(); - // 除系统自带角色之外，允许管理员-自己创建角色 - // private String userUid; - - // @Builder.Default - // private String platform = PlatformEnum.BYTEDESK.name(); - public void addAuthority(AuthorityEntity authority) { this.authorities.add(authority); } diff --git a/modules/core/src/main/java/com/bytedesk/core/rbac/user/UserEntity.java b/modules/core/src/main/java/com/bytedesk/core/rbac/user/UserEntity.java index d01146a1e2..d555fc8410 100644 --- a/modules/core/src/main/java/com/bytedesk/core/rbac/user/UserEntity.java +++ b/modules/core/src/main/java/com/bytedesk/core/rbac/user/UserEntity.java @@ -26,10 +26,8 @@ import jakarta.persistence.ManyToMany; import jakarta.persistence.ManyToOne; import jakarta.persistence.OneToMany; import jakarta.persistence.Table; -// import jakarta.persistence.UniqueConstraint; import jakarta.validation.constraints.Email; import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.Pattern; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -46,15 +44,7 @@ import lombok.experimental.SuperBuilder; @AllArgsConstructor @NoArgsConstructor @EntityListeners({ UserEntityListener.class }) -@Table(name = "bytedesk_core_user" -// 去掉表级别的唯一约束，使用代码级别的唯一约束 -// , uniqueConstraints = { -// @UniqueConstraint(columnNames = { "num", "platform", "is_deleted" }), -// @UniqueConstraint(columnNames = { "username", "platform", "is_deleted" }), -// @UniqueConstraint(columnNames = { "email", "platform", "is_deleted" }), -// @UniqueConstraint(columnNames = { "mobile", "platform", "is_deleted" }), -// } -) +@Table(name = "bytedesk_core_user") public class UserEntity extends BaseEntityNoOrg { private static final long serialVersionUID = 1L; @@ -80,11 +70,6 @@ public class UserEntity extends BaseEntityNoOrg { @Builder.Default private String country = "86"; - // @Digits(message = "phone length error", fraction = 0, integer = 11) - // @Column(unique = true) // mobile + platform unique - // only support chinese mobile number, - // TODO: support other country mobile number using libphonenumber library - @Pattern(regexp = "^1[3-9]\\d{9}$", message = "Invalid mobile number format") private String mobile; @Builder.Default @@ -113,13 +98,13 @@ public class UserEntity extends BaseEntityNoOrg { private boolean mobileVerified = false; // 同一时刻，用户只能在一个组织下，用户可以切换组织 - @ManyToOne(fetch = FetchType.LAZY) + @ManyToOne(fetch = FetchType.EAGER) @com.fasterxml.jackson.annotation.JsonManagedReference private OrganizationEntity currentOrganization; // 用户当前拥有的角色 @Builder.Default - @ManyToMany(fetch = FetchType.LAZY) + @ManyToMany(fetch = FetchType.EAGER) @JoinTable( name = "bytedesk_core_user_roles", joinColumns = @JoinColumn(name = "user_id"), diff --git a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/elastic/WebpageElasticService.java b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/elastic/WebpageElasticService.java index 423c904ae3..9e33f466f3 100644 --- a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/elastic/WebpageElasticService.java +++ b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/elastic/WebpageElasticService.java @@ -51,6 +51,31 @@ public class WebpageElasticService { private final WebpageRestService webpageRestService; + /** + * 更新单个网页的Elasticsearch索引 + * @param request 网页请求对象 + */ + public void updateIndex(WebpageRequest request) { + Optional webpageOpt = webpageRestService.findByUid(request.getUid()); + if (webpageOpt.isPresent()) { + WebpageEntity webpage = webpageOpt.get(); + indexWebpage(webpage); + } else { + throw new IllegalArgumentException("Webpage not found with UID: " + request.getUid()); + } + } + + /** + * 更新知识库下所有网页的Elasticsearch索引 + * @param request 网页请求对象，包含知识库UID + */ + public void updateAllIndex(WebpageRequest request) { + List webpageList = webpageRestService.findByKbUid(request.getKbUid()); + webpageList.forEach(webpage -> { + indexWebpage(webpage); + }); + } + /** * 索引网页实体到Elasticsearch * @param webpage 要索引的网页实体 @@ -99,11 +124,14 @@ public class WebpageElasticService { public void deleteWebpageIndexByKbUid(String kbUid) { log.info("批量删除知识库下所有网页索引: {}", kbUid); try { - // 创建删除查询 - DeleteQuery deleteQuery = DeleteQuery.builder() + // 创建查询条件 + Query query = NativeQuery.builder() .withQuery(QueryBuilders.term().field("kbUid").value(kbUid).build()._toQuery()) .build(); + // 创建删除查询 + DeleteQuery deleteQuery = DeleteQuery.builder(query).build(); + // 执行删除 elasticsearchOperations.delete(deleteQuery, WebpageElastic.class); diff --git a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageIndexConsumer.java b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageIndexConsumer.java index eb888484d4..30d54ccd66 100644 --- a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageIndexConsumer.java +++ b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageIndexConsumer.java @@ -24,6 +24,7 @@ import com.bytedesk.kbase.llm_webpage.WebpageEntity; import com.bytedesk.kbase.llm_webpage.WebpageRestService; import com.bytedesk.kbase.llm_webpage.elastic.WebpageElasticService; import com.bytedesk.kbase.llm_webpage.vector.WebpageVectorService; +import com.bytedesk.kbase.llm_webpage.service.WebpageCrawlerService; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -40,6 +41,7 @@ public class WebpageIndexConsumer { private final WebpageElasticService webpageElasticService; private final WebpageVectorService webpageVectorService; private final WebpageRestService webpageRestService; + private final WebpageCrawlerService webpageCrawlerService; private final Random random = new Random(); /** @@ -131,6 +133,23 @@ public class WebpageIndexConsumer { */ private void handleIndexOperation(WebpageEntity webpage, WebpageIndexMessage message) { try { + // 首先检查是否需要抓取网页内容 + if (webpageCrawlerService.needsCrawling(webpage)) { + log.info("网页需要抓取内容，开始抓取: {}", webpage.getUrl()); + webpage = webpageCrawlerService.crawlAndUpdateContent(webpage); + + // 验证抓取到的内容是否有效 + if (!webpageCrawlerService.isValidContent(webpage.getContent())) { + log.warn("网页内容无效或为空，跳过索引操作: {}", webpage.getUrl()); + return; + } + + log.info("成功抓取网页内容，继续索引操作: {} (内容长度: {})", + webpage.getUrl(), webpage.getContent().length()); + } else { + log.debug("网页已有内容或不需要抓取，直接进行索引: {}", webpage.getUrl()); + } + // 处理Elastic搜索索引 if (message.isUpdateElasticIndex()) { processElasticIndex(webpage); diff --git a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageIndexMessage.java b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageIndexMessage.java index 8e357fdb89..ea02cf6c12 100644 --- a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageIndexMessage.java +++ b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageIndexMessage.java @@ -51,4 +51,18 @@ public class WebpageIndexMessage implements Serializable { * 是否需要更新向量索引 */ private Boolean updateVectorIndex; + + /** + * 检查是否需要更新Elastic索引 + */ + public boolean isUpdateElasticIndex() { + return updateElasticIndex != null && updateElasticIndex; + } + + /** + * 检查是否需要更新向量索引 + */ + public boolean isUpdateVectorIndex() { + return updateVectorIndex != null && updateVectorIndex; + } } diff --git a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageMessageService.java b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageMessageService.java index e8c314b8c1..d35b2b1964 100644 --- a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageMessageService.java +++ b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/mq/WebpageMessageService.java @@ -136,32 +136,3 @@ public class WebpageMessageService { } } } - - /** - * 发送网页UID到索引队列，用于异步创建或更新索引 - * - * @param uid 网页UID - */ - public void sendToIndexQueue(String uid) { - try { - log.info("发送网页到索引队列: {}", uid); - amqpTemplate.convertAndSend(WEBPAGE_INDEX_QUEUE, uid); - } catch (Exception e) { - log.error("发送网页到索引队列失败: {}, 错误: {}", uid, e.getMessage(), e); - } - } - - /** - * 发送网页UID到删除队列，用于异步删除索引 - * - * @param uid 网页UID - */ - public void sendToDeleteQueue(String uid) { - try { - log.info("发送网页到删除队列: {}", uid); - amqpTemplate.convertAndSend(WEBPAGE_DELETE_QUEUE, uid); - } catch (Exception e) { - log.error("发送网页到删除队列失败: {}, 错误: {}", uid, e.getMessage(), e); - } - } -} diff --git a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/service/WebpageCrawlerService.java b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/service/WebpageCrawlerService.java new file mode 100644 index 0000000000..023426945b --- /dev/null +++ b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/service/WebpageCrawlerService.java @@ -0,0 +1,180 @@ +/* + * @Author: jackning 270580156@qq.com + * @Date: 2025-01-28 09:40:00 + * @LastEditors: jackning 270580156@qq.com + * @LastEditTime: 2025-05-31 12:09:09 + * @Description: bytedesk.com https://github.com/Bytedesk/bytedesk + * Please be aware of the BSL license restrictions before installing Bytedesk IM – + * selling, reselling, or hosting Bytedesk IM as a service is a breach of the terms and automatically terminates your rights under the license. + * Business Source License 1.1: https://github.com/Bytedesk/bytedesk/blob/main/LICENSE + * contact: 270580156@qq.com + * 联系：270580156@qq.com + * Copyright (c) 2024 by bytedesk.com, All Rights Reserved. + */ +package com.bytedesk.kbase.llm_webpage.service; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import com.bytedesk.kbase.llm_webpage.WebpageEntity; +import com.bytedesk.kbase.llm_webpage.WebpageRestService; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * 网页内容抓取服务 + * 负责从网页URL抓取内容并更新WebpageEntity的content字段 + */ +@Slf4j +@Service +@AllArgsConstructor +public class WebpageCrawlerService { + + private final WebpageRestService webpageRestService; + + /** + * 抓取网页内容并更新实体 + * + * @param webpage 网页实体 + * @return 更新后的网页实体，如果抓取失败返回原实体 + */ + public WebpageEntity crawlAndUpdateContent(WebpageEntity webpage) { + if (webpage == null || !StringUtils.hasText(webpage.getUrl())) { + log.warn("网页实体或URL为空，跳过内容抓取"); + return webpage; + } + + try { + // 检查是否已有内容 + if (StringUtils.hasText(webpage.getContent())) { + log.debug("网页已有内容，跳过抓取: {}", webpage.getUrl()); + return webpage; + } + + log.info("开始抓取网页内容: {}", webpage.getUrl()); + + // 使用JSoup抓取网页内容 + Document jsoupDoc = Jsoup.connect(webpage.getUrl()) + .timeout(10000) // 10秒超时 + .userAgent("Mozilla/5.0 (compatible; BytedeskBot/1.0; +https://www.bytedesk.com/bot)") + .get(); + + // 提取标题和内容 + String title = jsoupDoc.title(); + // 移除script、style等标签，只保留有用的文本 + jsoupDoc.select("script, style, meta, link").remove(); + String content = jsoupDoc.body().text(); + + // 验证内容有效性 + if (!StringUtils.hasText(content)) { + log.warn("抓取到的内容为空: {}", webpage.getUrl()); + return webpage; + } + + // 更新网页实体 + webpage.setContent(content); + + // 如果网页标题为空，使用抓取到的标题 + if (!StringUtils.hasText(webpage.getTitle()) && StringUtils.hasText(title)) { + webpage.setTitle(title); + } + + // 保存更新后的实体 + WebpageEntity savedWebpage = webpageRestService.save(webpage); + + log.info("成功抓取并更新网页内容: {} (内容长度: {})", + webpage.getUrl(), content.length()); + + return savedWebpage; + + } catch (Exception e) { + log.error("抓取网页内容失败: {}, 错误: {}", webpage.getUrl(), e.getMessage(), e); + return webpage; // 返回原实体，不中断后续处理 + } + } + + /** + * 仅抓取网页内容，不保存到数据库 + * + * @param url 网页URL + * @return 抓取到的内容，失败返回null + */ + public String crawlContent(String url) { + if (!StringUtils.hasText(url)) { + log.warn("URL为空，无法抓取内容"); + return null; + } + + try { + log.debug("开始抓取网页内容: {}", url); + + // 使用JSoup抓取网页内容 + Document jsoupDoc = Jsoup.connect(url) + .timeout(10000) // 10秒超时 + .userAgent("Mozilla/5.0 (compatible; BytedeskBot/1.0; +https://www.bytedesk.com/bot)") + .get(); + + // 移除script、style等标签，只保留有用的文本 + jsoupDoc.select("script, style, meta, link").remove(); + String content = jsoupDoc.body().text(); + + if (!StringUtils.hasText(content)) { + log.warn("抓取到的内容为空: {}", url); + return null; + } + + log.debug("成功抓取网页内容: {} (内容长度: {})", url, content.length()); + return content; + + } catch (Exception e) { + log.error("抓取网页内容失败: {}, 错误: {}", url, e.getMessage(), e); + return null; + } + } + + /** + * 检查网页是否需要抓取内容 + * + * @param webpage 网页实体 + * @return true如果需要抓取，false如果不需要 + */ + public boolean needsCrawling(WebpageEntity webpage) { + if (webpage == null) { + return false; + } + + // 没有URL的不需要抓取 + if (!StringUtils.hasText(webpage.getUrl())) { + return false; + } + + // 已有内容的不需要抓取 + if (StringUtils.hasText(webpage.getContent())) { + return false; + } + + return true; + } + + /** + * 验证抓取到的内容是否有效 + * + * @param content 内容 + * @return true如果内容有效，false如果无效 + */ + public boolean isValidContent(String content) { + if (!StringUtils.hasText(content)) { + return false; + } + + // 检查内容长度，太短的内容可能是错误页面 + if (content.trim().length() < 10) { + return false; + } + + return true; + } +} diff --git a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/vector/WebpageVectorService.java b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/vector/WebpageVectorService.java index 17c5e4161c..3387d2a575 100644 --- a/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/vector/WebpageVectorService.java +++ b/modules/kbase/src/main/java/com/bytedesk/kbase/llm_webpage/vector/WebpageVectorService.java @@ -19,12 +19,10 @@ import java.util.Optional; import java.util.Map; import org.springframework.ai.document.Document; -import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.elasticsearch.ElasticsearchVectorStore; import org.springframework.ai.vectorstore.filter.Filter.Expression; import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -226,13 +224,14 @@ public class WebpageVectorService { return new ArrayList<>(); } - // 创建过滤表达式生成器 - FilterExpressionBuilder expressionBuilder = FilterExpressionBuilder.create(); - FilterExpressionBuilder.Op finalOp = expressionBuilder.value(true); + // 创建过滤表达式构建器 + FilterExpressionBuilder expressionBuilder = new FilterExpressionBuilder(); - // 添加过滤条件：启用状态 + // 构建查询条件 FilterExpressionBuilder.Op enabledOp = expressionBuilder.eq("enabled", "true"); - finalOp = expressionBuilder.and(finalOp, enabledOp); + + // 添加可选的过滤条件 + FilterExpressionBuilder.Op finalOp = enabledOp; // 添加可选的过滤条件：知识库、分类、组织 if (kbUid != null && !kbUid.isEmpty()) { diff --git a/modules/team/src/main/java/com/bytedesk/team/group/GroupEventListener.java b/modules/team/src/main/java/com/bytedesk/team/group/GroupEventListener.java index dca420bbdb..49979c766e 100644 --- a/modules/team/src/main/java/com/bytedesk/team/group/GroupEventListener.java +++ b/modules/team/src/main/java/com/bytedesk/team/group/GroupEventListener.java @@ -2,7 +2,7 @@ * @Author: jackning 270580156@qq.com * @Date: 2024-06-27 23:00:43 * @LastEditors: jackning 270580156@qq.com - * @LastEditTime: 2025-04-05 15:30:52 + * @LastEditTime: 2025-05-31 12:39:31 * @Description: bytedesk.com https://github.com/Bytedesk/bytedesk * Please be aware of the BSL license restrictions before installing Bytedesk IM – * selling, reselling, or hosting Bytedesk IM as a service is a breach of the terms and automatically terminates your rights under the license. @@ -40,11 +40,10 @@ public class GroupEventListener { @EventListener public void onThreadCreateEvent(ThreadCreateEvent event) { ThreadEntity thread = event.getThread(); - // ThreadRequest request = event.getRequest(); - log.info("group ThreadCreateEvent: {}", thread.getUid()); // - if (thread.getType().equals(ThreadTypeEnum.GROUP.name())) { + if (ThreadTypeEnum.GROUP.name().equals(thread.getType())) { String topic = thread.getTopic(); + log.info("group ThreadCreateEvent: {}", topic); // 同事群组会话：org/group/{group_uid} String groupUid = topic.split("/")[2]; log.info("groupUid {}", groupUid); diff --git a/modules/team/src/main/java/com/bytedesk/team/member/MemberEventListener.java b/modules/team/src/main/java/com/bytedesk/team/member/MemberEventListener.java index 3096beee60..00db73418c 100644 --- a/modules/team/src/main/java/com/bytedesk/team/member/MemberEventListener.java +++ b/modules/team/src/main/java/com/bytedesk/team/member/MemberEventListener.java @@ -2,7 +2,7 @@ * @Author: jackning 270580156@qq.com * @Date: 2024-06-03 14:06:20 * @LastEditors: jackning 270580156@qq.com - * @LastEditTime: 2025-05-13 12:15:30 + * @LastEditTime: 2025-05-31 12:40:39 * @Description: bytedesk.com https://github.com/Bytedesk/bytedesk * Please be aware of the BSL license restrictions before installing Bytedesk IM – * selling, reselling, or hosting Bytedesk IM as a service is a breach of the terms and automatically terminates your rights under the license. @@ -107,9 +107,9 @@ public class MemberEventListener { @EventListener public void onThreadCreateEvent(ThreadCreateEvent event) { ThreadEntity thread = event.getThread(); - log.info("member ThreadCreateEvent: {}", thread.getUid()); // - if (thread.getType().equals(ThreadTypeEnum.MEMBER.name())) { + if (ThreadTypeEnum.MEMBER.name().equals(thread .getType())) { + log.info("member ThreadCreateEvent: {}", thread.getTopic()); memberService.createMemberReverseThread(thread); } } diff --git a/starter/src/main/java/com/bytedesk/starter/controller/ExcelDownloadController.java b/starter/src/main/java/com/bytedesk/starter/controller/ExcelDownloadController.java index fb6deb53b3..a8a321a9b6 100644 --- a/starter/src/main/java/com/bytedesk/starter/controller/ExcelDownloadController.java +++ b/starter/src/main/java/com/bytedesk/starter/controller/ExcelDownloadController.java @@ -74,10 +74,11 @@ public class ExcelDownloadController { return new FileInfo( resource.getFilename(), resource.contentLength(), - resource.lastModified() + resource.lastModified(), + getFileDescription(resource.getFilename()) ); } catch (IOException e) { - return new FileInfo(resource.getFilename(), 0, 0); + return new FileInfo(resource.getFilename(), 0, 0, getFileDescription(resource.getFilename())); } }) .collect(Collectors.toList()); @@ -90,16 +91,86 @@ public class ExcelDownloadController { } } + /** + * 根据文件名获取文件用途描述 + */ + private String getFileDescription(String filename) { + if (filename == null) return "未知"; + + String name = filename.toLowerCase(); + + // 精确匹配特定文件名 + if (name.equals("kbase_auto_reply_fixed.xlsx")) { + return "固定自动回复"; + } else if (name.equals("kbase_auto_reply_keyword.xlsx")) { + return "根据关键词自动回复"; + } else if (name.equals("kbase_faq.xlsx")) { + return "常见问题模板"; + } else if (name.equals("kbase_quick_reply.xlsx")) { + return "快捷回复模板"; + } else if (name.equals("kbase_replace.xlsx")) { + return "知识库替换模板"; + } else if (name.equals("kbase_taboo.xlsx")) { + return "敏感词模板"; + } else if (name.equals("kbase_text.xlsx")) { + return "大模型纯文本导入模板"; + } else if (name.equals("kbase_transfer.xlsx")) { + return "自动触发转人工模板"; + } else if (name.equals("team_member.xlsx")) { + return "导入组织成员模板"; + } + // 通用匹配规则（作为后备） + else if (name.contains("kbase") && name.contains("auto") && name.contains("reply")) { + return "知识库自动回复模板"; + } else if (name.contains("kbase") && name.contains("faq")) { + return "知识库常见问题模板"; + } else if (name.contains("kbase") && name.contains("quick")) { + return "知识库快捷回复模板"; + } else if (name.contains("kbase") && name.contains("taboo")) { + return "知识库敏感词模板"; + } else if (name.contains("kbase") && name.contains("text")) { + return "知识库文本导入模板"; + } else if (name.contains("kbase") && name.contains("transfer")) { + return "知识库转人工模板"; + } else if (name.contains("kbase") && name.contains("replace")) { + return "知识库替换模板"; + } else if (name.contains("team") && name.contains("member")) { + return "团队成员导入模板"; + } else if (name.contains("keyword")) { + return "关键词管理模板"; + } else if (name.contains("faq")) { + return "常见问题模板"; + } else if (name.contains("user") || name.contains("member")) { + return "用户信息导入模板"; + } else if (name.contains("agent") || name.contains("staff")) { + return "客服人员管理模板"; + } else if (name.contains("category") || name.contains("cate")) { + return "分类管理模板"; + } else if (name.contains("feedback")) { + return "反馈信息导入模板"; + } else if (name.contains("message") || name.contains("chat")) { + return "聊天记录导入模板"; + } else if (name.contains("statistics") || name.contains("stat")) { + return "统计数据导出模板"; + } else if (name.contains("config") || name.contains("setting")) { + return "配置信息模板"; + } else { + return "数据导入/导出模板"; + } + } + // 文件信息类 public static class FileInfo { private String name; private long size; private long lastModified; + private String description; - public FileInfo(String name, long size, long lastModified) { + public FileInfo(String name, long size, long lastModified, String description) { this.name = name; this.size = size; this.lastModified = lastModified; + this.description = description; } public String getName() { @@ -113,5 +184,9 @@ public class ExcelDownloadController { public long getLastModified() { return lastModified; } + + public String getDescription() { + return description; + } } } \ No newline at end of file diff --git a/starter/src/main/resources/templates/excel/list.html b/starter/src/main/resources/templates/excel/list.html index eda8b80974..1b2abffdeb 100644 --- a/starter/src/main/resources/templates/excel/list.html +++ b/starter/src/main/resources/templates/excel/list.html @@ -61,6 +61,7 @@ 文件名 + 文件用途 大小 最后修改时间 操作 @@ -71,6 +72,9 @@ + + +