From 094b185c495101800bfafcb4ba6c5593f8eec82a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Tue, 24 Feb 2026 11:19:53 +0800 Subject: [PATCH] =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E5=A2=9E=E5=BC=BA=EF=BC=8C=E6=94=AF=E6=8C=81Milvus=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E4=BC=98=E5=8C=96=E7=9B=B8=E5=85=B3=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ai/DocumentCollectionController.java | 15 +++-- .../ai/entity/DocumentCollection.java | 39 ++++++++++-- .../impl/DocumentCollectionServiceImpl.java | 24 ++++++- .../ai/service/impl/DocumentServiceImpl.java | 34 +++++++--- .../langs/en-US/documentCollection.json | 1 + .../langs/zh-CN/documentCollection.json | 1 + .../DocumentCollectionDataConfig.vue | 62 ++++++++++++++++--- .../DocumentCollectionModal.vue | 54 +++++++++++----- .../KnowledgeSearchConfig.vue | 23 ++++--- sql/02-easyflow-v2.data.sql | 2 +- 10 files changed, 196 insertions(+), 59 deletions(-) diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentCollectionController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentCollectionController.java index b9461c3..b2e3865 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentCollectionController.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentCollectionController.java @@ -4,6 +4,10 @@ import cn.dev33.satoken.annotation.SaCheckPermission; import com.easyagents.core.document.Document; import com.mybatisflex.core.query.QueryWrapper; import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; import tech.easyflow.ai.entity.BotDocumentCollection; import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.service.BotDocumentCollectionService; @@ -12,10 +16,6 @@ import tech.easyflow.ai.service.DocumentCollectionService; import tech.easyflow.ai.service.ModelService; import tech.easyflow.common.domain.Result; import tech.easyflow.common.web.controller.BaseCurdController; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import tech.easyflow.common.web.exceptions.BusinessException; import javax.annotation.Resource; @@ -70,11 +70,14 @@ public class DocumentCollectionController extends BaseCurdController options = new HashMap<>(); + Map options = entity.getOptions() == null + ? new HashMap<>() + : new HashMap<>(entity.getOptions()); if (entity.getSearchEngineEnable() == null){ entity.setSearchEngineEnable(false); } - options.put("canUpdateEmbeddingModel", true); + options.putIfAbsent(DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL, true); + options.putIfAbsent(DocumentCollection.KEY_RERANK_ENABLE, entity.getRerankModelId() != null); entity.setOptions(options); } return super.onSaveOrUpdateBefore(entity, isSave); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentCollection.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentCollection.java index 5063b33..b5e0a71 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentCollection.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentCollection.java @@ -6,6 +6,8 @@ import com.easyagents.store.aliyun.AliyunVectorStore; import com.easyagents.store.aliyun.AliyunVectorStoreConfig; import com.easyagents.store.elasticsearch.ElasticSearchVectorStore; import com.easyagents.store.elasticsearch.ElasticSearchVectorStoreConfig; +import com.easyagents.store.milvus.MilvusVectorStore; +import com.easyagents.store.milvus.MilvusVectorStoreConfig; import com.easyagents.store.opensearch.OpenSearchVectorStore; import com.easyagents.store.opensearch.OpenSearchVectorStoreConfig; import com.easyagents.store.qcloud.QCloudVectorStore; @@ -62,6 +64,11 @@ public class DocumentCollection extends DocumentCollectionBase { */ public static final String KEY_CAN_UPDATE_EMBEDDING_MODEL = "canUpdateEmbeddingModel"; + /** + * 是否启用重排模型 + */ + public static final String KEY_RERANK_ENABLE = "rerankEnable"; + public DocumentStore toDocumentStore() { String storeType = this.getVectorStoreType(); if (StringUtil.noText(storeType)) { @@ -73,8 +80,8 @@ public class DocumentCollection extends DocumentCollectionBase { switch (storeType.toLowerCase()) { case "redis": return redisStore(); -// case "milvus": -// return milvusStore(); + case "milvus": + return milvusStore(); case "opensearch": return openSearchStore(); case "elasticsearch": @@ -101,10 +108,13 @@ public class DocumentCollection extends DocumentCollectionBase { return new RedisVectorStore(redisVectorStoreConfig); } -// private DocumentStore milvusStore() { -// MilvusVectorStoreConfig milvusVectorStoreConfig = getStoreConfig(MilvusVectorStoreConfig.class); -// return new MilvusVectorStore(milvusVectorStoreConfig); -// } + private DocumentStore milvusStore() { + MilvusVectorStoreConfig milvusVectorStoreConfig = getStoreConfig(MilvusVectorStoreConfig.class); + if (milvusVectorStoreConfig != null && StringUtil.noText(milvusVectorStoreConfig.getDefaultCollectionName())) { + milvusVectorStoreConfig.setDefaultCollectionName(this.getVectorStoreCollection()); + } + return new MilvusVectorStore(milvusVectorStoreConfig); + } private DocumentStore openSearchStore() { OpenSearchVectorStoreConfig openSearchVectorStoreConfig = getStoreConfig(OpenSearchVectorStoreConfig.class); @@ -136,6 +146,23 @@ public class DocumentCollection extends DocumentCollectionBase { public Object getOptionsByKey(String key) { Map options = this.getOptions(); + if (KEY_RERANK_ENABLE.equals(key)) { + if (options == null || !options.containsKey(KEY_RERANK_ENABLE)) { + return this.getRerankModelId() != null; + } + Object value = options.get(key); + if (value instanceof Boolean) { + return value; + } + if (value instanceof Number) { + return ((Number) value).intValue() != 0; + } + if (value instanceof String) { + return Boolean.parseBoolean((String) value); + } + return false; + } + if (options == null) { return null; } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java index d3eb948..99673c3 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java @@ -2,6 +2,7 @@ package tech.easyflow.ai.service.impl; import com.easyagents.core.document.Document; +import com.easyagents.core.model.rerank.RerankException; import com.easyagents.core.model.rerank.RerankModel; import com.easyagents.core.store.DocumentStore; import com.easyagents.core.store.SearchWrapper; @@ -9,6 +10,8 @@ import com.easyagents.core.store.StoreOptions; import com.easyagents.search.engine.service.DocumentSearcher; import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.spring.service.impl.ServiceImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import tech.easyflow.ai.config.SearcherFactory; @@ -46,6 +49,8 @@ import static tech.easyflow.ai.entity.DocumentCollection.*; @Service public class DocumentCollectionServiceImpl extends ServiceImpl implements DocumentCollectionService { + private static final Logger LOG = LoggerFactory.getLogger(DocumentCollectionServiceImpl.class); + @Resource private ModelService llmService; @@ -128,20 +133,33 @@ public class DocumentCollectionServiceImpl extends ServiceImpl originalScores = new HashMap<>(); + searchDocuments.forEach(item -> originalScores.put(item.getId(), item.getScore())); searchDocuments.forEach(item -> item.setScore(null)); - List rerankDocuments = rerankModel.rerank(keyword, searchDocuments); - return formatDocuments(rerankDocuments, minSimilarity, docRecallMaxNum); + try { + List rerankDocuments = rerankModel.rerank(keyword, searchDocuments); + return formatDocuments(rerankDocuments, minSimilarity, docRecallMaxNum); + } catch (RerankException e) { + searchDocuments.forEach(item -> item.setScore(originalScores.get(item.getId()))); + LOG.warn("Rerank failed for collectionId={}, modelId={}, fallback to vector results. message={}", + documentCollection.getId(), documentCollection.getRerankModelId(), e.getMessage()); + return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum); + } } catch (InterruptedException | ExecutionException e) { Thread.currentThread().interrupt(); e.printStackTrace(); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java index d9b8a4b..bd9adda 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java @@ -25,16 +25,11 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import tech.easyflow.ai.config.SearcherFactory; import tech.easyflow.ai.entity.*; - -import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL; -import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE; -import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK; -import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT; import tech.easyflow.ai.mapper.DocumentChunkMapper; import tech.easyflow.ai.mapper.DocumentMapper; import tech.easyflow.ai.service.DocumentChunkService; -import tech.easyflow.ai.service.DocumentService; import tech.easyflow.ai.service.DocumentCollectionService; +import tech.easyflow.ai.service.DocumentService; import tech.easyflow.ai.service.ModelService; import tech.easyflow.common.ai.rag.ExcelDocumentSplitter; import tech.easyflow.common.domain.Result; @@ -50,6 +45,11 @@ import java.math.BigInteger; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; +import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL; +import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE; +import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK; +import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT; + /** * 服务层实现。 * @@ -228,11 +228,26 @@ public class DocumentServiceImpl extends ServiceImpl i @Override public Result saveTextResult(List documentChunks, Document document) { - Boolean result = storeDocument(document, documentChunks); + if (documentChunks == null || documentChunks.isEmpty()) { + return Result.fail(1, "切割结果为空，无法保存"); + } + + List validChunks = new ArrayList<>(); + for (DocumentChunk chunk : documentChunks) { + if (chunk != null && StringUtil.hasText(chunk.getContent())) { + validChunks.add(chunk); + } + } + + if (validChunks.isEmpty()) { + return Result.fail(1, "切割结果无有效文本，无法进行向量化"); + } + + Boolean result = storeDocument(document, validChunks); if (result) { this.getMapper().insert(document); AtomicInteger sort = new AtomicInteger(1); - documentChunks.forEach(item -> { + validChunks.forEach(item -> { item.setDocumentCollectionId(document.getCollectionId()); item.setSorting(sort.get()); item.setDocumentId(document.getId()); @@ -287,7 +302,8 @@ public class DocumentServiceImpl extends ServiceImpl i try { result = documentStore.store(documents, options); } catch (Exception e) { - Log.error(e.getMessage()); + Log.error("Vector store failed: knowledgeId={}, collection={}, chunkCount={}", + knowledge.getId(), options.getCollectionName(), documents.size(), e); throw new BusinessException("向量过程中发生错误，错误信息为：" + e.getMessage()); } if (result == null || !result.isSuccess()) { diff --git a/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json b/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json index 086894e..974108a 100644 --- a/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json +++ b/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json @@ -18,6 +18,7 @@ "modified": "Modified", "modifiedBy": "ModifiedBy", "options": "Options", + "rerankEnable": "Enable rerank model", "rerankLlmId": "RerankLlm", "searchEngineEnable": "SearchEngineEnable", "englishName": "EnglishName", diff --git a/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json b/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json index ce9eb80..41cd7ca 100644 --- a/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json +++ b/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json @@ -18,6 +18,7 @@ "modified": "最后一次修改时间", "modifiedBy": "最后一次修改用户ID", "options": "其他配置", + "rerankEnable": "是否启用重排模型", "rerankLlmId": "重排模型", "searchEngineEnable": "是否启用搜索引擎", "englishName": "英文名称", diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentCollectionDataConfig.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentCollectionDataConfig.vue index 5fc2058..9b42ab0 100644 --- a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentCollectionDataConfig.vue +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentCollectionDataConfig.vue @@ -1,9 +1,5 @@