feat: 收敛知识库检索调度与评分语义

- 固定 rag.engine 与 Milvus 配置,补齐启动期检索基础设施校验

- 支持调用方配置 retrievalMode,并统一知识库检索入口与结果来源展示

- 修正关键词检索 knowledgeId 过滤、混合检索评分归一化与本地 ES 默认配置
This commit is contained in:
2026-04-05 20:23:05 +08:00
parent 2592a1f09d
commit b5dd427920
41 changed files with 1260 additions and 600 deletions

View File

@@ -118,7 +118,32 @@ services:
minio-init: minio-init:
condition: service_completed_successfully condition: service_completed_successfully
ports: ports:
- "19530:19530" - "39530:19530"
- "9091:9091" - "9091:9091"
volumes: volumes:
- ./data/milvus:/var/lib/milvus - ./data/milvus:/var/lib/milvus
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:8.15.5
container_name: easyflow-elasticsearch
restart: unless-stopped
environment:
TZ: Asia/Shanghai
discovery.type: single-node
ELASTIC_PASSWORD: elastic
xpack.security.enabled: "true"
ES_JAVA_OPTS: -Xms1g -Xmx1g
ports:
- "39200:9200"
volumes:
- ./data/elasticsearch:/usr/share/elasticsearch/data
healthcheck:
test:
[
"CMD-SHELL",
"curl -fsS -u elastic:elastic http://127.0.0.1:9200 >/dev/null || exit 1",
]
interval: 15s
timeout: 10s
retries: 20
start_period: 60s

View File

@@ -1,6 +1,7 @@
package tech.easyflow.admin.controller.ai; package tech.easyflow.admin.controller.ai;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import tech.easyflow.ai.dto.BotKnowledgeBindingRequest;
import tech.easyflow.ai.entity.BotDocumentCollection; import tech.easyflow.ai.entity.BotDocumentCollection;
import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.permission.KnowledgeReadAccessSnapshot; import tech.easyflow.ai.permission.KnowledgeReadAccessSnapshot;
@@ -63,20 +64,21 @@ public class BotDocumentCollectionController extends BaseCurdController<BotDocum
} }
@PostMapping("updateBotKnowledgeIds") @PostMapping("updateBotKnowledgeIds")
public Result<?> save(@JsonBody("botId") BigInteger botId, @JsonBody("knowledgeIds") BigInteger [] knowledgeIds) { public Result<?> save(@JsonBody("botId") BigInteger botId,
if (knowledgeIds != null) { @JsonBody("knowledgeBindings") List<BotKnowledgeBindingRequest> knowledgeBindings) {
for (BigInteger knowledgeId : knowledgeIds) { if (knowledgeBindings != null) {
if (knowledgeId == null) { for (BotKnowledgeBindingRequest binding : knowledgeBindings) {
if (binding == null || binding.getKnowledgeId() == null) {
continue; continue;
} }
DocumentCollection collection = documentCollectionService.getById(knowledgeId); DocumentCollection collection = documentCollectionService.getById(binding.getKnowledgeId());
if (collection == null) { if (collection == null) {
continue; continue;
} }
resourceAccessService.assertAccess(CategoryResourceType.KNOWLEDGE, collection, ResourceAction.READ, "无权限绑定知识库"); resourceAccessService.assertAccess(CategoryResourceType.KNOWLEDGE, collection, ResourceAction.READ, "无权限绑定知识库");
} }
} }
service.saveBotAndKnowledge(botId, knowledgeIds); service.saveBotAndKnowledge(botId, knowledgeBindings);
return Result.ok(); return Result.ok();
} }
} }

View File

@@ -2,6 +2,7 @@ package tech.easyflow.admin.controller.ai;
import cn.dev33.satoken.annotation.SaCheckPermission; import cn.dev33.satoken.annotation.SaCheckPermission;
import com.easyagents.core.document.Document; import com.easyagents.core.document.Document;
import com.easyagents.rag.retrieval.RagRetrievalMetadataKeys;
import com.mybatisflex.core.paginate.Page; import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.query.QueryWrapper;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
@@ -12,9 +13,12 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import tech.easyflow.ai.permission.KnowledgeVisibilityQueryHelper; import tech.easyflow.ai.permission.KnowledgeVisibilityQueryHelper;
import tech.easyflow.ai.documentimport.DocumentImportDtos; import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
import tech.easyflow.ai.entity.BotDocumentCollection; import tech.easyflow.ai.entity.BotDocumentCollection;
import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.entity.Model; import tech.easyflow.ai.entity.Model;
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
import tech.easyflow.ai.service.BotDocumentCollectionService; import tech.easyflow.ai.service.BotDocumentCollectionService;
import tech.easyflow.ai.service.DocumentChunkService; import tech.easyflow.ai.service.DocumentChunkService;
import tech.easyflow.ai.service.DocumentCollectionService; import tech.easyflow.ai.service.DocumentCollectionService;
@@ -32,11 +36,15 @@ import tech.easyflow.system.service.ResourceAccessService;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.Serializable; import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger; import java.math.BigInteger;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID;
/** /**
* 控制层。 * 控制层。
@@ -105,13 +113,11 @@ public class DocumentCollectionController extends BaseCurdController<DocumentCol
Map<String, Object> options = entity.getOptions() == null Map<String, Object> options = entity.getOptions() == null
? new HashMap<>() ? new HashMap<>()
: new HashMap<>(entity.getOptions()); : new HashMap<>(entity.getOptions());
if (entity.getSearchEngineEnable() == null){
entity.setSearchEngineEnable(false);
}
options.putIfAbsent(DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL, true); options.putIfAbsent(DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL, true);
options.putIfAbsent(DocumentCollection.KEY_RERANK_ENABLE, entity.getRerankModelId() != null); options.putIfAbsent(DocumentCollection.KEY_RERANK_ENABLE, entity.getRerankModelId() != null);
entity.setOptions(options); entity.setOptions(options);
} }
normalizeInfrastructureFields(entity, isSave);
return super.onSaveOrUpdateBefore(entity, isSave); return super.onSaveOrUpdateBefore(entity, isSave);
} }
@@ -124,8 +130,16 @@ public class DocumentCollectionController extends BaseCurdController<DocumentCol
idExpr = "#knowledgeId", idExpr = "#knowledgeId",
denyMessage = "无权限访问知识库" denyMessage = "无权限访问知识库"
) )
public Result<List<Document>> search(@RequestParam BigInteger knowledgeId, @RequestParam String keyword) { public Result<List<KnowledgeSearchResultItem>> search(@RequestParam BigInteger knowledgeId,
return Result.ok(service.search(knowledgeId, keyword)); @RequestParam String keyword,
@RequestParam(required = false) String retrievalMode) {
KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
request.setKnowledgeId(knowledgeId);
request.setQuery(keyword);
request.setRetrievalMode(KnowledgeRetrievalModes.parse(retrievalMode));
request.setCallerType("API");
request.setCallerId(String.valueOf(knowledgeId));
return Result.ok(toKnowledgeSearchResult(service.search(request)));
} }
@@ -234,4 +248,85 @@ public class DocumentCollectionController extends BaseCurdController<DocumentCol
} }
return collection; return collection;
} }
private void normalizeInfrastructureFields(DocumentCollection entity, boolean isSave) {
if (entity == null) {
return;
}
if (isSave) {
entity.setVectorStoreEnable(true);
entity.setVectorStoreType("milvus");
entity.setSearchEngineEnable(true);
entity.setVectorStoreCollection(generateVectorCollectionName());
return;
}
if (entity.getVectorStoreEnable() != null) {
entity.setVectorStoreEnable(true);
}
if (entity.getVectorStoreType() != null) {
entity.setVectorStoreType("milvus");
}
if (entity.getSearchEngineEnable() != null) {
entity.setSearchEngineEnable(true);
}
}
private String generateVectorCollectionName() {
return "kb_" + UUID.randomUUID().toString().replace("-", "").substring(0, 28);
}
private List<KnowledgeSearchResultItem> toKnowledgeSearchResult(List<Document> documents) {
List<KnowledgeSearchResultItem> results = new ArrayList<>();
if (documents == null) {
return results;
}
for (int index = 0; index < documents.size(); index++) {
Document document = documents.get(index);
if (document == null) {
continue;
}
KnowledgeSearchResultItem item = new KnowledgeSearchResultItem();
item.setSorting(index + 1);
item.setContent(document.getContent());
item.setScore(roundScore(document.getScore()));
item.setHitSource(readMetadataAsString(document, RagRetrievalMetadataKeys.HIT_SOURCE));
item.setVectorScore(roundScore(readMetadataAsDouble(document, RagRetrievalMetadataKeys.VECTOR_SCORE)));
item.setKeywordScore(roundScore(readMetadataAsDouble(document, RagRetrievalMetadataKeys.KEYWORD_SCORE)));
results.add(item);
}
return results;
}
private String readMetadataAsString(Document document, String key) {
Object value = document == null ? null : document.getMetadata(key);
if (value == null) {
return null;
}
String text = String.valueOf(value);
return StringUtils.hasText(text) ? text : null;
}
private Double readMetadataAsDouble(Document document, String key) {
Object value = document == null ? null : document.getMetadata(key);
if (value instanceof Number) {
return ((Number) value).doubleValue();
}
if (value instanceof String && StringUtils.hasText((String) value)) {
try {
return Double.valueOf((String) value);
} catch (NumberFormatException ignore) {
return null;
}
}
return null;
}
private Double roundScore(Double value) {
if (value == null) {
return null;
}
return new BigDecimal(String.valueOf(value))
.setScale(4, RoundingMode.HALF_UP)
.doubleValue();
}
} }

View File

@@ -41,6 +41,10 @@
<groupId>com.easyagents</groupId> <groupId>com.easyagents</groupId>
<artifactId>easy-agents-support</artifactId> <artifactId>easy-agents-support</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-rag-retrieval</artifactId>
</dependency>
<dependency> <dependency>
<groupId>com.easyagents</groupId> <groupId>com.easyagents</groupId>
<artifactId>easy-agents-spring-boot-starter</artifactId> <artifactId>easy-agents-spring-boot-starter</artifactId>

View File

@@ -1,35 +1,10 @@
package tech.easyflow.ai.config; package tech.easyflow.ai.config;
import com.easyagents.engine.es.ESConfig; import com.easyagents.engine.es.ESConfig;
import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Component @Component
@ConfigurationProperties(prefix = "rag.searcher.elastic")
public class AiEsConfig extends ESConfig { public class AiEsConfig extends ESConfig {
@Value("${rag.searcher.elastic.host}")
@Override
public void setHost(String host) {
super.setHost(host);
}
@Value("${rag.searcher.elastic.userName}")
@Override
public void setUserName(String userName) {
super.setUserName(userName);
}
@Value("${rag.searcher.elastic.password}")
@Override
public void setPassword(String password) {
super.setPassword(password);
}
@Value("${rag.searcher.elastic.indexName}")
@Override
public void setIndexName(String indexName) {
super.setIndexName(indexName);
}
} }

View File

@@ -1,17 +1,10 @@
package tech.easyflow.ai.config; package tech.easyflow.ai.config;
import com.easyagents.search.engine.lucene.LuceneConfig; import com.easyagents.search.engine.lucene.LuceneConfig;
import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Component @Component
@ConfigurationProperties(prefix = "rag.searcher.lucene")
public class AiLuceneConfig extends LuceneConfig { public class AiLuceneConfig extends LuceneConfig {
@Value("${rag.searcher.lucene.indexDirPath}")
@Override
public void setIndexDirPath(String indexDirPath) {
super.setIndexDirPath(indexDirPath);
}
} }

View File

@@ -0,0 +1,22 @@
package tech.easyflow.ai.config;
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@ConfigurationProperties(prefix = "rag.milvus")
public class AiMilvusConfig extends MilvusVectorStoreConfig {
public MilvusVectorStoreConfig copyForCollection(String collectionName) {
MilvusVectorStoreConfig config = new MilvusVectorStoreConfig();
config.setUri(getUri());
config.setToken(getToken());
config.setDatabaseName(getDatabaseName());
config.setUsername(getUsername());
config.setPassword(getPassword());
config.setAutoCreateCollection(isAutoCreateCollection());
config.setDefaultCollectionName(collectionName);
return config;
}
}

View File

@@ -0,0 +1,103 @@
package tech.easyflow.ai.config;
import com.easyagents.engine.es.ElasticSearcher;
import com.easyagents.search.engine.service.DocumentSearcher;
import com.easyagents.store.milvus.MilvusVectorStore;
import org.springframework.beans.factory.SmartInitializingSingleton;
import org.springframework.stereotype.Component;
import tech.easyflow.ai.rag.KeywordEngineType;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.common.util.StringUtil;
import tech.easyflow.common.web.exceptions.BusinessException;
import javax.annotation.Resource;
import java.io.File;
@Component
public class RagInfrastructureValidator implements SmartInitializingSingleton {
private static final int STARTUP_CHECK_RETRY_TIMES = 10;
private static final long STARTUP_CHECK_RETRY_INTERVAL_MS = 1000L;
@Resource
private AiMilvusConfig aiMilvusConfig;
@Resource
private AiLuceneConfig aiLuceneConfig;
@Resource
private SearcherFactory searcherFactory;
@Override
public void afterSingletonsInstantiated() {
validateMilvus();
validateKeywordSearcher();
}
private void validateMilvus() {
Exception lastException = null;
for (int i = 0; i < STARTUP_CHECK_RETRY_TIMES; i++) {
try {
MilvusVectorStore vectorStore = new MilvusVectorStore(aiMilvusConfig.copyForCollection("__rag_boot_probe__"));
if (vectorStore.checkAvailable()) {
return;
}
} catch (Exception e) {
lastException = e;
}
sleepBeforeRetry();
}
if (lastException != null) {
throw new BusinessException("Milvus 服务不可用,项目启动失败,请检查 rag.milvus 配置与服务状态: " + lastException.getMessage());
}
throw new BusinessException("Milvus 服务不可用,项目启动失败,请检查 rag.milvus 配置与服务状态");
}
private void validateKeywordSearcher() {
KeywordEngineType engineType = KeywordEngineType.from(
SpringContextUtil.getProperty("rag.engine", "ES")
);
if (engineType == KeywordEngineType.LUCENE) {
validateLuceneDirectory();
return;
}
DocumentSearcher searcher = searcherFactory.getSearcher();
if (!(searcher instanceof ElasticSearcher) || !checkElasticAvailable((ElasticSearcher) searcher)) {
throw new BusinessException("ES 服务不可用,项目启动失败,请检查 rag.engine 与 rag.searcher.elastic 配置");
}
}
private boolean checkElasticAvailable(ElasticSearcher elasticSearcher) {
for (int i = 0; i < STARTUP_CHECK_RETRY_TIMES; i++) {
if (elasticSearcher.checkAvailable()) {
return true;
}
sleepBeforeRetry();
}
return false;
}
private void validateLuceneDirectory() {
String indexDirPath = aiLuceneConfig.getIndexDirPath();
if (StringUtil.noText(indexDirPath)) {
throw new BusinessException("Lucene 索引目录未配置,请检查 rag.searcher.lucene.indexDirPath");
}
File indexDir = new File(indexDirPath);
if (!indexDir.exists() && !indexDir.mkdirs()) {
throw new BusinessException("Lucene 索引目录创建失败: " + indexDirPath);
}
if (!indexDir.isDirectory() || !indexDir.canRead() || !indexDir.canWrite()) {
throw new BusinessException("Lucene 索引目录不可读写: " + indexDirPath);
}
}
private void sleepBeforeRetry() {
try {
Thread.sleep(STARTUP_CHECK_RETRY_INTERVAL_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new BusinessException("中间件启动校验被中断");
}
}
}

View File

@@ -3,40 +3,37 @@ package tech.easyflow.ai.config;
import com.easyagents.engine.es.ElasticSearcher; import com.easyagents.engine.es.ElasticSearcher;
import com.easyagents.search.engine.lucene.LuceneSearcher; import com.easyagents.search.engine.lucene.LuceneSearcher;
import com.easyagents.search.engine.service.DocumentSearcher; import com.easyagents.search.engine.service.DocumentSearcher;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.ObjectProvider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
@Configuration @Configuration
public class SearcherFactory { public class SearcherFactory {
@Autowired private final ObjectProvider<DocumentSearcher> documentSearcherProvider;
private AiLuceneConfig luceneConfig;
@Autowired public SearcherFactory(ObjectProvider<DocumentSearcher> documentSearcherProvider) {
private AiEsConfig aiEsConfig; this.documentSearcherProvider = documentSearcherProvider;
}
@Bean @Bean
public LuceneSearcher luceneSearcher() { @ConditionalOnProperty(prefix = "rag", name = "engine", havingValue = "LUCENE")
public LuceneSearcher luceneSearcher(AiLuceneConfig luceneConfig) {
return new LuceneSearcher(luceneConfig); return new LuceneSearcher(luceneConfig);
} }
@Bean @Bean
public ElasticSearcher elasticSearcher() { @ConditionalOnProperty(prefix = "rag", name = "engine", havingValue = "ES", matchIfMissing = true)
public ElasticSearcher elasticSearcher(AiEsConfig aiEsConfig) {
return new ElasticSearcher(aiEsConfig); return new ElasticSearcher(aiEsConfig);
} }
public DocumentSearcher getSearcher() {
return documentSearcherProvider.getIfAvailable();
}
public DocumentSearcher getSearcher(String defaultSearcherType) { public DocumentSearcher getSearcher(String ignored) {
if (defaultSearcherType == null) { return getSearcher();
defaultSearcherType = "lucene";
}
switch (defaultSearcherType) {
case "elasticSearch":
return new ElasticSearcher(aiEsConfig);
case "lucene":
default:
return new LuceneSearcher(luceneConfig);
}
} }
} }

View File

@@ -0,0 +1,32 @@
package tech.easyflow.ai.dto;
import com.easyagents.rag.retrieval.RetrievalMode;
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
import java.math.BigInteger;
public class BotKnowledgeBindingRequest {
private BigInteger knowledgeId;
private String retrievalMode;
public BigInteger getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
public String getRetrievalMode() {
return retrievalMode;
}
public void setRetrievalMode(String retrievalMode) {
this.retrievalMode = retrievalMode;
}
public RetrievalMode resolveRetrievalMode() {
return KnowledgeRetrievalModes.parse(retrievalMode);
}
}

View File

@@ -0,0 +1,59 @@
package tech.easyflow.ai.dto;
public class KnowledgeSearchResultItem {
private Integer sorting;
private String content;
private Double score;
private String hitSource;
private Double vectorScore;
private Double keywordScore;
public Integer getSorting() {
return sorting;
}
public void setSorting(Integer sorting) {
this.sorting = sorting;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public Double getScore() {
return score;
}
public void setScore(Double score) {
this.score = score;
}
public String getHitSource() {
return hitSource;
}
public void setHitSource(String hitSource) {
this.hitSource = hitSource;
}
public Double getVectorScore() {
return vectorScore;
}
public void setVectorScore(Double vectorScore) {
this.vectorScore = vectorScore;
}
public Double getKeywordScore() {
return keywordScore;
}
public void setKeywordScore(Double keywordScore) {
this.keywordScore = keywordScore;
}
}

View File

@@ -3,7 +3,9 @@ package tech.easyflow.ai.easyagents.tool;
import com.easyagents.core.document.Document; import com.easyagents.core.document.Document;
import com.easyagents.core.model.chat.tool.BaseTool; import com.easyagents.core.model.chat.tool.BaseTool;
import com.easyagents.core.model.chat.tool.Parameter; import com.easyagents.core.model.chat.tool.Parameter;
import com.easyagents.rag.retrieval.RetrievalMode;
import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
import tech.easyflow.ai.service.DocumentCollectionService; import tech.easyflow.ai.service.DocumentCollectionService;
import tech.easyflow.common.util.SpringContextUtil; import tech.easyflow.common.util.SpringContextUtil;
@@ -14,12 +16,18 @@ import java.util.Map;
public class DocumentCollectionTool extends BaseTool { public class DocumentCollectionTool extends BaseTool {
private BigInteger knowledgeId; private BigInteger knowledgeId;
private RetrievalMode retrievalMode = RetrievalMode.HYBRID;
public DocumentCollectionTool() { public DocumentCollectionTool() {
} }
public DocumentCollectionTool(DocumentCollection documentCollection, boolean needEnglishName) { public DocumentCollectionTool(DocumentCollection documentCollection, boolean needEnglishName) {
this(documentCollection, needEnglishName, RetrievalMode.HYBRID);
}
public DocumentCollectionTool(DocumentCollection documentCollection, boolean needEnglishName, RetrievalMode retrievalMode) {
this.knowledgeId = documentCollection.getId(); this.knowledgeId = documentCollection.getId();
this.retrievalMode = retrievalMode == null ? RetrievalMode.HYBRID : retrievalMode;
if (needEnglishName) { if (needEnglishName) {
this.name = documentCollection.getEnglishName(); this.name = documentCollection.getEnglishName();
} else { } else {
@@ -47,11 +55,25 @@ public class DocumentCollectionTool extends BaseTool {
this.knowledgeId = knowledgeId; this.knowledgeId = knowledgeId;
} }
public RetrievalMode getRetrievalMode() {
return retrievalMode;
}
public void setRetrievalMode(RetrievalMode retrievalMode) {
this.retrievalMode = retrievalMode == null ? RetrievalMode.HYBRID : retrievalMode;
}
@Override @Override
public Object invoke(Map<String, Object> argsMap) { public Object invoke(Map<String, Object> argsMap) {
DocumentCollectionService knowledgeService = SpringContextUtil.getBean(DocumentCollectionService.class); DocumentCollectionService knowledgeService = SpringContextUtil.getBean(DocumentCollectionService.class);
List<Document> documents = knowledgeService.search(this.knowledgeId, (String) argsMap.get("input")); KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
request.setKnowledgeId(this.knowledgeId);
request.setQuery((String) argsMap.get("input"));
request.setRetrievalMode(this.retrievalMode);
request.setCallerType("BOT_TOOL");
request.setCallerId(this.knowledgeId == null ? null : this.knowledgeId.toString());
List<Document> documents = knowledgeService.search(request);
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
if (documents != null) { if (documents != null) {

View File

@@ -7,6 +7,8 @@ import com.easyagents.flow.core.knowledge.Knowledge;
import com.easyagents.flow.core.knowledge.KnowledgeProvider; import com.easyagents.flow.core.knowledge.KnowledgeProvider;
import com.easyagents.flow.core.node.KnowledgeNode; import com.easyagents.flow.core.node.KnowledgeNode;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
import tech.easyflow.ai.service.DocumentCollectionService; import tech.easyflow.ai.service.DocumentCollectionService;
import javax.annotation.Resource; import javax.annotation.Resource;
@@ -30,7 +32,17 @@ public class KnowledgeProviderImpl implements KnowledgeProvider {
return new Knowledge() { return new Knowledge() {
@Override @Override
public List<Map<String, Object>> search(String keyword, int limit, KnowledgeNode knowledgeNode, Chain chain) { public List<Map<String, Object>> search(String keyword, int limit, KnowledgeNode knowledgeNode, Chain chain) {
List<Document> documents = documentCollectionService.search(new BigInteger(id.toString()), keyword); KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
request.setKnowledgeId(new BigInteger(id.toString()));
request.setQuery(keyword);
request.setLimit(limit);
request.setRetrievalMode(KnowledgeRetrievalModes.parse(knowledgeNode.getRetrievalMode()));
request.setCallerType("WORKFLOW");
request.setCallerId(knowledgeNode.getId());
List<Document> documents = documentCollectionService.search(request);
if (limit > 0 && documents.size() > limit) {
documents = new ArrayList<>(documents.subList(0, limit));
}
List<Map<String, Object>> res = new ArrayList<>(); List<Map<String, Object>> res = new ArrayList<>();
for (Document document : documents) { for (Document document : documents) {
res.add(JSONObject.from(document)); res.add(JSONObject.from(document));

View File

@@ -1,9 +1,14 @@
package tech.easyflow.ai.entity; package tech.easyflow.ai.entity;
import com.easyagents.rag.retrieval.RetrievalMode;
import tech.easyflow.ai.entity.base.BotDocumentCollectionBase; import tech.easyflow.ai.entity.base.BotDocumentCollectionBase;
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
import com.mybatisflex.annotation.RelationOneToOne; import com.mybatisflex.annotation.RelationOneToOne;
import com.mybatisflex.annotation.Table; import com.mybatisflex.annotation.Table;
import java.util.HashMap;
import java.util.Map;
/** /**
* 实体类。 * 实体类。
* *
@@ -14,6 +19,8 @@ import com.mybatisflex.annotation.Table;
@Table("tb_bot_document_collection") @Table("tb_bot_document_collection")
public class BotDocumentCollection extends BotDocumentCollectionBase { public class BotDocumentCollection extends BotDocumentCollectionBase {
public static final String OPTION_KEY_RETRIEVAL_MODE = "retrievalMode";
@RelationOneToOne(selfField = "documentCollectionId", targetField = "id") @RelationOneToOne(selfField = "documentCollectionId", targetField = "id")
private DocumentCollection knowledge; private DocumentCollection knowledge;
@@ -24,4 +31,21 @@ public class BotDocumentCollection extends BotDocumentCollectionBase {
public void setKnowledge(DocumentCollection knowledge) { public void setKnowledge(DocumentCollection knowledge) {
this.knowledge = knowledge; this.knowledge = knowledge;
} }
public RetrievalMode getRetrievalMode() {
Map<String, Object> options = getOptions();
if (options == null) {
return RetrievalMode.HYBRID;
}
Object value = options.get(OPTION_KEY_RETRIEVAL_MODE);
return KnowledgeRetrievalModes.parse(value == null ? null : String.valueOf(value));
}
public void setRetrievalMode(RetrievalMode retrievalMode) {
Map<String, Object> options = getOptions() == null
? new HashMap<>()
: new HashMap<>(getOptions());
options.put(OPTION_KEY_RETRIEVAL_MODE, (retrievalMode == null ? RetrievalMode.HYBRID : retrievalMode).name());
setOptions(options);
}
} }

View File

@@ -2,24 +2,16 @@ package tech.easyflow.ai.entity;
import com.easyagents.core.model.chat.tool.Tool; import com.easyagents.core.model.chat.tool.Tool;
import com.easyagents.core.store.DocumentStore; import com.easyagents.core.store.DocumentStore;
import com.easyagents.store.aliyun.AliyunVectorStore; import com.easyagents.rag.retrieval.RetrievalMode;
import com.easyagents.store.aliyun.AliyunVectorStoreConfig;
import com.easyagents.store.elasticsearch.ElasticSearchVectorStore;
import com.easyagents.store.elasticsearch.ElasticSearchVectorStoreConfig;
import com.easyagents.store.milvus.MilvusVectorStore; import com.easyagents.store.milvus.MilvusVectorStore;
import com.easyagents.store.milvus.MilvusVectorStoreConfig; import com.easyagents.store.milvus.MilvusVectorStoreConfig;
import com.easyagents.store.opensearch.OpenSearchVectorStore;
import com.easyagents.store.opensearch.OpenSearchVectorStoreConfig;
import com.easyagents.store.qcloud.QCloudVectorStore;
import com.easyagents.store.qcloud.QCloudVectorStoreConfig;
import com.easyagents.store.redis.RedisVectorStore;
import com.easyagents.store.redis.RedisVectorStoreConfig;
import com.mybatisflex.annotation.Table; import com.mybatisflex.annotation.Table;
import tech.easyflow.ai.config.AiMilvusConfig;
import tech.easyflow.ai.easyagents.tool.DocumentCollectionTool; import tech.easyflow.ai.easyagents.tool.DocumentCollectionTool;
import tech.easyflow.ai.entity.base.DocumentCollectionBase; import tech.easyflow.ai.entity.base.DocumentCollectionBase;
import tech.easyflow.common.util.PropertiesUtil; import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.common.util.StringUtil; import tech.easyflow.common.util.StringUtil;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.system.permission.resource.VisibilityResource; import tech.easyflow.system.permission.resource.VisibilityResource;
import java.math.BigDecimal; import java.math.BigDecimal;
@@ -58,11 +50,6 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
*/ */
public static final String KEY_SIMILARITY_THRESHOLD = "simThreshold"; public static final String KEY_SIMILARITY_THRESHOLD = "simThreshold";
/**
* 搜索引擎类型
*/
public static final String KEY_SEARCH_ENGINE_TYPE = "searchEngineType";
/** /**
* 是否允许更新向量模型 * 是否允许更新向量模型
*/ */
@@ -78,28 +65,10 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
public static final String KEY_SPLITTER_STRATEGY_PROFILES = "splitter.strategyProfiles"; public static final String KEY_SPLITTER_STRATEGY_PROFILES = "splitter.strategyProfiles";
public DocumentStore toDocumentStore() { public DocumentStore toDocumentStore() {
String storeType = this.getVectorStoreType(); if (StringUtil.noText(this.getVectorStoreCollection())) {
if (StringUtil.noText(storeType)) {
throw new BusinessException("向量数据库类型未设置");
}
if (storeType == null) {
return null; return null;
} }
switch (storeType.toLowerCase()) { return milvusStore();
case "redis":
return redisStore();
case "milvus":
return milvusStore();
case "opensearch":
return openSearchStore();
case "elasticsearch":
return elasticSearchStore();
case "aliyun":
return aliyunStore();
case "qcloud":
return qcloudStore();
}
return null;
} }
public boolean isVectorStoreEnabled() { public boolean isVectorStoreEnabled() {
@@ -115,53 +84,31 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
} }
public boolean isSearchEngineEnabled() { public boolean isSearchEngineEnabled() {
return this.getSearchEngineEnable() != null && this.getSearchEngineEnable(); return true;
}
private DocumentStore redisStore() {
RedisVectorStoreConfig redisVectorStoreConfig = getStoreConfig(RedisVectorStoreConfig.class);
return new RedisVectorStore(redisVectorStoreConfig);
} }
private DocumentStore milvusStore() { private DocumentStore milvusStore() {
MilvusVectorStoreConfig milvusVectorStoreConfig = getStoreConfig(MilvusVectorStoreConfig.class); AiMilvusConfig aiMilvusConfig = SpringContextUtil.getBean(AiMilvusConfig.class);
if (milvusVectorStoreConfig != null && StringUtil.noText(milvusVectorStoreConfig.getDefaultCollectionName())) { MilvusVectorStoreConfig milvusVectorStoreConfig = aiMilvusConfig.copyForCollection(this.getVectorStoreCollection());
milvusVectorStoreConfig.setDefaultCollectionName(this.getVectorStoreCollection());
}
return new MilvusVectorStore(milvusVectorStoreConfig); return new MilvusVectorStore(milvusVectorStoreConfig);
} }
private DocumentStore openSearchStore() {
OpenSearchVectorStoreConfig openSearchVectorStoreConfig = getStoreConfig(OpenSearchVectorStoreConfig.class);
return new OpenSearchVectorStore(openSearchVectorStoreConfig);
}
private DocumentStore elasticSearchStore() {
ElasticSearchVectorStoreConfig elasticSearchVectorStoreConfig = getStoreConfig(ElasticSearchVectorStoreConfig.class);
return new ElasticSearchVectorStore(elasticSearchVectorStoreConfig);
}
private DocumentStore aliyunStore() {
AliyunVectorStoreConfig aliyunVectorStoreConfig = getStoreConfig(AliyunVectorStoreConfig.class);
return new AliyunVectorStore(aliyunVectorStoreConfig);
}
private DocumentStore qcloudStore() {
QCloudVectorStoreConfig qCloudVectorStoreConfig = getStoreConfig(QCloudVectorStoreConfig.class);
return new QCloudVectorStore(qCloudVectorStoreConfig);
}
private <T> T getStoreConfig(Class<T> clazz) {
return PropertiesUtil.propertiesTextToEntity(this.getVectorStoreConfig(), clazz);
}
public Tool toFunction(boolean needEnglishName) { public Tool toFunction(boolean needEnglishName) {
return new DocumentCollectionTool(this, needEnglishName); return toFunction(needEnglishName, RetrievalMode.HYBRID.name());
}
public Tool toFunction(boolean needEnglishName, String retrievalMode) {
return new DocumentCollectionTool(this, needEnglishName, KnowledgeRetrievalModes.parse(retrievalMode));
} }
public Object getOptionsByKey(String key) { public Object getOptionsByKey(String key) {
Map<String, Object> options = this.getOptions(); Map<String, Object> options = this.getOptions();
if (KEY_DOC_RECALL_MAX_NUM.equals(key) && (options == null || !options.containsKey(KEY_DOC_RECALL_MAX_NUM))) {
return 5;
}
if (KEY_SIMILARITY_THRESHOLD.equals(key) && (options == null || !options.containsKey(KEY_SIMILARITY_THRESHOLD))) {
return 0.6f;
}
if (KEY_RERANK_ENABLE.equals(key)) { if (KEY_RERANK_ENABLE.equals(key)) {
if (options == null || !options.containsKey(KEY_RERANK_ENABLE)) { if (options == null || !options.containsKey(KEY_RERANK_ENABLE)) {
return this.getRerankModelId() != null; return this.getRerankModelId() != null;
@@ -182,21 +129,9 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
if (options == null) { if (options == null) {
return null; return null;
} }
if (KEY_DOC_RECALL_MAX_NUM.equals(key) && !options.containsKey(KEY_DOC_RECALL_MAX_NUM)) {
return 5;
}
if (KEY_SIMILARITY_THRESHOLD.equals(key)) { if (KEY_SIMILARITY_THRESHOLD.equals(key)) {
if (!options.containsKey(KEY_SIMILARITY_THRESHOLD)) { BigDecimal score = (BigDecimal) options.get(key);
return 0.6f; return (float) score.doubleValue();
} else {
BigDecimal score = (BigDecimal) options.get(key);
return (float) score.doubleValue();
}
}
if (KEY_SEARCH_ENGINE_TYPE.equals(key)) {
if (!options.containsKey(KEY_SEARCH_ENGINE_TYPE)) {
return "lucene";
}
} }
return options.get(key); return options.get(key);
} }

View File

@@ -0,0 +1,21 @@
package tech.easyflow.ai.rag;
import tech.easyflow.common.util.StringUtil;
import tech.easyflow.common.web.exceptions.BusinessException;
public enum KeywordEngineType {
ES,
LUCENE;
public static KeywordEngineType from(String value) {
if (StringUtil.noText(value)) {
return ES;
}
for (KeywordEngineType type : values()) {
if (type.name().equalsIgnoreCase(value.trim())) {
return type;
}
}
throw new BusinessException("不支持的关键词检索引擎: " + value);
}
}

View File

@@ -0,0 +1,18 @@
package tech.easyflow.ai.rag;
import com.easyagents.rag.retrieval.RetrievalMode;
import tech.easyflow.common.web.exceptions.BusinessException;
public final class KnowledgeRetrievalModes {
private KnowledgeRetrievalModes() {
}
public static RetrievalMode parse(String value) {
try {
return RetrievalMode.from(value);
} catch (IllegalArgumentException e) {
throw new BusinessException("不支持的检索方式: " + value);
}
}
}

View File

@@ -0,0 +1,63 @@
package tech.easyflow.ai.rag;
import com.easyagents.rag.retrieval.RetrievalMode;
import java.math.BigInteger;
public class KnowledgeRetrievalRequest {
private BigInteger knowledgeId;
private String query;
private Integer limit;
private RetrievalMode retrievalMode = RetrievalMode.HYBRID;
private String callerType;
private String callerId;
public BigInteger getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
public String getQuery() {
return query;
}
public void setQuery(String query) {
this.query = query;
}
public Integer getLimit() {
return limit;
}
public void setLimit(Integer limit) {
this.limit = limit;
}
public RetrievalMode getRetrievalMode() {
return retrievalMode;
}
public void setRetrievalMode(RetrievalMode retrievalMode) {
this.retrievalMode = retrievalMode == null ? RetrievalMode.HYBRID : retrievalMode;
}
public String getCallerType() {
return callerType;
}
public void setCallerType(String callerType) {
this.callerType = callerType;
}
public String getCallerId() {
return callerId;
}
public void setCallerId(String callerId) {
this.callerId = callerId;
}
}

View File

@@ -1,5 +1,6 @@
package tech.easyflow.ai.service; package tech.easyflow.ai.service;
import tech.easyflow.ai.dto.BotKnowledgeBindingRequest;
import tech.easyflow.ai.entity.BotDocumentCollection; import tech.easyflow.ai.entity.BotDocumentCollection;
import com.mybatisflex.core.service.IService; import com.mybatisflex.core.service.IService;
@@ -16,5 +17,5 @@ public interface BotDocumentCollectionService extends IService<BotDocumentCollec
List<BotDocumentCollection> listByBotId(BigInteger botId); List<BotDocumentCollection> listByBotId(BigInteger botId);
void saveBotAndKnowledge(BigInteger botId, BigInteger[] knowledgeIds); void saveBotAndKnowledge(BigInteger botId, List<BotKnowledgeBindingRequest> knowledgeBindings);
} }

View File

@@ -2,6 +2,7 @@ package tech.easyflow.ai.service;
import com.easyagents.core.document.Document; import com.easyagents.core.document.Document;
import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
import com.mybatisflex.core.service.IService; import com.mybatisflex.core.service.IService;
import java.math.BigInteger; import java.math.BigInteger;
@@ -17,6 +18,8 @@ public interface DocumentCollectionService extends IService<DocumentCollection>
List<Document> search(BigInteger id, String keyword); List<Document> search(BigInteger id, String keyword);
List<Document> search(KnowledgeRetrievalRequest request);
DocumentCollection getDetail(String idOrAlias); DocumentCollection getDetail(String idOrAlias);
DocumentCollection getByAlias(String idOrAlias); DocumentCollection getByAlias(String idOrAlias);

View File

@@ -1,5 +1,7 @@
package tech.easyflow.ai.service.impl; package tech.easyflow.ai.service.impl;
import com.easyagents.rag.retrieval.RetrievalMode;
import tech.easyflow.ai.dto.BotKnowledgeBindingRequest;
import tech.easyflow.ai.entity.BotDocumentCollection; import tech.easyflow.ai.entity.BotDocumentCollection;
import tech.easyflow.ai.mapper.BotDocumentCollectionMapper; import tech.easyflow.ai.mapper.BotDocumentCollectionMapper;
import tech.easyflow.ai.service.BotDocumentCollectionService; import tech.easyflow.ai.service.BotDocumentCollectionService;
@@ -11,9 +13,10 @@ import tech.easyflow.common.cache.RedisLockExecutor;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.math.BigInteger; import java.math.BigInteger;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashSet; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Map;
import java.time.Duration; import java.time.Duration;
import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.query.QueryWrapper;
@@ -44,26 +47,30 @@ public class BotDocumentCollectionServiceImpl extends ServiceImpl<BotDocumentCol
@Override @Override
@Transactional @Transactional
public void saveBotAndKnowledge(BigInteger botId, BigInteger[] knowledgeIds) { public void saveBotAndKnowledge(BigInteger botId, List<BotKnowledgeBindingRequest> knowledgeBindings) {
redisLockExecutor.executeWithLock(BOT_BINDING_LOCK_KEY_PREFIX + botId, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> { redisLockExecutor.executeWithLock(BOT_BINDING_LOCK_KEY_PREFIX + botId, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
this.remove(QueryWrapper.create().eq(BotDocumentCollection::getBotId, botId)); this.remove(QueryWrapper.create().eq(BotDocumentCollection::getBotId, botId));
Set<BigInteger> uniqueKnowledgeIds = new LinkedHashSet<>(); Map<BigInteger, RetrievalMode> bindingMap = new LinkedHashMap<>();
if (knowledgeIds != null) { if (knowledgeBindings != null) {
for (BigInteger knowledgeId : knowledgeIds) { for (BotKnowledgeBindingRequest binding : knowledgeBindings) {
if (knowledgeId != null) { if (binding == null || binding.getKnowledgeId() == null) {
uniqueKnowledgeIds.add(knowledgeId); continue;
} }
bindingMap.put(binding.getKnowledgeId(), binding.resolveRetrievalMode());
} }
} }
if (uniqueKnowledgeIds.isEmpty()) { if (bindingMap.isEmpty()) {
return; return;
} }
List<BotDocumentCollection> list = new ArrayList<>(uniqueKnowledgeIds.size()); List<BotDocumentCollection> list = new ArrayList<>(bindingMap.size());
for (BigInteger knowledgeId : uniqueKnowledgeIds) { for (Map.Entry<BigInteger, RetrievalMode> entry : bindingMap.entrySet()) {
BotDocumentCollection botDocumentCollection = new BotDocumentCollection(); BotDocumentCollection botDocumentCollection = new BotDocumentCollection();
botDocumentCollection.setBotId(botId); botDocumentCollection.setBotId(botId);
botDocumentCollection.setDocumentCollectionId(knowledgeId); botDocumentCollection.setDocumentCollectionId(entry.getKey());
Map<String, Object> options = new HashMap<>();
options.put(BotDocumentCollection.OPTION_KEY_RETRIEVAL_MODE, entry.getValue().name());
botDocumentCollection.setOptions(options);
list.add(botDocumentCollection); list.add(botDocumentCollection);
} }
this.saveBatch(list); this.saveBatch(list);

View File

@@ -401,7 +401,8 @@ public class BotServiceImpl extends ServiceImpl<BotMapper, Bot> implements BotSe
.selectListWithRelationsByQuery(queryWrapper); .selectListWithRelationsByQuery(queryWrapper);
if (botDocumentCollections != null && !botDocumentCollections.isEmpty()) { if (botDocumentCollections != null && !botDocumentCollections.isEmpty()) {
for (BotDocumentCollection botDocumentCollection : botDocumentCollections) { for (BotDocumentCollection botDocumentCollection : botDocumentCollections) {
Tool function = botDocumentCollection.getKnowledge().toFunction(needEnglishName); Tool function = botDocumentCollection.getKnowledge()
.toFunction(needEnglishName, botDocumentCollection.getRetrievalMode().name());
functionList.add(function); functionList.add(function);
} }
} }

View File

@@ -12,8 +12,6 @@ import org.springframework.stereotype.Service;
import java.math.BigInteger; import java.math.BigInteger;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE;
/** /**
* 服务层实现。 * 服务层实现。
* *
@@ -28,10 +26,9 @@ public class DocumentChunkServiceImpl extends ServiceImpl<DocumentChunkMapper, D
@Override @Override
public boolean removeChunk(DocumentCollection knowledge, BigInteger chunkId) { public boolean removeChunk(DocumentCollection knowledge, BigInteger chunkId) {
String searchEngineType = (String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE); DocumentSearcher searcher = searcherFactory.getSearcher();
DocumentSearcher searcher = searcherFactory.getSearcher(searchEngineType);
// 删除搜索引擎中的数据 // 删除搜索引擎中的数据
if (searcherFactory.getSearcher(searchEngineType) == null){ if (searcher == null){
return true; return true;
} }
return searcher.deleteDocument(chunkId); return searcher.deleteDocument(chunkId);

View File

@@ -1,13 +1,23 @@
package tech.easyflow.ai.service.impl; package tech.easyflow.ai.service.impl;
import com.easyagents.core.document.Document; import com.easyagents.core.document.Document;
import com.easyagents.core.model.rerank.RerankException; import com.easyagents.core.model.rerank.RerankException;
import com.easyagents.core.model.rerank.RerankModel; import com.easyagents.core.model.rerank.RerankModel;
import com.easyagents.core.store.DocumentStore; import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.SearchWrapper; import com.easyagents.core.store.SearchWrapper;
import com.easyagents.core.store.StoreOptions; import com.easyagents.core.store.StoreOptions;
import com.easyagents.rag.retrieval.HitSource;
import com.easyagents.rag.retrieval.KeywordRetriever;
import com.easyagents.rag.retrieval.RagHit;
import com.easyagents.rag.retrieval.RagQuery;
import com.easyagents.rag.retrieval.RagRetrievalExecutor;
import com.easyagents.rag.retrieval.RagScoreNormalizer;
import com.easyagents.rag.retrieval.RagRetrievalResult;
import com.easyagents.rag.retrieval.RetrievalMode;
import com.easyagents.rag.retrieval.RrfFusionStrategy;
import com.easyagents.rag.retrieval.VectorRetriever;
import com.easyagents.search.engine.service.DocumentSearcher; import com.easyagents.search.engine.service.DocumentSearcher;
import com.easyagents.search.engine.service.KeywordSearchRequest;
import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.spring.service.impl.ServiceImpl; import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -22,7 +32,7 @@ import tech.easyflow.ai.entity.Model;
import tech.easyflow.ai.mapper.DocumentChunkMapper; import tech.easyflow.ai.mapper.DocumentChunkMapper;
import tech.easyflow.ai.mapper.DocumentCollectionMapper; import tech.easyflow.ai.mapper.DocumentCollectionMapper;
import tech.easyflow.ai.mapper.FaqItemMapper; import tech.easyflow.ai.mapper.FaqItemMapper;
import tech.easyflow.ai.service.DocumentChunkService; import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
import tech.easyflow.ai.service.DocumentCollectionService; import tech.easyflow.ai.service.DocumentCollectionService;
import tech.easyflow.ai.service.ModelService; import tech.easyflow.ai.service.ModelService;
import tech.easyflow.ai.utils.CustomBeanUtils; import tech.easyflow.ai.utils.CustomBeanUtils;
@@ -35,12 +45,18 @@ import java.io.Serializable;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.math.BigInteger; import java.math.BigInteger;
import java.math.RoundingMode; import java.math.RoundingMode;
import java.util.*; import java.util.ArrayList;
import java.util.concurrent.CompletableFuture; import java.util.Collections;
import java.util.concurrent.ExecutionException; import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static tech.easyflow.ai.entity.DocumentCollection.*; import static tech.easyflow.ai.entity.DocumentCollection.KEY_DOC_RECALL_MAX_NUM;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_RERANK_ENABLE;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SIMILARITY_THRESHOLD;
/** /**
* 服务层实现。 * 服务层实现。
@@ -57,9 +73,6 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
@Resource @Resource
private ModelService llmService; private ModelService llmService;
@Resource
private DocumentChunkService chunkService;
@Autowired @Autowired
private SearcherFactory searcherFactory; private SearcherFactory searcherFactory;
@@ -69,14 +82,96 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
@Autowired @Autowired
private FaqItemMapper faqItemMapper; private FaqItemMapper faqItemMapper;
@Override @Override
public List<Document> search(BigInteger id, String keyword) { public List<Document> search(BigInteger id, String keyword) {
DocumentCollection documentCollection = getById(id); KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
request.setKnowledgeId(id);
request.setQuery(keyword);
request.setRetrievalMode(RetrievalMode.HYBRID);
return search(request);
}
@Override
public List<Document> search(KnowledgeRetrievalRequest request) {
if (request == null || request.getKnowledgeId() == null) {
throw new BusinessException("知识库ID不能为空");
}
String keyword = request.getQuery();
if (StringUtil.noText(keyword)) {
return Collections.emptyList();
}
RetrievalMode retrievalMode = request.getRetrievalMode() == null
? RetrievalMode.HYBRID
: request.getRetrievalMode();
DocumentCollection documentCollection = getById(request.getKnowledgeId());
if (documentCollection == null) { if (documentCollection == null) {
throw new BusinessException("知识库不存在"); throw new BusinessException("知识库不存在");
} }
int docRecallMaxNum = readIntegerOption(documentCollection, KEY_DOC_RECALL_MAX_NUM, 5);
float minSimilarity = readFloatOption(documentCollection, KEY_SIMILARITY_THRESHOLD, 0.6F);
RagQuery ragQuery = new RagQuery();
ragQuery.setQuery(keyword);
ragQuery.setRetrievalMode(retrievalMode);
ragQuery.setTopK(docRecallMaxNum);
ragQuery.setMinScore((double) minSimilarity);
RagRetrievalExecutor retrievalExecutor = new RagRetrievalExecutor(
buildVectorRetriever(documentCollection, docRecallMaxNum, retrievalMode == RetrievalMode.VECTOR ? minSimilarity : null),
buildKeywordRetriever(documentCollection, docRecallMaxNum),
new RrfFusionStrategy()
);
RagRetrievalResult retrievalResult = retrievalExecutor.retrieve(ragQuery);
List<Document> searchDocuments = toDocuments(retrievalResult.getHits());
fillSearchContent(documentCollection, searchDocuments);
if (searchDocuments.isEmpty()) {
return Collections.emptyList();
}
RerankModel rerankModel = resolveRerankModel(documentCollection);
boolean reranked = false;
if (rerankModel != null) {
try {
RagRetrievalResult rerankResult = retrievalExecutor.rerank(keyword, toRagHits(searchDocuments), rerankModel, docRecallMaxNum);
searchDocuments = toDocuments(rerankResult.getHits());
reranked = true;
} catch (RerankException e) {
LOG.warn("Rerank failed for collectionId={}, modelId={}, fallback to retrieved results. message={}",
documentCollection.getId(), documentCollection.getRerankModelId(), e.getMessage());
}
}
RagScoreNormalizer.normalize(searchDocuments, retrievalMode, reranked);
return formatDocuments(searchDocuments, shouldApplyMinSimilarityFilter(retrievalMode, reranked), minSimilarity, docRecallMaxNum);
}
private VectorRetriever buildVectorRetriever(DocumentCollection documentCollection,
int docRecallMaxNum,
Float minSimilarity) {
return new VectorRetriever() {
@Override
public List<RagHit> retrieve(RagQuery query) {
return adaptDocuments(searchVectorDocuments(documentCollection, query.getQuery(), docRecallMaxNum, minSimilarity), HitSource.VECTOR);
}
};
}
private KeywordRetriever buildKeywordRetriever(DocumentCollection documentCollection, int docRecallMaxNum) {
return new KeywordRetriever() {
@Override
public List<RagHit> retrieve(RagQuery query) {
return adaptDocuments(searchKeywordDocuments(documentCollection, query.getQuery(), docRecallMaxNum), HitSource.KEYWORD);
}
};
}
private List<Document> searchVectorDocuments(DocumentCollection documentCollection,
String keyword,
int docRecallMaxNum,
Float minSimilarity) {
DocumentStore documentStore = documentCollection.toDocumentStore(); DocumentStore documentStore = documentCollection.toDocumentStore();
if (documentStore == null) { if (documentStore == null) {
throw new BusinessException("知识库没有配置向量库"); throw new BusinessException("知识库没有配置向量库");
@@ -88,88 +183,92 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
} }
documentStore.setEmbeddingModel(model.toEmbeddingModel()); documentStore.setEmbeddingModel(model.toEmbeddingModel());
// 最大召回知识条数
Integer docRecallMaxNum = (Integer) documentCollection.getOptionsByKey(KEY_DOC_RECALL_MAX_NUM);
// 最低相似度
float minSimilarity = (float) documentCollection.getOptionsByKey(KEY_SIMILARITY_THRESHOLD);
SearchWrapper wrapper = new SearchWrapper(); SearchWrapper wrapper = new SearchWrapper();
wrapper.setMaxResults(docRecallMaxNum); wrapper.setMaxResults(docRecallMaxNum);
wrapper.setMinScore((double) minSimilarity); if (minSimilarity != null) {
wrapper.setMinScore((double) minSimilarity);
}
wrapper.setText(keyword); wrapper.setText(keyword);
StoreOptions options = StoreOptions.ofCollectionName(documentCollection.getVectorStoreCollection()); StoreOptions options = StoreOptions.ofCollectionName(documentCollection.getVectorStoreCollection());
options.setIndexName(documentCollection.getVectorStoreCollection()); options.setIndexName(documentCollection.getVectorStoreCollection());
List<Document> documents = documentStore.search(wrapper, options);
return documents == null ? Collections.<Document>emptyList() : documents;
}
// 并行查询:向量库 + 搜索引擎 private List<Document> searchKeywordDocuments(DocumentCollection documentCollection, String keyword, int docRecallMaxNum) {
CompletableFuture<List<Document>> vectorFuture = CompletableFuture.supplyAsync(() -> DocumentSearcher searcher = searcherFactory.getSearcher();
documentStore.search(wrapper, options) if (searcher == null) {
); return Collections.emptyList();
CompletableFuture<List<Document>> searcherFuture = CompletableFuture.supplyAsync(() -> {
DocumentSearcher searcher = searcherFactory.getSearcher((String) documentCollection.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
if (searcher == null || !documentCollection.isSearchEngineEnabled()) {
return Collections.emptyList();
}
List<Document> documents = searcher.searchDocuments(keyword);
return documents == null ? Collections.emptyList() : documents;
});
// 合并两个查询结果
CompletableFuture<Map<String, Document>> combinedFuture = vectorFuture.thenCombine(
searcherFuture,
(vectorDocs, searcherDocs) -> {
Map<String, Document> uniqueDocs = new HashMap<>();
vectorDocs.forEach(doc -> uniqueDocs.putIfAbsent(doc.getId().toString(), doc));
searcherDocs.forEach(doc -> uniqueDocs.putIfAbsent(doc.getId().toString(), doc));
return uniqueDocs;
}
);
try {
Map<String, Document> uniqueDocs = combinedFuture.get(); // 阻塞等待所有查询完成
List<Document> searchDocuments = new ArrayList<>(uniqueDocs.values());
searchDocuments.sort((doc1, doc2) -> Double.compare(doc2.getScore(), doc1.getScore()));
fillSearchContent(documentCollection, searchDocuments);
if (searchDocuments.isEmpty()) {
return Collections.emptyList();
}
boolean rerankEnable = Boolean.TRUE.equals(documentCollection.getOptionsByKey(KEY_RERANK_ENABLE));
if (!rerankEnable || documentCollection.getRerankModelId() == null) {
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
}
Model modelRerank = llmService.getModelInstance(documentCollection.getRerankModelId());
if (modelRerank == null) {
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
}
RerankModel rerankModel = modelRerank.toRerankModel();
if (rerankModel == null) {
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
}
Map<Object, Double> originalScores = new HashMap<>();
searchDocuments.forEach(item -> originalScores.put(item.getId(), item.getScore()));
searchDocuments.forEach(item -> item.setScore(null));
try {
List<Document> rerankDocuments = rerankModel.rerank(keyword, searchDocuments);
return formatDocuments(rerankDocuments, minSimilarity, docRecallMaxNum);
} catch (RerankException e) {
searchDocuments.forEach(item -> item.setScore(originalScores.get(item.getId())));
LOG.warn("Rerank failed for collectionId={}, modelId={}, fallback to vector results. message={}",
documentCollection.getId(), documentCollection.getRerankModelId(), e.getMessage());
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
}
} catch (InterruptedException | ExecutionException e) {
Thread.currentThread().interrupt();
e.printStackTrace();
throw new RuntimeException(e);
} }
KeywordSearchRequest request = KeywordSearchRequest.of(keyword, docRecallMaxNum);
request.setKnowledgeId(documentCollection == null || documentCollection.getId() == null
? null
: documentCollection.getId().toString());
List<Document> documents = searcher.searchDocuments(request);
return documents == null ? Collections.<Document>emptyList() : documents;
}
private List<RagHit> adaptDocuments(List<Document> documents, HitSource hitSource) {
List<RagHit> hits = new ArrayList<>();
if (documents == null) {
return hits;
}
for (Document document : documents) {
RagHit hit = RagHit.fromDocument(document, hitSource);
if (hit != null) {
hits.add(hit);
}
}
return hits;
}
private List<Document> toDocuments(List<RagHit> hits) {
List<Document> documents = new ArrayList<>();
if (hits == null) {
return documents;
}
for (RagHit hit : hits) {
if (hit == null) {
continue;
}
documents.add(hit.toDocument());
}
return documents;
}
private List<RagHit> toRagHits(List<Document> documents) {
List<RagHit> hits = new ArrayList<>();
if (documents == null) {
return hits;
}
for (Document document : documents) {
RagHit hit = RagHit.fromDocument(document);
if (hit != null) {
hits.add(hit);
}
}
return hits;
}
private RerankModel resolveRerankModel(DocumentCollection documentCollection) {
boolean rerankEnable = Boolean.TRUE.equals(documentCollection.getOptionsByKey(KEY_RERANK_ENABLE));
if (!rerankEnable || documentCollection.getRerankModelId() == null) {
return null;
}
Model modelRerank = llmService.getModelInstance(documentCollection.getRerankModelId());
if (modelRerank == null) {
return null;
}
return modelRerank.toRerankModel();
}
private boolean shouldApplyMinSimilarityFilter(RetrievalMode retrievalMode, boolean reranked) {
return !reranked && retrievalMode == RetrievalMode.VECTOR;
} }
@Override @Override
public DocumentCollection getDetail(String idOrAlias) { public DocumentCollection getDetail(String idOrAlias) {
DocumentCollection knowledge = null; DocumentCollection knowledge = null;
if (idOrAlias.matches(RegexUtils.ALL_NUMBER)) { if (idOrAlias.matches(RegexUtils.ALL_NUMBER)) {
@@ -188,15 +287,11 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
@Override @Override
public DocumentCollection getByAlias(String idOrAlias) { public DocumentCollection getByAlias(String idOrAlias) {
QueryWrapper queryWrapper = QueryWrapper.create(); QueryWrapper queryWrapper = QueryWrapper.create();
queryWrapper.eq(DocumentCollection::getAlias, idOrAlias); queryWrapper.eq(DocumentCollection::getAlias, idOrAlias);
return getOne(queryWrapper); return getOne(queryWrapper);
} }
@Override @Override
public boolean updateById(DocumentCollection entity) { public boolean updateById(DocumentCollection entity) {
DocumentCollection documentCollection = getById(entity.getId()); DocumentCollection documentCollection = getById(entity.getId());
@@ -210,38 +305,29 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
documentCollection.setAlias(null); documentCollection.setAlias(null);
} }
return super.updateById(documentCollection, false); return super.updateById(documentCollection, false);
} }
/** public List<Document> formatDocuments(List<Document> documents,
* 格式化文档列表 boolean applyMinSimilarity,
* float minSimilarity,
* @param documents 文档列表 int maxResults) {
* @param minSimilarity 最小相似度
* @return 格式化后的文档列表
*/
public List<Document> formatDocuments(List<Document> documents, float minSimilarity, int maxResults) {
return documents.stream() return documents.stream()
// 过滤掉分数为空 或 分数低于最小值的文档 .filter(Objects::nonNull)
.filter(document -> { .filter(document -> !applyMinSimilarity
Double score = document.getScore(); || (document.getScore() != null && document.getScore() >= minSimilarity))
return score != null && score >= minSimilarity; .map(this::roundDocumentScore)
}) .sorted(Comparator.comparing(Document::getScore, Comparator.nullsLast(Comparator.reverseOrder())))
// 格式化保留四位小数 .limit(maxResults)
.map(document -> { .collect(Collectors.toList());
Double score = document.getScore(); }
BigDecimal bd = new BigDecimal(score.toString());
bd = bd.setScale(4, RoundingMode.HALF_UP); private Document roundDocumentScore(Document document) {
Double roundedScore = bd.doubleValue(); if (document == null || document.getScore() == null) {
document.setScore(roundedScore); return document;
return document; }
}) document.setScore(roundDouble(document.getScore()));
// 按score降序排序分数最高的排前面 return document;
.sorted(Comparator.comparing(Document::getScore, Comparator.reverseOrder()))
// 限制只保留前maxResults条
.limit(maxResults)
.collect(Collectors.toList());
} }
private void fillSearchContent(DocumentCollection documentCollection, List<Document> searchDocuments) { private void fillSearchContent(DocumentCollection documentCollection, List<Document> searchDocuments) {
@@ -249,33 +335,42 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
return; return;
} }
List<Serializable> ids = searchDocuments.stream() List<Serializable> ids = searchDocuments.stream()
.map(item -> (Serializable) item.getId()) .map(item -> (Serializable) item.getId())
.collect(Collectors.toList()); .collect(Collectors.toList());
if (documentCollection.isFaqCollection()) { if (documentCollection.isFaqCollection()) {
Map<String, FaqItem> faqItemMap = faqItemMapper.selectListByIds(ids).stream() QueryWrapper queryWrapper = QueryWrapper.create();
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a)); queryWrapper.in(FaqItem::getId, ids);
queryWrapper.eq(FaqItem::getCollectionId, documentCollection.getId());
Map<String, FaqItem> faqItemMap = faqItemMapper.selectListByQuery(queryWrapper).stream()
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
searchDocuments.removeIf(item -> !faqItemMap.containsKey(String.valueOf(item.getId())));
searchDocuments.forEach(item -> { searchDocuments.forEach(item -> {
FaqItem faqItem = faqItemMap.get(String.valueOf(item.getId())); FaqItem faqItem = faqItemMap.get(String.valueOf(item.getId()));
if (faqItem != null) { if (faqItem == null) {
List<Map<String, String>> faqImages = readFaqImages(faqItem); return;
item.setContent(buildFaqPromptContent(faqItem, faqImages));
Map<String, Object> metadataMap = item.getMetadataMap() == null
? new HashMap<>()
: new HashMap<>(item.getMetadataMap());
List<String> imageUrls = faqImages.stream()
.map(image -> image.get("url"))
.filter(Objects::nonNull)
.collect(Collectors.toList());
metadataMap.put("imageUrls", imageUrls);
item.setMetadataMap(metadataMap);
} }
List<Map<String, String>> faqImages = readFaqImages(faqItem);
item.setContent(buildFaqPromptContent(faqItem, faqImages));
Map<String, Object> metadataMap = item.getMetadataMap() == null
? new HashMap<String, Object>()
: new HashMap<String, Object>(item.getMetadataMap());
List<String> imageUrls = faqImages.stream()
.map(image -> image.get("url"))
.filter(Objects::nonNull)
.collect(Collectors.toList());
metadataMap.put("imageUrls", imageUrls);
item.setMetadataMap(metadataMap);
}); });
return; return;
} }
Map<String, DocumentChunk> chunkMap = documentChunkMapper.selectListByIds(ids).stream() QueryWrapper queryWrapper = QueryWrapper.create();
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a)); queryWrapper.in(DocumentChunk::getId, ids);
queryWrapper.eq(DocumentChunk::getDocumentCollectionId, documentCollection.getId());
Map<String, DocumentChunk> chunkMap = documentChunkMapper.selectListByQuery(queryWrapper).stream()
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
searchDocuments.removeIf(item -> !chunkMap.containsKey(String.valueOf(item.getId())));
searchDocuments.forEach(item -> { searchDocuments.forEach(item -> {
DocumentChunk documentChunk = chunkMap.get(String.valueOf(item.getId())); DocumentChunk documentChunk = chunkMap.get(String.valueOf(item.getId()));
if (documentChunk != null && !StringUtil.noText(documentChunk.getContent())) { if (documentChunk != null && !StringUtil.noText(documentChunk.getContent())) {
@@ -354,4 +449,43 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
String text = String.valueOf(value).trim(); String text = String.valueOf(value).trim();
return text.isEmpty() ? null : text; return text.isEmpty() ? null : text;
} }
private int readIntegerOption(DocumentCollection documentCollection, String key, int defaultValue) {
Object value = documentCollection.getOptionsByKey(key);
if (value instanceof Number) {
return ((Number) value).intValue();
}
if (value instanceof String && StringUtil.hasText((String) value)) {
try {
return Integer.parseInt((String) value);
} catch (NumberFormatException ignore) {
return defaultValue;
}
}
return defaultValue;
}
private float readFloatOption(DocumentCollection documentCollection, String key, float defaultValue) {
Object value = documentCollection.getOptionsByKey(key);
if (value instanceof Number) {
return ((Number) value).floatValue();
}
if (value instanceof String && StringUtil.hasText((String) value)) {
try {
return Float.parseFloat((String) value);
} catch (NumberFormatException ignore) {
return defaultValue;
}
}
return defaultValue;
}
private Double roundDouble(Double value) {
if (value == null) {
return null;
}
return new BigDecimal(String.valueOf(value))
.setScale(4, RoundingMode.HALF_UP)
.doubleValue();
}
} }

View File

@@ -19,6 +19,7 @@ import com.easyagents.rag.ingestion.RagIngestionService;
import com.easyagents.rag.ingestion.model.AnalysisResult; import com.easyagents.rag.ingestion.model.AnalysisResult;
import com.easyagents.rag.ingestion.model.StrategyConfig; import com.easyagents.rag.ingestion.model.StrategyConfig;
import com.easyagents.search.engine.service.DocumentSearcher; import com.easyagents.search.engine.service.DocumentSearcher;
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
import com.mybatisflex.core.keygen.impl.FlexIDKeyGenerator; import com.mybatisflex.core.keygen.impl.FlexIDKeyGenerator;
import com.mybatisflex.core.paginate.Page; import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryMethods; import com.mybatisflex.core.query.QueryMethods;
@@ -56,7 +57,6 @@ import java.util.*;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL; import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE;
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK; import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT; import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT;
@@ -157,8 +157,8 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class); List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class);
documentStore.delete(chunkIds, options); documentStore.delete(chunkIds, options);
// 删除搜索引擎中的数据 // 删除搜索引擎中的数据
if (searcherFactory.getSearcher((String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE)) != null) { DocumentSearcher searcher = searcherFactory.getSearcher();
DocumentSearcher searcher = searcherFactory.getSearcher((String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE)); if (searcher != null) {
chunkIds.forEach(searcher::deleteDocument); chunkIds.forEach(searcher::deleteDocument);
} }
int ck = documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, id)); int ck = documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, id));
@@ -691,9 +691,7 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
options.setIndexName(options.getCollectionName()); options.setIndexName(options.getCollectionName());
DocumentSearcher searcher = null; DocumentSearcher searcher = null;
if (knowledge.isSearchEngineEnabled()) { searcher = searcherFactory.getSearcher();
searcher = searcherFactory.getSearcher((String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
}
return new StoreExecutionContext(knowledge, model, embeddingModel, documentStore, options, searcher); return new StoreExecutionContext(knowledge, model, embeddingModel, documentStore, options, searcher);
} }
@@ -703,6 +701,8 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
com.easyagents.core.document.Document document = new com.easyagents.core.document.Document(); com.easyagents.core.document.Document document = new com.easyagents.core.document.Document();
document.setId(item.getId()); document.setId(item.getId());
document.setContent(item.getContent()); document.setContent(item.getContent());
document.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID,
storeContext.knowledge.getId() == null ? null : storeContext.knowledge.getId().toString());
documents.add(document); documents.add(document);
} }

View File

@@ -15,6 +15,7 @@ import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.StoreOptions; import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult; import com.easyagents.core.store.StoreResult;
import com.easyagents.search.engine.service.DocumentSearcher; import com.easyagents.search.engine.service.DocumentSearcher;
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.spring.service.impl.ServiceImpl; import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
@@ -52,7 +53,6 @@ import java.util.*;
import java.util.function.Supplier; import java.util.function.Supplier;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL; import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE;
@Service @Service
public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> implements FaqItemService { public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> implements FaqItemService {
@@ -356,14 +356,12 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
throw new BusinessException("FAQ向量化失败"); throw new BusinessException("FAQ向量化失败");
} }
if (collection.isSearchEngineEnabled()) { DocumentSearcher searcher = searcherFactory.getSearcher();
DocumentSearcher searcher = searcherFactory.getSearcher((String) collection.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE)); if (searcher != null) {
if (searcher != null) { if (isUpdate) {
if (isUpdate) { searcher.deleteDocument(entity.getId());
searcher.deleteDocument(entity.getId());
}
searcher.addDocument(doc);
} }
searcher.addDocument(doc);
} }
markCollectionEmbedded(collection, preparedStore.embeddingModel); markCollectionEmbedded(collection, preparedStore.embeddingModel);
} }
@@ -375,14 +373,11 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
throw new BusinessException("FAQ向量删除失败"); throw new BusinessException("FAQ向量删除失败");
} }
if (collection.isSearchEngineEnabled()) { DocumentSearcher searcher = searcherFactory.getSearcher();
DocumentSearcher searcher = searcherFactory.getSearcher((String) collection.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE)); if (searcher != null) {
if (searcher != null) { boolean removed = searcher.deleteDocument(entity.getId());
boolean removed = searcher.deleteDocument(entity.getId()); if (!removed) {
if (!removed) { LOG.warn("Delete faq search index failed. faqId={}", entity.getId());
LOG.warn("Delete faq search index failed. faqId={}, searcherType={}",
entity.getId(), collection.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
}
} }
} }
} }
@@ -443,6 +438,7 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
metadata.put("answerText", entity.getAnswerText()); metadata.put("answerText", entity.getAnswerText());
metadata.put("categoryId", entity.getCategoryId()); metadata.put("categoryId", entity.getCategoryId());
metadata.put("imageUrls", readImageUrls(entity.getOptions())); metadata.put("imageUrls", readImageUrls(entity.getOptions()));
metadata.put(KeywordSearchMetadataKeys.KNOWLEDGE_ID, entity.getCollectionId() == null ? null : entity.getCollectionId().toString());
doc.setMetadataMap(metadata); doc.setMetadataMap(metadata);
return doc; return doc;
} }

View File

@@ -208,12 +208,19 @@ jetcache:
# 多路召回搜索引擎配置 # 多路召回搜索引擎配置
rag: rag:
engine: ES
milvus:
uri: http://127.0.0.1:39530
databaseName: default
token:
username: easyflowadmin
password: easyflowadmin123
autoCreateCollection: true
searcher: searcher:
# 搜索方式 默认lucene
lucene: lucene:
indexDirPath: /Users/slience/data/easyflow/luceneKnowledge indexDirPath: /Users/slience/data/easyflow/luceneKnowledge
elastic: elastic:
host: https://127.0.0.1:9200 host: http://127.0.0.1:39200
userName: elastic userName: elastic
password: elastic password: elastic
indexName: easyflow indexName: easyflow

View File

@@ -49,6 +49,12 @@
"publicChatCopySuccess": "Copied", "publicChatCopySuccess": "Copied",
"publicChatCopyFail": "Copy failed", "publicChatCopyFail": "Copy failed",
"basicInfo": "Basic Info", "basicInfo": "Basic Info",
"knowledgeRetrievalMode": "Retrieval Mode",
"retrievalModes": {
"hybrid": "Hybrid Retrieval",
"vector": "Vector Retrieval",
"keyword": "Keyword Retrieval"
},
"modal": { "modal": {
"createDescription": "Set the assistant appearance, identity and base availability.", "createDescription": "Set the assistant appearance, identity and base availability.",
"editDescription": "Update the assistant presentation and base availability.", "editDescription": "Update the assistant presentation and base availability.",

View File

@@ -159,7 +159,13 @@
"documentPreview": "DocumentPreview", "documentPreview": "DocumentPreview",
"total": "Total", "total": "Total",
"segments": "Segments", "segments": "Segments",
"similarityScore": "SimilarityScore", "similarityScore": "Relevance",
"searchFailed": "Search failed. Please try again later.",
"hitSources": {
"vector": "Vector Hit",
"keyword": "Keyword Hit",
"both": "Dual Hit"
},
"alibabaCloud": "AlibabaCloud", "alibabaCloud": "AlibabaCloud",
"tencentCloud": "tencentCloud", "tencentCloud": "tencentCloud",
"vectorEmbedModelTips": "After successful vector data, it is not allowed to modify the vector model", "vectorEmbedModelTips": "After successful vector data, it is not allowed to modify the vector model",

View File

@@ -11,6 +11,7 @@
"button": { "button": {
"save": "Save Configuration" "save": "Save Configuration"
}, },
"engineHint": "The keyword search engine is controlled by platform-level configuration and is no longer configured per knowledge base.",
"message": { "message": {
"saveSuccess": "Configuration saved successfully", "saveSuccess": "Configuration saved successfully",
"saveFailed": "Configuration saved failed" "saveFailed": "Configuration saved failed"

View File

@@ -49,6 +49,12 @@
"publicChatCopySuccess": "复制成功", "publicChatCopySuccess": "复制成功",
"publicChatCopyFail": "复制失败", "publicChatCopyFail": "复制失败",
"basicInfo": "基础信息", "basicInfo": "基础信息",
"knowledgeRetrievalMode": "检索方式",
"retrievalModes": {
"hybrid": "混合检索",
"vector": "向量检索",
"keyword": "关键词检索"
},
"modal": { "modal": {
"createDescription": "设置助手的外观、标识和基础发布状态。", "createDescription": "设置助手的外观、标识和基础发布状态。",
"editDescription": "更新助手的展示信息与基础状态。", "editDescription": "更新助手的展示信息与基础状态。",

View File

@@ -159,7 +159,13 @@
"documentPreview": "文档预览", "documentPreview": "文档预览",
"total": "共", "total": "共",
"segments": "个分段", "segments": "个分段",
"similarityScore": "相度", "similarityScore": "相度",
"searchFailed": "检索失败,请稍后重试",
"hitSources": {
"vector": "语义命中",
"keyword": "关键词命中",
"both": "双路命中"
},
"alibabaCloud": "阿里云", "alibabaCloud": "阿里云",
"tencentCloud": "腾讯云", "tencentCloud": "腾讯云",
"vectorEmbedModelTips": "成功向量数据之后不允许修改向量模型", "vectorEmbedModelTips": "成功向量数据之后不允许修改向量模型",

View File

@@ -11,6 +11,7 @@
"button": { "button": {
"save": "保存配置" "save": "保存配置"
}, },
"engineHint": "关键词检索引擎由平台全局配置决定,知识库侧不再单独配置。",
"message": { "message": {
"saveSuccess": "配置保存成功", "saveSuccess": "配置保存成功",
"saveFailed": "配置保存失败" "saveFailed": "配置保存失败"

View File

@@ -18,6 +18,7 @@ import {
import { useDebounceFn } from '@vueuse/core'; import { useDebounceFn } from '@vueuse/core';
import { import {
ElAlert, ElAlert,
ElAvatar,
ElButton, ElButton,
ElCol, ElCol,
ElCollapse, ElCollapse,
@@ -26,6 +27,7 @@ import {
ElInput, ElInput,
ElInputNumber, ElInputNumber,
ElMessage, ElMessage,
ElMessageBox,
ElOption, ElOption,
ElRow, ElRow,
ElSelect, ElSelect,
@@ -63,6 +65,13 @@ interface ApiKeyOption {
label: string; label: string;
} }
type RetrievalMode = 'HYBRID' | 'KEYWORD' | 'VECTOR';
interface BotKnowledgeBindingItem {
knowledgeId: string;
retrievalMode: RetrievalMode;
}
const props = defineProps<{ const props = defineProps<{
bot?: BotInfo; bot?: BotInfo;
hasSavePermission?: boolean; hasSavePermission?: boolean;
@@ -70,6 +79,7 @@ const props = defineProps<{
const botStore = useBotStore(); const botStore = useBotStore();
const route = useRoute(); const route = useRoute();
const botId = ref<string>((route.params.id as string) || ''); const botId = ref<string>((route.params.id as string) || '');
const fallbackAvatarUrl = `${import.meta.env.BASE_URL || '/'}favicon.svg`;
const options = ref<AiLlm[]>([]); const options = ref<AiLlm[]>([]);
const selectedId = ref<string>(''); const selectedId = ref<string>('');
const llmConfig = ref({ const llmConfig = ref({
@@ -170,11 +180,11 @@ const iframeCodeHighlighted = computed(() => {
} }
const escaped = escapeHtml(iframeCode.value); const escaped = escapeHtml(iframeCode.value);
return escaped return escaped
.replace( .replaceAll(
/(&lt;\/?)([a-zA-Z][\w-]*)/g, /(&lt;\/?)([a-zA-Z][\w-]*)/g,
'$1<span class="hljs-name">$2</span>', '$1<span class="hljs-name">$2</span>',
) )
.replace( .replaceAll(
/([:@a-zA-Z_][\w:-]*)=(&quot;.*?&quot;)/g, /([:@a-zA-Z_][\w:-]*)=(&quot;.*?&quot;)/g,
'<span class="hljs-attr">$1</span>=<span class="hljs-string">$2</span>', '<span class="hljs-attr">$1</span>=<span class="hljs-string">$2</span>',
); );
@@ -202,6 +212,32 @@ const mcpToolData = ref<any>([]);
const workflowData = ref<any[]>([]); const workflowData = ref<any[]>([]);
const knowledgeData = ref<any[]>([]); const knowledgeData = ref<any[]>([]);
const pluginToolData = ref<any[]>([]); const pluginToolData = ref<any[]>([]);
const retrievalModeOptions = computed(() => [
{ label: $t('bot.retrievalModes.hybrid'), value: 'HYBRID' },
{ label: $t('bot.retrievalModes.vector'), value: 'VECTOR' },
{ label: $t('bot.retrievalModes.keyword'), value: 'KEYWORD' },
]);
const normalizeRetrievalMode = (value?: string): RetrievalMode => {
if (value === 'VECTOR' || value === 'KEYWORD' || value === 'HYBRID') {
return value;
}
return 'HYBRID';
};
const buildKnowledgeBindingsPayload = (
selectedIds: Array<number | string>,
): BotKnowledgeBindingItem[] => {
return [...new Set((selectedIds || []).map(String).filter(Boolean))].map(
(knowledgeId) => {
const existing = knowledgeData.value.find(
(item) => String(item.id) === knowledgeId,
);
return {
knowledgeId,
retrievalMode: normalizeRetrievalMode(existing?.retrievalMode),
};
},
);
};
const getAiBotPluginToolList = async () => { const getAiBotPluginToolList = async () => {
api api
.post('/api/v1/pluginItem/tool/list', { botId: botId.value }) .post('/api/v1/pluginItem/tool/list', { botId: botId.value })
@@ -226,6 +262,7 @@ const getAiBotKnowledgeList = async () => {
knowledgeData.value = res.data.map((item: any) => { knowledgeData.value = res.data.map((item: any) => {
return { return {
recordId: item.id, recordId: item.id,
retrievalMode: normalizeRetrievalMode(item.options?.retrievalMode),
...item.knowledge, ...item.knowledge,
}; };
}); });
@@ -419,7 +456,7 @@ const copyText = async (value: string) => {
textarea.style.opacity = '0'; textarea.style.opacity = '0';
textarea.style.pointerEvents = 'none'; textarea.style.pointerEvents = 'none';
document.body.appendChild(textarea); document.body.append(textarea);
const selection = document.getSelection(); const selection = document.getSelection();
const previousRange = const previousRange =
@@ -430,7 +467,7 @@ const copyText = async (value: string) => {
textarea.setSelectionRange(0, textarea.value.length); textarea.setSelectionRange(0, textarea.value.length);
const copied = document.execCommand('copy'); const copied = document.execCommand('copy');
document.body.removeChild(textarea); textarea.remove();
if (selection) { if (selection) {
selection.removeAllRanges(); selection.removeAllRanges();
if (previousRange) { if (previousRange) {
@@ -543,7 +580,7 @@ const confirmUpdateAiBotKnowledge = (data: any) => {
api api
.post('/api/v1/botKnowledge/updateBotKnowledgeIds', { .post('/api/v1/botKnowledge/updateBotKnowledgeIds', {
botId: botId.value, botId: botId.value,
knowledgeIds: data, knowledgeBindings: buildKnowledgeBindingsPayload(data),
}) })
.then((res) => { .then((res) => {
if (res.errorCode === 0) { if (res.errorCode === 0) {
@@ -555,6 +592,31 @@ const confirmUpdateAiBotKnowledge = (data: any) => {
}); });
}; };
const updateKnowledgeBindings = (bindings: BotKnowledgeBindingItem[]) => {
return api.post('/api/v1/botKnowledge/updateBotKnowledgeIds', {
botId: botId.value,
knowledgeBindings: bindings,
});
};
const handleKnowledgeRetrievalModeChange = async (
item: any,
value: RetrievalMode | string,
) => {
item.retrievalMode = normalizeRetrievalMode(value);
const bindings = knowledgeData.value.map((knowledgeItem: any) => ({
knowledgeId: String(knowledgeItem.id),
retrievalMode: normalizeRetrievalMode(knowledgeItem.retrievalMode),
}));
const res = await updateKnowledgeBindings(bindings);
if (res.errorCode === 0) {
ElMessage.success($t('message.updateOkMessage'));
getAiBotKnowledgeList();
return;
}
ElMessage.error(res.message || $t('message.updateFailMessage'));
};
const confirmUpdateAiBotWorkflow = (data: any) => { const confirmUpdateAiBotWorkflow = (data: any) => {
api api
.post('/api/v1/botWorkflow/updateBotWorkflowIds', { .post('/api/v1/botWorkflow/updateBotWorkflowIds', {
@@ -616,7 +678,20 @@ const deleteBotMcpTool = (item: any) => {
}); });
}; };
const deleteKnowledge = (item: any) => { const deleteKnowledge = async (item: any) => {
try {
await ElMessageBox.confirm(
$t('message.deleteAlert'),
$t('message.noticeTitle'),
{
confirmButtonText: $t('button.confirm'),
cancelButtonText: $t('button.cancel'),
type: 'warning',
},
);
} catch {
return;
}
api api
.post('/api/v1/botKnowledge/remove', { .post('/api/v1/botKnowledge/remove', {
id: item.recordId, id: item.recordId,
@@ -840,10 +915,10 @@ const handleBasicInfoChange = async (
<div class="bot-avatar-panel"> <div class="bot-avatar-panel">
<span class="bot-avatar-label">{{ $t('common.avatar') }}</span> <span class="bot-avatar-label">{{ $t('common.avatar') }}</span>
<div <div
:class="[ class="bot-avatar-upload-wrap"
'bot-avatar-upload-wrap', :class="
!hasSavePermission || updatingBotIcon ? 'is-disabled' : '', !hasSavePermission || updatingBotIcon ? 'is-disabled' : ''
]" "
> >
<UploadAvatar <UploadAvatar
v-if="botInfo" v-if="botInfo"
@@ -1105,7 +1180,53 @@ const handleBasicInfoChange = async (
</div> </div>
</div> </div>
</template> </template>
<CollapseViewItem :data="knowledgeData" @delete="deleteKnowledge" /> <div class="knowledge-binding-list">
<div
v-for="item in knowledgeData"
:key="item.recordId"
class="knowledge-binding-item"
>
<div class="knowledge-binding-main">
<ElAvatar
:src="item.icon || fallbackAvatarUrl"
shape="circle"
/>
<div class="knowledge-binding-content">
<div class="knowledge-binding-title">
{{ item.title }}
</div>
<div class="knowledge-binding-description">
{{ item.description }}
</div>
</div>
</div>
<div class="knowledge-binding-actions" @click.stop>
<ElSelect
:model-value="item.retrievalMode"
class="knowledge-binding-select"
size="small"
@change="
(value) => handleKnowledgeRetrievalModeChange(item, value)
"
>
<ElOption
v-for="option in retrievalModeOptions"
:key="option.value"
:label="option.label"
:value="option.value"
/>
</ElSelect>
<ElIcon
class="knowledge-binding-delete"
color="var(--el-color-danger)"
size="20px"
@click="deleteKnowledge(item)"
>
<Delete />
</ElIcon>
</div>
</div>
</div>
</ElCollapseItem> </ElCollapseItem>
<ElCollapseItem> <ElCollapseItem>
<template #title> <template #title>
@@ -1779,4 +1900,64 @@ const handleBasicInfoChange = async (
height: 50px; height: 50px;
border-radius: 50%; border-radius: 50%;
} }
.knowledge-binding-list {
display: flex;
flex-direction: column;
gap: 12px;
padding: 10px;
background-color: var(--bot-collapse-itme-back);
}
.knowledge-binding-item {
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
padding: 12px;
border-radius: 8px;
background-color: hsl(var(--background));
}
.knowledge-binding-main {
display: flex;
align-items: center;
gap: 10px;
min-width: 0;
flex: 1;
}
.knowledge-binding-content {
min-width: 0;
}
.knowledge-binding-title {
font-size: 12px;
font-weight: 500;
line-height: 24px;
color: var(--el-text-color-primary);
}
.knowledge-binding-description {
font-size: 12px;
line-height: 20px;
color: var(--el-text-color-secondary);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.knowledge-binding-actions {
display: flex;
align-items: center;
gap: 12px;
}
.knowledge-binding-select {
width: 140px;
}
.knowledge-binding-delete {
cursor: pointer;
}
</style> </style>

View File

@@ -1,7 +1,7 @@
<script setup lang="ts"> <script setup lang="ts">
import type { FormInstance } from 'element-plus'; import type { FormInstance } from 'element-plus';
import { computed, onMounted, ref, watch } from 'vue'; import { onMounted, ref, watch } from 'vue';
import { InfoFilled } from '@element-plus/icons-vue'; import { InfoFilled } from '@element-plus/icons-vue';
import { import {
@@ -80,29 +80,6 @@ watch(
const embeddingLlmList = ref<any>([]); const embeddingLlmList = ref<any>([]);
const rerankerLlmList = ref<any>([]); const rerankerLlmList = ref<any>([]);
const vecotrDatabaseList = ref<any>([
{ value: 'milvus', label: 'Milvus' },
{ value: 'redis', label: 'Redis' },
{ value: 'opensearch', label: 'OpenSearch' },
{ value: 'elasticsearch', label: 'ElasticSearch' },
{ value: 'aliyun', label: $t('documentCollection.alibabaCloud') },
{ value: 'qcloud', label: $t('documentCollection.tencentCloud') },
]);
const milvusVectorStoreConfigPlaceholder =
'uri=http://127.0.0.1:19530\n' +
'databaseName=default\n' +
'token=\n' +
'username=\n' +
'password=\n' +
'autoCreateCollection=true';
const vectorStoreConfigPlaceholder = computed(() => {
return entity.value?.vectorStoreType === 'milvus'
? milvusVectorStoreConfigPlaceholder
: '';
});
const getEmbeddingLlmListData = async () => { const getEmbeddingLlmListData = async () => {
try { try {
const url = `/api/v1/documentCollection/modelList?modelType=embeddingModel`; const url = `/api/v1/documentCollection/modelList?modelType=embeddingModel`;
@@ -146,15 +123,6 @@ const rules = ref({
{ required: true, message: $t('message.required'), trigger: 'blur' }, { required: true, message: $t('message.required'), trigger: 'blur' },
], ],
title: [{ required: true, message: $t('message.required'), trigger: 'blur' }], title: [{ required: true, message: $t('message.required'), trigger: 'blur' }],
vectorStoreType: [
{ required: true, message: $t('message.required'), trigger: 'blur' },
],
vectorStoreCollection: [
{ required: true, message: $t('message.required'), trigger: 'blur' },
],
vectorStoreConfig: [
{ required: true, message: $t('message.required'), trigger: 'blur' },
],
vectorEmbedModelId: [ vectorEmbedModelId: [
{ required: true, message: $t('message.required'), trigger: 'blur' }, { required: true, message: $t('message.required'), trigger: 'blur' },
], ],
@@ -239,50 +207,6 @@ async function save() {
:placeholder="$t('documentCollection.placeholder.description')" :placeholder="$t('documentCollection.placeholder.description')"
/> />
</ElFormItem> </ElFormItem>
<!-- <ElFormItem
prop="vectorStoreEnable"
:label="$t('documentCollection.vectorStoreEnable')"
>
<ElSwitch v-model="entity.vectorStoreEnable" />
</ElFormItem>-->
<ElFormItem
prop="vectorStoreType"
:label="$t('documentCollection.vectorStoreType')"
>
<ElSelect
v-model="entity.vectorStoreType"
:placeholder="$t('documentCollection.placeholder.vectorStoreType')"
>
<ElOption
v-for="item in vecotrDatabaseList"
:key="item.value"
:label="item.label"
:value="item.value || ''"
/>
</ElSelect>
</ElFormItem>
<ElFormItem
prop="vectorStoreCollection"
:label="$t('documentCollection.vectorStoreCollection')"
>
<ElInput
v-model.trim="entity.vectorStoreCollection"
:placeholder="
$t('documentCollection.placeholder.vectorStoreCollection')
"
/>
</ElFormItem>
<ElFormItem
prop="vectorStoreConfig"
:label="$t('documentCollection.vectorStoreConfig')"
>
<ElInput
v-model.trim="entity.vectorStoreConfig"
:rows="4"
type="textarea"
:placeholder="vectorStoreConfigPlaceholder"
/>
</ElFormItem>
<ElFormItem prop="vectorEmbedModelId"> <ElFormItem prop="vectorEmbedModelId">
<template #label> <template #label>
<span style="display: flex; align-items: center"> <span style="display: flex; align-items: center">
@@ -373,12 +297,6 @@ async function save() {
/> />
</ElSelect> </ElSelect>
</ElFormItem> </ElFormItem>
<ElFormItem
prop="searchEngineEnable"
:label="$t('documentCollection.searchEngineEnable')"
>
<ElSwitch v-model="entity.searchEngineEnable" />
</ElFormItem>
<ElFormItem style="margin-top: 20px; text-align: right"> <ElFormItem style="margin-top: 20px; text-align: right">
<ElButton <ElButton
type="primary" type="primary"

View File

@@ -60,15 +60,6 @@ onMounted(async () => {
const saveForm = ref<FormInstance>(); const saveForm = ref<FormInstance>();
const dialogVisible = ref(false); const dialogVisible = ref(false);
const isAdd = ref(true); const isAdd = ref(true);
const vecotrDatabaseList = ref<any>([
{ value: 'milvus', label: 'Milvus' },
{ value: 'redis', label: 'Redis' },
{ value: 'opensearch', label: 'OpenSearch' },
{ value: 'elasticsearch', label: 'ElasticSearch' },
{ value: 'aliyun', label: $t('documentCollection.alibabaCloud') },
{ value: 'qcloud', label: $t('documentCollection.tencentCloud') },
]);
const defaultEntity = { const defaultEntity = {
collectionType: 'DOCUMENT', collectionType: 'DOCUMENT',
alias: '', alias: '',
@@ -125,15 +116,6 @@ const rules = ref({
{ required: true, message: $t('message.required'), trigger: 'blur' }, { required: true, message: $t('message.required'), trigger: 'blur' },
], ],
title: [{ required: true, message: $t('message.required'), trigger: 'blur' }], title: [{ required: true, message: $t('message.required'), trigger: 'blur' }],
vectorStoreType: [
{ required: true, message: $t('message.required'), trigger: 'blur' },
],
vectorStoreCollection: [
{ required: true, message: $t('message.required'), trigger: 'blur' },
],
vectorStoreConfig: [
{ required: true, message: $t('message.required'), trigger: 'blur' },
],
vectorEmbedModelId: [ vectorEmbedModelId: [
{ required: true, message: $t('message.required'), trigger: 'blur' }, { required: true, message: $t('message.required'), trigger: 'blur' },
], ],
@@ -307,49 +289,6 @@ defineExpose({
:placeholder="$t('documentCollection.placeholder.description')" :placeholder="$t('documentCollection.placeholder.description')"
/> />
</ElFormItem> </ElFormItem>
<!-- <ElFormItem
prop="vectorStoreEnable"
:label="$t('documentCollection.vectorStoreEnable')"
>
<ElSwitch v-model="entity.vectorStoreEnable" />
</ElFormItem>-->
<ElFormItem
prop="vectorStoreType"
:label="$t('documentCollection.vectorStoreType')"
>
<ElSelect
v-model="entity.vectorStoreType"
:placeholder="$t('documentCollection.placeholder.vectorStoreType')"
>
<ElOption
v-for="item in vecotrDatabaseList"
:key="item.value"
:label="item.label"
:value="item.value || ''"
/>
</ElSelect>
</ElFormItem>
<ElFormItem
prop="vectorStoreCollection"
:label="$t('documentCollection.vectorStoreCollection')"
>
<ElInput
v-model.trim="entity.vectorStoreCollection"
:placeholder="
$t('documentCollection.placeholder.vectorStoreCollection')
"
/>
</ElFormItem>
<ElFormItem
prop="vectorStoreConfig"
:label="$t('documentCollection.vectorStoreConfig')"
>
<ElInput
v-model.trim="entity.vectorStoreConfig"
:rows="4"
type="textarea"
/>
</ElFormItem>
<ElFormItem prop="vectorEmbedModelId"> <ElFormItem prop="vectorEmbedModelId">
<template #label> <template #label>
<span style="display: flex; align-items: center"> <span style="display: flex; align-items: center">
@@ -440,12 +379,6 @@ defineExpose({
/> />
</ElSelect> </ElSelect>
</ElFormItem> </ElFormItem>
<ElFormItem
prop="searchEngineEnable"
:label="$t('documentCollection.searchEngineEnable')"
>
<ElSwitch v-model="entity.searchEngineEnable" />
</ElFormItem>
</ElForm> </ElForm>
</EasyFlowFormModal> </EasyFlowFormModal>
</template> </template>

View File

@@ -3,20 +3,40 @@ import { ref } from 'vue';
import { $t } from '@easyflow/locales'; import { $t } from '@easyflow/locales';
import { ElButton, ElInput, ElMessage } from 'element-plus'; import { ElButton, ElInput, ElMessage, ElOption, ElSelect } from 'element-plus';
import { api } from '#/api/request'; import { api } from '#/api/request';
import PreviewSearchKnowledge from '#/views/ai/documentCollection/PreviewSearchKnowledge.vue'; import PreviewSearchKnowledge from '#/views/ai/documentCollection/PreviewSearchKnowledge.vue';
type RetrievalMode = 'HYBRID' | 'KEYWORD' | 'VECTOR';
interface SearchResultItem {
sorting: number;
content: string;
score?: number;
hitSource?: 'BOTH' | 'KEYWORD' | 'VECTOR';
vectorScore?: number;
keywordScore?: number;
}
const props = defineProps({ const props = defineProps({
knowledgeId: { knowledgeId: {
type: String, type: String,
required: true, required: true,
}, },
}); });
const searchDataList = ref([]);
const searchDataList = ref<SearchResultItem[]>([]);
const keyword = ref(''); const keyword = ref('');
const retrievalMode = ref<RetrievalMode>('HYBRID');
const previewSearchKnowledgeRef = ref(); const previewSearchKnowledgeRef = ref();
const retrievalModeOptions = [
{ label: $t('bot.retrievalModes.hybrid'), value: 'HYBRID' },
{ label: $t('bot.retrievalModes.vector'), value: 'VECTOR' },
{ label: $t('bot.retrievalModes.keyword'), value: 'KEYWORD' },
];
const handleSearch = () => { const handleSearch = () => {
if (!keyword.value) { if (!keyword.value) {
ElMessage.error($t('message.pleaseInputContent')); ElMessage.error($t('message.pleaseInputContent'));
@@ -24,12 +44,22 @@ const handleSearch = () => {
} }
previewSearchKnowledgeRef.value.loadingContent(true); previewSearchKnowledgeRef.value.loadingContent(true);
api api
.get( .get('/api/v1/documentCollection/search', {
`/api/v1/documentCollection/search?knowledgeId=${props.knowledgeId}&keyword=${keyword.value}`, params: {
) knowledgeId: props.knowledgeId,
keyword: keyword.value,
retrievalMode: retrievalMode.value,
},
})
.then((res) => { .then((res) => {
previewSearchKnowledgeRef.value.loadingContent(false);
searchDataList.value = res.data; searchDataList.value = res.data;
})
.catch(() => {
ElMessage.error($t('documentCollection.searchFailed'));
searchDataList.value = [];
})
.finally(() => {
previewSearchKnowledgeRef.value.loadingContent(false);
}); });
}; };
</script> </script>
@@ -40,7 +70,16 @@ const handleSearch = () => {
<ElInput <ElInput
v-model="keyword" v-model="keyword"
:placeholder="$t('common.searchPlaceholder')" :placeholder="$t('common.searchPlaceholder')"
@keyup.enter="handleSearch"
/> />
<ElSelect v-model="retrievalMode" class="retrieval-select">
<ElOption
v-for="item in retrievalModeOptions"
:key="item.value"
:label="item.label"
:value="item.value"
/>
</ElSelect>
<ElButton type="primary" @click="handleSearch"> <ElButton type="primary" @click="handleSearch">
{{ $t('button.query') }} {{ $t('button.query') }}
</ElButton> </ElButton>
@@ -48,6 +87,7 @@ const handleSearch = () => {
<div class="search-result"> <div class="search-result">
<PreviewSearchKnowledge <PreviewSearchKnowledge
:data="searchDataList" :data="searchDataList"
:retrieval-mode="retrievalMode"
ref="previewSearchKnowledgeRef" ref="previewSearchKnowledgeRef"
/> />
</div> </div>
@@ -68,6 +108,12 @@ const handleSearch = () => {
justify-content: space-between; justify-content: space-between;
gap: 12px; gap: 12px;
} }
.retrieval-select {
width: 180px;
flex-shrink: 0;
}
.search-result { .search-result {
padding-top: 20px; padding-top: 20px;
flex: 1; flex: 1;

View File

@@ -10,9 +10,6 @@ import {
ElFormItem, ElFormItem,
ElInputNumber, ElInputNumber,
ElMessage, ElMessage,
ElOption,
ElSelect,
ElSwitch,
ElTooltip, ElTooltip,
} from 'element-plus'; } from 'element-plus';
@@ -32,7 +29,6 @@ const props = defineProps({
onMounted(() => { onMounted(() => {
getDocumentCollectionConfig(); getDocumentCollectionConfig();
}); });
const searchEngineEnable = ref(false);
const baseOptions = ref<Record<string, any>>({}); const baseOptions = ref<Record<string, any>>({});
const getDocumentCollectionConfig = () => { const getDocumentCollectionConfig = () => {
api api
@@ -46,16 +42,13 @@ const getDocumentCollectionConfig = () => {
: 5; : 5;
searchConfig.simThreshold = options.simThreshold searchConfig.simThreshold = options.simThreshold
? Number(options.simThreshold) ? Number(options.simThreshold)
: 0.5; : 0.6;
searchConfig.searchEngineType = options.searchEngineType || 'lucene';
searchEngineEnable.value = !!data.searchEngineEnable;
}); });
}; };
const searchConfig = reactive({ const searchConfig = reactive({
docRecallMaxNum: 5, docRecallMaxNum: 5,
simThreshold: 0.5, simThreshold: 0.6,
searchEngineType: 'lucene',
}); });
const submitConfig = () => { const submitConfig = () => {
@@ -69,9 +62,7 @@ const submitConfig = () => {
...baseOptions.value, ...baseOptions.value,
docRecallMaxNum: searchConfig.docRecallMaxNum, docRecallMaxNum: searchConfig.docRecallMaxNum,
simThreshold: searchConfig.simThreshold, simThreshold: searchConfig.simThreshold,
searchEngineType: searchConfig.searchEngineType,
}, },
searchEngineEnable: searchEngineEnable.value,
}; };
api api
@@ -85,26 +76,6 @@ const submitConfig = () => {
console.error('保存配置失败:', error); console.error('保存配置失败:', error);
}); });
}; };
const searchEngineOptions = [
{
label: 'Lucene',
value: 'lucene',
},
{
label: 'ElasticSearch',
value: 'elasticSearch',
},
];
const handleSearchEngineEnableChange = () => {
if (!props.manageable) {
return;
}
api.post('/api/v1/documentCollection/update', {
id: props.documentCollectionId,
searchEngineEnable: searchEngineEnable.value,
});
};
</script> </script>
<template> <template>
@@ -176,71 +147,12 @@ const handleSearchEngineEnableChange = () => {
/> />
</div> </div>
</ElFormItem> </ElFormItem>
<!-- 搜索引擎启用开关 -->
<ElFormItem class="form-item">
<div class="form-item-label">
<span>{{
$t('documentCollectionSearch.searchEngineEnable.label')
}}</span>
<ElTooltip
:content="$t('documentCollectionSearch.searchEngineEnable.tooltip')"
placement="top"
effect="dark"
class="label-tooltip"
>
<InfoFilled class="info-icon" />
</ElTooltip>
</div>
<div class="form-item-content">
<ElSwitch
v-model="searchEngineEnable"
@change="handleSearchEngineEnableChange"
:active-text="$t('documentCollectionSearch.switch.on')"
:inactive-text="$t('documentCollectionSearch.switch.off')"
class="form-control switch-control"
/>
</div>
</ElFormItem>
<!-- 通过 searchEngineEnable 控制显示/隐藏 -->
<ElFormItem
v-if="searchEngineEnable"
prop="searchEngineType"
class="form-item"
>
<div class="form-item-label">
<span>{{
$t('documentCollectionSearch.searchEngineType.label')
}}</span>
<ElTooltip
:content="$t('documentCollectionSearch.searchEngineType.tooltip')"
placement="top"
effect="dark"
class="label-tooltip"
>
<InfoFilled class="info-icon" />
</ElTooltip>
</div>
<div class="form-item-content">
<ElSelect
v-model="searchConfig.searchEngineType"
:placeholder="
$t('documentCollectionSearch.searchEngineType.placeholder')
"
class="form-control"
>
<ElOption
v-for="option in searchEngineOptions"
:key="option.value"
:label="option.label"
:value="option.value"
/>
</ElSelect>
</div>
</ElFormItem>
</ElForm> </ElForm>
<div class="config-hint">
{{ $t('documentCollectionSearch.engineHint') }}
</div>
<div class="config-footer"> <div class="config-footer">
<ElButton <ElButton
type="primary" type="primary"
@@ -352,6 +264,13 @@ const handleSearchEngineEnableChange = () => {
padding: 8px 0; padding: 8px 0;
} }
.config-hint {
margin: 12px 0 4px;
font-size: 12px;
line-height: 1.6;
color: var(--el-text-color-secondary);
}
.config-footer { .config-footer {
position: sticky; position: sticky;
bottom: 0; bottom: 0;

View File

@@ -1,14 +1,21 @@
<script setup lang="ts"> <script setup lang="ts">
import { ref } from 'vue'; import { ref } from 'vue';
import { $t } from '@easyflow/locales';
import { Document } from '@element-plus/icons-vue'; import { Document } from '@element-plus/icons-vue';
import { ElButton, ElIcon } from 'element-plus'; import { ElButton, ElIcon, ElTag } from 'element-plus';
// 定义类型接口(与 React 版本一致)
type RetrievalMode = 'HYBRID' | 'KEYWORD' | 'VECTOR';
type HitSource = 'BOTH' | 'KEYWORD' | 'VECTOR';
interface PreviewItem { interface PreviewItem {
sorting: string; sorting: number | string;
content: string; content: string;
score: string; score?: number | string;
hitSource?: HitSource;
} }
const props = defineProps({ const props = defineProps({
hideScore: { hideScore: {
type: Boolean, type: Boolean,
@@ -46,8 +53,37 @@ const props = defineProps({
type: Boolean, type: Boolean,
default: false, default: false,
}, },
retrievalMode: {
type: String as () => RetrievalMode,
default: 'HYBRID',
},
}); });
const loadingStatus = ref(false); const loadingStatus = ref(false);
const resolveHitSourceLabel = (hitSource?: HitSource) => {
if (hitSource === 'VECTOR') {
return $t('documentCollection.hitSources.vector');
}
if (hitSource === 'KEYWORD') {
return $t('documentCollection.hitSources.keyword');
}
if (hitSource === 'BOTH') {
return $t('documentCollection.hitSources.both');
}
return '';
};
const resolveHitSourceType = (hitSource?: HitSource) => {
if (hitSource === 'VECTOR') {
return 'success';
}
if (hitSource === 'KEYWORD') {
return 'warning';
}
return 'info';
};
defineExpose({ defineExpose({
loadingContent: (state: boolean) => { loadingContent: (state: boolean) => {
loadingStatus.value = state; loadingStatus.value = state;
@@ -89,10 +125,22 @@ defineExpose({
{{ item.sorting ?? index + 1 }} {{ item.sorting ?? index + 1 }}
</div> </div>
<div class="el-list-item-meta"> <div class="el-list-item-meta">
<div v-if="!hideScore"> <div v-if="!hideScore" class="score-text">
{{ $t('documentCollection.similarityScore') }}: {{ item.score }} {{ $t('documentCollection.similarityScore') }}: {{ item.score }}
</div> </div>
<div class="content-desc">{{ item.content }}</div> <div class="content-desc">{{ item.content }}</div>
<div
v-if="retrievalMode === 'HYBRID' && item.hitSource"
class="hit-source-row"
>
<ElTag
size="small"
effect="plain"
:type="resolveHitSourceType(item.hitSource)"
>
{{ resolveHitSourceLabel(item.hitSource) }}
</ElTag>
</div>
</div> </div>
</div> </div>
</div> </div>
@@ -203,10 +251,20 @@ defineExpose({
} }
} }
.score-text {
color: var(--el-text-color-secondary);
}
.hit-source-row {
display: flex;
justify-content: flex-end;
width: 100%;
}
.el-list-item { .el-list-item {
display: flex; display: flex;
gap: 12px; gap: 12px;
align-items: center; align-items: flex-start;
padding: 18px; padding: 18px;
border-radius: 8px; border-radius: 8px;
} }

View File

@@ -24,6 +24,11 @@
}); });
const options = getOptions(); const options = getOptions();
const retrievalModeOptions: SelectItem[] = [
{ value: 'HYBRID', label: '混合检索' },
{ value: 'VECTOR', label: '向量检索' },
{ value: 'KEYWORD', label: '关键词检索' }
];
let knowledgeArray = $state<SelectItem[]>([]); let knowledgeArray = $state<SelectItem[]>([]);
onMount(async () => { onMount(async () => {
@@ -78,6 +83,16 @@
} }
}); });
$effect(() => {
if (!data.retrievalMode) {
updateNodeData(currentNodeId, () => {
return {
retrievalMode: 'HYBRID'
};
});
}
});
</script> </script>
@@ -134,6 +149,18 @@
/> />
</div> </div>
<div class="setting-title">检索方式</div>
<div class="setting-item">
<Select items={retrievalModeOptions} style="width: 100%" placeholder="请选择检索方式" onSelect={(item)=>{
const newValue = item.value;
updateNodeData(currentNodeId, ()=>{
return {
retrievalMode: newValue
}
})
}} value={data.retrievalMode ? [data.retrievalMode] : ['HYBRID']} />
</div>
<div class="setting-title">获取数据量</div> <div class="setting-title">获取数据量</div>
<div class="setting-item"> <div class="setting-item">
@@ -184,4 +211,3 @@
} }
</style> </style>

View File

@@ -182,6 +182,11 @@
<artifactId>easy-agents-support</artifactId> <artifactId>easy-agents-support</artifactId>
<version>${easy-agents.version}</version> <version>${easy-agents.version}</version>
</dependency> </dependency>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-rag-retrieval</artifactId>
<version>${easy-agents.version}</version>
</dependency>
<dependency> <dependency>
<groupId>com.easyagents</groupId> <groupId>com.easyagents</groupId>
<artifactId>easy-agents-spring-boot-starter</artifactId> <artifactId>easy-agents-spring-boot-starter</artifactId>