feat: 收敛知识库检索调度与评分语义
- 固定 rag.engine 与 Milvus 配置,补齐启动期检索基础设施校验 - 支持调用方配置 retrievalMode,并统一知识库检索入口与结果来源展示 - 修正关键词检索 knowledgeId 过滤、混合检索评分归一化与本地 ES 默认配置
This commit is contained in:
@@ -118,7 +118,32 @@ services:
|
|||||||
minio-init:
|
minio-init:
|
||||||
condition: service_completed_successfully
|
condition: service_completed_successfully
|
||||||
ports:
|
ports:
|
||||||
- "19530:19530"
|
- "39530:19530"
|
||||||
- "9091:9091"
|
- "9091:9091"
|
||||||
volumes:
|
volumes:
|
||||||
- ./data/milvus:/var/lib/milvus
|
- ./data/milvus:/var/lib/milvus
|
||||||
|
|
||||||
|
elasticsearch:
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:8.15.5
|
||||||
|
container_name: easyflow-elasticsearch
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
TZ: Asia/Shanghai
|
||||||
|
discovery.type: single-node
|
||||||
|
ELASTIC_PASSWORD: elastic
|
||||||
|
xpack.security.enabled: "true"
|
||||||
|
ES_JAVA_OPTS: -Xms1g -Xmx1g
|
||||||
|
ports:
|
||||||
|
- "39200:9200"
|
||||||
|
volumes:
|
||||||
|
- ./data/elasticsearch:/usr/share/elasticsearch/data
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
"CMD-SHELL",
|
||||||
|
"curl -fsS -u elastic:elastic http://127.0.0.1:9200 >/dev/null || exit 1",
|
||||||
|
]
|
||||||
|
interval: 15s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 20
|
||||||
|
start_period: 60s
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package tech.easyflow.admin.controller.ai;
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import tech.easyflow.ai.dto.BotKnowledgeBindingRequest;
|
||||||
import tech.easyflow.ai.entity.BotDocumentCollection;
|
import tech.easyflow.ai.entity.BotDocumentCollection;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.permission.KnowledgeReadAccessSnapshot;
|
import tech.easyflow.ai.permission.KnowledgeReadAccessSnapshot;
|
||||||
@@ -63,20 +64,21 @@ public class BotDocumentCollectionController extends BaseCurdController<BotDocum
|
|||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("updateBotKnowledgeIds")
|
@PostMapping("updateBotKnowledgeIds")
|
||||||
public Result<?> save(@JsonBody("botId") BigInteger botId, @JsonBody("knowledgeIds") BigInteger [] knowledgeIds) {
|
public Result<?> save(@JsonBody("botId") BigInteger botId,
|
||||||
if (knowledgeIds != null) {
|
@JsonBody("knowledgeBindings") List<BotKnowledgeBindingRequest> knowledgeBindings) {
|
||||||
for (BigInteger knowledgeId : knowledgeIds) {
|
if (knowledgeBindings != null) {
|
||||||
if (knowledgeId == null) {
|
for (BotKnowledgeBindingRequest binding : knowledgeBindings) {
|
||||||
|
if (binding == null || binding.getKnowledgeId() == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
DocumentCollection collection = documentCollectionService.getById(knowledgeId);
|
DocumentCollection collection = documentCollectionService.getById(binding.getKnowledgeId());
|
||||||
if (collection == null) {
|
if (collection == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
resourceAccessService.assertAccess(CategoryResourceType.KNOWLEDGE, collection, ResourceAction.READ, "无权限绑定知识库");
|
resourceAccessService.assertAccess(CategoryResourceType.KNOWLEDGE, collection, ResourceAction.READ, "无权限绑定知识库");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
service.saveBotAndKnowledge(botId, knowledgeIds);
|
service.saveBotAndKnowledge(botId, knowledgeBindings);
|
||||||
return Result.ok();
|
return Result.ok();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package tech.easyflow.admin.controller.ai;
|
|||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
import com.easyagents.core.document.Document;
|
import com.easyagents.core.document.Document;
|
||||||
|
import com.easyagents.rag.retrieval.RagRetrievalMetadataKeys;
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
@@ -12,9 +13,12 @@ import org.springframework.web.bind.annotation.RequestParam;
|
|||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import tech.easyflow.ai.permission.KnowledgeVisibilityQueryHelper;
|
import tech.easyflow.ai.permission.KnowledgeVisibilityQueryHelper;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||||
|
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
|
||||||
import tech.easyflow.ai.entity.BotDocumentCollection;
|
import tech.easyflow.ai.entity.BotDocumentCollection;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.entity.Model;
|
import tech.easyflow.ai.entity.Model;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
||||||
import tech.easyflow.ai.service.BotDocumentCollectionService;
|
import tech.easyflow.ai.service.BotDocumentCollectionService;
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
@@ -32,11 +36,15 @@ import tech.easyflow.system.service.ResourceAccessService;
|
|||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
import java.math.BigDecimal;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
|
import java.math.RoundingMode;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 控制层。
|
* 控制层。
|
||||||
@@ -105,13 +113,11 @@ public class DocumentCollectionController extends BaseCurdController<DocumentCol
|
|||||||
Map<String, Object> options = entity.getOptions() == null
|
Map<String, Object> options = entity.getOptions() == null
|
||||||
? new HashMap<>()
|
? new HashMap<>()
|
||||||
: new HashMap<>(entity.getOptions());
|
: new HashMap<>(entity.getOptions());
|
||||||
if (entity.getSearchEngineEnable() == null){
|
|
||||||
entity.setSearchEngineEnable(false);
|
|
||||||
}
|
|
||||||
options.putIfAbsent(DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL, true);
|
options.putIfAbsent(DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL, true);
|
||||||
options.putIfAbsent(DocumentCollection.KEY_RERANK_ENABLE, entity.getRerankModelId() != null);
|
options.putIfAbsent(DocumentCollection.KEY_RERANK_ENABLE, entity.getRerankModelId() != null);
|
||||||
entity.setOptions(options);
|
entity.setOptions(options);
|
||||||
}
|
}
|
||||||
|
normalizeInfrastructureFields(entity, isSave);
|
||||||
return super.onSaveOrUpdateBefore(entity, isSave);
|
return super.onSaveOrUpdateBefore(entity, isSave);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,8 +130,16 @@ public class DocumentCollectionController extends BaseCurdController<DocumentCol
|
|||||||
idExpr = "#knowledgeId",
|
idExpr = "#knowledgeId",
|
||||||
denyMessage = "无权限访问知识库"
|
denyMessage = "无权限访问知识库"
|
||||||
)
|
)
|
||||||
public Result<List<Document>> search(@RequestParam BigInteger knowledgeId, @RequestParam String keyword) {
|
public Result<List<KnowledgeSearchResultItem>> search(@RequestParam BigInteger knowledgeId,
|
||||||
return Result.ok(service.search(knowledgeId, keyword));
|
@RequestParam String keyword,
|
||||||
|
@RequestParam(required = false) String retrievalMode) {
|
||||||
|
KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
|
||||||
|
request.setKnowledgeId(knowledgeId);
|
||||||
|
request.setQuery(keyword);
|
||||||
|
request.setRetrievalMode(KnowledgeRetrievalModes.parse(retrievalMode));
|
||||||
|
request.setCallerType("API");
|
||||||
|
request.setCallerId(String.valueOf(knowledgeId));
|
||||||
|
return Result.ok(toKnowledgeSearchResult(service.search(request)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -234,4 +248,85 @@ public class DocumentCollectionController extends BaseCurdController<DocumentCol
|
|||||||
}
|
}
|
||||||
return collection;
|
return collection;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void normalizeInfrastructureFields(DocumentCollection entity, boolean isSave) {
|
||||||
|
if (entity == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isSave) {
|
||||||
|
entity.setVectorStoreEnable(true);
|
||||||
|
entity.setVectorStoreType("milvus");
|
||||||
|
entity.setSearchEngineEnable(true);
|
||||||
|
entity.setVectorStoreCollection(generateVectorCollectionName());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (entity.getVectorStoreEnable() != null) {
|
||||||
|
entity.setVectorStoreEnable(true);
|
||||||
|
}
|
||||||
|
if (entity.getVectorStoreType() != null) {
|
||||||
|
entity.setVectorStoreType("milvus");
|
||||||
|
}
|
||||||
|
if (entity.getSearchEngineEnable() != null) {
|
||||||
|
entity.setSearchEngineEnable(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String generateVectorCollectionName() {
|
||||||
|
return "kb_" + UUID.randomUUID().toString().replace("-", "").substring(0, 28);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<KnowledgeSearchResultItem> toKnowledgeSearchResult(List<Document> documents) {
|
||||||
|
List<KnowledgeSearchResultItem> results = new ArrayList<>();
|
||||||
|
if (documents == null) {
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
for (int index = 0; index < documents.size(); index++) {
|
||||||
|
Document document = documents.get(index);
|
||||||
|
if (document == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
KnowledgeSearchResultItem item = new KnowledgeSearchResultItem();
|
||||||
|
item.setSorting(index + 1);
|
||||||
|
item.setContent(document.getContent());
|
||||||
|
item.setScore(roundScore(document.getScore()));
|
||||||
|
item.setHitSource(readMetadataAsString(document, RagRetrievalMetadataKeys.HIT_SOURCE));
|
||||||
|
item.setVectorScore(roundScore(readMetadataAsDouble(document, RagRetrievalMetadataKeys.VECTOR_SCORE)));
|
||||||
|
item.setKeywordScore(roundScore(readMetadataAsDouble(document, RagRetrievalMetadataKeys.KEYWORD_SCORE)));
|
||||||
|
results.add(item);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String readMetadataAsString(Document document, String key) {
|
||||||
|
Object value = document == null ? null : document.getMetadata(key);
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String text = String.valueOf(value);
|
||||||
|
return StringUtils.hasText(text) ? text : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Double readMetadataAsDouble(Document document, String key) {
|
||||||
|
Object value = document == null ? null : document.getMetadata(key);
|
||||||
|
if (value instanceof Number) {
|
||||||
|
return ((Number) value).doubleValue();
|
||||||
|
}
|
||||||
|
if (value instanceof String && StringUtils.hasText((String) value)) {
|
||||||
|
try {
|
||||||
|
return Double.valueOf((String) value);
|
||||||
|
} catch (NumberFormatException ignore) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Double roundScore(Double value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new BigDecimal(String.valueOf(value))
|
||||||
|
.setScale(4, RoundingMode.HALF_UP)
|
||||||
|
.doubleValue();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,6 +41,10 @@
|
|||||||
<groupId>com.easyagents</groupId>
|
<groupId>com.easyagents</groupId>
|
||||||
<artifactId>easy-agents-support</artifactId>
|
<artifactId>easy-agents-support</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.easyagents</groupId>
|
||||||
|
<artifactId>easy-agents-rag-retrieval</artifactId>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.easyagents</groupId>
|
<groupId>com.easyagents</groupId>
|
||||||
<artifactId>easy-agents-spring-boot-starter</artifactId>
|
<artifactId>easy-agents-spring-boot-starter</artifactId>
|
||||||
|
|||||||
@@ -1,35 +1,10 @@
|
|||||||
package tech.easyflow.ai.config;
|
package tech.easyflow.ai.config;
|
||||||
|
|
||||||
import com.easyagents.engine.es.ESConfig;
|
import com.easyagents.engine.es.ESConfig;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
|
@ConfigurationProperties(prefix = "rag.searcher.elastic")
|
||||||
public class AiEsConfig extends ESConfig {
|
public class AiEsConfig extends ESConfig {
|
||||||
|
|
||||||
@Value("${rag.searcher.elastic.host}")
|
|
||||||
@Override
|
|
||||||
public void setHost(String host) {
|
|
||||||
super.setHost(host);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Value("${rag.searcher.elastic.userName}")
|
|
||||||
@Override
|
|
||||||
public void setUserName(String userName) {
|
|
||||||
super.setUserName(userName);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Value("${rag.searcher.elastic.password}")
|
|
||||||
@Override
|
|
||||||
public void setPassword(String password) {
|
|
||||||
super.setPassword(password);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Value("${rag.searcher.elastic.indexName}")
|
|
||||||
@Override
|
|
||||||
public void setIndexName(String indexName) {
|
|
||||||
super.setIndexName(indexName);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,17 +1,10 @@
|
|||||||
package tech.easyflow.ai.config;
|
package tech.easyflow.ai.config;
|
||||||
|
|
||||||
import com.easyagents.search.engine.lucene.LuceneConfig;
|
import com.easyagents.search.engine.lucene.LuceneConfig;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
|
@ConfigurationProperties(prefix = "rag.searcher.lucene")
|
||||||
public class AiLuceneConfig extends LuceneConfig {
|
public class AiLuceneConfig extends LuceneConfig {
|
||||||
|
|
||||||
@Value("${rag.searcher.lucene.indexDirPath}")
|
|
||||||
@Override
|
|
||||||
public void setIndexDirPath(String indexDirPath) {
|
|
||||||
super.setIndexDirPath(indexDirPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package tech.easyflow.ai.config;
|
||||||
|
|
||||||
|
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
@ConfigurationProperties(prefix = "rag.milvus")
|
||||||
|
public class AiMilvusConfig extends MilvusVectorStoreConfig {
|
||||||
|
|
||||||
|
public MilvusVectorStoreConfig copyForCollection(String collectionName) {
|
||||||
|
MilvusVectorStoreConfig config = new MilvusVectorStoreConfig();
|
||||||
|
config.setUri(getUri());
|
||||||
|
config.setToken(getToken());
|
||||||
|
config.setDatabaseName(getDatabaseName());
|
||||||
|
config.setUsername(getUsername());
|
||||||
|
config.setPassword(getPassword());
|
||||||
|
config.setAutoCreateCollection(isAutoCreateCollection());
|
||||||
|
config.setDefaultCollectionName(collectionName);
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package tech.easyflow.ai.config;
|
||||||
|
|
||||||
|
import com.easyagents.engine.es.ElasticSearcher;
|
||||||
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
|
import com.easyagents.store.milvus.MilvusVectorStore;
|
||||||
|
import org.springframework.beans.factory.SmartInitializingSingleton;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.ai.rag.KeywordEngineType;
|
||||||
|
import tech.easyflow.common.util.SpringContextUtil;
|
||||||
|
import tech.easyflow.common.util.StringUtil;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.io.File;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class RagInfrastructureValidator implements SmartInitializingSingleton {
|
||||||
|
|
||||||
|
private static final int STARTUP_CHECK_RETRY_TIMES = 10;
|
||||||
|
private static final long STARTUP_CHECK_RETRY_INTERVAL_MS = 1000L;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private AiMilvusConfig aiMilvusConfig;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private AiLuceneConfig aiLuceneConfig;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private SearcherFactory searcherFactory;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void afterSingletonsInstantiated() {
|
||||||
|
validateMilvus();
|
||||||
|
validateKeywordSearcher();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateMilvus() {
|
||||||
|
Exception lastException = null;
|
||||||
|
for (int i = 0; i < STARTUP_CHECK_RETRY_TIMES; i++) {
|
||||||
|
try {
|
||||||
|
MilvusVectorStore vectorStore = new MilvusVectorStore(aiMilvusConfig.copyForCollection("__rag_boot_probe__"));
|
||||||
|
if (vectorStore.checkAvailable()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
lastException = e;
|
||||||
|
}
|
||||||
|
sleepBeforeRetry();
|
||||||
|
}
|
||||||
|
if (lastException != null) {
|
||||||
|
throw new BusinessException("Milvus 服务不可用,项目启动失败,请检查 rag.milvus 配置与服务状态: " + lastException.getMessage());
|
||||||
|
}
|
||||||
|
throw new BusinessException("Milvus 服务不可用,项目启动失败,请检查 rag.milvus 配置与服务状态");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateKeywordSearcher() {
|
||||||
|
KeywordEngineType engineType = KeywordEngineType.from(
|
||||||
|
SpringContextUtil.getProperty("rag.engine", "ES")
|
||||||
|
);
|
||||||
|
if (engineType == KeywordEngineType.LUCENE) {
|
||||||
|
validateLuceneDirectory();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
|
if (!(searcher instanceof ElasticSearcher) || !checkElasticAvailable((ElasticSearcher) searcher)) {
|
||||||
|
throw new BusinessException("ES 服务不可用,项目启动失败,请检查 rag.engine 与 rag.searcher.elastic 配置");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean checkElasticAvailable(ElasticSearcher elasticSearcher) {
|
||||||
|
for (int i = 0; i < STARTUP_CHECK_RETRY_TIMES; i++) {
|
||||||
|
if (elasticSearcher.checkAvailable()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
sleepBeforeRetry();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateLuceneDirectory() {
|
||||||
|
String indexDirPath = aiLuceneConfig.getIndexDirPath();
|
||||||
|
if (StringUtil.noText(indexDirPath)) {
|
||||||
|
throw new BusinessException("Lucene 索引目录未配置,请检查 rag.searcher.lucene.indexDirPath");
|
||||||
|
}
|
||||||
|
File indexDir = new File(indexDirPath);
|
||||||
|
if (!indexDir.exists() && !indexDir.mkdirs()) {
|
||||||
|
throw new BusinessException("Lucene 索引目录创建失败: " + indexDirPath);
|
||||||
|
}
|
||||||
|
if (!indexDir.isDirectory() || !indexDir.canRead() || !indexDir.canWrite()) {
|
||||||
|
throw new BusinessException("Lucene 索引目录不可读写: " + indexDirPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void sleepBeforeRetry() {
|
||||||
|
try {
|
||||||
|
Thread.sleep(STARTUP_CHECK_RETRY_INTERVAL_MS);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new BusinessException("中间件启动校验被中断");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,40 +3,37 @@ package tech.easyflow.ai.config;
|
|||||||
import com.easyagents.engine.es.ElasticSearcher;
|
import com.easyagents.engine.es.ElasticSearcher;
|
||||||
import com.easyagents.search.engine.lucene.LuceneSearcher;
|
import com.easyagents.search.engine.lucene.LuceneSearcher;
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
import org.springframework.context.annotation.Bean;
|
import org.springframework.context.annotation.Bean;
|
||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
public class SearcherFactory {
|
public class SearcherFactory {
|
||||||
|
|
||||||
@Autowired
|
private final ObjectProvider<DocumentSearcher> documentSearcherProvider;
|
||||||
private AiLuceneConfig luceneConfig;
|
|
||||||
|
|
||||||
@Autowired
|
public SearcherFactory(ObjectProvider<DocumentSearcher> documentSearcherProvider) {
|
||||||
private AiEsConfig aiEsConfig;
|
this.documentSearcherProvider = documentSearcherProvider;
|
||||||
|
}
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
public LuceneSearcher luceneSearcher() {
|
@ConditionalOnProperty(prefix = "rag", name = "engine", havingValue = "LUCENE")
|
||||||
|
public LuceneSearcher luceneSearcher(AiLuceneConfig luceneConfig) {
|
||||||
return new LuceneSearcher(luceneConfig);
|
return new LuceneSearcher(luceneConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
public ElasticSearcher elasticSearcher() {
|
@ConditionalOnProperty(prefix = "rag", name = "engine", havingValue = "ES", matchIfMissing = true)
|
||||||
|
public ElasticSearcher elasticSearcher(AiEsConfig aiEsConfig) {
|
||||||
return new ElasticSearcher(aiEsConfig);
|
return new ElasticSearcher(aiEsConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public DocumentSearcher getSearcher() {
|
||||||
|
return documentSearcherProvider.getIfAvailable();
|
||||||
|
}
|
||||||
|
|
||||||
public DocumentSearcher getSearcher(String defaultSearcherType) {
|
public DocumentSearcher getSearcher(String ignored) {
|
||||||
if (defaultSearcherType == null) {
|
return getSearcher();
|
||||||
defaultSearcherType = "lucene";
|
|
||||||
}
|
|
||||||
switch (defaultSearcherType) {
|
|
||||||
case "elasticSearch":
|
|
||||||
return new ElasticSearcher(aiEsConfig);
|
|
||||||
case "lucene":
|
|
||||||
default:
|
|
||||||
return new LuceneSearcher(luceneConfig);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import com.easyagents.rag.retrieval.RetrievalMode;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public class BotKnowledgeBindingRequest {
|
||||||
|
|
||||||
|
private BigInteger knowledgeId;
|
||||||
|
private String retrievalMode;
|
||||||
|
|
||||||
|
public BigInteger getKnowledgeId() {
|
||||||
|
return knowledgeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||||
|
this.knowledgeId = knowledgeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRetrievalMode() {
|
||||||
|
return retrievalMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRetrievalMode(String retrievalMode) {
|
||||||
|
this.retrievalMode = retrievalMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RetrievalMode resolveRetrievalMode() {
|
||||||
|
return KnowledgeRetrievalModes.parse(retrievalMode);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
public class KnowledgeSearchResultItem {
|
||||||
|
|
||||||
|
private Integer sorting;
|
||||||
|
private String content;
|
||||||
|
private Double score;
|
||||||
|
private String hitSource;
|
||||||
|
private Double vectorScore;
|
||||||
|
private Double keywordScore;
|
||||||
|
|
||||||
|
public Integer getSorting() {
|
||||||
|
return sorting;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSorting(Integer sorting) {
|
||||||
|
this.sorting = sorting;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getContent() {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setContent(String content) {
|
||||||
|
this.content = content;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Double getScore() {
|
||||||
|
return score;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setScore(Double score) {
|
||||||
|
this.score = score;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getHitSource() {
|
||||||
|
return hitSource;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setHitSource(String hitSource) {
|
||||||
|
this.hitSource = hitSource;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Double getVectorScore() {
|
||||||
|
return vectorScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setVectorScore(Double vectorScore) {
|
||||||
|
this.vectorScore = vectorScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Double getKeywordScore() {
|
||||||
|
return keywordScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setKeywordScore(Double keywordScore) {
|
||||||
|
this.keywordScore = keywordScore;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,9 @@ package tech.easyflow.ai.easyagents.tool;
|
|||||||
import com.easyagents.core.document.Document;
|
import com.easyagents.core.document.Document;
|
||||||
import com.easyagents.core.model.chat.tool.BaseTool;
|
import com.easyagents.core.model.chat.tool.BaseTool;
|
||||||
import com.easyagents.core.model.chat.tool.Parameter;
|
import com.easyagents.core.model.chat.tool.Parameter;
|
||||||
|
import com.easyagents.rag.retrieval.RetrievalMode;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
import tech.easyflow.common.util.SpringContextUtil;
|
import tech.easyflow.common.util.SpringContextUtil;
|
||||||
|
|
||||||
@@ -14,12 +16,18 @@ import java.util.Map;
|
|||||||
public class DocumentCollectionTool extends BaseTool {
|
public class DocumentCollectionTool extends BaseTool {
|
||||||
|
|
||||||
private BigInteger knowledgeId;
|
private BigInteger knowledgeId;
|
||||||
|
private RetrievalMode retrievalMode = RetrievalMode.HYBRID;
|
||||||
|
|
||||||
public DocumentCollectionTool() {
|
public DocumentCollectionTool() {
|
||||||
}
|
}
|
||||||
|
|
||||||
public DocumentCollectionTool(DocumentCollection documentCollection, boolean needEnglishName) {
|
public DocumentCollectionTool(DocumentCollection documentCollection, boolean needEnglishName) {
|
||||||
|
this(documentCollection, needEnglishName, RetrievalMode.HYBRID);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DocumentCollectionTool(DocumentCollection documentCollection, boolean needEnglishName, RetrievalMode retrievalMode) {
|
||||||
this.knowledgeId = documentCollection.getId();
|
this.knowledgeId = documentCollection.getId();
|
||||||
|
this.retrievalMode = retrievalMode == null ? RetrievalMode.HYBRID : retrievalMode;
|
||||||
if (needEnglishName) {
|
if (needEnglishName) {
|
||||||
this.name = documentCollection.getEnglishName();
|
this.name = documentCollection.getEnglishName();
|
||||||
} else {
|
} else {
|
||||||
@@ -47,11 +55,25 @@ public class DocumentCollectionTool extends BaseTool {
|
|||||||
this.knowledgeId = knowledgeId;
|
this.knowledgeId = knowledgeId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public RetrievalMode getRetrievalMode() {
|
||||||
|
return retrievalMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRetrievalMode(RetrievalMode retrievalMode) {
|
||||||
|
this.retrievalMode = retrievalMode == null ? RetrievalMode.HYBRID : retrievalMode;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Object invoke(Map<String, Object> argsMap) {
|
public Object invoke(Map<String, Object> argsMap) {
|
||||||
|
|
||||||
DocumentCollectionService knowledgeService = SpringContextUtil.getBean(DocumentCollectionService.class);
|
DocumentCollectionService knowledgeService = SpringContextUtil.getBean(DocumentCollectionService.class);
|
||||||
List<Document> documents = knowledgeService.search(this.knowledgeId, (String) argsMap.get("input"));
|
KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
|
||||||
|
request.setKnowledgeId(this.knowledgeId);
|
||||||
|
request.setQuery((String) argsMap.get("input"));
|
||||||
|
request.setRetrievalMode(this.retrievalMode);
|
||||||
|
request.setCallerType("BOT_TOOL");
|
||||||
|
request.setCallerId(this.knowledgeId == null ? null : this.knowledgeId.toString());
|
||||||
|
List<Document> documents = knowledgeService.search(request);
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
if (documents != null) {
|
if (documents != null) {
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import com.easyagents.flow.core.knowledge.Knowledge;
|
|||||||
import com.easyagents.flow.core.knowledge.KnowledgeProvider;
|
import com.easyagents.flow.core.knowledge.KnowledgeProvider;
|
||||||
import com.easyagents.flow.core.node.KnowledgeNode;
|
import com.easyagents.flow.core.node.KnowledgeNode;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
@@ -30,7 +32,17 @@ public class KnowledgeProviderImpl implements KnowledgeProvider {
|
|||||||
return new Knowledge() {
|
return new Knowledge() {
|
||||||
@Override
|
@Override
|
||||||
public List<Map<String, Object>> search(String keyword, int limit, KnowledgeNode knowledgeNode, Chain chain) {
|
public List<Map<String, Object>> search(String keyword, int limit, KnowledgeNode knowledgeNode, Chain chain) {
|
||||||
List<Document> documents = documentCollectionService.search(new BigInteger(id.toString()), keyword);
|
KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
|
||||||
|
request.setKnowledgeId(new BigInteger(id.toString()));
|
||||||
|
request.setQuery(keyword);
|
||||||
|
request.setLimit(limit);
|
||||||
|
request.setRetrievalMode(KnowledgeRetrievalModes.parse(knowledgeNode.getRetrievalMode()));
|
||||||
|
request.setCallerType("WORKFLOW");
|
||||||
|
request.setCallerId(knowledgeNode.getId());
|
||||||
|
List<Document> documents = documentCollectionService.search(request);
|
||||||
|
if (limit > 0 && documents.size() > limit) {
|
||||||
|
documents = new ArrayList<>(documents.subList(0, limit));
|
||||||
|
}
|
||||||
List<Map<String, Object>> res = new ArrayList<>();
|
List<Map<String, Object>> res = new ArrayList<>();
|
||||||
for (Document document : documents) {
|
for (Document document : documents) {
|
||||||
res.add(JSONObject.from(document));
|
res.add(JSONObject.from(document));
|
||||||
|
|||||||
@@ -1,9 +1,14 @@
|
|||||||
package tech.easyflow.ai.entity;
|
package tech.easyflow.ai.entity;
|
||||||
|
|
||||||
|
import com.easyagents.rag.retrieval.RetrievalMode;
|
||||||
import tech.easyflow.ai.entity.base.BotDocumentCollectionBase;
|
import tech.easyflow.ai.entity.base.BotDocumentCollectionBase;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
||||||
import com.mybatisflex.annotation.RelationOneToOne;
|
import com.mybatisflex.annotation.RelationOneToOne;
|
||||||
import com.mybatisflex.annotation.Table;
|
import com.mybatisflex.annotation.Table;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 实体类。
|
* 实体类。
|
||||||
*
|
*
|
||||||
@@ -14,6 +19,8 @@ import com.mybatisflex.annotation.Table;
|
|||||||
@Table("tb_bot_document_collection")
|
@Table("tb_bot_document_collection")
|
||||||
public class BotDocumentCollection extends BotDocumentCollectionBase {
|
public class BotDocumentCollection extends BotDocumentCollectionBase {
|
||||||
|
|
||||||
|
public static final String OPTION_KEY_RETRIEVAL_MODE = "retrievalMode";
|
||||||
|
|
||||||
@RelationOneToOne(selfField = "documentCollectionId", targetField = "id")
|
@RelationOneToOne(selfField = "documentCollectionId", targetField = "id")
|
||||||
private DocumentCollection knowledge;
|
private DocumentCollection knowledge;
|
||||||
|
|
||||||
@@ -24,4 +31,21 @@ public class BotDocumentCollection extends BotDocumentCollectionBase {
|
|||||||
public void setKnowledge(DocumentCollection knowledge) {
|
public void setKnowledge(DocumentCollection knowledge) {
|
||||||
this.knowledge = knowledge;
|
this.knowledge = knowledge;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public RetrievalMode getRetrievalMode() {
|
||||||
|
Map<String, Object> options = getOptions();
|
||||||
|
if (options == null) {
|
||||||
|
return RetrievalMode.HYBRID;
|
||||||
|
}
|
||||||
|
Object value = options.get(OPTION_KEY_RETRIEVAL_MODE);
|
||||||
|
return KnowledgeRetrievalModes.parse(value == null ? null : String.valueOf(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRetrievalMode(RetrievalMode retrievalMode) {
|
||||||
|
Map<String, Object> options = getOptions() == null
|
||||||
|
? new HashMap<>()
|
||||||
|
: new HashMap<>(getOptions());
|
||||||
|
options.put(OPTION_KEY_RETRIEVAL_MODE, (retrievalMode == null ? RetrievalMode.HYBRID : retrievalMode).name());
|
||||||
|
setOptions(options);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,24 +2,16 @@ package tech.easyflow.ai.entity;
|
|||||||
|
|
||||||
import com.easyagents.core.model.chat.tool.Tool;
|
import com.easyagents.core.model.chat.tool.Tool;
|
||||||
import com.easyagents.core.store.DocumentStore;
|
import com.easyagents.core.store.DocumentStore;
|
||||||
import com.easyagents.store.aliyun.AliyunVectorStore;
|
import com.easyagents.rag.retrieval.RetrievalMode;
|
||||||
import com.easyagents.store.aliyun.AliyunVectorStoreConfig;
|
|
||||||
import com.easyagents.store.elasticsearch.ElasticSearchVectorStore;
|
|
||||||
import com.easyagents.store.elasticsearch.ElasticSearchVectorStoreConfig;
|
|
||||||
import com.easyagents.store.milvus.MilvusVectorStore;
|
import com.easyagents.store.milvus.MilvusVectorStore;
|
||||||
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
||||||
import com.easyagents.store.opensearch.OpenSearchVectorStore;
|
|
||||||
import com.easyagents.store.opensearch.OpenSearchVectorStoreConfig;
|
|
||||||
import com.easyagents.store.qcloud.QCloudVectorStore;
|
|
||||||
import com.easyagents.store.qcloud.QCloudVectorStoreConfig;
|
|
||||||
import com.easyagents.store.redis.RedisVectorStore;
|
|
||||||
import com.easyagents.store.redis.RedisVectorStoreConfig;
|
|
||||||
import com.mybatisflex.annotation.Table;
|
import com.mybatisflex.annotation.Table;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusConfig;
|
||||||
import tech.easyflow.ai.easyagents.tool.DocumentCollectionTool;
|
import tech.easyflow.ai.easyagents.tool.DocumentCollectionTool;
|
||||||
import tech.easyflow.ai.entity.base.DocumentCollectionBase;
|
import tech.easyflow.ai.entity.base.DocumentCollectionBase;
|
||||||
import tech.easyflow.common.util.PropertiesUtil;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
||||||
|
import tech.easyflow.common.util.SpringContextUtil;
|
||||||
import tech.easyflow.common.util.StringUtil;
|
import tech.easyflow.common.util.StringUtil;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
import tech.easyflow.system.permission.resource.VisibilityResource;
|
import tech.easyflow.system.permission.resource.VisibilityResource;
|
||||||
|
|
||||||
import java.math.BigDecimal;
|
import java.math.BigDecimal;
|
||||||
@@ -58,11 +50,6 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
|
|||||||
*/
|
*/
|
||||||
public static final String KEY_SIMILARITY_THRESHOLD = "simThreshold";
|
public static final String KEY_SIMILARITY_THRESHOLD = "simThreshold";
|
||||||
|
|
||||||
/**
|
|
||||||
* 搜索引擎类型
|
|
||||||
*/
|
|
||||||
public static final String KEY_SEARCH_ENGINE_TYPE = "searchEngineType";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 是否允许更新向量模型
|
* 是否允许更新向量模型
|
||||||
*/
|
*/
|
||||||
@@ -78,28 +65,10 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
|
|||||||
public static final String KEY_SPLITTER_STRATEGY_PROFILES = "splitter.strategyProfiles";
|
public static final String KEY_SPLITTER_STRATEGY_PROFILES = "splitter.strategyProfiles";
|
||||||
|
|
||||||
public DocumentStore toDocumentStore() {
|
public DocumentStore toDocumentStore() {
|
||||||
String storeType = this.getVectorStoreType();
|
if (StringUtil.noText(this.getVectorStoreCollection())) {
|
||||||
if (StringUtil.noText(storeType)) {
|
|
||||||
throw new BusinessException("向量数据库类型未设置");
|
|
||||||
}
|
|
||||||
if (storeType == null) {
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
switch (storeType.toLowerCase()) {
|
|
||||||
case "redis":
|
|
||||||
return redisStore();
|
|
||||||
case "milvus":
|
|
||||||
return milvusStore();
|
return milvusStore();
|
||||||
case "opensearch":
|
|
||||||
return openSearchStore();
|
|
||||||
case "elasticsearch":
|
|
||||||
return elasticSearchStore();
|
|
||||||
case "aliyun":
|
|
||||||
return aliyunStore();
|
|
||||||
case "qcloud":
|
|
||||||
return qcloudStore();
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isVectorStoreEnabled() {
|
public boolean isVectorStoreEnabled() {
|
||||||
@@ -115,53 +84,31 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
|
|||||||
}
|
}
|
||||||
|
|
||||||
public boolean isSearchEngineEnabled() {
|
public boolean isSearchEngineEnabled() {
|
||||||
return this.getSearchEngineEnable() != null && this.getSearchEngineEnable();
|
return true;
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private DocumentStore redisStore() {
|
|
||||||
RedisVectorStoreConfig redisVectorStoreConfig = getStoreConfig(RedisVectorStoreConfig.class);
|
|
||||||
return new RedisVectorStore(redisVectorStoreConfig);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private DocumentStore milvusStore() {
|
private DocumentStore milvusStore() {
|
||||||
MilvusVectorStoreConfig milvusVectorStoreConfig = getStoreConfig(MilvusVectorStoreConfig.class);
|
AiMilvusConfig aiMilvusConfig = SpringContextUtil.getBean(AiMilvusConfig.class);
|
||||||
if (milvusVectorStoreConfig != null && StringUtil.noText(milvusVectorStoreConfig.getDefaultCollectionName())) {
|
MilvusVectorStoreConfig milvusVectorStoreConfig = aiMilvusConfig.copyForCollection(this.getVectorStoreCollection());
|
||||||
milvusVectorStoreConfig.setDefaultCollectionName(this.getVectorStoreCollection());
|
|
||||||
}
|
|
||||||
return new MilvusVectorStore(milvusVectorStoreConfig);
|
return new MilvusVectorStore(milvusVectorStoreConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
private DocumentStore openSearchStore() {
|
|
||||||
OpenSearchVectorStoreConfig openSearchVectorStoreConfig = getStoreConfig(OpenSearchVectorStoreConfig.class);
|
|
||||||
return new OpenSearchVectorStore(openSearchVectorStoreConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
private DocumentStore elasticSearchStore() {
|
|
||||||
ElasticSearchVectorStoreConfig elasticSearchVectorStoreConfig = getStoreConfig(ElasticSearchVectorStoreConfig.class);
|
|
||||||
return new ElasticSearchVectorStore(elasticSearchVectorStoreConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
private DocumentStore aliyunStore() {
|
|
||||||
AliyunVectorStoreConfig aliyunVectorStoreConfig = getStoreConfig(AliyunVectorStoreConfig.class);
|
|
||||||
return new AliyunVectorStore(aliyunVectorStoreConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
private DocumentStore qcloudStore() {
|
|
||||||
QCloudVectorStoreConfig qCloudVectorStoreConfig = getStoreConfig(QCloudVectorStoreConfig.class);
|
|
||||||
return new QCloudVectorStore(qCloudVectorStoreConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
private <T> T getStoreConfig(Class<T> clazz) {
|
|
||||||
return PropertiesUtil.propertiesTextToEntity(this.getVectorStoreConfig(), clazz);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Tool toFunction(boolean needEnglishName) {
|
public Tool toFunction(boolean needEnglishName) {
|
||||||
return new DocumentCollectionTool(this, needEnglishName);
|
return toFunction(needEnglishName, RetrievalMode.HYBRID.name());
|
||||||
|
}
|
||||||
|
|
||||||
|
public Tool toFunction(boolean needEnglishName, String retrievalMode) {
|
||||||
|
return new DocumentCollectionTool(this, needEnglishName, KnowledgeRetrievalModes.parse(retrievalMode));
|
||||||
}
|
}
|
||||||
|
|
||||||
public Object getOptionsByKey(String key) {
|
public Object getOptionsByKey(String key) {
|
||||||
Map<String, Object> options = this.getOptions();
|
Map<String, Object> options = this.getOptions();
|
||||||
|
if (KEY_DOC_RECALL_MAX_NUM.equals(key) && (options == null || !options.containsKey(KEY_DOC_RECALL_MAX_NUM))) {
|
||||||
|
return 5;
|
||||||
|
}
|
||||||
|
if (KEY_SIMILARITY_THRESHOLD.equals(key) && (options == null || !options.containsKey(KEY_SIMILARITY_THRESHOLD))) {
|
||||||
|
return 0.6f;
|
||||||
|
}
|
||||||
if (KEY_RERANK_ENABLE.equals(key)) {
|
if (KEY_RERANK_ENABLE.equals(key)) {
|
||||||
if (options == null || !options.containsKey(KEY_RERANK_ENABLE)) {
|
if (options == null || !options.containsKey(KEY_RERANK_ENABLE)) {
|
||||||
return this.getRerankModelId() != null;
|
return this.getRerankModelId() != null;
|
||||||
@@ -182,22 +129,10 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
|
|||||||
if (options == null) {
|
if (options == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (KEY_DOC_RECALL_MAX_NUM.equals(key) && !options.containsKey(KEY_DOC_RECALL_MAX_NUM)) {
|
|
||||||
return 5;
|
|
||||||
}
|
|
||||||
if (KEY_SIMILARITY_THRESHOLD.equals(key)) {
|
if (KEY_SIMILARITY_THRESHOLD.equals(key)) {
|
||||||
if (!options.containsKey(KEY_SIMILARITY_THRESHOLD)) {
|
|
||||||
return 0.6f;
|
|
||||||
} else {
|
|
||||||
BigDecimal score = (BigDecimal) options.get(key);
|
BigDecimal score = (BigDecimal) options.get(key);
|
||||||
return (float) score.doubleValue();
|
return (float) score.doubleValue();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if (KEY_SEARCH_ENGINE_TYPE.equals(key)) {
|
|
||||||
if (!options.containsKey(KEY_SEARCH_ENGINE_TYPE)) {
|
|
||||||
return "lucene";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return options.get(key);
|
return options.get(key);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
package tech.easyflow.ai.rag;
|
||||||
|
|
||||||
|
import tech.easyflow.common.util.StringUtil;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
|
public enum KeywordEngineType {
|
||||||
|
ES,
|
||||||
|
LUCENE;
|
||||||
|
|
||||||
|
public static KeywordEngineType from(String value) {
|
||||||
|
if (StringUtil.noText(value)) {
|
||||||
|
return ES;
|
||||||
|
}
|
||||||
|
for (KeywordEngineType type : values()) {
|
||||||
|
if (type.name().equalsIgnoreCase(value.trim())) {
|
||||||
|
return type;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new BusinessException("不支持的关键词检索引擎: " + value);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package tech.easyflow.ai.rag;
|
||||||
|
|
||||||
|
import com.easyagents.rag.retrieval.RetrievalMode;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
|
public final class KnowledgeRetrievalModes {
|
||||||
|
|
||||||
|
private KnowledgeRetrievalModes() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static RetrievalMode parse(String value) {
|
||||||
|
try {
|
||||||
|
return RetrievalMode.from(value);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
throw new BusinessException("不支持的检索方式: " + value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package tech.easyflow.ai.rag;
|
||||||
|
|
||||||
|
import com.easyagents.rag.retrieval.RetrievalMode;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public class KnowledgeRetrievalRequest {
|
||||||
|
|
||||||
|
private BigInteger knowledgeId;
|
||||||
|
private String query;
|
||||||
|
private Integer limit;
|
||||||
|
private RetrievalMode retrievalMode = RetrievalMode.HYBRID;
|
||||||
|
private String callerType;
|
||||||
|
private String callerId;
|
||||||
|
|
||||||
|
public BigInteger getKnowledgeId() {
|
||||||
|
return knowledgeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||||
|
this.knowledgeId = knowledgeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getQuery() {
|
||||||
|
return query;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQuery(String query) {
|
||||||
|
this.query = query;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getLimit() {
|
||||||
|
return limit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setLimit(Integer limit) {
|
||||||
|
this.limit = limit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RetrievalMode getRetrievalMode() {
|
||||||
|
return retrievalMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRetrievalMode(RetrievalMode retrievalMode) {
|
||||||
|
this.retrievalMode = retrievalMode == null ? RetrievalMode.HYBRID : retrievalMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCallerType() {
|
||||||
|
return callerType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCallerType(String callerType) {
|
||||||
|
this.callerType = callerType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCallerId() {
|
||||||
|
return callerId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCallerId(String callerId) {
|
||||||
|
this.callerId = callerId;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
package tech.easyflow.ai.service;
|
package tech.easyflow.ai.service;
|
||||||
|
|
||||||
|
import tech.easyflow.ai.dto.BotKnowledgeBindingRequest;
|
||||||
import tech.easyflow.ai.entity.BotDocumentCollection;
|
import tech.easyflow.ai.entity.BotDocumentCollection;
|
||||||
import com.mybatisflex.core.service.IService;
|
import com.mybatisflex.core.service.IService;
|
||||||
|
|
||||||
@@ -16,5 +17,5 @@ public interface BotDocumentCollectionService extends IService<BotDocumentCollec
|
|||||||
|
|
||||||
List<BotDocumentCollection> listByBotId(BigInteger botId);
|
List<BotDocumentCollection> listByBotId(BigInteger botId);
|
||||||
|
|
||||||
void saveBotAndKnowledge(BigInteger botId, BigInteger[] knowledgeIds);
|
void saveBotAndKnowledge(BigInteger botId, List<BotKnowledgeBindingRequest> knowledgeBindings);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package tech.easyflow.ai.service;
|
|||||||
|
|
||||||
import com.easyagents.core.document.Document;
|
import com.easyagents.core.document.Document;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
import com.mybatisflex.core.service.IService;
|
import com.mybatisflex.core.service.IService;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
@@ -17,6 +18,8 @@ public interface DocumentCollectionService extends IService<DocumentCollection>
|
|||||||
|
|
||||||
List<Document> search(BigInteger id, String keyword);
|
List<Document> search(BigInteger id, String keyword);
|
||||||
|
|
||||||
|
List<Document> search(KnowledgeRetrievalRequest request);
|
||||||
|
|
||||||
DocumentCollection getDetail(String idOrAlias);
|
DocumentCollection getDetail(String idOrAlias);
|
||||||
|
|
||||||
DocumentCollection getByAlias(String idOrAlias);
|
DocumentCollection getByAlias(String idOrAlias);
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
package tech.easyflow.ai.service.impl;
|
package tech.easyflow.ai.service.impl;
|
||||||
|
|
||||||
|
import com.easyagents.rag.retrieval.RetrievalMode;
|
||||||
|
import tech.easyflow.ai.dto.BotKnowledgeBindingRequest;
|
||||||
import tech.easyflow.ai.entity.BotDocumentCollection;
|
import tech.easyflow.ai.entity.BotDocumentCollection;
|
||||||
import tech.easyflow.ai.mapper.BotDocumentCollectionMapper;
|
import tech.easyflow.ai.mapper.BotDocumentCollectionMapper;
|
||||||
import tech.easyflow.ai.service.BotDocumentCollectionService;
|
import tech.easyflow.ai.service.BotDocumentCollectionService;
|
||||||
@@ -11,9 +13,10 @@ import tech.easyflow.common.cache.RedisLockExecutor;
|
|||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Map;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
|
|
||||||
@@ -44,26 +47,30 @@ public class BotDocumentCollectionServiceImpl extends ServiceImpl<BotDocumentCol
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
@Transactional
|
@Transactional
|
||||||
public void saveBotAndKnowledge(BigInteger botId, BigInteger[] knowledgeIds) {
|
public void saveBotAndKnowledge(BigInteger botId, List<BotKnowledgeBindingRequest> knowledgeBindings) {
|
||||||
redisLockExecutor.executeWithLock(BOT_BINDING_LOCK_KEY_PREFIX + botId, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
|
redisLockExecutor.executeWithLock(BOT_BINDING_LOCK_KEY_PREFIX + botId, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
|
||||||
this.remove(QueryWrapper.create().eq(BotDocumentCollection::getBotId, botId));
|
this.remove(QueryWrapper.create().eq(BotDocumentCollection::getBotId, botId));
|
||||||
Set<BigInteger> uniqueKnowledgeIds = new LinkedHashSet<>();
|
Map<BigInteger, RetrievalMode> bindingMap = new LinkedHashMap<>();
|
||||||
if (knowledgeIds != null) {
|
if (knowledgeBindings != null) {
|
||||||
for (BigInteger knowledgeId : knowledgeIds) {
|
for (BotKnowledgeBindingRequest binding : knowledgeBindings) {
|
||||||
if (knowledgeId != null) {
|
if (binding == null || binding.getKnowledgeId() == null) {
|
||||||
uniqueKnowledgeIds.add(knowledgeId);
|
continue;
|
||||||
|
}
|
||||||
|
bindingMap.put(binding.getKnowledgeId(), binding.resolveRetrievalMode());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
if (bindingMap.isEmpty()) {
|
||||||
if (uniqueKnowledgeIds.isEmpty()) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
List<BotDocumentCollection> list = new ArrayList<>(uniqueKnowledgeIds.size());
|
List<BotDocumentCollection> list = new ArrayList<>(bindingMap.size());
|
||||||
for (BigInteger knowledgeId : uniqueKnowledgeIds) {
|
for (Map.Entry<BigInteger, RetrievalMode> entry : bindingMap.entrySet()) {
|
||||||
BotDocumentCollection botDocumentCollection = new BotDocumentCollection();
|
BotDocumentCollection botDocumentCollection = new BotDocumentCollection();
|
||||||
botDocumentCollection.setBotId(botId);
|
botDocumentCollection.setBotId(botId);
|
||||||
botDocumentCollection.setDocumentCollectionId(knowledgeId);
|
botDocumentCollection.setDocumentCollectionId(entry.getKey());
|
||||||
|
Map<String, Object> options = new HashMap<>();
|
||||||
|
options.put(BotDocumentCollection.OPTION_KEY_RETRIEVAL_MODE, entry.getValue().name());
|
||||||
|
botDocumentCollection.setOptions(options);
|
||||||
list.add(botDocumentCollection);
|
list.add(botDocumentCollection);
|
||||||
}
|
}
|
||||||
this.saveBatch(list);
|
this.saveBatch(list);
|
||||||
|
|||||||
@@ -401,7 +401,8 @@ public class BotServiceImpl extends ServiceImpl<BotMapper, Bot> implements BotSe
|
|||||||
.selectListWithRelationsByQuery(queryWrapper);
|
.selectListWithRelationsByQuery(queryWrapper);
|
||||||
if (botDocumentCollections != null && !botDocumentCollections.isEmpty()) {
|
if (botDocumentCollections != null && !botDocumentCollections.isEmpty()) {
|
||||||
for (BotDocumentCollection botDocumentCollection : botDocumentCollections) {
|
for (BotDocumentCollection botDocumentCollection : botDocumentCollections) {
|
||||||
Tool function = botDocumentCollection.getKnowledge().toFunction(needEnglishName);
|
Tool function = botDocumentCollection.getKnowledge()
|
||||||
|
.toFunction(needEnglishName, botDocumentCollection.getRetrievalMode().name());
|
||||||
functionList.add(function);
|
functionList.add(function);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,8 +12,6 @@ import org.springframework.stereotype.Service;
|
|||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 服务层实现。
|
* 服务层实现。
|
||||||
*
|
*
|
||||||
@@ -28,10 +26,9 @@ public class DocumentChunkServiceImpl extends ServiceImpl<DocumentChunkMapper, D
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean removeChunk(DocumentCollection knowledge, BigInteger chunkId) {
|
public boolean removeChunk(DocumentCollection knowledge, BigInteger chunkId) {
|
||||||
String searchEngineType = (String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE);
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher(searchEngineType);
|
|
||||||
// 删除搜索引擎中的数据
|
// 删除搜索引擎中的数据
|
||||||
if (searcherFactory.getSearcher(searchEngineType) == null){
|
if (searcher == null){
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return searcher.deleteDocument(chunkId);
|
return searcher.deleteDocument(chunkId);
|
||||||
|
|||||||
@@ -1,13 +1,23 @@
|
|||||||
package tech.easyflow.ai.service.impl;
|
package tech.easyflow.ai.service.impl;
|
||||||
|
|
||||||
|
|
||||||
import com.easyagents.core.document.Document;
|
import com.easyagents.core.document.Document;
|
||||||
import com.easyagents.core.model.rerank.RerankException;
|
import com.easyagents.core.model.rerank.RerankException;
|
||||||
import com.easyagents.core.model.rerank.RerankModel;
|
import com.easyagents.core.model.rerank.RerankModel;
|
||||||
import com.easyagents.core.store.DocumentStore;
|
import com.easyagents.core.store.DocumentStore;
|
||||||
import com.easyagents.core.store.SearchWrapper;
|
import com.easyagents.core.store.SearchWrapper;
|
||||||
import com.easyagents.core.store.StoreOptions;
|
import com.easyagents.core.store.StoreOptions;
|
||||||
|
import com.easyagents.rag.retrieval.HitSource;
|
||||||
|
import com.easyagents.rag.retrieval.KeywordRetriever;
|
||||||
|
import com.easyagents.rag.retrieval.RagHit;
|
||||||
|
import com.easyagents.rag.retrieval.RagQuery;
|
||||||
|
import com.easyagents.rag.retrieval.RagRetrievalExecutor;
|
||||||
|
import com.easyagents.rag.retrieval.RagScoreNormalizer;
|
||||||
|
import com.easyagents.rag.retrieval.RagRetrievalResult;
|
||||||
|
import com.easyagents.rag.retrieval.RetrievalMode;
|
||||||
|
import com.easyagents.rag.retrieval.RrfFusionStrategy;
|
||||||
|
import com.easyagents.rag.retrieval.VectorRetriever;
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
|
import com.easyagents.search.engine.service.KeywordSearchRequest;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
@@ -22,7 +32,7 @@ import tech.easyflow.ai.entity.Model;
|
|||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentCollectionMapper;
|
import tech.easyflow.ai.mapper.DocumentCollectionMapper;
|
||||||
import tech.easyflow.ai.mapper.FaqItemMapper;
|
import tech.easyflow.ai.mapper.FaqItemMapper;
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.ai.utils.CustomBeanUtils;
|
import tech.easyflow.ai.utils.CustomBeanUtils;
|
||||||
@@ -35,12 +45,18 @@ import java.io.Serializable;
|
|||||||
import java.math.BigDecimal;
|
import java.math.BigDecimal;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.math.RoundingMode;
|
import java.math.RoundingMode;
|
||||||
import java.util.*;
|
import java.util.ArrayList;
|
||||||
import java.util.concurrent.CompletableFuture;
|
import java.util.Collections;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.Comparator;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.*;
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_DOC_RECALL_MAX_NUM;
|
||||||
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_RERANK_ENABLE;
|
||||||
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SIMILARITY_THRESHOLD;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 服务层实现。
|
* 服务层实现。
|
||||||
@@ -57,9 +73,6 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
@Resource
|
@Resource
|
||||||
private ModelService llmService;
|
private ModelService llmService;
|
||||||
|
|
||||||
@Resource
|
|
||||||
private DocumentChunkService chunkService;
|
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private SearcherFactory searcherFactory;
|
private SearcherFactory searcherFactory;
|
||||||
|
|
||||||
@@ -69,14 +82,96 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
@Autowired
|
@Autowired
|
||||||
private FaqItemMapper faqItemMapper;
|
private FaqItemMapper faqItemMapper;
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<Document> search(BigInteger id, String keyword) {
|
public List<Document> search(BigInteger id, String keyword) {
|
||||||
DocumentCollection documentCollection = getById(id);
|
KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
|
||||||
|
request.setKnowledgeId(id);
|
||||||
|
request.setQuery(keyword);
|
||||||
|
request.setRetrievalMode(RetrievalMode.HYBRID);
|
||||||
|
return search(request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Document> search(KnowledgeRetrievalRequest request) {
|
||||||
|
if (request == null || request.getKnowledgeId() == null) {
|
||||||
|
throw new BusinessException("知识库ID不能为空");
|
||||||
|
}
|
||||||
|
String keyword = request.getQuery();
|
||||||
|
if (StringUtil.noText(keyword)) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
|
||||||
|
RetrievalMode retrievalMode = request.getRetrievalMode() == null
|
||||||
|
? RetrievalMode.HYBRID
|
||||||
|
: request.getRetrievalMode();
|
||||||
|
DocumentCollection documentCollection = getById(request.getKnowledgeId());
|
||||||
if (documentCollection == null) {
|
if (documentCollection == null) {
|
||||||
throw new BusinessException("知识库不存在");
|
throw new BusinessException("知识库不存在");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int docRecallMaxNum = readIntegerOption(documentCollection, KEY_DOC_RECALL_MAX_NUM, 5);
|
||||||
|
float minSimilarity = readFloatOption(documentCollection, KEY_SIMILARITY_THRESHOLD, 0.6F);
|
||||||
|
|
||||||
|
RagQuery ragQuery = new RagQuery();
|
||||||
|
ragQuery.setQuery(keyword);
|
||||||
|
ragQuery.setRetrievalMode(retrievalMode);
|
||||||
|
ragQuery.setTopK(docRecallMaxNum);
|
||||||
|
ragQuery.setMinScore((double) minSimilarity);
|
||||||
|
|
||||||
|
RagRetrievalExecutor retrievalExecutor = new RagRetrievalExecutor(
|
||||||
|
buildVectorRetriever(documentCollection, docRecallMaxNum, retrievalMode == RetrievalMode.VECTOR ? minSimilarity : null),
|
||||||
|
buildKeywordRetriever(documentCollection, docRecallMaxNum),
|
||||||
|
new RrfFusionStrategy()
|
||||||
|
);
|
||||||
|
|
||||||
|
RagRetrievalResult retrievalResult = retrievalExecutor.retrieve(ragQuery);
|
||||||
|
List<Document> searchDocuments = toDocuments(retrievalResult.getHits());
|
||||||
|
fillSearchContent(documentCollection, searchDocuments);
|
||||||
|
if (searchDocuments.isEmpty()) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
|
||||||
|
RerankModel rerankModel = resolveRerankModel(documentCollection);
|
||||||
|
boolean reranked = false;
|
||||||
|
if (rerankModel != null) {
|
||||||
|
try {
|
||||||
|
RagRetrievalResult rerankResult = retrievalExecutor.rerank(keyword, toRagHits(searchDocuments), rerankModel, docRecallMaxNum);
|
||||||
|
searchDocuments = toDocuments(rerankResult.getHits());
|
||||||
|
reranked = true;
|
||||||
|
} catch (RerankException e) {
|
||||||
|
LOG.warn("Rerank failed for collectionId={}, modelId={}, fallback to retrieved results. message={}",
|
||||||
|
documentCollection.getId(), documentCollection.getRerankModelId(), e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RagScoreNormalizer.normalize(searchDocuments, retrievalMode, reranked);
|
||||||
|
return formatDocuments(searchDocuments, shouldApplyMinSimilarityFilter(retrievalMode, reranked), minSimilarity, docRecallMaxNum);
|
||||||
|
}
|
||||||
|
|
||||||
|
private VectorRetriever buildVectorRetriever(DocumentCollection documentCollection,
|
||||||
|
int docRecallMaxNum,
|
||||||
|
Float minSimilarity) {
|
||||||
|
return new VectorRetriever() {
|
||||||
|
@Override
|
||||||
|
public List<RagHit> retrieve(RagQuery query) {
|
||||||
|
return adaptDocuments(searchVectorDocuments(documentCollection, query.getQuery(), docRecallMaxNum, minSimilarity), HitSource.VECTOR);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private KeywordRetriever buildKeywordRetriever(DocumentCollection documentCollection, int docRecallMaxNum) {
|
||||||
|
return new KeywordRetriever() {
|
||||||
|
@Override
|
||||||
|
public List<RagHit> retrieve(RagQuery query) {
|
||||||
|
return adaptDocuments(searchKeywordDocuments(documentCollection, query.getQuery(), docRecallMaxNum), HitSource.KEYWORD);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Document> searchVectorDocuments(DocumentCollection documentCollection,
|
||||||
|
String keyword,
|
||||||
|
int docRecallMaxNum,
|
||||||
|
Float minSimilarity) {
|
||||||
DocumentStore documentStore = documentCollection.toDocumentStore();
|
DocumentStore documentStore = documentCollection.toDocumentStore();
|
||||||
if (documentStore == null) {
|
if (documentStore == null) {
|
||||||
throw new BusinessException("知识库没有配置向量库");
|
throw new BusinessException("知识库没有配置向量库");
|
||||||
@@ -88,88 +183,92 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
}
|
}
|
||||||
|
|
||||||
documentStore.setEmbeddingModel(model.toEmbeddingModel());
|
documentStore.setEmbeddingModel(model.toEmbeddingModel());
|
||||||
// 最大召回知识条数
|
|
||||||
Integer docRecallMaxNum = (Integer) documentCollection.getOptionsByKey(KEY_DOC_RECALL_MAX_NUM);
|
|
||||||
// 最低相似度
|
|
||||||
float minSimilarity = (float) documentCollection.getOptionsByKey(KEY_SIMILARITY_THRESHOLD);
|
|
||||||
SearchWrapper wrapper = new SearchWrapper();
|
SearchWrapper wrapper = new SearchWrapper();
|
||||||
wrapper.setMaxResults(docRecallMaxNum);
|
wrapper.setMaxResults(docRecallMaxNum);
|
||||||
|
if (minSimilarity != null) {
|
||||||
wrapper.setMinScore((double) minSimilarity);
|
wrapper.setMinScore((double) minSimilarity);
|
||||||
|
}
|
||||||
wrapper.setText(keyword);
|
wrapper.setText(keyword);
|
||||||
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(documentCollection.getVectorStoreCollection());
|
StoreOptions options = StoreOptions.ofCollectionName(documentCollection.getVectorStoreCollection());
|
||||||
options.setIndexName(documentCollection.getVectorStoreCollection());
|
options.setIndexName(documentCollection.getVectorStoreCollection());
|
||||||
|
List<Document> documents = documentStore.search(wrapper, options);
|
||||||
|
return documents == null ? Collections.<Document>emptyList() : documents;
|
||||||
|
}
|
||||||
|
|
||||||
// 并行查询:向量库 + 搜索引擎
|
private List<Document> searchKeywordDocuments(DocumentCollection documentCollection, String keyword, int docRecallMaxNum) {
|
||||||
CompletableFuture<List<Document>> vectorFuture = CompletableFuture.supplyAsync(() ->
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
documentStore.search(wrapper, options)
|
if (searcher == null) {
|
||||||
);
|
|
||||||
|
|
||||||
CompletableFuture<List<Document>> searcherFuture = CompletableFuture.supplyAsync(() -> {
|
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher((String) documentCollection.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
|
|
||||||
if (searcher == null || !documentCollection.isSearchEngineEnabled()) {
|
|
||||||
return Collections.emptyList();
|
return Collections.emptyList();
|
||||||
}
|
}
|
||||||
List<Document> documents = searcher.searchDocuments(keyword);
|
KeywordSearchRequest request = KeywordSearchRequest.of(keyword, docRecallMaxNum);
|
||||||
return documents == null ? Collections.emptyList() : documents;
|
request.setKnowledgeId(documentCollection == null || documentCollection.getId() == null
|
||||||
});
|
? null
|
||||||
|
: documentCollection.getId().toString());
|
||||||
// 合并两个查询结果
|
List<Document> documents = searcher.searchDocuments(request);
|
||||||
CompletableFuture<Map<String, Document>> combinedFuture = vectorFuture.thenCombine(
|
return documents == null ? Collections.<Document>emptyList() : documents;
|
||||||
searcherFuture,
|
|
||||||
(vectorDocs, searcherDocs) -> {
|
|
||||||
Map<String, Document> uniqueDocs = new HashMap<>();
|
|
||||||
vectorDocs.forEach(doc -> uniqueDocs.putIfAbsent(doc.getId().toString(), doc));
|
|
||||||
searcherDocs.forEach(doc -> uniqueDocs.putIfAbsent(doc.getId().toString(), doc));
|
|
||||||
return uniqueDocs;
|
|
||||||
}
|
}
|
||||||
);
|
|
||||||
|
|
||||||
try {
|
private List<RagHit> adaptDocuments(List<Document> documents, HitSource hitSource) {
|
||||||
Map<String, Document> uniqueDocs = combinedFuture.get(); // 阻塞等待所有查询完成
|
List<RagHit> hits = new ArrayList<>();
|
||||||
List<Document> searchDocuments = new ArrayList<>(uniqueDocs.values());
|
if (documents == null) {
|
||||||
searchDocuments.sort((doc1, doc2) -> Double.compare(doc2.getScore(), doc1.getScore()));
|
return hits;
|
||||||
fillSearchContent(documentCollection, searchDocuments);
|
|
||||||
if (searchDocuments.isEmpty()) {
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
}
|
||||||
|
for (Document document : documents) {
|
||||||
|
RagHit hit = RagHit.fromDocument(document, hitSource);
|
||||||
|
if (hit != null) {
|
||||||
|
hits.add(hit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hits;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Document> toDocuments(List<RagHit> hits) {
|
||||||
|
List<Document> documents = new ArrayList<>();
|
||||||
|
if (hits == null) {
|
||||||
|
return documents;
|
||||||
|
}
|
||||||
|
for (RagHit hit : hits) {
|
||||||
|
if (hit == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
documents.add(hit.toDocument());
|
||||||
|
}
|
||||||
|
return documents;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<RagHit> toRagHits(List<Document> documents) {
|
||||||
|
List<RagHit> hits = new ArrayList<>();
|
||||||
|
if (documents == null) {
|
||||||
|
return hits;
|
||||||
|
}
|
||||||
|
for (Document document : documents) {
|
||||||
|
RagHit hit = RagHit.fromDocument(document);
|
||||||
|
if (hit != null) {
|
||||||
|
hits.add(hit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hits;
|
||||||
|
}
|
||||||
|
|
||||||
|
private RerankModel resolveRerankModel(DocumentCollection documentCollection) {
|
||||||
boolean rerankEnable = Boolean.TRUE.equals(documentCollection.getOptionsByKey(KEY_RERANK_ENABLE));
|
boolean rerankEnable = Boolean.TRUE.equals(documentCollection.getOptionsByKey(KEY_RERANK_ENABLE));
|
||||||
if (!rerankEnable || documentCollection.getRerankModelId() == null) {
|
if (!rerankEnable || documentCollection.getRerankModelId() == null) {
|
||||||
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
Model modelRerank = llmService.getModelInstance(documentCollection.getRerankModelId());
|
Model modelRerank = llmService.getModelInstance(documentCollection.getRerankModelId());
|
||||||
if (modelRerank == null) {
|
if (modelRerank == null) {
|
||||||
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
return null;
|
||||||
|
}
|
||||||
|
return modelRerank.toRerankModel();
|
||||||
}
|
}
|
||||||
|
|
||||||
RerankModel rerankModel = modelRerank.toRerankModel();
|
private boolean shouldApplyMinSimilarityFilter(RetrievalMode retrievalMode, boolean reranked) {
|
||||||
if (rerankModel == null) {
|
return !reranked && retrievalMode == RetrievalMode.VECTOR;
|
||||||
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<Object, Double> originalScores = new HashMap<>();
|
|
||||||
searchDocuments.forEach(item -> originalScores.put(item.getId(), item.getScore()));
|
|
||||||
searchDocuments.forEach(item -> item.setScore(null));
|
|
||||||
try {
|
|
||||||
List<Document> rerankDocuments = rerankModel.rerank(keyword, searchDocuments);
|
|
||||||
return formatDocuments(rerankDocuments, minSimilarity, docRecallMaxNum);
|
|
||||||
} catch (RerankException e) {
|
|
||||||
searchDocuments.forEach(item -> item.setScore(originalScores.get(item.getId())));
|
|
||||||
LOG.warn("Rerank failed for collectionId={}, modelId={}, fallback to vector results. message={}",
|
|
||||||
documentCollection.getId(), documentCollection.getRerankModelId(), e.getMessage());
|
|
||||||
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
|
||||||
}
|
|
||||||
} catch (InterruptedException | ExecutionException e) {
|
|
||||||
Thread.currentThread().interrupt();
|
|
||||||
e.printStackTrace();
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DocumentCollection getDetail(String idOrAlias) {
|
public DocumentCollection getDetail(String idOrAlias) {
|
||||||
|
|
||||||
DocumentCollection knowledge = null;
|
DocumentCollection knowledge = null;
|
||||||
|
|
||||||
if (idOrAlias.matches(RegexUtils.ALL_NUMBER)) {
|
if (idOrAlias.matches(RegexUtils.ALL_NUMBER)) {
|
||||||
@@ -188,15 +287,11 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DocumentCollection getByAlias(String idOrAlias) {
|
public DocumentCollection getByAlias(String idOrAlias) {
|
||||||
|
|
||||||
QueryWrapper queryWrapper = QueryWrapper.create();
|
QueryWrapper queryWrapper = QueryWrapper.create();
|
||||||
queryWrapper.eq(DocumentCollection::getAlias, idOrAlias);
|
queryWrapper.eq(DocumentCollection::getAlias, idOrAlias);
|
||||||
|
|
||||||
return getOne(queryWrapper);
|
return getOne(queryWrapper);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean updateById(DocumentCollection entity) {
|
public boolean updateById(DocumentCollection entity) {
|
||||||
DocumentCollection documentCollection = getById(entity.getId());
|
DocumentCollection documentCollection = getById(entity.getId());
|
||||||
@@ -210,40 +305,31 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
documentCollection.setAlias(null);
|
documentCollection.setAlias(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return super.updateById(documentCollection, false);
|
return super.updateById(documentCollection, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
public List<Document> formatDocuments(List<Document> documents,
|
||||||
* 格式化文档列表
|
boolean applyMinSimilarity,
|
||||||
*
|
float minSimilarity,
|
||||||
* @param documents 文档列表
|
int maxResults) {
|
||||||
* @param minSimilarity 最小相似度
|
|
||||||
* @return 格式化后的文档列表
|
|
||||||
*/
|
|
||||||
public List<Document> formatDocuments(List<Document> documents, float minSimilarity, int maxResults) {
|
|
||||||
return documents.stream()
|
return documents.stream()
|
||||||
// 过滤掉分数为空 或 分数低于最小值的文档
|
.filter(Objects::nonNull)
|
||||||
.filter(document -> {
|
.filter(document -> !applyMinSimilarity
|
||||||
Double score = document.getScore();
|
|| (document.getScore() != null && document.getScore() >= minSimilarity))
|
||||||
return score != null && score >= minSimilarity;
|
.map(this::roundDocumentScore)
|
||||||
})
|
.sorted(Comparator.comparing(Document::getScore, Comparator.nullsLast(Comparator.reverseOrder())))
|
||||||
// 格式化保留四位小数
|
|
||||||
.map(document -> {
|
|
||||||
Double score = document.getScore();
|
|
||||||
BigDecimal bd = new BigDecimal(score.toString());
|
|
||||||
bd = bd.setScale(4, RoundingMode.HALF_UP);
|
|
||||||
Double roundedScore = bd.doubleValue();
|
|
||||||
document.setScore(roundedScore);
|
|
||||||
return document;
|
|
||||||
})
|
|
||||||
// 按score降序排序(分数最高的排前面)
|
|
||||||
.sorted(Comparator.comparing(Document::getScore, Comparator.reverseOrder()))
|
|
||||||
// 限制只保留前maxResults条
|
|
||||||
.limit(maxResults)
|
.limit(maxResults)
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Document roundDocumentScore(Document document) {
|
||||||
|
if (document == null || document.getScore() == null) {
|
||||||
|
return document;
|
||||||
|
}
|
||||||
|
document.setScore(roundDouble(document.getScore()));
|
||||||
|
return document;
|
||||||
|
}
|
||||||
|
|
||||||
private void fillSearchContent(DocumentCollection documentCollection, List<Document> searchDocuments) {
|
private void fillSearchContent(DocumentCollection documentCollection, List<Document> searchDocuments) {
|
||||||
if (searchDocuments == null || searchDocuments.isEmpty()) {
|
if (searchDocuments == null || searchDocuments.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
@@ -252,30 +338,39 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
.map(item -> (Serializable) item.getId())
|
.map(item -> (Serializable) item.getId())
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
if (documentCollection.isFaqCollection()) {
|
if (documentCollection.isFaqCollection()) {
|
||||||
Map<String, FaqItem> faqItemMap = faqItemMapper.selectListByIds(ids).stream()
|
QueryWrapper queryWrapper = QueryWrapper.create();
|
||||||
|
queryWrapper.in(FaqItem::getId, ids);
|
||||||
|
queryWrapper.eq(FaqItem::getCollectionId, documentCollection.getId());
|
||||||
|
Map<String, FaqItem> faqItemMap = faqItemMapper.selectListByQuery(queryWrapper).stream()
|
||||||
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
|
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
|
||||||
|
searchDocuments.removeIf(item -> !faqItemMap.containsKey(String.valueOf(item.getId())));
|
||||||
searchDocuments.forEach(item -> {
|
searchDocuments.forEach(item -> {
|
||||||
FaqItem faqItem = faqItemMap.get(String.valueOf(item.getId()));
|
FaqItem faqItem = faqItemMap.get(String.valueOf(item.getId()));
|
||||||
if (faqItem != null) {
|
if (faqItem == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
List<Map<String, String>> faqImages = readFaqImages(faqItem);
|
List<Map<String, String>> faqImages = readFaqImages(faqItem);
|
||||||
item.setContent(buildFaqPromptContent(faqItem, faqImages));
|
item.setContent(buildFaqPromptContent(faqItem, faqImages));
|
||||||
|
|
||||||
Map<String, Object> metadataMap = item.getMetadataMap() == null
|
Map<String, Object> metadataMap = item.getMetadataMap() == null
|
||||||
? new HashMap<>()
|
? new HashMap<String, Object>()
|
||||||
: new HashMap<>(item.getMetadataMap());
|
: new HashMap<String, Object>(item.getMetadataMap());
|
||||||
List<String> imageUrls = faqImages.stream()
|
List<String> imageUrls = faqImages.stream()
|
||||||
.map(image -> image.get("url"))
|
.map(image -> image.get("url"))
|
||||||
.filter(Objects::nonNull)
|
.filter(Objects::nonNull)
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
metadataMap.put("imageUrls", imageUrls);
|
metadataMap.put("imageUrls", imageUrls);
|
||||||
item.setMetadataMap(metadataMap);
|
item.setMetadataMap(metadataMap);
|
||||||
}
|
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, DocumentChunk> chunkMap = documentChunkMapper.selectListByIds(ids).stream()
|
QueryWrapper queryWrapper = QueryWrapper.create();
|
||||||
|
queryWrapper.in(DocumentChunk::getId, ids);
|
||||||
|
queryWrapper.eq(DocumentChunk::getDocumentCollectionId, documentCollection.getId());
|
||||||
|
Map<String, DocumentChunk> chunkMap = documentChunkMapper.selectListByQuery(queryWrapper).stream()
|
||||||
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
|
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
|
||||||
|
searchDocuments.removeIf(item -> !chunkMap.containsKey(String.valueOf(item.getId())));
|
||||||
searchDocuments.forEach(item -> {
|
searchDocuments.forEach(item -> {
|
||||||
DocumentChunk documentChunk = chunkMap.get(String.valueOf(item.getId()));
|
DocumentChunk documentChunk = chunkMap.get(String.valueOf(item.getId()));
|
||||||
if (documentChunk != null && !StringUtil.noText(documentChunk.getContent())) {
|
if (documentChunk != null && !StringUtil.noText(documentChunk.getContent())) {
|
||||||
@@ -354,4 +449,43 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
String text = String.valueOf(value).trim();
|
String text = String.valueOf(value).trim();
|
||||||
return text.isEmpty() ? null : text;
|
return text.isEmpty() ? null : text;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private int readIntegerOption(DocumentCollection documentCollection, String key, int defaultValue) {
|
||||||
|
Object value = documentCollection.getOptionsByKey(key);
|
||||||
|
if (value instanceof Number) {
|
||||||
|
return ((Number) value).intValue();
|
||||||
|
}
|
||||||
|
if (value instanceof String && StringUtil.hasText((String) value)) {
|
||||||
|
try {
|
||||||
|
return Integer.parseInt((String) value);
|
||||||
|
} catch (NumberFormatException ignore) {
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
private float readFloatOption(DocumentCollection documentCollection, String key, float defaultValue) {
|
||||||
|
Object value = documentCollection.getOptionsByKey(key);
|
||||||
|
if (value instanceof Number) {
|
||||||
|
return ((Number) value).floatValue();
|
||||||
|
}
|
||||||
|
if (value instanceof String && StringUtil.hasText((String) value)) {
|
||||||
|
try {
|
||||||
|
return Float.parseFloat((String) value);
|
||||||
|
} catch (NumberFormatException ignore) {
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Double roundDouble(Double value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new BigDecimal(String.valueOf(value))
|
||||||
|
.setScale(4, RoundingMode.HALF_UP)
|
||||||
|
.doubleValue();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import com.easyagents.rag.ingestion.RagIngestionService;
|
|||||||
import com.easyagents.rag.ingestion.model.AnalysisResult;
|
import com.easyagents.rag.ingestion.model.AnalysisResult;
|
||||||
import com.easyagents.rag.ingestion.model.StrategyConfig;
|
import com.easyagents.rag.ingestion.model.StrategyConfig;
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
|
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
|
||||||
import com.mybatisflex.core.keygen.impl.FlexIDKeyGenerator;
|
import com.mybatisflex.core.keygen.impl.FlexIDKeyGenerator;
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.query.QueryMethods;
|
import com.mybatisflex.core.query.QueryMethods;
|
||||||
@@ -56,7 +57,6 @@ import java.util.*;
|
|||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE;
|
|
||||||
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
|
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
|
||||||
import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT;
|
import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT;
|
||||||
|
|
||||||
@@ -157,8 +157,8 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class);
|
List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class);
|
||||||
documentStore.delete(chunkIds, options);
|
documentStore.delete(chunkIds, options);
|
||||||
// 删除搜索引擎中的数据
|
// 删除搜索引擎中的数据
|
||||||
if (searcherFactory.getSearcher((String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE)) != null) {
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher((String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
|
if (searcher != null) {
|
||||||
chunkIds.forEach(searcher::deleteDocument);
|
chunkIds.forEach(searcher::deleteDocument);
|
||||||
}
|
}
|
||||||
int ck = documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, id));
|
int ck = documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, id));
|
||||||
@@ -691,9 +691,7 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
options.setIndexName(options.getCollectionName());
|
options.setIndexName(options.getCollectionName());
|
||||||
|
|
||||||
DocumentSearcher searcher = null;
|
DocumentSearcher searcher = null;
|
||||||
if (knowledge.isSearchEngineEnabled()) {
|
searcher = searcherFactory.getSearcher();
|
||||||
searcher = searcherFactory.getSearcher((String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
|
|
||||||
}
|
|
||||||
return new StoreExecutionContext(knowledge, model, embeddingModel, documentStore, options, searcher);
|
return new StoreExecutionContext(knowledge, model, embeddingModel, documentStore, options, searcher);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -703,6 +701,8 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
com.easyagents.core.document.Document document = new com.easyagents.core.document.Document();
|
com.easyagents.core.document.Document document = new com.easyagents.core.document.Document();
|
||||||
document.setId(item.getId());
|
document.setId(item.getId());
|
||||||
document.setContent(item.getContent());
|
document.setContent(item.getContent());
|
||||||
|
document.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID,
|
||||||
|
storeContext.knowledge.getId() == null ? null : storeContext.knowledge.getId().toString());
|
||||||
documents.add(document);
|
documents.add(document);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import com.easyagents.core.store.DocumentStore;
|
|||||||
import com.easyagents.core.store.StoreOptions;
|
import com.easyagents.core.store.StoreOptions;
|
||||||
import com.easyagents.core.store.StoreResult;
|
import com.easyagents.core.store.StoreResult;
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
|
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||||
import org.jsoup.Jsoup;
|
import org.jsoup.Jsoup;
|
||||||
@@ -52,7 +53,6 @@ import java.util.*;
|
|||||||
import java.util.function.Supplier;
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE;
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> implements FaqItemService {
|
public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> implements FaqItemService {
|
||||||
@@ -356,15 +356,13 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
|
|||||||
throw new BusinessException("FAQ向量化失败");
|
throw new BusinessException("FAQ向量化失败");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (collection.isSearchEngineEnabled()) {
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher((String) collection.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
|
|
||||||
if (searcher != null) {
|
if (searcher != null) {
|
||||||
if (isUpdate) {
|
if (isUpdate) {
|
||||||
searcher.deleteDocument(entity.getId());
|
searcher.deleteDocument(entity.getId());
|
||||||
}
|
}
|
||||||
searcher.addDocument(doc);
|
searcher.addDocument(doc);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
markCollectionEmbedded(collection, preparedStore.embeddingModel);
|
markCollectionEmbedded(collection, preparedStore.embeddingModel);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -375,14 +373,11 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
|
|||||||
throw new BusinessException("FAQ向量删除失败");
|
throw new BusinessException("FAQ向量删除失败");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (collection.isSearchEngineEnabled()) {
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher((String) collection.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
|
|
||||||
if (searcher != null) {
|
if (searcher != null) {
|
||||||
boolean removed = searcher.deleteDocument(entity.getId());
|
boolean removed = searcher.deleteDocument(entity.getId());
|
||||||
if (!removed) {
|
if (!removed) {
|
||||||
LOG.warn("Delete faq search index failed. faqId={}, searcherType={}",
|
LOG.warn("Delete faq search index failed. faqId={}", entity.getId());
|
||||||
entity.getId(), collection.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -443,6 +438,7 @@ public class FaqItemServiceImpl extends ServiceImpl<FaqItemMapper, FaqItem> impl
|
|||||||
metadata.put("answerText", entity.getAnswerText());
|
metadata.put("answerText", entity.getAnswerText());
|
||||||
metadata.put("categoryId", entity.getCategoryId());
|
metadata.put("categoryId", entity.getCategoryId());
|
||||||
metadata.put("imageUrls", readImageUrls(entity.getOptions()));
|
metadata.put("imageUrls", readImageUrls(entity.getOptions()));
|
||||||
|
metadata.put(KeywordSearchMetadataKeys.KNOWLEDGE_ID, entity.getCollectionId() == null ? null : entity.getCollectionId().toString());
|
||||||
doc.setMetadataMap(metadata);
|
doc.setMetadataMap(metadata);
|
||||||
return doc;
|
return doc;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -208,12 +208,19 @@ jetcache:
|
|||||||
|
|
||||||
# 多路召回搜索引擎配置
|
# 多路召回搜索引擎配置
|
||||||
rag:
|
rag:
|
||||||
|
engine: ES
|
||||||
|
milvus:
|
||||||
|
uri: http://127.0.0.1:39530
|
||||||
|
databaseName: default
|
||||||
|
token:
|
||||||
|
username: easyflowadmin
|
||||||
|
password: easyflowadmin123
|
||||||
|
autoCreateCollection: true
|
||||||
searcher:
|
searcher:
|
||||||
# 搜索方式 默认lucene
|
|
||||||
lucene:
|
lucene:
|
||||||
indexDirPath: /Users/slience/data/easyflow/luceneKnowledge
|
indexDirPath: /Users/slience/data/easyflow/luceneKnowledge
|
||||||
elastic:
|
elastic:
|
||||||
host: https://127.0.0.1:9200
|
host: http://127.0.0.1:39200
|
||||||
userName: elastic
|
userName: elastic
|
||||||
password: elastic
|
password: elastic
|
||||||
indexName: easyflow
|
indexName: easyflow
|
||||||
|
|||||||
@@ -49,6 +49,12 @@
|
|||||||
"publicChatCopySuccess": "Copied",
|
"publicChatCopySuccess": "Copied",
|
||||||
"publicChatCopyFail": "Copy failed",
|
"publicChatCopyFail": "Copy failed",
|
||||||
"basicInfo": "Basic Info",
|
"basicInfo": "Basic Info",
|
||||||
|
"knowledgeRetrievalMode": "Retrieval Mode",
|
||||||
|
"retrievalModes": {
|
||||||
|
"hybrid": "Hybrid Retrieval",
|
||||||
|
"vector": "Vector Retrieval",
|
||||||
|
"keyword": "Keyword Retrieval"
|
||||||
|
},
|
||||||
"modal": {
|
"modal": {
|
||||||
"createDescription": "Set the assistant appearance, identity and base availability.",
|
"createDescription": "Set the assistant appearance, identity and base availability.",
|
||||||
"editDescription": "Update the assistant presentation and base availability.",
|
"editDescription": "Update the assistant presentation and base availability.",
|
||||||
|
|||||||
@@ -159,7 +159,13 @@
|
|||||||
"documentPreview": "DocumentPreview",
|
"documentPreview": "DocumentPreview",
|
||||||
"total": "Total",
|
"total": "Total",
|
||||||
"segments": "Segments",
|
"segments": "Segments",
|
||||||
"similarityScore": "SimilarityScore",
|
"similarityScore": "Relevance",
|
||||||
|
"searchFailed": "Search failed. Please try again later.",
|
||||||
|
"hitSources": {
|
||||||
|
"vector": "Vector Hit",
|
||||||
|
"keyword": "Keyword Hit",
|
||||||
|
"both": "Dual Hit"
|
||||||
|
},
|
||||||
"alibabaCloud": "AlibabaCloud",
|
"alibabaCloud": "AlibabaCloud",
|
||||||
"tencentCloud": "tencentCloud",
|
"tencentCloud": "tencentCloud",
|
||||||
"vectorEmbedModelTips": "After successful vector data, it is not allowed to modify the vector model",
|
"vectorEmbedModelTips": "After successful vector data, it is not allowed to modify the vector model",
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
"button": {
|
"button": {
|
||||||
"save": "Save Configuration"
|
"save": "Save Configuration"
|
||||||
},
|
},
|
||||||
|
"engineHint": "The keyword search engine is controlled by platform-level configuration and is no longer configured per knowledge base.",
|
||||||
"message": {
|
"message": {
|
||||||
"saveSuccess": "Configuration saved successfully",
|
"saveSuccess": "Configuration saved successfully",
|
||||||
"saveFailed": "Configuration saved failed"
|
"saveFailed": "Configuration saved failed"
|
||||||
|
|||||||
@@ -49,6 +49,12 @@
|
|||||||
"publicChatCopySuccess": "复制成功",
|
"publicChatCopySuccess": "复制成功",
|
||||||
"publicChatCopyFail": "复制失败",
|
"publicChatCopyFail": "复制失败",
|
||||||
"basicInfo": "基础信息",
|
"basicInfo": "基础信息",
|
||||||
|
"knowledgeRetrievalMode": "检索方式",
|
||||||
|
"retrievalModes": {
|
||||||
|
"hybrid": "混合检索",
|
||||||
|
"vector": "向量检索",
|
||||||
|
"keyword": "关键词检索"
|
||||||
|
},
|
||||||
"modal": {
|
"modal": {
|
||||||
"createDescription": "设置助手的外观、标识和基础发布状态。",
|
"createDescription": "设置助手的外观、标识和基础发布状态。",
|
||||||
"editDescription": "更新助手的展示信息与基础状态。",
|
"editDescription": "更新助手的展示信息与基础状态。",
|
||||||
|
|||||||
@@ -159,7 +159,13 @@
|
|||||||
"documentPreview": "文档预览",
|
"documentPreview": "文档预览",
|
||||||
"total": "共",
|
"total": "共",
|
||||||
"segments": "个分段",
|
"segments": "个分段",
|
||||||
"similarityScore": "相似度",
|
"similarityScore": "相关度",
|
||||||
|
"searchFailed": "检索失败,请稍后重试",
|
||||||
|
"hitSources": {
|
||||||
|
"vector": "语义命中",
|
||||||
|
"keyword": "关键词命中",
|
||||||
|
"both": "双路命中"
|
||||||
|
},
|
||||||
"alibabaCloud": "阿里云",
|
"alibabaCloud": "阿里云",
|
||||||
"tencentCloud": "腾讯云",
|
"tencentCloud": "腾讯云",
|
||||||
"vectorEmbedModelTips": "成功向量数据之后不允许修改向量模型",
|
"vectorEmbedModelTips": "成功向量数据之后不允许修改向量模型",
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
"button": {
|
"button": {
|
||||||
"save": "保存配置"
|
"save": "保存配置"
|
||||||
},
|
},
|
||||||
|
"engineHint": "关键词检索引擎由平台全局配置决定,知识库侧不再单独配置。",
|
||||||
"message": {
|
"message": {
|
||||||
"saveSuccess": "配置保存成功",
|
"saveSuccess": "配置保存成功",
|
||||||
"saveFailed": "配置保存失败"
|
"saveFailed": "配置保存失败"
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import {
|
|||||||
import { useDebounceFn } from '@vueuse/core';
|
import { useDebounceFn } from '@vueuse/core';
|
||||||
import {
|
import {
|
||||||
ElAlert,
|
ElAlert,
|
||||||
|
ElAvatar,
|
||||||
ElButton,
|
ElButton,
|
||||||
ElCol,
|
ElCol,
|
||||||
ElCollapse,
|
ElCollapse,
|
||||||
@@ -26,6 +27,7 @@ import {
|
|||||||
ElInput,
|
ElInput,
|
||||||
ElInputNumber,
|
ElInputNumber,
|
||||||
ElMessage,
|
ElMessage,
|
||||||
|
ElMessageBox,
|
||||||
ElOption,
|
ElOption,
|
||||||
ElRow,
|
ElRow,
|
||||||
ElSelect,
|
ElSelect,
|
||||||
@@ -63,6 +65,13 @@ interface ApiKeyOption {
|
|||||||
label: string;
|
label: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type RetrievalMode = 'HYBRID' | 'KEYWORD' | 'VECTOR';
|
||||||
|
|
||||||
|
interface BotKnowledgeBindingItem {
|
||||||
|
knowledgeId: string;
|
||||||
|
retrievalMode: RetrievalMode;
|
||||||
|
}
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = defineProps<{
|
||||||
bot?: BotInfo;
|
bot?: BotInfo;
|
||||||
hasSavePermission?: boolean;
|
hasSavePermission?: boolean;
|
||||||
@@ -70,6 +79,7 @@ const props = defineProps<{
|
|||||||
const botStore = useBotStore();
|
const botStore = useBotStore();
|
||||||
const route = useRoute();
|
const route = useRoute();
|
||||||
const botId = ref<string>((route.params.id as string) || '');
|
const botId = ref<string>((route.params.id as string) || '');
|
||||||
|
const fallbackAvatarUrl = `${import.meta.env.BASE_URL || '/'}favicon.svg`;
|
||||||
const options = ref<AiLlm[]>([]);
|
const options = ref<AiLlm[]>([]);
|
||||||
const selectedId = ref<string>('');
|
const selectedId = ref<string>('');
|
||||||
const llmConfig = ref({
|
const llmConfig = ref({
|
||||||
@@ -170,11 +180,11 @@ const iframeCodeHighlighted = computed(() => {
|
|||||||
}
|
}
|
||||||
const escaped = escapeHtml(iframeCode.value);
|
const escaped = escapeHtml(iframeCode.value);
|
||||||
return escaped
|
return escaped
|
||||||
.replace(
|
.replaceAll(
|
||||||
/(<\/?)([a-zA-Z][\w-]*)/g,
|
/(<\/?)([a-zA-Z][\w-]*)/g,
|
||||||
'$1<span class="hljs-name">$2</span>',
|
'$1<span class="hljs-name">$2</span>',
|
||||||
)
|
)
|
||||||
.replace(
|
.replaceAll(
|
||||||
/([:@a-zA-Z_][\w:-]*)=(".*?")/g,
|
/([:@a-zA-Z_][\w:-]*)=(".*?")/g,
|
||||||
'<span class="hljs-attr">$1</span>=<span class="hljs-string">$2</span>',
|
'<span class="hljs-attr">$1</span>=<span class="hljs-string">$2</span>',
|
||||||
);
|
);
|
||||||
@@ -202,6 +212,32 @@ const mcpToolData = ref<any>([]);
|
|||||||
const workflowData = ref<any[]>([]);
|
const workflowData = ref<any[]>([]);
|
||||||
const knowledgeData = ref<any[]>([]);
|
const knowledgeData = ref<any[]>([]);
|
||||||
const pluginToolData = ref<any[]>([]);
|
const pluginToolData = ref<any[]>([]);
|
||||||
|
const retrievalModeOptions = computed(() => [
|
||||||
|
{ label: $t('bot.retrievalModes.hybrid'), value: 'HYBRID' },
|
||||||
|
{ label: $t('bot.retrievalModes.vector'), value: 'VECTOR' },
|
||||||
|
{ label: $t('bot.retrievalModes.keyword'), value: 'KEYWORD' },
|
||||||
|
]);
|
||||||
|
const normalizeRetrievalMode = (value?: string): RetrievalMode => {
|
||||||
|
if (value === 'VECTOR' || value === 'KEYWORD' || value === 'HYBRID') {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
return 'HYBRID';
|
||||||
|
};
|
||||||
|
const buildKnowledgeBindingsPayload = (
|
||||||
|
selectedIds: Array<number | string>,
|
||||||
|
): BotKnowledgeBindingItem[] => {
|
||||||
|
return [...new Set((selectedIds || []).map(String).filter(Boolean))].map(
|
||||||
|
(knowledgeId) => {
|
||||||
|
const existing = knowledgeData.value.find(
|
||||||
|
(item) => String(item.id) === knowledgeId,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
knowledgeId,
|
||||||
|
retrievalMode: normalizeRetrievalMode(existing?.retrievalMode),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
};
|
||||||
const getAiBotPluginToolList = async () => {
|
const getAiBotPluginToolList = async () => {
|
||||||
api
|
api
|
||||||
.post('/api/v1/pluginItem/tool/list', { botId: botId.value })
|
.post('/api/v1/pluginItem/tool/list', { botId: botId.value })
|
||||||
@@ -226,6 +262,7 @@ const getAiBotKnowledgeList = async () => {
|
|||||||
knowledgeData.value = res.data.map((item: any) => {
|
knowledgeData.value = res.data.map((item: any) => {
|
||||||
return {
|
return {
|
||||||
recordId: item.id,
|
recordId: item.id,
|
||||||
|
retrievalMode: normalizeRetrievalMode(item.options?.retrievalMode),
|
||||||
...item.knowledge,
|
...item.knowledge,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
@@ -419,7 +456,7 @@ const copyText = async (value: string) => {
|
|||||||
textarea.style.opacity = '0';
|
textarea.style.opacity = '0';
|
||||||
textarea.style.pointerEvents = 'none';
|
textarea.style.pointerEvents = 'none';
|
||||||
|
|
||||||
document.body.appendChild(textarea);
|
document.body.append(textarea);
|
||||||
|
|
||||||
const selection = document.getSelection();
|
const selection = document.getSelection();
|
||||||
const previousRange =
|
const previousRange =
|
||||||
@@ -430,7 +467,7 @@ const copyText = async (value: string) => {
|
|||||||
textarea.setSelectionRange(0, textarea.value.length);
|
textarea.setSelectionRange(0, textarea.value.length);
|
||||||
const copied = document.execCommand('copy');
|
const copied = document.execCommand('copy');
|
||||||
|
|
||||||
document.body.removeChild(textarea);
|
textarea.remove();
|
||||||
if (selection) {
|
if (selection) {
|
||||||
selection.removeAllRanges();
|
selection.removeAllRanges();
|
||||||
if (previousRange) {
|
if (previousRange) {
|
||||||
@@ -543,7 +580,7 @@ const confirmUpdateAiBotKnowledge = (data: any) => {
|
|||||||
api
|
api
|
||||||
.post('/api/v1/botKnowledge/updateBotKnowledgeIds', {
|
.post('/api/v1/botKnowledge/updateBotKnowledgeIds', {
|
||||||
botId: botId.value,
|
botId: botId.value,
|
||||||
knowledgeIds: data,
|
knowledgeBindings: buildKnowledgeBindingsPayload(data),
|
||||||
})
|
})
|
||||||
.then((res) => {
|
.then((res) => {
|
||||||
if (res.errorCode === 0) {
|
if (res.errorCode === 0) {
|
||||||
@@ -555,6 +592,31 @@ const confirmUpdateAiBotKnowledge = (data: any) => {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const updateKnowledgeBindings = (bindings: BotKnowledgeBindingItem[]) => {
|
||||||
|
return api.post('/api/v1/botKnowledge/updateBotKnowledgeIds', {
|
||||||
|
botId: botId.value,
|
||||||
|
knowledgeBindings: bindings,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleKnowledgeRetrievalModeChange = async (
|
||||||
|
item: any,
|
||||||
|
value: RetrievalMode | string,
|
||||||
|
) => {
|
||||||
|
item.retrievalMode = normalizeRetrievalMode(value);
|
||||||
|
const bindings = knowledgeData.value.map((knowledgeItem: any) => ({
|
||||||
|
knowledgeId: String(knowledgeItem.id),
|
||||||
|
retrievalMode: normalizeRetrievalMode(knowledgeItem.retrievalMode),
|
||||||
|
}));
|
||||||
|
const res = await updateKnowledgeBindings(bindings);
|
||||||
|
if (res.errorCode === 0) {
|
||||||
|
ElMessage.success($t('message.updateOkMessage'));
|
||||||
|
getAiBotKnowledgeList();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ElMessage.error(res.message || $t('message.updateFailMessage'));
|
||||||
|
};
|
||||||
|
|
||||||
const confirmUpdateAiBotWorkflow = (data: any) => {
|
const confirmUpdateAiBotWorkflow = (data: any) => {
|
||||||
api
|
api
|
||||||
.post('/api/v1/botWorkflow/updateBotWorkflowIds', {
|
.post('/api/v1/botWorkflow/updateBotWorkflowIds', {
|
||||||
@@ -616,7 +678,20 @@ const deleteBotMcpTool = (item: any) => {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const deleteKnowledge = (item: any) => {
|
const deleteKnowledge = async (item: any) => {
|
||||||
|
try {
|
||||||
|
await ElMessageBox.confirm(
|
||||||
|
$t('message.deleteAlert'),
|
||||||
|
$t('message.noticeTitle'),
|
||||||
|
{
|
||||||
|
confirmButtonText: $t('button.confirm'),
|
||||||
|
cancelButtonText: $t('button.cancel'),
|
||||||
|
type: 'warning',
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
api
|
api
|
||||||
.post('/api/v1/botKnowledge/remove', {
|
.post('/api/v1/botKnowledge/remove', {
|
||||||
id: item.recordId,
|
id: item.recordId,
|
||||||
@@ -840,10 +915,10 @@ const handleBasicInfoChange = async (
|
|||||||
<div class="bot-avatar-panel">
|
<div class="bot-avatar-panel">
|
||||||
<span class="bot-avatar-label">{{ $t('common.avatar') }}</span>
|
<span class="bot-avatar-label">{{ $t('common.avatar') }}</span>
|
||||||
<div
|
<div
|
||||||
:class="[
|
class="bot-avatar-upload-wrap"
|
||||||
'bot-avatar-upload-wrap',
|
:class="
|
||||||
!hasSavePermission || updatingBotIcon ? 'is-disabled' : '',
|
!hasSavePermission || updatingBotIcon ? 'is-disabled' : ''
|
||||||
]"
|
"
|
||||||
>
|
>
|
||||||
<UploadAvatar
|
<UploadAvatar
|
||||||
v-if="botInfo"
|
v-if="botInfo"
|
||||||
@@ -1105,7 +1180,53 @@ const handleBasicInfoChange = async (
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
<CollapseViewItem :data="knowledgeData" @delete="deleteKnowledge" />
|
<div class="knowledge-binding-list">
|
||||||
|
<div
|
||||||
|
v-for="item in knowledgeData"
|
||||||
|
:key="item.recordId"
|
||||||
|
class="knowledge-binding-item"
|
||||||
|
>
|
||||||
|
<div class="knowledge-binding-main">
|
||||||
|
<ElAvatar
|
||||||
|
:src="item.icon || fallbackAvatarUrl"
|
||||||
|
shape="circle"
|
||||||
|
/>
|
||||||
|
<div class="knowledge-binding-content">
|
||||||
|
<div class="knowledge-binding-title">
|
||||||
|
{{ item.title }}
|
||||||
|
</div>
|
||||||
|
<div class="knowledge-binding-description">
|
||||||
|
{{ item.description }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="knowledge-binding-actions" @click.stop>
|
||||||
|
<ElSelect
|
||||||
|
:model-value="item.retrievalMode"
|
||||||
|
class="knowledge-binding-select"
|
||||||
|
size="small"
|
||||||
|
@change="
|
||||||
|
(value) => handleKnowledgeRetrievalModeChange(item, value)
|
||||||
|
"
|
||||||
|
>
|
||||||
|
<ElOption
|
||||||
|
v-for="option in retrievalModeOptions"
|
||||||
|
:key="option.value"
|
||||||
|
:label="option.label"
|
||||||
|
:value="option.value"
|
||||||
|
/>
|
||||||
|
</ElSelect>
|
||||||
|
<ElIcon
|
||||||
|
class="knowledge-binding-delete"
|
||||||
|
color="var(--el-color-danger)"
|
||||||
|
size="20px"
|
||||||
|
@click="deleteKnowledge(item)"
|
||||||
|
>
|
||||||
|
<Delete />
|
||||||
|
</ElIcon>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</ElCollapseItem>
|
</ElCollapseItem>
|
||||||
<ElCollapseItem>
|
<ElCollapseItem>
|
||||||
<template #title>
|
<template #title>
|
||||||
@@ -1779,4 +1900,64 @@ const handleBasicInfoChange = async (
|
|||||||
height: 50px;
|
height: 50px;
|
||||||
border-radius: 50%;
|
border-radius: 50%;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-list {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 12px;
|
||||||
|
padding: 10px;
|
||||||
|
background-color: var(--bot-collapse-itme-back);
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-item {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 16px;
|
||||||
|
padding: 12px;
|
||||||
|
border-radius: 8px;
|
||||||
|
background-color: hsl(var(--background));
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-main {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
min-width: 0;
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-content {
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-title {
|
||||||
|
font-size: 12px;
|
||||||
|
font-weight: 500;
|
||||||
|
line-height: 24px;
|
||||||
|
color: var(--el-text-color-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-description {
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 20px;
|
||||||
|
color: var(--el-text-color-secondary);
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-actions {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-select {
|
||||||
|
width: 140px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-binding-delete {
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import type { FormInstance } from 'element-plus';
|
import type { FormInstance } from 'element-plus';
|
||||||
|
|
||||||
import { computed, onMounted, ref, watch } from 'vue';
|
import { onMounted, ref, watch } from 'vue';
|
||||||
|
|
||||||
import { InfoFilled } from '@element-plus/icons-vue';
|
import { InfoFilled } from '@element-plus/icons-vue';
|
||||||
import {
|
import {
|
||||||
@@ -80,29 +80,6 @@ watch(
|
|||||||
|
|
||||||
const embeddingLlmList = ref<any>([]);
|
const embeddingLlmList = ref<any>([]);
|
||||||
const rerankerLlmList = ref<any>([]);
|
const rerankerLlmList = ref<any>([]);
|
||||||
const vecotrDatabaseList = ref<any>([
|
|
||||||
{ value: 'milvus', label: 'Milvus' },
|
|
||||||
{ value: 'redis', label: 'Redis' },
|
|
||||||
{ value: 'opensearch', label: 'OpenSearch' },
|
|
||||||
{ value: 'elasticsearch', label: 'ElasticSearch' },
|
|
||||||
{ value: 'aliyun', label: $t('documentCollection.alibabaCloud') },
|
|
||||||
{ value: 'qcloud', label: $t('documentCollection.tencentCloud') },
|
|
||||||
]);
|
|
||||||
|
|
||||||
const milvusVectorStoreConfigPlaceholder =
|
|
||||||
'uri=http://127.0.0.1:19530\n' +
|
|
||||||
'databaseName=default\n' +
|
|
||||||
'token=\n' +
|
|
||||||
'username=\n' +
|
|
||||||
'password=\n' +
|
|
||||||
'autoCreateCollection=true';
|
|
||||||
|
|
||||||
const vectorStoreConfigPlaceholder = computed(() => {
|
|
||||||
return entity.value?.vectorStoreType === 'milvus'
|
|
||||||
? milvusVectorStoreConfigPlaceholder
|
|
||||||
: '';
|
|
||||||
});
|
|
||||||
|
|
||||||
const getEmbeddingLlmListData = async () => {
|
const getEmbeddingLlmListData = async () => {
|
||||||
try {
|
try {
|
||||||
const url = `/api/v1/documentCollection/modelList?modelType=embeddingModel`;
|
const url = `/api/v1/documentCollection/modelList?modelType=embeddingModel`;
|
||||||
@@ -146,15 +123,6 @@ const rules = ref({
|
|||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
||||||
],
|
],
|
||||||
title: [{ required: true, message: $t('message.required'), trigger: 'blur' }],
|
title: [{ required: true, message: $t('message.required'), trigger: 'blur' }],
|
||||||
vectorStoreType: [
|
|
||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
|
||||||
],
|
|
||||||
vectorStoreCollection: [
|
|
||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
|
||||||
],
|
|
||||||
vectorStoreConfig: [
|
|
||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
|
||||||
],
|
|
||||||
vectorEmbedModelId: [
|
vectorEmbedModelId: [
|
||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
||||||
],
|
],
|
||||||
@@ -239,50 +207,6 @@ async function save() {
|
|||||||
:placeholder="$t('documentCollection.placeholder.description')"
|
:placeholder="$t('documentCollection.placeholder.description')"
|
||||||
/>
|
/>
|
||||||
</ElFormItem>
|
</ElFormItem>
|
||||||
<!-- <ElFormItem
|
|
||||||
prop="vectorStoreEnable"
|
|
||||||
:label="$t('documentCollection.vectorStoreEnable')"
|
|
||||||
>
|
|
||||||
<ElSwitch v-model="entity.vectorStoreEnable" />
|
|
||||||
</ElFormItem>-->
|
|
||||||
<ElFormItem
|
|
||||||
prop="vectorStoreType"
|
|
||||||
:label="$t('documentCollection.vectorStoreType')"
|
|
||||||
>
|
|
||||||
<ElSelect
|
|
||||||
v-model="entity.vectorStoreType"
|
|
||||||
:placeholder="$t('documentCollection.placeholder.vectorStoreType')"
|
|
||||||
>
|
|
||||||
<ElOption
|
|
||||||
v-for="item in vecotrDatabaseList"
|
|
||||||
:key="item.value"
|
|
||||||
:label="item.label"
|
|
||||||
:value="item.value || ''"
|
|
||||||
/>
|
|
||||||
</ElSelect>
|
|
||||||
</ElFormItem>
|
|
||||||
<ElFormItem
|
|
||||||
prop="vectorStoreCollection"
|
|
||||||
:label="$t('documentCollection.vectorStoreCollection')"
|
|
||||||
>
|
|
||||||
<ElInput
|
|
||||||
v-model.trim="entity.vectorStoreCollection"
|
|
||||||
:placeholder="
|
|
||||||
$t('documentCollection.placeholder.vectorStoreCollection')
|
|
||||||
"
|
|
||||||
/>
|
|
||||||
</ElFormItem>
|
|
||||||
<ElFormItem
|
|
||||||
prop="vectorStoreConfig"
|
|
||||||
:label="$t('documentCollection.vectorStoreConfig')"
|
|
||||||
>
|
|
||||||
<ElInput
|
|
||||||
v-model.trim="entity.vectorStoreConfig"
|
|
||||||
:rows="4"
|
|
||||||
type="textarea"
|
|
||||||
:placeholder="vectorStoreConfigPlaceholder"
|
|
||||||
/>
|
|
||||||
</ElFormItem>
|
|
||||||
<ElFormItem prop="vectorEmbedModelId">
|
<ElFormItem prop="vectorEmbedModelId">
|
||||||
<template #label>
|
<template #label>
|
||||||
<span style="display: flex; align-items: center">
|
<span style="display: flex; align-items: center">
|
||||||
@@ -373,12 +297,6 @@ async function save() {
|
|||||||
/>
|
/>
|
||||||
</ElSelect>
|
</ElSelect>
|
||||||
</ElFormItem>
|
</ElFormItem>
|
||||||
<ElFormItem
|
|
||||||
prop="searchEngineEnable"
|
|
||||||
:label="$t('documentCollection.searchEngineEnable')"
|
|
||||||
>
|
|
||||||
<ElSwitch v-model="entity.searchEngineEnable" />
|
|
||||||
</ElFormItem>
|
|
||||||
<ElFormItem style="margin-top: 20px; text-align: right">
|
<ElFormItem style="margin-top: 20px; text-align: right">
|
||||||
<ElButton
|
<ElButton
|
||||||
type="primary"
|
type="primary"
|
||||||
|
|||||||
@@ -60,15 +60,6 @@ onMounted(async () => {
|
|||||||
const saveForm = ref<FormInstance>();
|
const saveForm = ref<FormInstance>();
|
||||||
const dialogVisible = ref(false);
|
const dialogVisible = ref(false);
|
||||||
const isAdd = ref(true);
|
const isAdd = ref(true);
|
||||||
const vecotrDatabaseList = ref<any>([
|
|
||||||
{ value: 'milvus', label: 'Milvus' },
|
|
||||||
{ value: 'redis', label: 'Redis' },
|
|
||||||
{ value: 'opensearch', label: 'OpenSearch' },
|
|
||||||
{ value: 'elasticsearch', label: 'ElasticSearch' },
|
|
||||||
{ value: 'aliyun', label: $t('documentCollection.alibabaCloud') },
|
|
||||||
{ value: 'qcloud', label: $t('documentCollection.tencentCloud') },
|
|
||||||
]);
|
|
||||||
|
|
||||||
const defaultEntity = {
|
const defaultEntity = {
|
||||||
collectionType: 'DOCUMENT',
|
collectionType: 'DOCUMENT',
|
||||||
alias: '',
|
alias: '',
|
||||||
@@ -125,15 +116,6 @@ const rules = ref({
|
|||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
||||||
],
|
],
|
||||||
title: [{ required: true, message: $t('message.required'), trigger: 'blur' }],
|
title: [{ required: true, message: $t('message.required'), trigger: 'blur' }],
|
||||||
vectorStoreType: [
|
|
||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
|
||||||
],
|
|
||||||
vectorStoreCollection: [
|
|
||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
|
||||||
],
|
|
||||||
vectorStoreConfig: [
|
|
||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
|
||||||
],
|
|
||||||
vectorEmbedModelId: [
|
vectorEmbedModelId: [
|
||||||
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
{ required: true, message: $t('message.required'), trigger: 'blur' },
|
||||||
],
|
],
|
||||||
@@ -307,49 +289,6 @@ defineExpose({
|
|||||||
:placeholder="$t('documentCollection.placeholder.description')"
|
:placeholder="$t('documentCollection.placeholder.description')"
|
||||||
/>
|
/>
|
||||||
</ElFormItem>
|
</ElFormItem>
|
||||||
<!-- <ElFormItem
|
|
||||||
prop="vectorStoreEnable"
|
|
||||||
:label="$t('documentCollection.vectorStoreEnable')"
|
|
||||||
>
|
|
||||||
<ElSwitch v-model="entity.vectorStoreEnable" />
|
|
||||||
</ElFormItem>-->
|
|
||||||
<ElFormItem
|
|
||||||
prop="vectorStoreType"
|
|
||||||
:label="$t('documentCollection.vectorStoreType')"
|
|
||||||
>
|
|
||||||
<ElSelect
|
|
||||||
v-model="entity.vectorStoreType"
|
|
||||||
:placeholder="$t('documentCollection.placeholder.vectorStoreType')"
|
|
||||||
>
|
|
||||||
<ElOption
|
|
||||||
v-for="item in vecotrDatabaseList"
|
|
||||||
:key="item.value"
|
|
||||||
:label="item.label"
|
|
||||||
:value="item.value || ''"
|
|
||||||
/>
|
|
||||||
</ElSelect>
|
|
||||||
</ElFormItem>
|
|
||||||
<ElFormItem
|
|
||||||
prop="vectorStoreCollection"
|
|
||||||
:label="$t('documentCollection.vectorStoreCollection')"
|
|
||||||
>
|
|
||||||
<ElInput
|
|
||||||
v-model.trim="entity.vectorStoreCollection"
|
|
||||||
:placeholder="
|
|
||||||
$t('documentCollection.placeholder.vectorStoreCollection')
|
|
||||||
"
|
|
||||||
/>
|
|
||||||
</ElFormItem>
|
|
||||||
<ElFormItem
|
|
||||||
prop="vectorStoreConfig"
|
|
||||||
:label="$t('documentCollection.vectorStoreConfig')"
|
|
||||||
>
|
|
||||||
<ElInput
|
|
||||||
v-model.trim="entity.vectorStoreConfig"
|
|
||||||
:rows="4"
|
|
||||||
type="textarea"
|
|
||||||
/>
|
|
||||||
</ElFormItem>
|
|
||||||
<ElFormItem prop="vectorEmbedModelId">
|
<ElFormItem prop="vectorEmbedModelId">
|
||||||
<template #label>
|
<template #label>
|
||||||
<span style="display: flex; align-items: center">
|
<span style="display: flex; align-items: center">
|
||||||
@@ -440,12 +379,6 @@ defineExpose({
|
|||||||
/>
|
/>
|
||||||
</ElSelect>
|
</ElSelect>
|
||||||
</ElFormItem>
|
</ElFormItem>
|
||||||
<ElFormItem
|
|
||||||
prop="searchEngineEnable"
|
|
||||||
:label="$t('documentCollection.searchEngineEnable')"
|
|
||||||
>
|
|
||||||
<ElSwitch v-model="entity.searchEngineEnable" />
|
|
||||||
</ElFormItem>
|
|
||||||
</ElForm>
|
</ElForm>
|
||||||
</EasyFlowFormModal>
|
</EasyFlowFormModal>
|
||||||
</template>
|
</template>
|
||||||
|
|||||||
@@ -3,20 +3,40 @@ import { ref } from 'vue';
|
|||||||
|
|
||||||
import { $t } from '@easyflow/locales';
|
import { $t } from '@easyflow/locales';
|
||||||
|
|
||||||
import { ElButton, ElInput, ElMessage } from 'element-plus';
|
import { ElButton, ElInput, ElMessage, ElOption, ElSelect } from 'element-plus';
|
||||||
|
|
||||||
import { api } from '#/api/request';
|
import { api } from '#/api/request';
|
||||||
import PreviewSearchKnowledge from '#/views/ai/documentCollection/PreviewSearchKnowledge.vue';
|
import PreviewSearchKnowledge from '#/views/ai/documentCollection/PreviewSearchKnowledge.vue';
|
||||||
|
|
||||||
|
type RetrievalMode = 'HYBRID' | 'KEYWORD' | 'VECTOR';
|
||||||
|
|
||||||
|
interface SearchResultItem {
|
||||||
|
sorting: number;
|
||||||
|
content: string;
|
||||||
|
score?: number;
|
||||||
|
hitSource?: 'BOTH' | 'KEYWORD' | 'VECTOR';
|
||||||
|
vectorScore?: number;
|
||||||
|
keywordScore?: number;
|
||||||
|
}
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
knowledgeId: {
|
knowledgeId: {
|
||||||
type: String,
|
type: String,
|
||||||
required: true,
|
required: true,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const searchDataList = ref([]);
|
|
||||||
|
const searchDataList = ref<SearchResultItem[]>([]);
|
||||||
const keyword = ref('');
|
const keyword = ref('');
|
||||||
|
const retrievalMode = ref<RetrievalMode>('HYBRID');
|
||||||
const previewSearchKnowledgeRef = ref();
|
const previewSearchKnowledgeRef = ref();
|
||||||
|
|
||||||
|
const retrievalModeOptions = [
|
||||||
|
{ label: $t('bot.retrievalModes.hybrid'), value: 'HYBRID' },
|
||||||
|
{ label: $t('bot.retrievalModes.vector'), value: 'VECTOR' },
|
||||||
|
{ label: $t('bot.retrievalModes.keyword'), value: 'KEYWORD' },
|
||||||
|
];
|
||||||
|
|
||||||
const handleSearch = () => {
|
const handleSearch = () => {
|
||||||
if (!keyword.value) {
|
if (!keyword.value) {
|
||||||
ElMessage.error($t('message.pleaseInputContent'));
|
ElMessage.error($t('message.pleaseInputContent'));
|
||||||
@@ -24,12 +44,22 @@ const handleSearch = () => {
|
|||||||
}
|
}
|
||||||
previewSearchKnowledgeRef.value.loadingContent(true);
|
previewSearchKnowledgeRef.value.loadingContent(true);
|
||||||
api
|
api
|
||||||
.get(
|
.get('/api/v1/documentCollection/search', {
|
||||||
`/api/v1/documentCollection/search?knowledgeId=${props.knowledgeId}&keyword=${keyword.value}`,
|
params: {
|
||||||
)
|
knowledgeId: props.knowledgeId,
|
||||||
|
keyword: keyword.value,
|
||||||
|
retrievalMode: retrievalMode.value,
|
||||||
|
},
|
||||||
|
})
|
||||||
.then((res) => {
|
.then((res) => {
|
||||||
previewSearchKnowledgeRef.value.loadingContent(false);
|
|
||||||
searchDataList.value = res.data;
|
searchDataList.value = res.data;
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
ElMessage.error($t('documentCollection.searchFailed'));
|
||||||
|
searchDataList.value = [];
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
previewSearchKnowledgeRef.value.loadingContent(false);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
</script>
|
</script>
|
||||||
@@ -40,7 +70,16 @@ const handleSearch = () => {
|
|||||||
<ElInput
|
<ElInput
|
||||||
v-model="keyword"
|
v-model="keyword"
|
||||||
:placeholder="$t('common.searchPlaceholder')"
|
:placeholder="$t('common.searchPlaceholder')"
|
||||||
|
@keyup.enter="handleSearch"
|
||||||
/>
|
/>
|
||||||
|
<ElSelect v-model="retrievalMode" class="retrieval-select">
|
||||||
|
<ElOption
|
||||||
|
v-for="item in retrievalModeOptions"
|
||||||
|
:key="item.value"
|
||||||
|
:label="item.label"
|
||||||
|
:value="item.value"
|
||||||
|
/>
|
||||||
|
</ElSelect>
|
||||||
<ElButton type="primary" @click="handleSearch">
|
<ElButton type="primary" @click="handleSearch">
|
||||||
{{ $t('button.query') }}
|
{{ $t('button.query') }}
|
||||||
</ElButton>
|
</ElButton>
|
||||||
@@ -48,6 +87,7 @@ const handleSearch = () => {
|
|||||||
<div class="search-result">
|
<div class="search-result">
|
||||||
<PreviewSearchKnowledge
|
<PreviewSearchKnowledge
|
||||||
:data="searchDataList"
|
:data="searchDataList"
|
||||||
|
:retrieval-mode="retrievalMode"
|
||||||
ref="previewSearchKnowledgeRef"
|
ref="previewSearchKnowledgeRef"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
@@ -68,6 +108,12 @@ const handleSearch = () => {
|
|||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
gap: 12px;
|
gap: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.retrieval-select {
|
||||||
|
width: 180px;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
.search-result {
|
.search-result {
|
||||||
padding-top: 20px;
|
padding-top: 20px;
|
||||||
flex: 1;
|
flex: 1;
|
||||||
|
|||||||
@@ -10,9 +10,6 @@ import {
|
|||||||
ElFormItem,
|
ElFormItem,
|
||||||
ElInputNumber,
|
ElInputNumber,
|
||||||
ElMessage,
|
ElMessage,
|
||||||
ElOption,
|
|
||||||
ElSelect,
|
|
||||||
ElSwitch,
|
|
||||||
ElTooltip,
|
ElTooltip,
|
||||||
} from 'element-plus';
|
} from 'element-plus';
|
||||||
|
|
||||||
@@ -32,7 +29,6 @@ const props = defineProps({
|
|||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
getDocumentCollectionConfig();
|
getDocumentCollectionConfig();
|
||||||
});
|
});
|
||||||
const searchEngineEnable = ref(false);
|
|
||||||
const baseOptions = ref<Record<string, any>>({});
|
const baseOptions = ref<Record<string, any>>({});
|
||||||
const getDocumentCollectionConfig = () => {
|
const getDocumentCollectionConfig = () => {
|
||||||
api
|
api
|
||||||
@@ -46,16 +42,13 @@ const getDocumentCollectionConfig = () => {
|
|||||||
: 5;
|
: 5;
|
||||||
searchConfig.simThreshold = options.simThreshold
|
searchConfig.simThreshold = options.simThreshold
|
||||||
? Number(options.simThreshold)
|
? Number(options.simThreshold)
|
||||||
: 0.5;
|
: 0.6;
|
||||||
searchConfig.searchEngineType = options.searchEngineType || 'lucene';
|
|
||||||
searchEngineEnable.value = !!data.searchEngineEnable;
|
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const searchConfig = reactive({
|
const searchConfig = reactive({
|
||||||
docRecallMaxNum: 5,
|
docRecallMaxNum: 5,
|
||||||
simThreshold: 0.5,
|
simThreshold: 0.6,
|
||||||
searchEngineType: 'lucene',
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const submitConfig = () => {
|
const submitConfig = () => {
|
||||||
@@ -69,9 +62,7 @@ const submitConfig = () => {
|
|||||||
...baseOptions.value,
|
...baseOptions.value,
|
||||||
docRecallMaxNum: searchConfig.docRecallMaxNum,
|
docRecallMaxNum: searchConfig.docRecallMaxNum,
|
||||||
simThreshold: searchConfig.simThreshold,
|
simThreshold: searchConfig.simThreshold,
|
||||||
searchEngineType: searchConfig.searchEngineType,
|
|
||||||
},
|
},
|
||||||
searchEngineEnable: searchEngineEnable.value,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
api
|
api
|
||||||
@@ -85,26 +76,6 @@ const submitConfig = () => {
|
|||||||
console.error('保存配置失败:', error);
|
console.error('保存配置失败:', error);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const searchEngineOptions = [
|
|
||||||
{
|
|
||||||
label: 'Lucene',
|
|
||||||
value: 'lucene',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
label: 'ElasticSearch',
|
|
||||||
value: 'elasticSearch',
|
|
||||||
},
|
|
||||||
];
|
|
||||||
const handleSearchEngineEnableChange = () => {
|
|
||||||
if (!props.manageable) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
api.post('/api/v1/documentCollection/update', {
|
|
||||||
id: props.documentCollectionId,
|
|
||||||
searchEngineEnable: searchEngineEnable.value,
|
|
||||||
});
|
|
||||||
};
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
@@ -176,71 +147,12 @@ const handleSearchEngineEnableChange = () => {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</ElFormItem>
|
</ElFormItem>
|
||||||
|
|
||||||
<!-- 搜索引擎启用开关 -->
|
|
||||||
<ElFormItem class="form-item">
|
|
||||||
<div class="form-item-label">
|
|
||||||
<span>{{
|
|
||||||
$t('documentCollectionSearch.searchEngineEnable.label')
|
|
||||||
}}</span>
|
|
||||||
<ElTooltip
|
|
||||||
:content="$t('documentCollectionSearch.searchEngineEnable.tooltip')"
|
|
||||||
placement="top"
|
|
||||||
effect="dark"
|
|
||||||
class="label-tooltip"
|
|
||||||
>
|
|
||||||
<InfoFilled class="info-icon" />
|
|
||||||
</ElTooltip>
|
|
||||||
</div>
|
|
||||||
<div class="form-item-content">
|
|
||||||
<ElSwitch
|
|
||||||
v-model="searchEngineEnable"
|
|
||||||
@change="handleSearchEngineEnableChange"
|
|
||||||
:active-text="$t('documentCollectionSearch.switch.on')"
|
|
||||||
:inactive-text="$t('documentCollectionSearch.switch.off')"
|
|
||||||
class="form-control switch-control"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
</ElFormItem>
|
|
||||||
|
|
||||||
<!-- 通过 searchEngineEnable 控制显示/隐藏 -->
|
|
||||||
<ElFormItem
|
|
||||||
v-if="searchEngineEnable"
|
|
||||||
prop="searchEngineType"
|
|
||||||
class="form-item"
|
|
||||||
>
|
|
||||||
<div class="form-item-label">
|
|
||||||
<span>{{
|
|
||||||
$t('documentCollectionSearch.searchEngineType.label')
|
|
||||||
}}</span>
|
|
||||||
<ElTooltip
|
|
||||||
:content="$t('documentCollectionSearch.searchEngineType.tooltip')"
|
|
||||||
placement="top"
|
|
||||||
effect="dark"
|
|
||||||
class="label-tooltip"
|
|
||||||
>
|
|
||||||
<InfoFilled class="info-icon" />
|
|
||||||
</ElTooltip>
|
|
||||||
</div>
|
|
||||||
<div class="form-item-content">
|
|
||||||
<ElSelect
|
|
||||||
v-model="searchConfig.searchEngineType"
|
|
||||||
:placeholder="
|
|
||||||
$t('documentCollectionSearch.searchEngineType.placeholder')
|
|
||||||
"
|
|
||||||
class="form-control"
|
|
||||||
>
|
|
||||||
<ElOption
|
|
||||||
v-for="option in searchEngineOptions"
|
|
||||||
:key="option.value"
|
|
||||||
:label="option.label"
|
|
||||||
:value="option.value"
|
|
||||||
/>
|
|
||||||
</ElSelect>
|
|
||||||
</div>
|
|
||||||
</ElFormItem>
|
|
||||||
</ElForm>
|
</ElForm>
|
||||||
|
|
||||||
|
<div class="config-hint">
|
||||||
|
{{ $t('documentCollectionSearch.engineHint') }}
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="config-footer">
|
<div class="config-footer">
|
||||||
<ElButton
|
<ElButton
|
||||||
type="primary"
|
type="primary"
|
||||||
@@ -352,6 +264,13 @@ const handleSearchEngineEnableChange = () => {
|
|||||||
padding: 8px 0;
|
padding: 8px 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.config-hint {
|
||||||
|
margin: 12px 0 4px;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.6;
|
||||||
|
color: var(--el-text-color-secondary);
|
||||||
|
}
|
||||||
|
|
||||||
.config-footer {
|
.config-footer {
|
||||||
position: sticky;
|
position: sticky;
|
||||||
bottom: 0;
|
bottom: 0;
|
||||||
|
|||||||
@@ -1,14 +1,21 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { ref } from 'vue';
|
import { ref } from 'vue';
|
||||||
|
|
||||||
|
import { $t } from '@easyflow/locales';
|
||||||
|
|
||||||
import { Document } from '@element-plus/icons-vue';
|
import { Document } from '@element-plus/icons-vue';
|
||||||
import { ElButton, ElIcon } from 'element-plus';
|
import { ElButton, ElIcon, ElTag } from 'element-plus';
|
||||||
// 定义类型接口(与 React 版本一致)
|
|
||||||
|
type RetrievalMode = 'HYBRID' | 'KEYWORD' | 'VECTOR';
|
||||||
|
type HitSource = 'BOTH' | 'KEYWORD' | 'VECTOR';
|
||||||
|
|
||||||
interface PreviewItem {
|
interface PreviewItem {
|
||||||
sorting: string;
|
sorting: number | string;
|
||||||
content: string;
|
content: string;
|
||||||
score: string;
|
score?: number | string;
|
||||||
|
hitSource?: HitSource;
|
||||||
}
|
}
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
hideScore: {
|
hideScore: {
|
||||||
type: Boolean,
|
type: Boolean,
|
||||||
@@ -46,8 +53,37 @@ const props = defineProps({
|
|||||||
type: Boolean,
|
type: Boolean,
|
||||||
default: false,
|
default: false,
|
||||||
},
|
},
|
||||||
|
retrievalMode: {
|
||||||
|
type: String as () => RetrievalMode,
|
||||||
|
default: 'HYBRID',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const loadingStatus = ref(false);
|
const loadingStatus = ref(false);
|
||||||
|
|
||||||
|
const resolveHitSourceLabel = (hitSource?: HitSource) => {
|
||||||
|
if (hitSource === 'VECTOR') {
|
||||||
|
return $t('documentCollection.hitSources.vector');
|
||||||
|
}
|
||||||
|
if (hitSource === 'KEYWORD') {
|
||||||
|
return $t('documentCollection.hitSources.keyword');
|
||||||
|
}
|
||||||
|
if (hitSource === 'BOTH') {
|
||||||
|
return $t('documentCollection.hitSources.both');
|
||||||
|
}
|
||||||
|
return '';
|
||||||
|
};
|
||||||
|
|
||||||
|
const resolveHitSourceType = (hitSource?: HitSource) => {
|
||||||
|
if (hitSource === 'VECTOR') {
|
||||||
|
return 'success';
|
||||||
|
}
|
||||||
|
if (hitSource === 'KEYWORD') {
|
||||||
|
return 'warning';
|
||||||
|
}
|
||||||
|
return 'info';
|
||||||
|
};
|
||||||
|
|
||||||
defineExpose({
|
defineExpose({
|
||||||
loadingContent: (state: boolean) => {
|
loadingContent: (state: boolean) => {
|
||||||
loadingStatus.value = state;
|
loadingStatus.value = state;
|
||||||
@@ -89,10 +125,22 @@ defineExpose({
|
|||||||
{{ item.sorting ?? index + 1 }}
|
{{ item.sorting ?? index + 1 }}
|
||||||
</div>
|
</div>
|
||||||
<div class="el-list-item-meta">
|
<div class="el-list-item-meta">
|
||||||
<div v-if="!hideScore">
|
<div v-if="!hideScore" class="score-text">
|
||||||
{{ $t('documentCollection.similarityScore') }}: {{ item.score }}
|
{{ $t('documentCollection.similarityScore') }}: {{ item.score }}
|
||||||
</div>
|
</div>
|
||||||
<div class="content-desc">{{ item.content }}</div>
|
<div class="content-desc">{{ item.content }}</div>
|
||||||
|
<div
|
||||||
|
v-if="retrievalMode === 'HYBRID' && item.hitSource"
|
||||||
|
class="hit-source-row"
|
||||||
|
>
|
||||||
|
<ElTag
|
||||||
|
size="small"
|
||||||
|
effect="plain"
|
||||||
|
:type="resolveHitSourceType(item.hitSource)"
|
||||||
|
>
|
||||||
|
{{ resolveHitSourceLabel(item.hitSource) }}
|
||||||
|
</ElTag>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -203,10 +251,20 @@ defineExpose({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.score-text {
|
||||||
|
color: var(--el-text-color-secondary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.hit-source-row {
|
||||||
|
display: flex;
|
||||||
|
justify-content: flex-end;
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
.el-list-item {
|
.el-list-item {
|
||||||
display: flex;
|
display: flex;
|
||||||
gap: 12px;
|
gap: 12px;
|
||||||
align-items: center;
|
align-items: flex-start;
|
||||||
padding: 18px;
|
padding: 18px;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,11 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
const options = getOptions();
|
const options = getOptions();
|
||||||
|
const retrievalModeOptions: SelectItem[] = [
|
||||||
|
{ value: 'HYBRID', label: '混合检索' },
|
||||||
|
{ value: 'VECTOR', label: '向量检索' },
|
||||||
|
{ value: 'KEYWORD', label: '关键词检索' }
|
||||||
|
];
|
||||||
|
|
||||||
let knowledgeArray = $state<SelectItem[]>([]);
|
let knowledgeArray = $state<SelectItem[]>([]);
|
||||||
onMount(async () => {
|
onMount(async () => {
|
||||||
@@ -78,6 +83,16 @@
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
$effect(() => {
|
||||||
|
if (!data.retrievalMode) {
|
||||||
|
updateNodeData(currentNodeId, () => {
|
||||||
|
return {
|
||||||
|
retrievalMode: 'HYBRID'
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|
||||||
@@ -134,6 +149,18 @@
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="setting-title">检索方式</div>
|
||||||
|
<div class="setting-item">
|
||||||
|
<Select items={retrievalModeOptions} style="width: 100%" placeholder="请选择检索方式" onSelect={(item)=>{
|
||||||
|
const newValue = item.value;
|
||||||
|
updateNodeData(currentNodeId, ()=>{
|
||||||
|
return {
|
||||||
|
retrievalMode: newValue
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}} value={data.retrievalMode ? [data.retrievalMode] : ['HYBRID']} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
<div class="setting-title">获取数据量</div>
|
<div class="setting-title">获取数据量</div>
|
||||||
<div class="setting-item">
|
<div class="setting-item">
|
||||||
@@ -184,4 +211,3 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
</style>
|
</style>
|
||||||
|
|
||||||
|
|||||||
5
pom.xml
5
pom.xml
@@ -182,6 +182,11 @@
|
|||||||
<artifactId>easy-agents-support</artifactId>
|
<artifactId>easy-agents-support</artifactId>
|
||||||
<version>${easy-agents.version}</version>
|
<version>${easy-agents.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.easyagents</groupId>
|
||||||
|
<artifactId>easy-agents-rag-retrieval</artifactId>
|
||||||
|
<version>${easy-agents.version}</version>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.easyagents</groupId>
|
<groupId>com.easyagents</groupId>
|
||||||
<artifactId>easy-agents-spring-boot-starter</artifactId>
|
<artifactId>easy-agents-spring-boot-starter</artifactId>
|
||||||
|
|||||||
Reference in New Issue
Block a user