知识库功能增强,支持Milvus,并优化相关逻辑
This commit is contained in:
@@ -4,6 +4,10 @@ import cn.dev33.satoken.annotation.SaCheckPermission;
|
|||||||
import com.easyagents.core.document.Document;
|
import com.easyagents.core.document.Document;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import tech.easyflow.ai.entity.BotDocumentCollection;
|
import tech.easyflow.ai.entity.BotDocumentCollection;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.service.BotDocumentCollectionService;
|
import tech.easyflow.ai.service.BotDocumentCollectionService;
|
||||||
@@ -12,10 +16,6 @@ import tech.easyflow.ai.service.DocumentCollectionService;
|
|||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
import tech.easyflow.common.web.controller.BaseCurdController;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
@@ -70,11 +70,14 @@ public class DocumentCollectionController extends BaseCurdController<DocumentCol
|
|||||||
|
|
||||||
|
|
||||||
if (isSave){
|
if (isSave){
|
||||||
Map<String, Object> options = new HashMap<>();
|
Map<String, Object> options = entity.getOptions() == null
|
||||||
|
? new HashMap<>()
|
||||||
|
: new HashMap<>(entity.getOptions());
|
||||||
if (entity.getSearchEngineEnable() == null){
|
if (entity.getSearchEngineEnable() == null){
|
||||||
entity.setSearchEngineEnable(false);
|
entity.setSearchEngineEnable(false);
|
||||||
}
|
}
|
||||||
options.put("canUpdateEmbeddingModel", true);
|
options.putIfAbsent(DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL, true);
|
||||||
|
options.putIfAbsent(DocumentCollection.KEY_RERANK_ENABLE, entity.getRerankModelId() != null);
|
||||||
entity.setOptions(options);
|
entity.setOptions(options);
|
||||||
}
|
}
|
||||||
return super.onSaveOrUpdateBefore(entity, isSave);
|
return super.onSaveOrUpdateBefore(entity, isSave);
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import com.easyagents.store.aliyun.AliyunVectorStore;
|
|||||||
import com.easyagents.store.aliyun.AliyunVectorStoreConfig;
|
import com.easyagents.store.aliyun.AliyunVectorStoreConfig;
|
||||||
import com.easyagents.store.elasticsearch.ElasticSearchVectorStore;
|
import com.easyagents.store.elasticsearch.ElasticSearchVectorStore;
|
||||||
import com.easyagents.store.elasticsearch.ElasticSearchVectorStoreConfig;
|
import com.easyagents.store.elasticsearch.ElasticSearchVectorStoreConfig;
|
||||||
|
import com.easyagents.store.milvus.MilvusVectorStore;
|
||||||
|
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
||||||
import com.easyagents.store.opensearch.OpenSearchVectorStore;
|
import com.easyagents.store.opensearch.OpenSearchVectorStore;
|
||||||
import com.easyagents.store.opensearch.OpenSearchVectorStoreConfig;
|
import com.easyagents.store.opensearch.OpenSearchVectorStoreConfig;
|
||||||
import com.easyagents.store.qcloud.QCloudVectorStore;
|
import com.easyagents.store.qcloud.QCloudVectorStore;
|
||||||
@@ -62,6 +64,11 @@ public class DocumentCollection extends DocumentCollectionBase {
|
|||||||
*/
|
*/
|
||||||
public static final String KEY_CAN_UPDATE_EMBEDDING_MODEL = "canUpdateEmbeddingModel";
|
public static final String KEY_CAN_UPDATE_EMBEDDING_MODEL = "canUpdateEmbeddingModel";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否启用重排模型
|
||||||
|
*/
|
||||||
|
public static final String KEY_RERANK_ENABLE = "rerankEnable";
|
||||||
|
|
||||||
public DocumentStore toDocumentStore() {
|
public DocumentStore toDocumentStore() {
|
||||||
String storeType = this.getVectorStoreType();
|
String storeType = this.getVectorStoreType();
|
||||||
if (StringUtil.noText(storeType)) {
|
if (StringUtil.noText(storeType)) {
|
||||||
@@ -73,8 +80,8 @@ public class DocumentCollection extends DocumentCollectionBase {
|
|||||||
switch (storeType.toLowerCase()) {
|
switch (storeType.toLowerCase()) {
|
||||||
case "redis":
|
case "redis":
|
||||||
return redisStore();
|
return redisStore();
|
||||||
// case "milvus":
|
case "milvus":
|
||||||
// return milvusStore();
|
return milvusStore();
|
||||||
case "opensearch":
|
case "opensearch":
|
||||||
return openSearchStore();
|
return openSearchStore();
|
||||||
case "elasticsearch":
|
case "elasticsearch":
|
||||||
@@ -101,10 +108,13 @@ public class DocumentCollection extends DocumentCollectionBase {
|
|||||||
return new RedisVectorStore(redisVectorStoreConfig);
|
return new RedisVectorStore(redisVectorStoreConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
// private DocumentStore milvusStore() {
|
private DocumentStore milvusStore() {
|
||||||
// MilvusVectorStoreConfig milvusVectorStoreConfig = getStoreConfig(MilvusVectorStoreConfig.class);
|
MilvusVectorStoreConfig milvusVectorStoreConfig = getStoreConfig(MilvusVectorStoreConfig.class);
|
||||||
// return new MilvusVectorStore(milvusVectorStoreConfig);
|
if (milvusVectorStoreConfig != null && StringUtil.noText(milvusVectorStoreConfig.getDefaultCollectionName())) {
|
||||||
// }
|
milvusVectorStoreConfig.setDefaultCollectionName(this.getVectorStoreCollection());
|
||||||
|
}
|
||||||
|
return new MilvusVectorStore(milvusVectorStoreConfig);
|
||||||
|
}
|
||||||
|
|
||||||
private DocumentStore openSearchStore() {
|
private DocumentStore openSearchStore() {
|
||||||
OpenSearchVectorStoreConfig openSearchVectorStoreConfig = getStoreConfig(OpenSearchVectorStoreConfig.class);
|
OpenSearchVectorStoreConfig openSearchVectorStoreConfig = getStoreConfig(OpenSearchVectorStoreConfig.class);
|
||||||
@@ -136,6 +146,23 @@ public class DocumentCollection extends DocumentCollectionBase {
|
|||||||
|
|
||||||
public Object getOptionsByKey(String key) {
|
public Object getOptionsByKey(String key) {
|
||||||
Map<String, Object> options = this.getOptions();
|
Map<String, Object> options = this.getOptions();
|
||||||
|
if (KEY_RERANK_ENABLE.equals(key)) {
|
||||||
|
if (options == null || !options.containsKey(KEY_RERANK_ENABLE)) {
|
||||||
|
return this.getRerankModelId() != null;
|
||||||
|
}
|
||||||
|
Object value = options.get(key);
|
||||||
|
if (value instanceof Boolean) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
if (value instanceof Number) {
|
||||||
|
return ((Number) value).intValue() != 0;
|
||||||
|
}
|
||||||
|
if (value instanceof String) {
|
||||||
|
return Boolean.parseBoolean((String) value);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (options == null) {
|
if (options == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package tech.easyflow.ai.service.impl;
|
|||||||
|
|
||||||
|
|
||||||
import com.easyagents.core.document.Document;
|
import com.easyagents.core.document.Document;
|
||||||
|
import com.easyagents.core.model.rerank.RerankException;
|
||||||
import com.easyagents.core.model.rerank.RerankModel;
|
import com.easyagents.core.model.rerank.RerankModel;
|
||||||
import com.easyagents.core.store.DocumentStore;
|
import com.easyagents.core.store.DocumentStore;
|
||||||
import com.easyagents.core.store.SearchWrapper;
|
import com.easyagents.core.store.SearchWrapper;
|
||||||
@@ -9,6 +10,8 @@ import com.easyagents.core.store.StoreOptions;
|
|||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import tech.easyflow.ai.config.SearcherFactory;
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
@@ -46,6 +49,8 @@ import static tech.easyflow.ai.entity.DocumentCollection.*;
|
|||||||
@Service
|
@Service
|
||||||
public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectionMapper, DocumentCollection> implements DocumentCollectionService {
|
public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectionMapper, DocumentCollection> implements DocumentCollectionService {
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentCollectionServiceImpl.class);
|
||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
private ModelService llmService;
|
private ModelService llmService;
|
||||||
|
|
||||||
@@ -128,20 +133,33 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
if (searchDocuments.isEmpty()) {
|
if (searchDocuments.isEmpty()) {
|
||||||
return Collections.emptyList();
|
return Collections.emptyList();
|
||||||
}
|
}
|
||||||
if (documentCollection.getRerankModelId() == null) {
|
boolean rerankEnable = Boolean.TRUE.equals(documentCollection.getOptionsByKey(KEY_RERANK_ENABLE));
|
||||||
|
if (!rerankEnable || documentCollection.getRerankModelId() == null) {
|
||||||
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
||||||
}
|
}
|
||||||
|
|
||||||
Model modelRerank = llmService.getModelInstance(documentCollection.getRerankModelId());
|
Model modelRerank = llmService.getModelInstance(documentCollection.getRerankModelId());
|
||||||
|
if (modelRerank == null) {
|
||||||
|
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
||||||
|
}
|
||||||
|
|
||||||
RerankModel rerankModel = modelRerank.toRerankModel();
|
RerankModel rerankModel = modelRerank.toRerankModel();
|
||||||
if (rerankModel == null) {
|
if (rerankModel == null) {
|
||||||
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Map<Object, Double> originalScores = new HashMap<>();
|
||||||
|
searchDocuments.forEach(item -> originalScores.put(item.getId(), item.getScore()));
|
||||||
searchDocuments.forEach(item -> item.setScore(null));
|
searchDocuments.forEach(item -> item.setScore(null));
|
||||||
|
try {
|
||||||
List<Document> rerankDocuments = rerankModel.rerank(keyword, searchDocuments);
|
List<Document> rerankDocuments = rerankModel.rerank(keyword, searchDocuments);
|
||||||
return formatDocuments(rerankDocuments, minSimilarity, docRecallMaxNum);
|
return formatDocuments(rerankDocuments, minSimilarity, docRecallMaxNum);
|
||||||
|
} catch (RerankException e) {
|
||||||
|
searchDocuments.forEach(item -> item.setScore(originalScores.get(item.getId())));
|
||||||
|
LOG.warn("Rerank failed for collectionId={}, modelId={}, fallback to vector results. message={}",
|
||||||
|
documentCollection.getId(), documentCollection.getRerankModelId(), e.getMessage());
|
||||||
|
return formatDocuments(searchDocuments, minSimilarity, docRecallMaxNum);
|
||||||
|
}
|
||||||
} catch (InterruptedException | ExecutionException e) {
|
} catch (InterruptedException | ExecutionException e) {
|
||||||
Thread.currentThread().interrupt();
|
Thread.currentThread().interrupt();
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
|
|||||||
@@ -25,16 +25,11 @@ import org.springframework.stereotype.Service;
|
|||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import tech.easyflow.ai.config.SearcherFactory;
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
import tech.easyflow.ai.entity.*;
|
import tech.easyflow.ai.entity.*;
|
||||||
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE;
|
|
||||||
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
|
|
||||||
import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT;
|
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
import tech.easyflow.ai.service.DocumentService;
|
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.ai.service.DocumentService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.common.ai.rag.ExcelDocumentSplitter;
|
import tech.easyflow.common.ai.rag.ExcelDocumentSplitter;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
@@ -50,6 +45,11 @@ import java.math.BigInteger;
|
|||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
||||||
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_SEARCH_ENGINE_TYPE;
|
||||||
|
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
|
||||||
|
import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 服务层实现。
|
* 服务层实现。
|
||||||
*
|
*
|
||||||
@@ -228,11 +228,26 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Result<?> saveTextResult(List<DocumentChunk> documentChunks, Document document) {
|
public Result<?> saveTextResult(List<DocumentChunk> documentChunks, Document document) {
|
||||||
Boolean result = storeDocument(document, documentChunks);
|
if (documentChunks == null || documentChunks.isEmpty()) {
|
||||||
|
return Result.fail(1, "切割结果为空,无法保存");
|
||||||
|
}
|
||||||
|
|
||||||
|
List<DocumentChunk> validChunks = new ArrayList<>();
|
||||||
|
for (DocumentChunk chunk : documentChunks) {
|
||||||
|
if (chunk != null && StringUtil.hasText(chunk.getContent())) {
|
||||||
|
validChunks.add(chunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (validChunks.isEmpty()) {
|
||||||
|
return Result.fail(1, "切割结果无有效文本,无法进行向量化");
|
||||||
|
}
|
||||||
|
|
||||||
|
Boolean result = storeDocument(document, validChunks);
|
||||||
if (result) {
|
if (result) {
|
||||||
this.getMapper().insert(document);
|
this.getMapper().insert(document);
|
||||||
AtomicInteger sort = new AtomicInteger(1);
|
AtomicInteger sort = new AtomicInteger(1);
|
||||||
documentChunks.forEach(item -> {
|
validChunks.forEach(item -> {
|
||||||
item.setDocumentCollectionId(document.getCollectionId());
|
item.setDocumentCollectionId(document.getCollectionId());
|
||||||
item.setSorting(sort.get());
|
item.setSorting(sort.get());
|
||||||
item.setDocumentId(document.getId());
|
item.setDocumentId(document.getId());
|
||||||
@@ -287,7 +302,8 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
try {
|
try {
|
||||||
result = documentStore.store(documents, options);
|
result = documentStore.store(documents, options);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
Log.error(e.getMessage());
|
Log.error("Vector store failed: knowledgeId={}, collection={}, chunkCount={}",
|
||||||
|
knowledge.getId(), options.getCollectionName(), documents.size(), e);
|
||||||
throw new BusinessException("向量过程中发生错误,错误信息为:" + e.getMessage());
|
throw new BusinessException("向量过程中发生错误,错误信息为:" + e.getMessage());
|
||||||
}
|
}
|
||||||
if (result == null || !result.isSuccess()) {
|
if (result == null || !result.isSuccess()) {
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
"modified": "Modified",
|
"modified": "Modified",
|
||||||
"modifiedBy": "ModifiedBy",
|
"modifiedBy": "ModifiedBy",
|
||||||
"options": "Options",
|
"options": "Options",
|
||||||
|
"rerankEnable": "Enable rerank model",
|
||||||
"rerankLlmId": "RerankLlm",
|
"rerankLlmId": "RerankLlm",
|
||||||
"searchEngineEnable": "SearchEngineEnable",
|
"searchEngineEnable": "SearchEngineEnable",
|
||||||
"englishName": "EnglishName",
|
"englishName": "EnglishName",
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
"modified": "最后一次修改时间",
|
"modified": "最后一次修改时间",
|
||||||
"modifiedBy": "最后一次修改用户ID",
|
"modifiedBy": "最后一次修改用户ID",
|
||||||
"options": "其他配置",
|
"options": "其他配置",
|
||||||
|
"rerankEnable": "是否启用重排模型",
|
||||||
"rerankLlmId": "重排模型",
|
"rerankLlmId": "重排模型",
|
||||||
"searchEngineEnable": "是否启用搜索引擎",
|
"searchEngineEnable": "是否启用搜索引擎",
|
||||||
"englishName": "英文名称",
|
"englishName": "英文名称",
|
||||||
|
|||||||
@@ -1,9 +1,5 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import type { FormInstance } from 'element-plus';
|
import type {FormInstance} from 'element-plus';
|
||||||
|
|
||||||
import { onMounted, ref, watch } from 'vue';
|
|
||||||
|
|
||||||
import { InfoFilled } from '@element-plus/icons-vue';
|
|
||||||
import {
|
import {
|
||||||
ElButton,
|
ElButton,
|
||||||
ElForm,
|
ElForm,
|
||||||
@@ -17,9 +13,13 @@ import {
|
|||||||
ElTooltip,
|
ElTooltip,
|
||||||
} from 'element-plus';
|
} from 'element-plus';
|
||||||
|
|
||||||
import { api } from '#/api/request';
|
import {computed, onMounted, ref, watch} from 'vue';
|
||||||
|
|
||||||
|
import {InfoFilled} from '@element-plus/icons-vue';
|
||||||
|
|
||||||
|
import {api} from '#/api/request';
|
||||||
import UploadAvatar from '#/components/upload/UploadAvatar.vue';
|
import UploadAvatar from '#/components/upload/UploadAvatar.vue';
|
||||||
import { $t } from '#/locales';
|
import {$t} from '#/locales';
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
detailData: {
|
detailData: {
|
||||||
@@ -39,6 +39,7 @@ const props = defineProps({
|
|||||||
vectorEmbedModelId: '',
|
vectorEmbedModelId: '',
|
||||||
options: {
|
options: {
|
||||||
canUpdateEmbeddingModel: true,
|
canUpdateEmbeddingModel: true,
|
||||||
|
rerankEnable: false,
|
||||||
},
|
},
|
||||||
rerankModelId: '',
|
rerankModelId: '',
|
||||||
searchEngineEnable: false,
|
searchEngineEnable: false,
|
||||||
@@ -50,12 +51,26 @@ const props = defineProps({
|
|||||||
|
|
||||||
const emit = defineEmits(['reload']);
|
const emit = defineEmits(['reload']);
|
||||||
|
|
||||||
const entity = ref<any>({ ...props.detailData });
|
const normalizeEntity = (raw: any) => {
|
||||||
|
const options = {
|
||||||
|
canUpdateEmbeddingModel: true,
|
||||||
|
...(raw?.options || {}),
|
||||||
|
};
|
||||||
|
if (options.rerankEnable === undefined || options.rerankEnable === null) {
|
||||||
|
options.rerankEnable = !!raw?.rerankModelId;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...raw,
|
||||||
|
options,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const entity = ref<any>(normalizeEntity(props.detailData));
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
() => props.detailData,
|
() => props.detailData,
|
||||||
(newVal) => {
|
(newVal) => {
|
||||||
entity.value = { ...newVal };
|
entity.value = normalizeEntity(newVal);
|
||||||
},
|
},
|
||||||
{ immediate: true, deep: true },
|
{ immediate: true, deep: true },
|
||||||
);
|
);
|
||||||
@@ -63,7 +78,7 @@ watch(
|
|||||||
const embeddingLlmList = ref<any>([]);
|
const embeddingLlmList = ref<any>([]);
|
||||||
const rerankerLlmList = ref<any>([]);
|
const rerankerLlmList = ref<any>([]);
|
||||||
const vecotrDatabaseList = ref<any>([
|
const vecotrDatabaseList = ref<any>([
|
||||||
// { value: 'milvus', label: 'Milvus' },
|
{ value: 'milvus', label: 'Milvus' },
|
||||||
{ value: 'redis', label: 'Redis' },
|
{ value: 'redis', label: 'Redis' },
|
||||||
{ value: 'opensearch', label: 'OpenSearch' },
|
{ value: 'opensearch', label: 'OpenSearch' },
|
||||||
{ value: 'elasticsearch', label: 'ElasticSearch' },
|
{ value: 'elasticsearch', label: 'ElasticSearch' },
|
||||||
@@ -71,6 +86,20 @@ const vecotrDatabaseList = ref<any>([
|
|||||||
{ value: 'qcloud', label: $t('documentCollection.tencentCloud') },
|
{ value: 'qcloud', label: $t('documentCollection.tencentCloud') },
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
const milvusVectorStoreConfigPlaceholder =
|
||||||
|
'uri=http://127.0.0.1:19530\n' +
|
||||||
|
'databaseName=default\n' +
|
||||||
|
'token=\n' +
|
||||||
|
'username=\n' +
|
||||||
|
'password=\n' +
|
||||||
|
'autoCreateCollection=true';
|
||||||
|
|
||||||
|
const vectorStoreConfigPlaceholder = computed(() => {
|
||||||
|
return entity.value?.vectorStoreType === 'milvus'
|
||||||
|
? milvusVectorStoreConfigPlaceholder
|
||||||
|
: '';
|
||||||
|
});
|
||||||
|
|
||||||
const getEmbeddingLlmListData = async () => {
|
const getEmbeddingLlmListData = async () => {
|
||||||
try {
|
try {
|
||||||
const url = `/api/v1/model/list?modelType=embeddingModel`;
|
const url = `/api/v1/model/list?modelType=embeddingModel`;
|
||||||
@@ -131,6 +160,11 @@ async function save() {
|
|||||||
const valid = await saveForm.value?.validate();
|
const valid = await saveForm.value?.validate();
|
||||||
if (!valid) return;
|
if (!valid) return;
|
||||||
|
|
||||||
|
if (!entity.value.options) {
|
||||||
|
entity.value.options = {};
|
||||||
|
}
|
||||||
|
entity.value.options.rerankEnable = !!entity.value.options.rerankEnable;
|
||||||
|
|
||||||
btnLoading.value = true;
|
btnLoading.value = true;
|
||||||
const res = await api.post(
|
const res = await api.post(
|
||||||
'/api/v1/documentCollection/update',
|
'/api/v1/documentCollection/update',
|
||||||
@@ -233,6 +267,7 @@ async function save() {
|
|||||||
v-model.trim="entity.vectorStoreConfig"
|
v-model.trim="entity.vectorStoreConfig"
|
||||||
:rows="4"
|
:rows="4"
|
||||||
type="textarea"
|
type="textarea"
|
||||||
|
:placeholder="vectorStoreConfigPlaceholder"
|
||||||
/>
|
/>
|
||||||
</ElFormItem>
|
</ElFormItem>
|
||||||
<ElFormItem prop="vectorEmbedModelId">
|
<ElFormItem prop="vectorEmbedModelId">
|
||||||
@@ -302,12 +337,19 @@ async function save() {
|
|||||||
type="number"
|
type="number"
|
||||||
/>
|
/>
|
||||||
</ElFormItem>
|
</ElFormItem>
|
||||||
|
<ElFormItem
|
||||||
|
prop="options.rerankEnable"
|
||||||
|
:label="$t('documentCollection.rerankEnable')"
|
||||||
|
>
|
||||||
|
<ElSwitch v-model="entity.options.rerankEnable" />
|
||||||
|
</ElFormItem>
|
||||||
<ElFormItem
|
<ElFormItem
|
||||||
prop="rerankModelId"
|
prop="rerankModelId"
|
||||||
:label="$t('documentCollection.rerankLlmId')"
|
:label="$t('documentCollection.rerankLlmId')"
|
||||||
>
|
>
|
||||||
<ElSelect
|
<ElSelect
|
||||||
v-model="entity.rerankModelId"
|
v-model="entity.rerankModelId"
|
||||||
|
clearable
|
||||||
:placeholder="$t('documentCollection.placeholder.rerankLlm')"
|
:placeholder="$t('documentCollection.placeholder.rerankLlm')"
|
||||||
>
|
>
|
||||||
<ElOption
|
<ElOption
|
||||||
|
|||||||
@@ -1,9 +1,5 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import type { FormInstance } from 'element-plus';
|
import type {FormInstance} from 'element-plus';
|
||||||
|
|
||||||
import { onMounted, ref } from 'vue';
|
|
||||||
|
|
||||||
import { InfoFilled } from '@element-plus/icons-vue';
|
|
||||||
import {
|
import {
|
||||||
ElButton,
|
ElButton,
|
||||||
ElDialog,
|
ElDialog,
|
||||||
@@ -18,10 +14,14 @@ import {
|
|||||||
ElTooltip,
|
ElTooltip,
|
||||||
} from 'element-plus';
|
} from 'element-plus';
|
||||||
|
|
||||||
import { api } from '#/api/request';
|
import {onMounted, ref} from 'vue';
|
||||||
|
|
||||||
|
import {InfoFilled} from '@element-plus/icons-vue';
|
||||||
|
|
||||||
|
import {api} from '#/api/request';
|
||||||
import DictSelect from '#/components/dict/DictSelect.vue';
|
import DictSelect from '#/components/dict/DictSelect.vue';
|
||||||
import UploadAvatar from '#/components/upload/UploadAvatar.vue';
|
import UploadAvatar from '#/components/upload/UploadAvatar.vue';
|
||||||
import { $t } from '#/locales';
|
import {$t} from '#/locales';
|
||||||
|
|
||||||
const emit = defineEmits(['reload']);
|
const emit = defineEmits(['reload']);
|
||||||
const embeddingLlmList = ref<any>([]);
|
const embeddingLlmList = ref<any>([]);
|
||||||
@@ -83,12 +83,28 @@ const defaultEntity = {
|
|||||||
dimensionOfVectorModel: undefined,
|
dimensionOfVectorModel: undefined,
|
||||||
options: {
|
options: {
|
||||||
canUpdateEmbeddingModel: true,
|
canUpdateEmbeddingModel: true,
|
||||||
|
rerankEnable: false,
|
||||||
},
|
},
|
||||||
rerankModelId: '',
|
rerankModelId: '',
|
||||||
searchEngineEnable: '',
|
searchEngineEnable: '',
|
||||||
englishName: '',
|
englishName: '',
|
||||||
};
|
};
|
||||||
const entity = ref<any>({ ...defaultEntity });
|
const normalizeEntity = (raw: any = {}) => {
|
||||||
|
const options = {
|
||||||
|
canUpdateEmbeddingModel: true,
|
||||||
|
...(raw.options || {}),
|
||||||
|
};
|
||||||
|
if (options.rerankEnable === undefined || options.rerankEnable === null) {
|
||||||
|
options.rerankEnable = !!raw.rerankModelId;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...defaultEntity,
|
||||||
|
...raw,
|
||||||
|
options,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const entity = ref<any>(normalizeEntity(defaultEntity));
|
||||||
|
|
||||||
const btnLoading = ref(false);
|
const btnLoading = ref(false);
|
||||||
const rules = ref({
|
const rules = ref({
|
||||||
@@ -119,14 +135,10 @@ const rules = ref({
|
|||||||
function openDialog(row: any = {}) {
|
function openDialog(row: any = {}) {
|
||||||
if (row.id) {
|
if (row.id) {
|
||||||
isAdd.value = false;
|
isAdd.value = false;
|
||||||
entity.value = {
|
entity.value = normalizeEntity(row);
|
||||||
...defaultEntity,
|
|
||||||
...row,
|
|
||||||
options: { ...defaultEntity.options, ...row.options },
|
|
||||||
};
|
|
||||||
} else {
|
} else {
|
||||||
isAdd.value = true;
|
isAdd.value = true;
|
||||||
entity.value = { ...defaultEntity };
|
entity.value = normalizeEntity(defaultEntity);
|
||||||
}
|
}
|
||||||
dialogVisible.value = true;
|
dialogVisible.value = true;
|
||||||
}
|
}
|
||||||
@@ -136,6 +148,11 @@ async function save() {
|
|||||||
const valid = await saveForm.value?.validate();
|
const valid = await saveForm.value?.validate();
|
||||||
if (!valid) return;
|
if (!valid) return;
|
||||||
|
|
||||||
|
if (!entity.value.options) {
|
||||||
|
entity.value.options = {};
|
||||||
|
}
|
||||||
|
entity.value.options.rerankEnable = !!entity.value.options.rerankEnable;
|
||||||
|
|
||||||
btnLoading.value = true;
|
btnLoading.value = true;
|
||||||
const res = await api.post(
|
const res = await api.post(
|
||||||
isAdd.value
|
isAdd.value
|
||||||
@@ -162,7 +179,7 @@ async function save() {
|
|||||||
function closeDialog() {
|
function closeDialog() {
|
||||||
saveForm.value?.resetFields();
|
saveForm.value?.resetFields();
|
||||||
isAdd.value = true;
|
isAdd.value = true;
|
||||||
entity.value = { ...defaultEntity };
|
entity.value = normalizeEntity(defaultEntity);
|
||||||
dialogVisible.value = false;
|
dialogVisible.value = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -339,12 +356,19 @@ defineExpose({
|
|||||||
type="number"
|
type="number"
|
||||||
/>
|
/>
|
||||||
</ElFormItem>
|
</ElFormItem>
|
||||||
|
<ElFormItem
|
||||||
|
prop="options.rerankEnable"
|
||||||
|
:label="$t('documentCollection.rerankEnable')"
|
||||||
|
>
|
||||||
|
<ElSwitch v-model="entity.options.rerankEnable" />
|
||||||
|
</ElFormItem>
|
||||||
<ElFormItem
|
<ElFormItem
|
||||||
prop="rerankModelId"
|
prop="rerankModelId"
|
||||||
:label="$t('documentCollection.rerankLlmId')"
|
:label="$t('documentCollection.rerankLlmId')"
|
||||||
>
|
>
|
||||||
<ElSelect
|
<ElSelect
|
||||||
v-model="entity.rerankModelId"
|
v-model="entity.rerankModelId"
|
||||||
|
clearable
|
||||||
:placeholder="$t('documentCollection.placeholder.rerankLlm')"
|
:placeholder="$t('documentCollection.placeholder.rerankLlm')"
|
||||||
>
|
>
|
||||||
<ElOption
|
<ElOption
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { onMounted, reactive, ref } from 'vue';
|
import {onMounted, reactive, ref} from 'vue';
|
||||||
|
|
||||||
import { $t } from '@easyflow/locales';
|
import {$t} from '@easyflow/locales';
|
||||||
|
|
||||||
import { InfoFilled } from '@element-plus/icons-vue';
|
import {InfoFilled} from '@element-plus/icons-vue';
|
||||||
import {
|
import {
|
||||||
ElButton,
|
ElButton,
|
||||||
ElForm,
|
ElForm,
|
||||||
@@ -16,7 +16,7 @@ import {
|
|||||||
ElTooltip,
|
ElTooltip,
|
||||||
} from 'element-plus';
|
} from 'element-plus';
|
||||||
|
|
||||||
import { api } from '#/api/request';
|
import {api} from '#/api/request';
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
documentCollectionId: {
|
documentCollectionId: {
|
||||||
@@ -29,18 +29,21 @@ onMounted(() => {
|
|||||||
getDocumentCollectionConfig();
|
getDocumentCollectionConfig();
|
||||||
});
|
});
|
||||||
const searchEngineEnable = ref(false);
|
const searchEngineEnable = ref(false);
|
||||||
|
const baseOptions = ref<Record<string, any>>({});
|
||||||
const getDocumentCollectionConfig = () => {
|
const getDocumentCollectionConfig = () => {
|
||||||
api
|
api
|
||||||
.get(`/api/v1/documentCollection/detail?id=${props.documentCollectionId}`)
|
.get(`/api/v1/documentCollection/detail?id=${props.documentCollectionId}`)
|
||||||
.then((res) => {
|
.then((res) => {
|
||||||
const { data } = res;
|
const { data } = res;
|
||||||
searchConfig.docRecallMaxNum = data.options.docRecallMaxNum
|
const options = data.options || {};
|
||||||
? Number(data.options.docRecallMaxNum)
|
baseOptions.value = { ...options };
|
||||||
|
searchConfig.docRecallMaxNum = options.docRecallMaxNum
|
||||||
|
? Number(options.docRecallMaxNum)
|
||||||
: 5;
|
: 5;
|
||||||
searchConfig.simThreshold = data.options.simThreshold
|
searchConfig.simThreshold = options.simThreshold
|
||||||
? Number(data.options.simThreshold)
|
? Number(options.simThreshold)
|
||||||
: 0.5;
|
: 0.5;
|
||||||
searchConfig.searchEngineType = data.options.searchEngineType || 'lucene';
|
searchConfig.searchEngineType = options.searchEngineType || 'lucene';
|
||||||
searchEngineEnable.value = !!data.searchEngineEnable;
|
searchEngineEnable.value = !!data.searchEngineEnable;
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
@@ -55,6 +58,7 @@ const submitConfig = () => {
|
|||||||
const submitData = {
|
const submitData = {
|
||||||
id: props.documentCollectionId,
|
id: props.documentCollectionId,
|
||||||
options: {
|
options: {
|
||||||
|
...baseOptions.value,
|
||||||
docRecallMaxNum: searchConfig.docRecallMaxNum,
|
docRecallMaxNum: searchConfig.docRecallMaxNum,
|
||||||
simThreshold: searchConfig.simThreshold,
|
simThreshold: searchConfig.simThreshold,
|
||||||
searchEngineType: searchConfig.searchEngineType,
|
searchEngineType: searchConfig.searchEngineType,
|
||||||
@@ -65,6 +69,7 @@ const submitConfig = () => {
|
|||||||
api
|
api
|
||||||
.post('/api/v1/documentCollection/update', submitData)
|
.post('/api/v1/documentCollection/update', submitData)
|
||||||
.then(() => {
|
.then(() => {
|
||||||
|
baseOptions.value = { ...submitData.options };
|
||||||
ElMessage.success($t('documentCollectionSearch.message.saveSuccess'));
|
ElMessage.success($t('documentCollectionSearch.message.saveSuccess'));
|
||||||
})
|
})
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
|
|||||||
@@ -180,7 +180,7 @@ INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`,
|
|||||||
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359109019710914560, '百度千帆', 'baidu', '', '', 'https://qianfan.baidubce.com', '/v2/chat/completions', '/v2/embeddings', '', '2025-12-18 11:52:02', 1, '2025-12-18 11:52:02', 1);
|
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359109019710914560, '百度千帆', 'baidu', '', '', 'https://qianfan.baidubce.com', '/v2/chat/completions', '/v2/embeddings', '', '2025-12-18 11:52:02', 1, '2025-12-18 11:52:02', 1);
|
||||||
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359110565693620224, '火山引擎', 'volcengine', '', '', 'https://ark.cn-beijing.volces.com', '/api/v3/chat/completions', '/api/v3/embeddings', '', '2025-12-18 11:58:11', 1, '2025-12-18 11:58:11', 1);
|
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359110565693620224, '火山引擎', 'volcengine', '', '', 'https://ark.cn-beijing.volces.com', '/api/v3/chat/completions', '/api/v3/embeddings', '', '2025-12-18 11:58:11', 1, '2025-12-18 11:58:11', 1);
|
||||||
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359110640402563072, '星火大模型', 'spark', '', '', 'https://spark-api-open.xf-yun.com', '/v1/chat/completions', NULL, '', '2025-12-18 11:58:29', 1, '2025-12-18 11:58:29', 1);
|
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359110640402563072, '星火大模型', 'spark', '', '', 'https://spark-api-open.xf-yun.com', '/v1/chat/completions', NULL, '', '2025-12-18 11:58:29', 1, '2025-12-18 11:58:29', 1);
|
||||||
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359110667376132096, '硅基流动', 'siliconlow', '', '', 'https://api.siliconflow.cn', '/v1/chat/completions', '/v1/embeddings', '', '2025-12-18 11:58:35', 1, '2025-12-18 11:58:35', 1);
|
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359110667376132096, '硅基流动', 'siliconlow', '', '', 'https://api.siliconflow.cn', '/v1/chat/completions', '/v1/embeddings', '/v1/rerank', '2025-12-18 11:58:35', 1, '2025-12-18 11:58:35', 1);
|
||||||
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359110690079899648, 'Ollama', 'ollama', '', '', NULL, NULL, NULL, '', '2025-12-18 11:58:40', 1, '2025-12-18 11:58:40', 1);
|
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359110690079899648, 'Ollama', 'ollama', '', '', NULL, NULL, NULL, '', '2025-12-18 11:58:40', 1, '2025-12-18 11:58:40', 1);
|
||||||
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359111120310632448, 'DeepSeek', 'deepseek', '', '', 'https://api.deepseek.com', '/compatible-mode/v1/chat/completions', '/compatible-mode/v1/embeddings', '', '2025-12-18 12:00:23', 1, '2025-12-18 12:00:23', 1);
|
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359111120310632448, 'DeepSeek', 'deepseek', '', '', 'https://api.deepseek.com', '/compatible-mode/v1/chat/completions', '/compatible-mode/v1/embeddings', '', '2025-12-18 12:00:23', 1, '2025-12-18 12:00:23', 1);
|
||||||
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359111228158771200, 'Open AI', 'openai', '', '', 'https://api.openai.com', '/v1/chat/completions', '/v1/embeddings', '', '2025-12-18 12:00:49', 1, '2025-12-18 12:00:49', 1);
|
INSERT INTO `tb_model_provider` (`id`, `provider_name`, `provider_type`, `icon`, `api_key`, `endpoint`, `chat_path`, `embed_path`, `rerank_path`, `created`, `created_by`, `modified`, `modified_by`) VALUES (359111228158771200, 'Open AI', 'openai', '', '', 'https://api.openai.com', '/v1/chat/completions', '/v1/embeddings', '', '2025-12-18 12:00:49', 1, '2025-12-18 12:00:49', 1);
|
||||||
|
|||||||
Reference in New Issue
Block a user