feat: 增强知识库分块策略流程
- 增加导入分析预览提交与预览态缓存键 - 支持知识库分块策略配置与分块预览 - 重构知识库导入与确认导入前端流程
This commit is contained in:
@@ -0,0 +1,555 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
import com.easyagents.rag.core.RagChunk;
|
||||
import com.easyagents.rag.ingestion.model.AnalysisResult;
|
||||
import com.easyagents.rag.ingestion.model.StrategyConfig;
|
||||
import tech.easyflow.ai.entity.Document;
|
||||
import tech.easyflow.ai.entity.DocumentChunk;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public final class DocumentImportDtos {
|
||||
|
||||
private DocumentImportDtos() {
|
||||
}
|
||||
|
||||
public static class FileItem implements Serializable {
|
||||
private String filePath;
|
||||
private String fileName;
|
||||
|
||||
public String getFilePath() {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
public void setFilePath(String filePath) {
|
||||
this.filePath = filePath;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
}
|
||||
|
||||
public static class AnalyzeRequest implements Serializable {
|
||||
private BigInteger knowledgeId;
|
||||
private List<FileItem> files = new ArrayList<FileItem>();
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public List<FileItem> getFiles() {
|
||||
return files;
|
||||
}
|
||||
|
||||
public void setFiles(List<FileItem> files) {
|
||||
this.files = files;
|
||||
}
|
||||
}
|
||||
|
||||
public static class PreviewFileRequest implements Serializable {
|
||||
private String filePath;
|
||||
private String fileName;
|
||||
private StrategyConfig strategyConfig = StrategyConfig.defaults();
|
||||
|
||||
public String getFilePath() {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
public void setFilePath(String filePath) {
|
||||
this.filePath = filePath;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
public StrategyConfig getStrategyConfig() {
|
||||
return strategyConfig;
|
||||
}
|
||||
|
||||
public void setStrategyConfig(StrategyConfig strategyConfig) {
|
||||
this.strategyConfig = strategyConfig;
|
||||
}
|
||||
}
|
||||
|
||||
public static class PreviewRequest implements Serializable {
|
||||
private BigInteger knowledgeId;
|
||||
private List<PreviewFileRequest> files = new ArrayList<PreviewFileRequest>();
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public List<PreviewFileRequest> getFiles() {
|
||||
return files;
|
||||
}
|
||||
|
||||
public void setFiles(List<PreviewFileRequest> files) {
|
||||
this.files = files;
|
||||
}
|
||||
}
|
||||
|
||||
public static class CommitRequest implements Serializable {
|
||||
private BigInteger knowledgeId;
|
||||
private List<String> previewSessionIds = new ArrayList<String>();
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public List<String> getPreviewSessionIds() {
|
||||
return previewSessionIds;
|
||||
}
|
||||
|
||||
public void setPreviewSessionIds(List<String> previewSessionIds) {
|
||||
this.previewSessionIds = previewSessionIds;
|
||||
}
|
||||
}
|
||||
|
||||
public static class SplitterProfileSaveRequest implements Serializable {
|
||||
private BigInteger knowledgeId;
|
||||
private String defaultStrategyCode;
|
||||
private Boolean autoRecommendEnabled;
|
||||
private String fallbackStrategyCode;
|
||||
private Map<String, Object> strategyProfiles = new LinkedHashMap<String, Object>();
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public String getDefaultStrategyCode() {
|
||||
return defaultStrategyCode;
|
||||
}
|
||||
|
||||
public void setDefaultStrategyCode(String defaultStrategyCode) {
|
||||
this.defaultStrategyCode = defaultStrategyCode;
|
||||
}
|
||||
|
||||
public Boolean getAutoRecommendEnabled() {
|
||||
return autoRecommendEnabled;
|
||||
}
|
||||
|
||||
public void setAutoRecommendEnabled(Boolean autoRecommendEnabled) {
|
||||
this.autoRecommendEnabled = autoRecommendEnabled;
|
||||
}
|
||||
|
||||
public String getFallbackStrategyCode() {
|
||||
return fallbackStrategyCode;
|
||||
}
|
||||
|
||||
public void setFallbackStrategyCode(String fallbackStrategyCode) {
|
||||
this.fallbackStrategyCode = fallbackStrategyCode;
|
||||
}
|
||||
|
||||
public Map<String, Object> getStrategyProfiles() {
|
||||
return strategyProfiles;
|
||||
}
|
||||
|
||||
public void setStrategyProfiles(Map<String, Object> strategyProfiles) {
|
||||
this.strategyProfiles = strategyProfiles;
|
||||
}
|
||||
}
|
||||
|
||||
public static class AnalyzeItem implements Serializable {
|
||||
private String filePath;
|
||||
private String fileName;
|
||||
private AnalysisResult analysis;
|
||||
private StrategyConfig strategyConfig = StrategyConfig.defaults();
|
||||
|
||||
public String getFilePath() {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
public void setFilePath(String filePath) {
|
||||
this.filePath = filePath;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
public AnalysisResult getAnalysis() {
|
||||
return analysis;
|
||||
}
|
||||
|
||||
public void setAnalysis(AnalysisResult analysis) {
|
||||
this.analysis = analysis;
|
||||
}
|
||||
|
||||
public StrategyConfig getStrategyConfig() {
|
||||
return strategyConfig;
|
||||
}
|
||||
|
||||
public void setStrategyConfig(StrategyConfig strategyConfig) {
|
||||
this.strategyConfig = strategyConfig;
|
||||
}
|
||||
}
|
||||
|
||||
public static class AnalyzeResponse implements Serializable {
|
||||
private Integer total;
|
||||
private List<AnalyzeItem> items = new ArrayList<AnalyzeItem>();
|
||||
|
||||
public Integer getTotal() {
|
||||
return total;
|
||||
}
|
||||
|
||||
public void setTotal(Integer total) {
|
||||
this.total = total;
|
||||
}
|
||||
|
||||
public List<AnalyzeItem> getItems() {
|
||||
return items;
|
||||
}
|
||||
|
||||
public void setItems(List<AnalyzeItem> items) {
|
||||
this.items = items;
|
||||
}
|
||||
}
|
||||
|
||||
public static class PreviewFileResult implements Serializable {
|
||||
private String previewSessionId;
|
||||
private String filePath;
|
||||
private String fileName;
|
||||
private String strategyCode;
|
||||
private String strategyLabel;
|
||||
private AnalysisResult analysis;
|
||||
private Integer totalChunks;
|
||||
private Integer totalWarnings;
|
||||
private List<RagChunk> chunks = new ArrayList<RagChunk>();
|
||||
|
||||
public String getPreviewSessionId() {
|
||||
return previewSessionId;
|
||||
}
|
||||
|
||||
public void setPreviewSessionId(String previewSessionId) {
|
||||
this.previewSessionId = previewSessionId;
|
||||
}
|
||||
|
||||
public String getFilePath() {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
public void setFilePath(String filePath) {
|
||||
this.filePath = filePath;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
public String getStrategyCode() {
|
||||
return strategyCode;
|
||||
}
|
||||
|
||||
public void setStrategyCode(String strategyCode) {
|
||||
this.strategyCode = strategyCode;
|
||||
}
|
||||
|
||||
public String getStrategyLabel() {
|
||||
return strategyLabel;
|
||||
}
|
||||
|
||||
public void setStrategyLabel(String strategyLabel) {
|
||||
this.strategyLabel = strategyLabel;
|
||||
}
|
||||
|
||||
public AnalysisResult getAnalysis() {
|
||||
return analysis;
|
||||
}
|
||||
|
||||
public void setAnalysis(AnalysisResult analysis) {
|
||||
this.analysis = analysis;
|
||||
}
|
||||
|
||||
public Integer getTotalChunks() {
|
||||
return totalChunks;
|
||||
}
|
||||
|
||||
public void setTotalChunks(Integer totalChunks) {
|
||||
this.totalChunks = totalChunks;
|
||||
}
|
||||
|
||||
public Integer getTotalWarnings() {
|
||||
return totalWarnings;
|
||||
}
|
||||
|
||||
public void setTotalWarnings(Integer totalWarnings) {
|
||||
this.totalWarnings = totalWarnings;
|
||||
}
|
||||
|
||||
public List<RagChunk> getChunks() {
|
||||
return chunks;
|
||||
}
|
||||
|
||||
public void setChunks(List<RagChunk> chunks) {
|
||||
this.chunks = chunks;
|
||||
}
|
||||
}
|
||||
|
||||
public static class PreviewResponse implements Serializable {
|
||||
private Integer totalFiles;
|
||||
private Integer totalChunks;
|
||||
private List<PreviewFileResult> items = new ArrayList<PreviewFileResult>();
|
||||
|
||||
public Integer getTotalFiles() {
|
||||
return totalFiles;
|
||||
}
|
||||
|
||||
public void setTotalFiles(Integer totalFiles) {
|
||||
this.totalFiles = totalFiles;
|
||||
}
|
||||
|
||||
public Integer getTotalChunks() {
|
||||
return totalChunks;
|
||||
}
|
||||
|
||||
public void setTotalChunks(Integer totalChunks) {
|
||||
this.totalChunks = totalChunks;
|
||||
}
|
||||
|
||||
public List<PreviewFileResult> getItems() {
|
||||
return items;
|
||||
}
|
||||
|
||||
public void setItems(List<PreviewFileResult> items) {
|
||||
this.items = items;
|
||||
}
|
||||
}
|
||||
|
||||
public static class CommitFileResult implements Serializable {
|
||||
private String previewSessionId;
|
||||
private String fileName;
|
||||
private Boolean success;
|
||||
private String reason;
|
||||
private BigInteger documentId;
|
||||
private Integer chunkCount;
|
||||
|
||||
public String getPreviewSessionId() {
|
||||
return previewSessionId;
|
||||
}
|
||||
|
||||
public void setPreviewSessionId(String previewSessionId) {
|
||||
this.previewSessionId = previewSessionId;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
public Boolean getSuccess() {
|
||||
return success;
|
||||
}
|
||||
|
||||
public void setSuccess(Boolean success) {
|
||||
this.success = success;
|
||||
}
|
||||
|
||||
public String getReason() {
|
||||
return reason;
|
||||
}
|
||||
|
||||
public void setReason(String reason) {
|
||||
this.reason = reason;
|
||||
}
|
||||
|
||||
public BigInteger getDocumentId() {
|
||||
return documentId;
|
||||
}
|
||||
|
||||
public void setDocumentId(BigInteger documentId) {
|
||||
this.documentId = documentId;
|
||||
}
|
||||
|
||||
public Integer getChunkCount() {
|
||||
return chunkCount;
|
||||
}
|
||||
|
||||
public void setChunkCount(Integer chunkCount) {
|
||||
this.chunkCount = chunkCount;
|
||||
}
|
||||
}
|
||||
|
||||
public static class CommitResponse implements Serializable {
|
||||
private Integer totalFiles;
|
||||
private Integer successCount;
|
||||
private Integer errorCount;
|
||||
private List<CommitFileResult> results = new ArrayList<CommitFileResult>();
|
||||
|
||||
public Integer getTotalFiles() {
|
||||
return totalFiles;
|
||||
}
|
||||
|
||||
public void setTotalFiles(Integer totalFiles) {
|
||||
this.totalFiles = totalFiles;
|
||||
}
|
||||
|
||||
public Integer getSuccessCount() {
|
||||
return successCount;
|
||||
}
|
||||
|
||||
public void setSuccessCount(Integer successCount) {
|
||||
this.successCount = successCount;
|
||||
}
|
||||
|
||||
public Integer getErrorCount() {
|
||||
return errorCount;
|
||||
}
|
||||
|
||||
public void setErrorCount(Integer errorCount) {
|
||||
this.errorCount = errorCount;
|
||||
}
|
||||
|
||||
public List<CommitFileResult> getResults() {
|
||||
return results;
|
||||
}
|
||||
|
||||
public void setResults(List<CommitFileResult> results) {
|
||||
this.results = results;
|
||||
}
|
||||
}
|
||||
|
||||
public static class PreviewSession implements Serializable {
|
||||
private String sessionId;
|
||||
private BigInteger knowledgeId;
|
||||
private String filePath;
|
||||
private String fileName;
|
||||
private String sourceFormat;
|
||||
private StrategyConfig strategyConfig;
|
||||
private AnalysisResult analysis;
|
||||
private Document document;
|
||||
private List<DocumentChunk> documentChunks = new ArrayList<DocumentChunk>();
|
||||
private List<RagChunk> previewChunks = new ArrayList<RagChunk>();
|
||||
private Date createdAt;
|
||||
|
||||
public String getSessionId() {
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
public void setSessionId(String sessionId) {
|
||||
this.sessionId = sessionId;
|
||||
}
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public String getFilePath() {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
public void setFilePath(String filePath) {
|
||||
this.filePath = filePath;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
public String getSourceFormat() {
|
||||
return sourceFormat;
|
||||
}
|
||||
|
||||
public void setSourceFormat(String sourceFormat) {
|
||||
this.sourceFormat = sourceFormat;
|
||||
}
|
||||
|
||||
public StrategyConfig getStrategyConfig() {
|
||||
return strategyConfig;
|
||||
}
|
||||
|
||||
public void setStrategyConfig(StrategyConfig strategyConfig) {
|
||||
this.strategyConfig = strategyConfig;
|
||||
}
|
||||
|
||||
public AnalysisResult getAnalysis() {
|
||||
return analysis;
|
||||
}
|
||||
|
||||
public void setAnalysis(AnalysisResult analysis) {
|
||||
this.analysis = analysis;
|
||||
}
|
||||
|
||||
public Document getDocument() {
|
||||
return document;
|
||||
}
|
||||
|
||||
public void setDocument(Document document) {
|
||||
this.document = document;
|
||||
}
|
||||
|
||||
public List<DocumentChunk> getDocumentChunks() {
|
||||
return documentChunks;
|
||||
}
|
||||
|
||||
public void setDocumentChunks(List<DocumentChunk> documentChunks) {
|
||||
this.documentChunks = documentChunks;
|
||||
}
|
||||
|
||||
public List<RagChunk> getPreviewChunks() {
|
||||
return previewChunks;
|
||||
}
|
||||
|
||||
public void setPreviewChunks(List<RagChunk> previewChunks) {
|
||||
this.previewChunks = previewChunks;
|
||||
}
|
||||
|
||||
public Date getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
|
||||
public void setCreatedAt(Date createdAt) {
|
||||
this.createdAt = createdAt;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
public final class DocumentImportKeys {
|
||||
|
||||
private DocumentImportKeys() {
|
||||
}
|
||||
|
||||
public static final String CACHE_KEY_PREFIX = "easyflow:document:import:preview:";
|
||||
|
||||
public static final String KEY_SPLITTER_DEFAULT_STRATEGY = "splitter.defaultStrategyCode";
|
||||
public static final String KEY_SPLITTER_AUTO_RECOMMEND_ENABLED = "splitter.autoRecommendEnabled";
|
||||
public static final String KEY_SPLITTER_FALLBACK_STRATEGY = "splitter.fallbackStrategyCode";
|
||||
public static final String KEY_SPLITTER_STRATEGY_PROFILES = "splitter.strategyProfiles";
|
||||
|
||||
public static final String KEY_DOCUMENT_STRATEGY_CODE = "splitter.strategyCode";
|
||||
public static final String KEY_DOCUMENT_STRATEGY_LABEL = "splitter.strategyLabel";
|
||||
public static final String KEY_DOCUMENT_STRATEGY_SNAPSHOT = "splitter.strategySnapshot";
|
||||
public static final String KEY_DOCUMENT_ANALYSIS_SUMMARY = "splitter.analysisSummary";
|
||||
public static final String KEY_DOCUMENT_SOURCE_FILE_EXT = "splitter.sourceFileExt";
|
||||
public static final String KEY_DOCUMENT_PREVIEW_VERSION = "splitter.previewVersion";
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
import com.alicp.jetcache.Cache;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
public class DocumentImportPreviewService {
|
||||
|
||||
private static final Duration SESSION_TTL = Duration.ofMinutes(30);
|
||||
|
||||
private final Cache<String, Object> defaultCache;
|
||||
|
||||
public DocumentImportPreviewService(@Qualifier("defaultCache") Cache<String, Object> defaultCache) {
|
||||
this.defaultCache = defaultCache;
|
||||
}
|
||||
|
||||
public String put(DocumentImportDtos.PreviewSession session) {
|
||||
String sessionId = UUID.randomUUID().toString().replace("-", "");
|
||||
session.setSessionId(sessionId);
|
||||
defaultCache.put(buildKey(sessionId), session, SESSION_TTL.toMinutes(), TimeUnit.MINUTES);
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
public DocumentImportDtos.PreviewSession getRequired(String sessionId) {
|
||||
Object cached = defaultCache.get(buildKey(sessionId));
|
||||
if (!(cached instanceof DocumentImportDtos.PreviewSession)) {
|
||||
throw new BusinessException("预览会话已失效，请重新生成预览");
|
||||
}
|
||||
return (DocumentImportDtos.PreviewSession) cached;
|
||||
}
|
||||
|
||||
public void remove(String sessionId) {
|
||||
defaultCache.remove(buildKey(sessionId));
|
||||
}
|
||||
|
||||
private String buildKey(String sessionId) {
|
||||
return DocumentImportKeys.CACHE_KEY_PREFIX + sessionId;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package tech.easyflow.ai.service;
|
||||
import tech.easyflow.ai.entity.Document;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.service.IService;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||
import tech.easyflow.ai.entity.DocumentChunk;
|
||||
import tech.easyflow.ai.entity.DocumentCollectionSplitParams;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
@@ -25,4 +26,10 @@ public interface DocumentService extends IService<Document> {
|
||||
Result textSplit(DocumentCollectionSplitParams documentCollectionSplitParams);
|
||||
|
||||
Result saveTextResult(List<DocumentChunk> documentChunks, Document document);
|
||||
|
||||
Result<DocumentImportDtos.AnalyzeResponse> analyzeImport(DocumentImportDtos.AnalyzeRequest request);
|
||||
|
||||
Result<DocumentImportDtos.PreviewResponse> previewImport(DocumentImportDtos.PreviewRequest request);
|
||||
|
||||
Result<DocumentImportDtos.CommitResponse> commitImport(DocumentImportDtos.CommitRequest request);
|
||||
}
|
||||
|
||||
@@ -12,6 +12,12 @@ import com.easyagents.core.model.embedding.EmbeddingOptions;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.rag.core.RagChunk;
|
||||
import com.easyagents.rag.core.RagDefaults;
|
||||
import com.easyagents.rag.core.RagStrategyCodes;
|
||||
import com.easyagents.rag.ingestion.RagIngestionService;
|
||||
import com.easyagents.rag.ingestion.model.AnalysisResult;
|
||||
import com.easyagents.rag.ingestion.model.StrategyConfig;
|
||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||
import com.mybatisflex.core.keygen.impl.FlexIDKeyGenerator;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
@@ -24,6 +30,9 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import tech.easyflow.ai.config.SearcherFactory;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportPreviewService;
|
||||
import tech.easyflow.ai.entity.*;
|
||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||
@@ -42,6 +51,7 @@ import javax.annotation.Resource;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.math.BigDecimal;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@@ -81,6 +91,12 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
@Autowired
|
||||
private SearcherFactory searcherFactory;
|
||||
|
||||
@Autowired
|
||||
private RagIngestionService ragIngestionService;
|
||||
|
||||
@Autowired
|
||||
private DocumentImportPreviewService documentImportPreviewService;
|
||||
|
||||
@Override
|
||||
public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) {
|
||||
QueryWrapper queryWrapper=QueryWrapper.create()
|
||||
@@ -250,23 +266,397 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
return Result.fail(1, "切割结果无有效文本，无法进行向量化");
|
||||
}
|
||||
|
||||
Boolean result = storeDocument(document, validChunks);
|
||||
if (result) {
|
||||
this.getMapper().insert(document);
|
||||
AtomicInteger sort = new AtomicInteger(1);
|
||||
validChunks.forEach(item -> {
|
||||
item.setDocumentCollectionId(document.getCollectionId());
|
||||
item.setSorting(sort.get());
|
||||
item.setDocumentId(document.getId());
|
||||
sort.getAndIncrement();
|
||||
documentChunkService.save(item);
|
||||
});
|
||||
StoreExecutionContext storeContext = prepareStoreContext(document);
|
||||
storeDocumentChunks(storeContext, validChunks);
|
||||
try {
|
||||
persistDocumentWithChunks(document, validChunks);
|
||||
updateKnowledgeAfterStore(storeContext);
|
||||
return Result.ok();
|
||||
} catch (Exception e) {
|
||||
cleanupPersistedDocument(document);
|
||||
rollbackStoredChunks(storeContext, validChunks);
|
||||
Log.error("保存文档失败: documentId={}, title={}", document.getId(), document.getTitle(), e);
|
||||
throw new BusinessException("保存失败：" + e.getMessage());
|
||||
}
|
||||
return Result.fail(1, "保存失败");
|
||||
}
|
||||
|
||||
protected Boolean storeDocument(Document entity, List<DocumentChunk> documentChunks) {
|
||||
StoreExecutionContext storeContext = prepareStoreContext(entity);
|
||||
storeDocumentChunks(storeContext, documentChunks);
|
||||
updateKnowledgeAfterStore(storeContext);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<DocumentImportDtos.AnalyzeResponse> analyzeImport(DocumentImportDtos.AnalyzeRequest request) {
|
||||
DocumentCollection knowledge = assertDocumentCollection(request.getKnowledgeId());
|
||||
if (request.getFiles() == null || request.getFiles().isEmpty()) {
|
||||
throw new BusinessException("请先上传文件");
|
||||
}
|
||||
|
||||
List<DocumentImportDtos.AnalyzeItem> items = new ArrayList<>();
|
||||
for (DocumentImportDtos.FileItem file : request.getFiles()) {
|
||||
AnalysisResult analysis = analyzeSingleFile(file.getFilePath(), file.getFileName());
|
||||
StrategyConfig strategyConfig = resolveStrategyConfig(
|
||||
knowledge,
|
||||
null,
|
||||
analysis
|
||||
);
|
||||
|
||||
DocumentImportDtos.AnalyzeItem item = new DocumentImportDtos.AnalyzeItem();
|
||||
item.setFilePath(file.getFilePath());
|
||||
item.setFileName(file.getFileName());
|
||||
item.setAnalysis(analysis);
|
||||
item.setStrategyConfig(strategyConfig);
|
||||
items.add(item);
|
||||
}
|
||||
|
||||
DocumentImportDtos.AnalyzeResponse response = new DocumentImportDtos.AnalyzeResponse();
|
||||
response.setItems(items);
|
||||
response.setTotal(items.size());
|
||||
return Result.ok(response);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<DocumentImportDtos.PreviewResponse> previewImport(DocumentImportDtos.PreviewRequest request) {
|
||||
DocumentCollection knowledge = assertDocumentCollection(request.getKnowledgeId());
|
||||
if (request.getFiles() == null || request.getFiles().isEmpty()) {
|
||||
throw new BusinessException("请先上传文件");
|
||||
}
|
||||
|
||||
List<DocumentImportDtos.PreviewFileResult> items = new ArrayList<>();
|
||||
int totalChunks = 0;
|
||||
for (DocumentImportDtos.PreviewFileRequest file : request.getFiles()) {
|
||||
DocumentImportDtos.PreviewSession session = buildPreviewSession(knowledge, file);
|
||||
String sessionId = documentImportPreviewService.put(session);
|
||||
|
||||
DocumentImportDtos.PreviewFileResult item = new DocumentImportDtos.PreviewFileResult();
|
||||
item.setPreviewSessionId(sessionId);
|
||||
item.setFilePath(file.getFilePath());
|
||||
item.setFileName(file.getFileName());
|
||||
item.setStrategyCode(session.getStrategyConfig().getStrategyCode());
|
||||
item.setStrategyLabel(ragIngestionService.toStrategyLabel(session.getStrategyConfig().getStrategyCode()));
|
||||
item.setAnalysis(session.getAnalysis());
|
||||
item.setChunks(session.getPreviewChunks());
|
||||
item.setTotalChunks(session.getPreviewChunks().size());
|
||||
item.setTotalWarnings(countWarnings(session.getPreviewChunks()));
|
||||
items.add(item);
|
||||
totalChunks += session.getPreviewChunks().size();
|
||||
}
|
||||
|
||||
DocumentImportDtos.PreviewResponse response = new DocumentImportDtos.PreviewResponse();
|
||||
response.setItems(items);
|
||||
response.setTotalFiles(items.size());
|
||||
response.setTotalChunks(totalChunks);
|
||||
return Result.ok(response);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<DocumentImportDtos.CommitResponse> commitImport(DocumentImportDtos.CommitRequest request) {
|
||||
DocumentCollection knowledge = assertDocumentCollection(request.getKnowledgeId());
|
||||
if (request.getPreviewSessionIds() == null || request.getPreviewSessionIds().isEmpty()) {
|
||||
throw new BusinessException("请选择需要提交的预览结果");
|
||||
}
|
||||
|
||||
List<DocumentImportDtos.CommitFileResult> results = new ArrayList<>();
|
||||
int successCount = 0;
|
||||
int errorCount = 0;
|
||||
for (String previewSessionId : request.getPreviewSessionIds()) {
|
||||
DocumentImportDtos.CommitFileResult result = new DocumentImportDtos.CommitFileResult();
|
||||
result.setPreviewSessionId(previewSessionId);
|
||||
try {
|
||||
DocumentImportDtos.PreviewSession session = documentImportPreviewService.getRequired(previewSessionId);
|
||||
if (!Objects.equals(session.getKnowledgeId(), knowledge.getId())) {
|
||||
throw new BusinessException("预览会话与当前知识库不匹配");
|
||||
}
|
||||
commitSingleSession(session);
|
||||
result.setSuccess(true);
|
||||
result.setFileName(session.getFileName());
|
||||
result.setDocumentId(session.getDocument().getId());
|
||||
result.setChunkCount(session.getDocumentChunks().size());
|
||||
documentImportPreviewService.remove(previewSessionId);
|
||||
successCount++;
|
||||
} catch (Exception e) {
|
||||
result.setSuccess(false);
|
||||
result.setReason(e.getMessage());
|
||||
errorCount++;
|
||||
}
|
||||
results.add(result);
|
||||
}
|
||||
|
||||
DocumentImportDtos.CommitResponse response = new DocumentImportDtos.CommitResponse();
|
||||
response.setTotalFiles(results.size());
|
||||
response.setSuccessCount(successCount);
|
||||
response.setErrorCount(errorCount);
|
||||
response.setResults(results);
|
||||
return Result.ok(response);
|
||||
}
|
||||
|
||||
private void commitSingleSession(DocumentImportDtos.PreviewSession session) {
|
||||
Document document = session.getDocument();
|
||||
document.setCreated(new Date());
|
||||
document.setModified(new Date());
|
||||
document.setCreatedBy(BigInteger.valueOf(StpUtil.getLoginIdAsLong()));
|
||||
document.setModifiedBy(BigInteger.valueOf(StpUtil.getLoginIdAsLong()));
|
||||
for (DocumentChunk chunk : session.getDocumentChunks()) {
|
||||
chunk.setDocumentId(document.getId());
|
||||
chunk.setDocumentCollectionId(document.getCollectionId());
|
||||
}
|
||||
|
||||
StoreExecutionContext storeContext = prepareStoreContext(document);
|
||||
storeDocumentChunks(storeContext, session.getDocumentChunks());
|
||||
try {
|
||||
persistDocumentWithChunks(document, session.getDocumentChunks());
|
||||
updateKnowledgeAfterStore(storeContext);
|
||||
} catch (Exception e) {
|
||||
cleanupPersistedDocument(document);
|
||||
rollbackStoredChunks(storeContext, session.getDocumentChunks());
|
||||
throw new BusinessException("提交导入失败：" + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private DocumentImportDtos.PreviewSession buildPreviewSession(DocumentCollection knowledge,
|
||||
DocumentImportDtos.PreviewFileRequest fileRequest) {
|
||||
AnalysisResult analysis = analyzeSingleFile(fileRequest.getFilePath(), fileRequest.getFileName());
|
||||
StrategyConfig strategyConfig = resolveStrategyConfig(knowledge, fileRequest.getStrategyConfig(), analysis);
|
||||
List<RagChunk> previewChunks = ragIngestionService.split(analysis, strategyConfig);
|
||||
if (previewChunks.isEmpty()) {
|
||||
throw new BusinessException("未生成有效分块，请调整策略后重试");
|
||||
}
|
||||
|
||||
FlexIDKeyGenerator flexIDKeyGenerator = new FlexIDKeyGenerator();
|
||||
Document document = buildPreviewDocument(flexIDKeyGenerator, knowledge, fileRequest, analysis, strategyConfig);
|
||||
List<DocumentChunk> documentChunks = buildDocumentChunks(flexIDKeyGenerator, document, previewChunks);
|
||||
|
||||
DocumentImportDtos.PreviewSession session = new DocumentImportDtos.PreviewSession();
|
||||
session.setKnowledgeId(knowledge.getId());
|
||||
session.setFilePath(fileRequest.getFilePath());
|
||||
session.setFileName(fileRequest.getFileName());
|
||||
session.setSourceFormat(analysis.getSourceFormat());
|
||||
session.setStrategyConfig(strategyConfig);
|
||||
session.setAnalysis(analysis);
|
||||
session.setDocument(document);
|
||||
session.setDocumentChunks(documentChunks);
|
||||
session.setPreviewChunks(previewChunks);
|
||||
session.setCreatedAt(new Date());
|
||||
return session;
|
||||
}
|
||||
|
||||
private Document buildPreviewDocument(FlexIDKeyGenerator flexIDKeyGenerator,
|
||||
DocumentCollection knowledge,
|
||||
DocumentImportDtos.PreviewFileRequest fileRequest,
|
||||
AnalysisResult analysis,
|
||||
StrategyConfig strategyConfig) {
|
||||
Document document = new Document();
|
||||
document.setId(new BigInteger(String.valueOf(flexIDKeyGenerator.generate(document, null))));
|
||||
document.setCollectionId(knowledge.getId());
|
||||
document.setDocumentType(analysis.getSourceFormat());
|
||||
document.setDocumentPath(fileRequest.getFilePath());
|
||||
document.setTitle(fileRequest.getFileName());
|
||||
document.setContent(analysis.getNormalizedContent());
|
||||
document.setCreated(new Date());
|
||||
document.setModified(new Date());
|
||||
document.setModifiedBy(BigInteger.valueOf(StpUtil.getLoginIdAsLong()));
|
||||
|
||||
Map<String, Object> options = new LinkedHashMap<>();
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_STRATEGY_CODE, strategyConfig.getStrategyCode());
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_STRATEGY_LABEL, ragIngestionService.toStrategyLabel(strategyConfig.getStrategyCode()));
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_STRATEGY_SNAPSHOT, strategyConfigToMap(strategyConfig));
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_ANALYSIS_SUMMARY, analysis.getFeatures());
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_SOURCE_FILE_EXT, analysis.getSourceFormat());
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_PREVIEW_VERSION, "v1");
|
||||
document.setOptions(options);
|
||||
return document;
|
||||
}
|
||||
|
||||
private List<DocumentChunk> buildDocumentChunks(FlexIDKeyGenerator flexIDKeyGenerator,
|
||||
Document document,
|
||||
List<RagChunk> previewChunks) {
|
||||
List<DocumentChunk> chunks = new ArrayList<>();
|
||||
for (int i = 0; i < previewChunks.size(); i++) {
|
||||
RagChunk previewChunk = previewChunks.get(i);
|
||||
DocumentChunk chunk = new DocumentChunk();
|
||||
chunk.setId(new BigInteger(String.valueOf(flexIDKeyGenerator.generate(chunk, null))));
|
||||
chunk.setDocumentId(document.getId());
|
||||
chunk.setDocumentCollectionId(document.getCollectionId());
|
||||
chunk.setContent(previewChunk.getContent());
|
||||
chunk.setSorting(i + 1);
|
||||
|
||||
Map<String, Object> options = new LinkedHashMap<>(previewChunk.getOptions());
|
||||
options.put("chunkType", previewChunk.getChunkType());
|
||||
options.put("sourceLabel", previewChunk.getSourceLabel());
|
||||
options.put("headingPath", previewChunk.getHeadingPath());
|
||||
options.put("charCount", previewChunk.getCharCount());
|
||||
options.put("tokenEstimate", previewChunk.getTokenEstimate());
|
||||
options.put("qaQuestion", previewChunk.getQuestion());
|
||||
options.put("qaAnswer", previewChunk.getAnswer());
|
||||
options.put("partNo", previewChunk.getPartNo());
|
||||
options.put("partTotal", previewChunk.getPartTotal());
|
||||
options.put("warnings", previewChunk.getWarnings());
|
||||
chunk.setOptions(options);
|
||||
chunks.add(chunk);
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
private AnalysisResult analyzeSingleFile(String filePath, String fileName) {
|
||||
String fileExt = normalizeFileExtension(fileName, filePath);
|
||||
assertSupportedImportFile(fileExt);
|
||||
String content = readFileContent(filePath, fileName);
|
||||
return ragIngestionService.analyze(content, fileExt);
|
||||
}
|
||||
|
||||
private String readFileContent(String filePath, String fileName) {
|
||||
try (InputStream inputStream = storageService.readStream(filePath)) {
|
||||
return File2TextUtil.readFromStream(inputStream, fileName, null);
|
||||
} catch (IOException e) {
|
||||
Log.error("读取导入文件失败: filePath={}, fileName={}", filePath, fileName, e);
|
||||
throw new BusinessException("文件解析失败：" + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void assertSupportedImportFile(String fileExt) {
|
||||
if (!Arrays.asList("pdf", "docx", "txt", "md").contains(fileExt)) {
|
||||
throw new BusinessException("当前仅支持 pdf/docx/txt/md 文档导入");
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeFileExtension(String fileName, String filePath) {
|
||||
String target = StringUtil.hasText(fileName) ? fileName : filePath;
|
||||
String ext = FileUtil.getFileTypeByExtension(target);
|
||||
return ext == null ? "" : ext.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private DocumentCollection assertDocumentCollection(BigInteger knowledgeId) {
|
||||
DocumentCollection knowledge = knowledgeService.getById(knowledgeId);
|
||||
if (knowledge == null) {
|
||||
throw new BusinessException("知识库不存在");
|
||||
}
|
||||
if (knowledge.isFaqCollection()) {
|
||||
throw new BusinessException("FAQ知识库不支持文档上传");
|
||||
}
|
||||
return knowledge;
|
||||
}
|
||||
|
||||
private StrategyConfig resolveStrategyConfig(DocumentCollection knowledge,
|
||||
StrategyConfig requestConfig,
|
||||
AnalysisResult analysisResult) {
|
||||
Map<String, Object> options = knowledge.getOptions() == null
|
||||
? Collections.emptyMap()
|
||||
: knowledge.getOptions();
|
||||
String recommended = analysisResult.getRecommendedStrategyCode();
|
||||
String defaultStrategyCode = asString(options.get(DocumentImportKeys.KEY_SPLITTER_DEFAULT_STRATEGY));
|
||||
String fallbackStrategyCode = asString(options.get(DocumentImportKeys.KEY_SPLITTER_FALLBACK_STRATEGY));
|
||||
Boolean autoRecommendEnabled = asBoolean(options.get(DocumentImportKeys.KEY_SPLITTER_AUTO_RECOMMEND_ENABLED), true);
|
||||
|
||||
StrategyConfig config = readProfileConfig(options, defaultStrategyCode);
|
||||
if (config == null) {
|
||||
config = StrategyConfig.defaults();
|
||||
}
|
||||
String requestedStrategyCode = requestConfig == null ? null : requestConfig.getStrategyCode();
|
||||
|
||||
String strategyCode = StringUtil.hasText(requestedStrategyCode)
|
||||
? requestedStrategyCode
|
||||
: config.getStrategyCode();
|
||||
if (!StringUtil.hasText(strategyCode) || RagStrategyCodes.AUTO.equals(strategyCode)) {
|
||||
strategyCode = Boolean.TRUE.equals(autoRecommendEnabled)
|
||||
? recommended
|
||||
: (StringUtil.hasText(defaultStrategyCode) ? defaultStrategyCode : recommended);
|
||||
}
|
||||
if (!StringUtil.hasText(strategyCode)) {
|
||||
strategyCode = StringUtil.hasText(fallbackStrategyCode)
|
||||
? fallbackStrategyCode
|
||||
: RagStrategyCodes.PARAGRAPH_LENGTH;
|
||||
}
|
||||
|
||||
StrategyConfig profileConfig = readProfileConfig(options, strategyCode);
|
||||
if (profileConfig != null) {
|
||||
mergeStrategyConfig(config, profileConfig);
|
||||
}
|
||||
if (requestConfig != null) {
|
||||
mergeStrategyConfig(config, requestConfig);
|
||||
}
|
||||
config.setStrategyCode(strategyCode);
|
||||
if (config.getChunkSize() == null || config.getChunkSize() <= 0) {
|
||||
config.setChunkSize(RagDefaults.CHUNK_SIZE);
|
||||
}
|
||||
if (config.getOverlapSize() == null || config.getOverlapSize() < 0) {
|
||||
config.setOverlapSize(RagDefaults.OVERLAP_SIZE);
|
||||
}
|
||||
if (config.getMdSplitterLevel() == null || config.getMdSplitterLevel() <= 0) {
|
||||
config.setMdSplitterLevel(RagDefaults.MD_SPLITTER_LEVEL);
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private StrategyConfig readProfileConfig(Map<String, Object> options, String strategyCode) {
|
||||
if (!StringUtil.hasText(strategyCode)) {
|
||||
return null;
|
||||
}
|
||||
Object profileObject = options.get(DocumentImportKeys.KEY_SPLITTER_STRATEGY_PROFILES);
|
||||
if (!(profileObject instanceof Map)) {
|
||||
return null;
|
||||
}
|
||||
Object strategyObject = ((Map<String, Object>) profileObject).get(strategyCode);
|
||||
if (!(strategyObject instanceof Map)) {
|
||||
return null;
|
||||
}
|
||||
Map<String, Object> rawProfile = (Map<String, Object>) strategyObject;
|
||||
StrategyConfig config = StrategyConfig.defaults();
|
||||
config.setStrategyCode(strategyCode);
|
||||
config.setChunkSize(asInteger(rawProfile.get("chunkSize"), config.getChunkSize()));
|
||||
config.setOverlapSize(asInteger(rawProfile.get("overlapSize"), config.getOverlapSize()));
|
||||
config.setRegex(asString(rawProfile.get("regex")));
|
||||
config.setRowsPerChunk(asInteger(rawProfile.get("rowsPerChunk"), config.getRowsPerChunk()));
|
||||
config.setMdSplitterLevel(asInteger(rawProfile.get("mdSplitterLevel"), config.getMdSplitterLevel()));
|
||||
return config;
|
||||
}
|
||||
|
||||
private void mergeStrategyConfig(StrategyConfig target, StrategyConfig source) {
|
||||
if (source == null) {
|
||||
return;
|
||||
}
|
||||
if (StringUtil.hasText(source.getStrategyCode())) {
|
||||
target.setStrategyCode(source.getStrategyCode());
|
||||
}
|
||||
if (source.getChunkSize() != null) {
|
||||
target.setChunkSize(source.getChunkSize());
|
||||
}
|
||||
if (source.getOverlapSize() != null) {
|
||||
target.setOverlapSize(source.getOverlapSize());
|
||||
}
|
||||
if (StringUtil.hasText(source.getRegex())) {
|
||||
target.setRegex(source.getRegex());
|
||||
}
|
||||
if (source.getRowsPerChunk() != null) {
|
||||
target.setRowsPerChunk(source.getRowsPerChunk());
|
||||
}
|
||||
if (source.getMdSplitterLevel() != null) {
|
||||
target.setMdSplitterLevel(source.getMdSplitterLevel());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> strategyConfigToMap(StrategyConfig strategyConfig) {
|
||||
Map<String, Object> map = new LinkedHashMap<>();
|
||||
map.put("strategyCode", strategyConfig.getStrategyCode());
|
||||
map.put("chunkSize", strategyConfig.getChunkSize());
|
||||
map.put("overlapSize", strategyConfig.getOverlapSize());
|
||||
map.put("regex", strategyConfig.getRegex());
|
||||
map.put("rowsPerChunk", strategyConfig.getRowsPerChunk());
|
||||
map.put("mdSplitterLevel", strategyConfig.getMdSplitterLevel());
|
||||
return map;
|
||||
}
|
||||
|
||||
private int countWarnings(List<RagChunk> chunks) {
|
||||
int total = 0;
|
||||
for (RagChunk chunk : chunks) {
|
||||
total += chunk.getWarnings() == null ? 0 : chunk.getWarnings().size();
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private StoreExecutionContext prepareStoreContext(Document entity) {
|
||||
DocumentCollection knowledge = knowledgeService.getById(entity.getCollectionId());
|
||||
if (knowledge == null) {
|
||||
throw new BusinessException("知识库不存在");
|
||||
@@ -274,23 +664,22 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
if (knowledge.isFaqCollection()) {
|
||||
throw new BusinessException("FAQ知识库不支持文档上传");
|
||||
}
|
||||
DocumentStore documentStore = null;
|
||||
|
||||
DocumentStore documentStore;
|
||||
try {
|
||||
documentStore = knowledge.toDocumentStore();
|
||||
} catch (Exception e) {
|
||||
Log.error(e.getMessage());
|
||||
Log.error("向量库配置错误: knowledgeId={}", knowledge.getId(), e);
|
||||
throw new BusinessException("向量数据库配置错误");
|
||||
}
|
||||
|
||||
if (documentStore == null) {
|
||||
throw new BusinessException("向量数据库配置错误");
|
||||
}
|
||||
// 设置向量模型
|
||||
|
||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
||||
if (model == null) {
|
||||
throw new BusinessException("该知识库未配置大模型");
|
||||
}
|
||||
// 设置向量模型
|
||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
||||
documentStore.setEmbeddingModel(embeddingModel);
|
||||
|
||||
@@ -300,46 +689,152 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
embeddingOptions.setDimensions(knowledge.getDimensionOfVectorModel());
|
||||
options.setEmbeddingOptions(embeddingOptions);
|
||||
options.setIndexName(options.getCollectionName());
|
||||
|
||||
DocumentSearcher searcher = null;
|
||||
if (knowledge.isSearchEngineEnabled()) {
|
||||
searcher = searcherFactory.getSearcher((String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
|
||||
}
|
||||
return new StoreExecutionContext(knowledge, model, embeddingModel, documentStore, options, searcher);
|
||||
}
|
||||
|
||||
private void storeDocumentChunks(StoreExecutionContext storeContext, List<DocumentChunk> documentChunks) {
|
||||
List<com.easyagents.core.document.Document> documents = new ArrayList<>();
|
||||
documentChunks.forEach(item -> {
|
||||
for (DocumentChunk item : documentChunks) {
|
||||
com.easyagents.core.document.Document document = new com.easyagents.core.document.Document();
|
||||
document.setId(item.getId());
|
||||
document.setContent(item.getContent());
|
||||
documents.add(document);
|
||||
}
|
||||
);
|
||||
StoreResult result = null;
|
||||
|
||||
StoreResult result;
|
||||
try {
|
||||
result = documentStore.store(documents, options);
|
||||
result = storeContext.documentStore.store(documents, storeContext.options);
|
||||
} catch (Exception e) {
|
||||
Log.error("Vector store failed: knowledgeId={}, collection={}, chunkCount={}",
|
||||
knowledge.getId(), options.getCollectionName(), documents.size(), e);
|
||||
storeContext.knowledge.getId(),
|
||||
storeContext.options.getCollectionName(),
|
||||
documents.size(),
|
||||
e);
|
||||
throw new BusinessException("向量过程中发生错误，错误信息为：" + e.getMessage());
|
||||
}
|
||||
if (result == null || !result.isSuccess()) {
|
||||
Log.error("DocumentStore.store failed: " + result);
|
||||
Log.error("DocumentStore.store failed: {}", result);
|
||||
throw new BusinessException("DocumentStore.store failed");
|
||||
}
|
||||
|
||||
if (knowledge.isSearchEngineEnabled()) {
|
||||
// 获取搜索引擎
|
||||
DocumentSearcher searcher = searcherFactory.getSearcher((String) knowledge.getOptionsByKey(KEY_SEARCH_ENGINE_TYPE));
|
||||
// 添加到搜索引擎
|
||||
documents.forEach(searcher::addDocument);
|
||||
if (storeContext.searcher != null) {
|
||||
for (com.easyagents.core.document.Document document : documents) {
|
||||
storeContext.searcher.addDocument(document);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void rollbackStoredChunks(StoreExecutionContext storeContext, List<DocumentChunk> documentChunks) {
|
||||
try {
|
||||
List<BigInteger> chunkIds = new ArrayList<>();
|
||||
for (DocumentChunk chunk : documentChunks) {
|
||||
chunkIds.add(chunk.getId());
|
||||
}
|
||||
storeContext.documentStore.delete(chunkIds, storeContext.options);
|
||||
if (storeContext.searcher != null) {
|
||||
for (BigInteger chunkId : chunkIds) {
|
||||
storeContext.searcher.deleteDocument(chunkId);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.error("回滚向量文档失败: knowledgeId={}", storeContext.knowledge.getId(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateKnowledgeAfterStore(StoreExecutionContext storeContext) {
|
||||
DocumentCollection documentCollection = new DocumentCollection();
|
||||
documentCollection.setId(entity.getCollectionId());
|
||||
Map<String, Object> knowledgeOptions = knowledge.getOptions();
|
||||
documentCollection.setId(storeContext.knowledge.getId());
|
||||
Map<String, Object> knowledgeOptions = storeContext.knowledge.getOptions() == null
|
||||
? new HashMap<>()
|
||||
: new HashMap<>(storeContext.knowledge.getOptions());
|
||||
knowledgeOptions.put(KEY_CAN_UPDATE_EMBEDDING_MODEL, false);
|
||||
documentCollection.setOptions(knowledgeOptions);
|
||||
knowledgeService.updateById(documentCollection);
|
||||
if (knowledge.getDimensionOfVectorModel() == null) {
|
||||
int dimension = Model.getEmbeddingDimension(embeddingModel);
|
||||
knowledge.setDimensionOfVectorModel(dimension);
|
||||
knowledgeService.updateById(knowledge);
|
||||
|
||||
if (storeContext.knowledge.getDimensionOfVectorModel() == null) {
|
||||
int dimension = Model.getEmbeddingDimension(storeContext.embeddingModel);
|
||||
DocumentCollection update = new DocumentCollection();
|
||||
update.setId(storeContext.knowledge.getId());
|
||||
update.setDimensionOfVectorModel(dimension);
|
||||
knowledgeService.updateById(update);
|
||||
}
|
||||
}
|
||||
|
||||
private void persistDocumentWithChunks(Document document, List<DocumentChunk> chunks) {
|
||||
this.getMapper().insert(document);
|
||||
AtomicInteger sort = new AtomicInteger(1);
|
||||
for (DocumentChunk item : chunks) {
|
||||
item.setDocumentCollectionId(document.getCollectionId());
|
||||
item.setDocumentId(document.getId());
|
||||
item.setSorting(sort.getAndIncrement());
|
||||
documentChunkService.save(item);
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupPersistedDocument(Document document) {
|
||||
if (document == null || document.getId() == null) {
|
||||
return;
|
||||
}
|
||||
documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, document.getId()));
|
||||
this.getMapper().deleteById(document.getId());
|
||||
}
|
||||
|
||||
private String asString(Object value) {
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
private Integer asInteger(Object value, Integer defaultValue) {
|
||||
if (value == null) {
|
||||
return defaultValue;
|
||||
}
|
||||
if (value instanceof Number) {
|
||||
return ((Number) value).intValue();
|
||||
}
|
||||
if (value instanceof String && StringUtil.hasText((String) value)) {
|
||||
return Integer.parseInt((String) value);
|
||||
}
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
private Boolean asBoolean(Object value, boolean defaultValue) {
|
||||
if (value == null) {
|
||||
return defaultValue;
|
||||
}
|
||||
if (value instanceof Boolean) {
|
||||
return (Boolean) value;
|
||||
}
|
||||
if (value instanceof Number) {
|
||||
return ((Number) value).intValue() != 0;
|
||||
}
|
||||
return Boolean.parseBoolean(String.valueOf(value));
|
||||
}
|
||||
|
||||
private static class StoreExecutionContext {
|
||||
private final DocumentCollection knowledge;
|
||||
private final Model model;
|
||||
private final EmbeddingModel embeddingModel;
|
||||
private final DocumentStore documentStore;
|
||||
private final StoreOptions options;
|
||||
private final DocumentSearcher searcher;
|
||||
|
||||
private StoreExecutionContext(DocumentCollection knowledge,
|
||||
Model model,
|
||||
EmbeddingModel embeddingModel,
|
||||
DocumentStore documentStore,
|
||||
StoreOptions options,
|
||||
DocumentSearcher searcher) {
|
||||
this.knowledge = knowledge;
|
||||
this.model = model;
|
||||
this.embeddingModel = embeddingModel;
|
||||
this.documentStore = documentStore;
|
||||
this.options = options;
|
||||
this.searcher = searcher;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public DocumentSplitter getDocumentSplitter(DocumentCollectionSplitParams params) {
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
SET NAMES utf8mb4;
|
||||
|
||||
ALTER TABLE `tb_document_chunk`
|
||||
ADD COLUMN `options` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '扩展元信息' AFTER `sorting`;
|
||||
@@ -1,99 +1,141 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, watch } from 'vue';
|
||||
import { useRoute } from 'vue-router';
|
||||
import { computed } from 'vue';
|
||||
|
||||
import { $t } from '@easyflow/locales';
|
||||
|
||||
import { ElTable, ElTableColumn, ElTag } from 'element-plus';
|
||||
import {
|
||||
ElCard,
|
||||
ElDescriptions,
|
||||
ElDescriptionsItem,
|
||||
ElEmpty,
|
||||
ElTable,
|
||||
ElTableColumn,
|
||||
ElTag,
|
||||
} from 'element-plus';
|
||||
|
||||
import { api } from '#/api/request';
|
||||
|
||||
const props = defineProps({
|
||||
filesList: {
|
||||
default: () => [],
|
||||
type: Array<any>,
|
||||
},
|
||||
splitterParams: {
|
||||
default: () => {},
|
||||
type: Object,
|
||||
},
|
||||
});
|
||||
|
||||
const emit = defineEmits(['loadingFinish']);
|
||||
|
||||
const route = useRoute();
|
||||
|
||||
const knowledgeIdRef = ref<string>((route.query.id as string) || '');
|
||||
const localFilesList = ref<any[]>([]);
|
||||
watch(
|
||||
() => props.filesList,
|
||||
(newVal) => {
|
||||
localFilesList.value = [...newVal];
|
||||
},
|
||||
{ immediate: true },
|
||||
);
|
||||
defineExpose({
|
||||
handleSave() {
|
||||
localFilesList.value.forEach((file, index) => {
|
||||
localFilesList.value[index].progressUpload = 'loading';
|
||||
saveDoc(file.filePath, 'saveText', file.fileName, index);
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
function saveDoc(
|
||||
filePath: string,
|
||||
operation: string,
|
||||
fileOriginName: string,
|
||||
index: number,
|
||||
) {
|
||||
api
|
||||
.post('/api/v1/document/saveText', {
|
||||
filePath,
|
||||
operation,
|
||||
knowledgeId: knowledgeIdRef.value,
|
||||
fileOriginName,
|
||||
...props.splitterParams,
|
||||
})
|
||||
.then((res) => {
|
||||
if (res.errorCode === 0) {
|
||||
localFilesList.value[index].progressUpload = 'success';
|
||||
emit('loadingFinish');
|
||||
}
|
||||
/* if (index === localFilesList.value.length - 1) {
|
||||
emit('loadingFinish');
|
||||
}*/
|
||||
});
|
||||
interface PreviewItem {
|
||||
fileName: string;
|
||||
previewSessionId: string;
|
||||
totalChunks?: number;
|
||||
}
|
||||
|
||||
interface CommitResultItem {
|
||||
chunkCount?: number;
|
||||
fileName?: string;
|
||||
reason?: string;
|
||||
success?: boolean;
|
||||
}
|
||||
|
||||
const props = defineProps<{
|
||||
commitResults?: CommitResultItem[];
|
||||
loading?: boolean;
|
||||
previewItems?: PreviewItem[];
|
||||
}>();
|
||||
|
||||
const summary = computed(() => {
|
||||
const results = props.commitResults ?? [];
|
||||
const successCount = results.filter((item) => item.success).length;
|
||||
const errorCount = results.length - successCount;
|
||||
let totalCount = 0;
|
||||
if (results.length > 0) {
|
||||
totalCount = results.length;
|
||||
} else if (props.previewItems && props.previewItems.length > 0) {
|
||||
totalCount = props.previewItems.length;
|
||||
}
|
||||
return {
|
||||
errorCount,
|
||||
successCount,
|
||||
totalCount,
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="import-doc-file-list">
|
||||
<ElTable :data="localFilesList" size="large" style="width: 100%">
|
||||
<div class="confirm-shell">
|
||||
<ElCard shadow="never" class="confirm-card">
|
||||
<ElDescriptions :column="3" border>
|
||||
<ElDescriptionsItem
|
||||
:label="$t('documentCollection.faq.import.totalCount')"
|
||||
>
|
||||
{{ summary.totalCount }}
|
||||
</ElDescriptionsItem>
|
||||
<ElDescriptionsItem
|
||||
:label="$t('documentCollection.faq.import.successCount')"
|
||||
>
|
||||
{{ summary.successCount }}
|
||||
</ElDescriptionsItem>
|
||||
<ElDescriptionsItem
|
||||
:label="$t('documentCollection.faq.import.errorCount')"
|
||||
>
|
||||
{{ summary.errorCount }}
|
||||
</ElDescriptionsItem>
|
||||
</ElDescriptions>
|
||||
</ElCard>
|
||||
|
||||
<ElEmpty
|
||||
v-if="!previewItems || previewItems.length === 0"
|
||||
:description="$t('documentCollection.importDoc.resultEmpty')"
|
||||
/>
|
||||
|
||||
<ElTable
|
||||
v-else
|
||||
:data="
|
||||
commitResults && commitResults.length > 0 ? commitResults : previewItems
|
||||
"
|
||||
size="large"
|
||||
>
|
||||
<ElTableColumn
|
||||
prop="fileName"
|
||||
:label="$t('documentCollection.importDoc.fileName')"
|
||||
width="250"
|
||||
min-width="260"
|
||||
/>
|
||||
<ElTableColumn
|
||||
prop="progressUpload"
|
||||
:label="$t('documentCollection.splitterDoc.uploadStatus')"
|
||||
prop="chunkCount"
|
||||
:label="$t('documentCollection.importDoc.chunkCount')"
|
||||
width="120"
|
||||
>
|
||||
<template #default="{ row }">
|
||||
<ElTag type="success" v-if="row.progressUpload === 'success'">
|
||||
{{ row.chunkCount ?? row.totalChunks ?? '-' }}
|
||||
</template>
|
||||
</ElTableColumn>
|
||||
<ElTableColumn
|
||||
:label="$t('documentCollection.splitterDoc.uploadStatus')"
|
||||
width="140"
|
||||
>
|
||||
<template #default="{ row }">
|
||||
<ElTag v-if="row.success === true" type="success" effect="plain">
|
||||
{{ $t('documentCollection.splitterDoc.completed') }}
|
||||
</ElTag>
|
||||
<ElTag type="primary" v-else>
|
||||
{{ $t('documentCollection.splitterDoc.pendingUpload') }}
|
||||
<ElTag v-else-if="row.success === false" type="danger" effect="plain">
|
||||
{{ $t('documentCollection.importDoc.importFailed') }}
|
||||
</ElTag>
|
||||
<ElTag v-else type="info" effect="plain">
|
||||
{{
|
||||
loading
|
||||
? $t('documentCollection.splitterDoc.uploading')
|
||||
: $t('documentCollection.splitterDoc.pendingUpload')
|
||||
}}
|
||||
</ElTag>
|
||||
</template>
|
||||
</ElTableColumn>
|
||||
<ElTableColumn
|
||||
prop="reason"
|
||||
:label="$t('documentCollection.faq.import.reason')"
|
||||
min-width="280"
|
||||
/>
|
||||
</ElTable>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.import-doc-file-list {
|
||||
width: 100%;
|
||||
.confirm-shell {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.confirm-card {
|
||||
border: 1px solid var(--el-border-color-light);
|
||||
border-radius: 16px;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,189 +1,215 @@
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue';
|
||||
import { computed, ref } from 'vue';
|
||||
import { useRoute } from 'vue-router';
|
||||
|
||||
import { $t } from '@easyflow/locales';
|
||||
|
||||
import { Back } from '@element-plus/icons-vue';
|
||||
import {
|
||||
ElButton,
|
||||
ElMessage,
|
||||
ElPagination,
|
||||
ElStep,
|
||||
ElSteps,
|
||||
} from 'element-plus';
|
||||
import { ElButton, ElMessage, ElStep, ElSteps } from 'element-plus';
|
||||
|
||||
import { api } from '#/api/request';
|
||||
import ComfirmImportDocument from '#/views/ai/documentCollection/ComfirmImportDocument.vue';
|
||||
import ImportKnowledgeFileContainer from '#/views/ai/documentCollection/ImportKnowledgeFileContainer.vue';
|
||||
import SegmenterDoc from '#/views/ai/documentCollection/SegmenterDoc.vue';
|
||||
import SplitterDocPreview from '#/views/ai/documentCollection/SplitterDocPreview.vue';
|
||||
|
||||
interface UploadFileItem {
|
||||
fileName: string;
|
||||
filePath: string;
|
||||
}
|
||||
|
||||
interface AnalyzeItem {
|
||||
fileName: string;
|
||||
filePath: string;
|
||||
strategyConfig: Record<string, any>;
|
||||
}
|
||||
|
||||
interface PreviewItem {
|
||||
fileName: string;
|
||||
previewSessionId: string;
|
||||
totalChunks?: number;
|
||||
}
|
||||
|
||||
const emits = defineEmits(['importBack']);
|
||||
const back = () => {
|
||||
emits('importBack');
|
||||
};
|
||||
const files = ref([]);
|
||||
const splitterParams = ref({});
|
||||
const route = useRoute();
|
||||
const knowledgeId = computed(() => (route.query.id as string) || '');
|
||||
|
||||
const fileUploadRef = ref<InstanceType<typeof ImportKnowledgeFileContainer>>();
|
||||
const segmenterDocRef = ref<InstanceType<typeof SegmenterDoc>>();
|
||||
|
||||
const activeStep = ref(0);
|
||||
const fileUploadRef = ref();
|
||||
const confirmImportRef = ref();
|
||||
const segmenterDocRef = ref();
|
||||
const pagination = ref({
|
||||
pageSize: 10,
|
||||
currentPage: 1,
|
||||
total: 0,
|
||||
});
|
||||
const goToNextStep = () => {
|
||||
const files = ref<UploadFileItem[]>([]);
|
||||
const analysisItems = ref<AnalyzeItem[]>([]);
|
||||
const previewItems = ref<PreviewItem[]>([]);
|
||||
const commitResults = ref<any[]>([]);
|
||||
|
||||
const analyzing = ref(false);
|
||||
const previewing = ref(false);
|
||||
const committing = ref(false);
|
||||
|
||||
const canGoPrevious = computed(() => activeStep.value > 0 && !committing.value);
|
||||
|
||||
function back() {
|
||||
emits('importBack');
|
||||
}
|
||||
|
||||
function getUploadedFiles() {
|
||||
return fileUploadRef.value?.getFilesData?.() || [];
|
||||
}
|
||||
|
||||
async function goToNextStep() {
|
||||
if (activeStep.value === 0) {
|
||||
if (fileUploadRef.value.getFilesData().length === 0) {
|
||||
const currentFiles = getUploadedFiles();
|
||||
if (currentFiles.length === 0) {
|
||||
ElMessage.error($t('message.uploadFileFirst'));
|
||||
return;
|
||||
}
|
||||
files.value = fileUploadRef.value.getFilesData();
|
||||
files.value = currentFiles;
|
||||
await runAnalyze();
|
||||
activeStep.value = 1;
|
||||
return;
|
||||
}
|
||||
if (activeStep.value === 1 && segmenterDocRef.value) {
|
||||
splitterParams.value = segmenterDocRef.value.getSplitterFormValues();
|
||||
|
||||
if (activeStep.value === 1) {
|
||||
await runPreview();
|
||||
activeStep.value = 2;
|
||||
return;
|
||||
}
|
||||
|
||||
if (activeStep.value === 2) {
|
||||
activeStep.value = 3;
|
||||
}
|
||||
}
|
||||
|
||||
function goToPreviousStep() {
|
||||
if (!canGoPrevious.value) {
|
||||
return;
|
||||
}
|
||||
activeStep.value += 1;
|
||||
};
|
||||
const goToPreviousStep = () => {
|
||||
activeStep.value -= 1;
|
||||
};
|
||||
const handleSizeChange = (val: number) => {
|
||||
pagination.value.pageSize = val;
|
||||
};
|
||||
const handleCurrentChange = (val: number) => {
|
||||
pagination.value.currentPage = val;
|
||||
};
|
||||
const handleTotalUpdate = (newTotal: number) => {
|
||||
pagination.value.total = newTotal; // 同步到父组件的 pagination.total
|
||||
};
|
||||
const loadingSave = ref(false);
|
||||
const confirmImport = () => {
|
||||
loadingSave.value = true;
|
||||
// 确认导入
|
||||
confirmImportRef.value.handleSave();
|
||||
};
|
||||
const finishImport = () => {
|
||||
loadingSave.value = false;
|
||||
}
|
||||
|
||||
async function runAnalyze() {
|
||||
analyzing.value = true;
|
||||
try {
|
||||
const res = await api.post('/api/v1/document/import/analyze', {
|
||||
files: files.value.map((item) => ({
|
||||
fileName: item.fileName,
|
||||
filePath: item.filePath,
|
||||
})),
|
||||
knowledgeId: knowledgeId.value,
|
||||
});
|
||||
analysisItems.value = res.data?.items || [];
|
||||
} finally {
|
||||
analyzing.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runPreview() {
|
||||
const previewRequestItems =
|
||||
segmenterDocRef.value?.getPreviewRequestItems?.() || [];
|
||||
if (previewRequestItems.length === 0) {
|
||||
ElMessage.error($t('documentCollection.importDoc.previewEmpty'));
|
||||
return;
|
||||
}
|
||||
previewing.value = true;
|
||||
try {
|
||||
const res = await api.post('/api/v1/document/import/preview', {
|
||||
files: previewRequestItems,
|
||||
knowledgeId: knowledgeId.value,
|
||||
});
|
||||
previewItems.value = res.data?.items || [];
|
||||
commitResults.value = [];
|
||||
} finally {
|
||||
previewing.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmImport() {
|
||||
if (previewItems.value.length === 0) {
|
||||
ElMessage.error($t('documentCollection.importDoc.previewEmpty'));
|
||||
return;
|
||||
}
|
||||
committing.value = true;
|
||||
try {
|
||||
const res = await api.post('/api/v1/document/import/commit', {
|
||||
knowledgeId: knowledgeId.value,
|
||||
previewSessionIds: previewItems.value.map(
|
||||
(item) => item.previewSessionId,
|
||||
),
|
||||
});
|
||||
commitResults.value = res.data?.results || [];
|
||||
if ((res.data?.errorCount || 0) === 0) {
|
||||
ElMessage.success($t('documentCollection.splitterDoc.importSuccess'));
|
||||
emits('importBack');
|
||||
};
|
||||
}
|
||||
} finally {
|
||||
committing.value = false;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="imp-doc-kno-container">
|
||||
<div class="imp-doc-header">
|
||||
<ElButton @click="back" :icon="Back">
|
||||
<ElButton :icon="Back" @click="back">
|
||||
{{ $t('button.back') }}
|
||||
</ElButton>
|
||||
</div>
|
||||
|
||||
<div class="imp-doc-kno-content">
|
||||
<div class="rounded-lg bg-[var(--table-header-bg-color)] py-5">
|
||||
<div class="step-card">
|
||||
<ElSteps :active="activeStep" align-center>
|
||||
<ElStep>
|
||||
<template #icon>
|
||||
<div class="flex items-center gap-2">
|
||||
<div class="h-8 w-8 rounded-full bg-[var(--step-item-bg)]">
|
||||
<span class="text-accent-foreground text-sm/8">1</span>
|
||||
</div>
|
||||
<span class="text-base">{{
|
||||
$t('documentCollection.importDoc.fileUpload')
|
||||
}}</span>
|
||||
</div>
|
||||
</template>
|
||||
</ElStep>
|
||||
<ElStep>
|
||||
<template #icon>
|
||||
<div class="flex items-center gap-2">
|
||||
<div class="h-8 w-8 rounded-full bg-[var(--step-item-bg)]">
|
||||
<span class="text-accent-foreground text-sm/8">2</span>
|
||||
</div>
|
||||
<span class="text-base">{{
|
||||
$t('documentCollection.importDoc.parameterSettings')
|
||||
}}</span>
|
||||
</div>
|
||||
</template>
|
||||
</ElStep>
|
||||
<ElStep>
|
||||
<template #icon>
|
||||
<div class="flex items-center gap-2">
|
||||
<div class="h-8 w-8 rounded-full bg-[var(--step-item-bg)]">
|
||||
<span class="text-accent-foreground text-sm/8">3</span>
|
||||
</div>
|
||||
<span class="text-base">{{
|
||||
$t('documentCollection.importDoc.segmentedPreview')
|
||||
}}</span>
|
||||
</div>
|
||||
</template>
|
||||
</ElStep>
|
||||
<ElStep>
|
||||
<template #icon>
|
||||
<div class="flex items-center gap-2">
|
||||
<div class="h-8 w-8 rounded-full bg-[var(--step-item-bg)]">
|
||||
<span class="text-accent-foreground text-sm/8">4</span>
|
||||
</div>
|
||||
<span class="text-base">{{
|
||||
$t('documentCollection.importDoc.confirmImport')
|
||||
}}</span>
|
||||
</div>
|
||||
</template>
|
||||
</ElStep>
|
||||
<ElStep :title="$t('documentCollection.importDoc.fileUpload')" />
|
||||
<ElStep
|
||||
:title="$t('documentCollection.importDoc.strategyAnalysis')"
|
||||
/>
|
||||
<ElStep
|
||||
:title="$t('documentCollection.importDoc.segmentedPreview')"
|
||||
/>
|
||||
<ElStep :title="$t('documentCollection.importDoc.confirmImport')" />
|
||||
</ElSteps>
|
||||
</div>
|
||||
|
||||
<div style="margin-top: 20px">
|
||||
<!-- 文件上传导入-->
|
||||
<div class="knw-file-upload" v-if="activeStep === 0">
|
||||
<ImportKnowledgeFileContainer ref="fileUploadRef" />
|
||||
</div>
|
||||
<!-- 分割参数设置-->
|
||||
<div class="knw-file-splitter" v-if="activeStep === 1">
|
||||
<SegmenterDoc ref="segmenterDocRef" />
|
||||
</div>
|
||||
<!-- 分割预览-->
|
||||
<div class="knw-file-preview" v-if="activeStep === 2">
|
||||
<div class="step-body">
|
||||
<ImportKnowledgeFileContainer
|
||||
v-if="activeStep === 0"
|
||||
ref="fileUploadRef"
|
||||
/>
|
||||
<SegmenterDoc
|
||||
v-else-if="activeStep === 1"
|
||||
ref="segmenterDocRef"
|
||||
:analysis-items="analysisItems"
|
||||
/>
|
||||
<SplitterDocPreview
|
||||
:flies-list="files"
|
||||
:splitter-params="splitterParams"
|
||||
:page-number="pagination.currentPage"
|
||||
:page-size="pagination.pageSize"
|
||||
@update-total="handleTotalUpdate"
|
||||
v-else-if="activeStep === 2"
|
||||
:preview-items="previewItems"
|
||||
/>
|
||||
</div>
|
||||
<!-- 确认导入-->
|
||||
<div class="knw-file-confirm" v-if="activeStep === 3">
|
||||
<ComfirmImportDocument
|
||||
:splitter-params="splitterParams"
|
||||
:files-list="files"
|
||||
ref="confirmImportRef"
|
||||
@loading-finish="finishImport"
|
||||
v-else
|
||||
:preview-items="previewItems"
|
||||
:commit-results="commitResults"
|
||||
:loading="committing"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div style="height: 40px"></div>
|
||||
|
||||
<div class="imp-doc-footer">
|
||||
<div v-if="activeStep === 2" class="imp-doc-page-container">
|
||||
<ElPagination
|
||||
:page-sizes="[10, 20]"
|
||||
layout="total, sizes, prev, pager, next, jumper"
|
||||
:total="pagination.total"
|
||||
@size-change="handleSizeChange"
|
||||
@current-change="handleCurrentChange"
|
||||
/>
|
||||
</div>
|
||||
<ElButton @click="goToPreviousStep" type="primary" v-if="activeStep >= 1">
|
||||
<ElButton v-if="canGoPrevious" @click="goToPreviousStep">
|
||||
{{ $t('button.previousStep') }}
|
||||
</ElButton>
|
||||
<ElButton @click="goToNextStep" type="primary" v-if="activeStep < 3">
|
||||
<ElButton
|
||||
v-if="activeStep < 3"
|
||||
type="primary"
|
||||
:loading="analyzing || previewing"
|
||||
@click="goToNextStep"
|
||||
>
|
||||
{{ $t('button.nextStep') }}
|
||||
</ElButton>
|
||||
<ElButton
|
||||
@click="confirmImport"
|
||||
v-else
|
||||
type="primary"
|
||||
v-if="activeStep === 3"
|
||||
:loading="loadingSave"
|
||||
:disabled="loadingSave"
|
||||
:loading="committing"
|
||||
:disabled="committing"
|
||||
@click="confirmImport"
|
||||
>
|
||||
{{ $t('button.startImport') }}
|
||||
</ElButton>
|
||||
@@ -194,60 +220,41 @@ const finishImport = () => {
|
||||
<style scoped>
|
||||
.imp-doc-kno-container {
|
||||
position: relative;
|
||||
height: 100%;
|
||||
background-color: var(--el-bg-color);
|
||||
border-radius: 12px;
|
||||
padding: 20px;
|
||||
display: flex;
|
||||
height: 100%;
|
||||
flex-direction: column;
|
||||
padding: 24px;
|
||||
border-radius: 16px;
|
||||
background: var(--el-bg-color);
|
||||
}
|
||||
|
||||
.imp-doc-kno-content {
|
||||
flex: 1;
|
||||
padding-top: 20px;
|
||||
overflow: auto;
|
||||
}
|
||||
.imp-doc-footer {
|
||||
position: absolute;
|
||||
bottom: 20px;
|
||||
right: 20px;
|
||||
display: flex;
|
||||
height: 40px;
|
||||
background-color: var(--el-bg-color);
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
.knw-file-preview {
|
||||
flex: 1;
|
||||
flex-direction: column;
|
||||
gap: 20px;
|
||||
padding-top: 16px;
|
||||
overflow: auto;
|
||||
}
|
||||
.imp-doc-page-container {
|
||||
margin-right: 12px;
|
||||
}
|
||||
.knw-file-confirm {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
:deep(.el-step__head) {
|
||||
--step-item-bg: rgba(0, 0, 0, 0.06);
|
||||
--step-item-solid-bg: rgba(0, 0, 0, 0.15);
|
||||
--accent-foreground: rgba(0, 0, 0, 0.45);
|
||||
.step-card {
|
||||
padding: 20px 24px;
|
||||
border: 1px solid var(--el-border-color-light);
|
||||
border-radius: 16px;
|
||||
background: var(--el-fill-color-blank);
|
||||
}
|
||||
:deep(.el-step__head:where(.dark, .dark *)) {
|
||||
--step-item-bg: var(--el-text-color-placeholder);
|
||||
--step-item-solid-bg: var(--el-text-color-placeholder);
|
||||
--accent-foreground: var(--primary-foreground);
|
||||
|
||||
.step-body {
|
||||
flex: 1;
|
||||
padding-bottom: 72px;
|
||||
}
|
||||
:deep(.el-step__head.is-finish) {
|
||||
--step-item-bg: hsl(var(--primary));
|
||||
--step-item-solid-bg: hsl(var(--primary));
|
||||
--accent-foreground: var(--primary-foreground);
|
||||
}
|
||||
:deep(.el-step__icon.is-icon) {
|
||||
width: 120px;
|
||||
background-color: var(--table-header-bg-color);
|
||||
}
|
||||
:deep(.el-step__line) {
|
||||
background-color: var(--step-item-solid-bg);
|
||||
|
||||
.imp-doc-footer {
|
||||
position: absolute;
|
||||
right: 24px;
|
||||
bottom: 24px;
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
align-items: center;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -20,7 +20,7 @@ const fileData = ref<FileInfo[]>([]);
|
||||
const filesPath = ref([]);
|
||||
defineExpose({
|
||||
getFilesData() {
|
||||
return fileData.value;
|
||||
return fileData.value.filter((item) => item.filePath);
|
||||
},
|
||||
});
|
||||
function handleSuccess(response: any) {
|
||||
|
||||
@@ -1,189 +1,373 @@
|
||||
<script setup lang="ts">
|
||||
import { reactive, ref } from 'vue';
|
||||
import { computed, reactive, watch } from 'vue';
|
||||
|
||||
import { $t } from '@easyflow/locales';
|
||||
|
||||
import {
|
||||
ElAlert,
|
||||
ElCard,
|
||||
ElCol,
|
||||
ElForm,
|
||||
ElFormItem,
|
||||
ElInput,
|
||||
ElOption,
|
||||
ElRow,
|
||||
ElSelect,
|
||||
ElSlider,
|
||||
ElTag,
|
||||
} from 'element-plus';
|
||||
|
||||
const formRef = ref();
|
||||
const form = reactive({
|
||||
fileType: 'doc',
|
||||
splitterName: 'SimpleDocumentSplitter',
|
||||
chunkSize: 512,
|
||||
overlapSize: 128,
|
||||
regex: '',
|
||||
rowsPerChunk: 0,
|
||||
mdSplitterLevel: 1,
|
||||
});
|
||||
const fileTypes = [
|
||||
interface StrategyConfig {
|
||||
chunkSize?: number;
|
||||
mdSplitterLevel?: number;
|
||||
overlapSize?: number;
|
||||
regex?: string;
|
||||
rowsPerChunk?: number;
|
||||
strategyCode?: string;
|
||||
}
|
||||
|
||||
interface StrategyCandidate {
|
||||
score?: number;
|
||||
strategyCode: string;
|
||||
strategyLabel: string;
|
||||
}
|
||||
|
||||
interface AnalysisResult {
|
||||
candidateStrategies?: StrategyCandidate[];
|
||||
confidence?: number;
|
||||
reasons?: string[];
|
||||
recommendedStrategyCode?: string;
|
||||
recommendedStrategyLabel?: string;
|
||||
recommendedStructureType?: string;
|
||||
}
|
||||
|
||||
interface AnalyzeItem {
|
||||
analysis?: AnalysisResult;
|
||||
fileName: string;
|
||||
filePath: string;
|
||||
strategyConfig?: StrategyConfig;
|
||||
}
|
||||
|
||||
const props = defineProps<{
|
||||
analysisItems?: AnalyzeItem[];
|
||||
}>();
|
||||
|
||||
const strategyOptions = [
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.document'),
|
||||
value: 'doc',
|
||||
label: $t('documentCollection.splitterDoc.autoStrategy'),
|
||||
value: 'AUTO',
|
||||
},
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.markdownSection'),
|
||||
value: 'MARKDOWN_SECTION',
|
||||
},
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.outlineSection'),
|
||||
value: 'OUTLINE_SECTION',
|
||||
},
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.qaPair'),
|
||||
value: 'QA_PAIR',
|
||||
},
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.paragraphLength'),
|
||||
value: 'PARAGRAPH_LENGTH',
|
||||
},
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.customRegex'),
|
||||
value: 'CUSTOM_REGEX',
|
||||
},
|
||||
];
|
||||
const splitterNames = [
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.simpleDocumentSplitter'),
|
||||
value: 'SimpleDocumentSplitter',
|
||||
|
||||
const mdLevels = [1, 2, 3, 4, 5, 6];
|
||||
|
||||
const formMap = reactive<Record<string, StrategyConfig>>({});
|
||||
|
||||
watch(
|
||||
() => props.analysisItems,
|
||||
(items) => {
|
||||
for (const item of items || []) {
|
||||
formMap[item.filePath] = {
|
||||
chunkSize: item.strategyConfig?.chunkSize ?? 512,
|
||||
mdSplitterLevel: item.strategyConfig?.mdSplitterLevel ?? 2,
|
||||
overlapSize: item.strategyConfig?.overlapSize ?? 128,
|
||||
regex: item.strategyConfig?.regex ?? '',
|
||||
rowsPerChunk: item.strategyConfig?.rowsPerChunk ?? 1,
|
||||
strategyCode:
|
||||
item.strategyConfig?.strategyCode ||
|
||||
item.analysis?.recommendedStrategyCode ||
|
||||
'AUTO',
|
||||
};
|
||||
}
|
||||
},
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.simpleTokenizeSplitter'),
|
||||
value: 'SimpleTokenizeSplitter',
|
||||
},
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.regexDocumentSplitter'),
|
||||
value: 'RegexDocumentSplitter',
|
||||
},
|
||||
{
|
||||
label: $t('documentCollection.splitterDoc.markdownHeaderSplitter'),
|
||||
value: 'MarkdownHeaderSplitter',
|
||||
},
|
||||
];
|
||||
const mdSplitterLevel = [
|
||||
{
|
||||
label: '#',
|
||||
value: 1,
|
||||
},
|
||||
{
|
||||
label: '##',
|
||||
value: 2,
|
||||
},
|
||||
{
|
||||
label: '###',
|
||||
value: 3,
|
||||
},
|
||||
{
|
||||
label: '####',
|
||||
value: 4,
|
||||
},
|
||||
{
|
||||
label: '#####',
|
||||
value: 5,
|
||||
},
|
||||
{
|
||||
label: '######',
|
||||
value: 6,
|
||||
},
|
||||
];
|
||||
const rules = {
|
||||
name: [
|
||||
{ required: true, message: 'Please input Activity name', trigger: 'blur' },
|
||||
],
|
||||
region: [
|
||||
{
|
||||
required: true,
|
||||
message: 'Please select Activity zone',
|
||||
trigger: 'change',
|
||||
},
|
||||
],
|
||||
};
|
||||
{ immediate: true },
|
||||
);
|
||||
|
||||
const items = computed(() => props.analysisItems ?? []);
|
||||
|
||||
defineExpose({
|
||||
getSplitterFormValues() {
|
||||
return form;
|
||||
getPreviewRequestItems() {
|
||||
return items.value.map((item) => ({
|
||||
fileName: item.fileName,
|
||||
filePath: item.filePath,
|
||||
strategyConfig: {
|
||||
...formMap[item.filePath],
|
||||
},
|
||||
}));
|
||||
},
|
||||
});
|
||||
|
||||
function showLengthSettings(strategyCode?: string) {
|
||||
return [
|
||||
'AUTO',
|
||||
'MARKDOWN_SECTION',
|
||||
'OUTLINE_SECTION',
|
||||
'PARAGRAPH_LENGTH',
|
||||
].includes(strategyCode || '');
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="splitter-doc-container">
|
||||
<div class="strategy-container">
|
||||
<ElAlert
|
||||
:title="$t('documentCollection.importDoc.analysisTip')"
|
||||
type="info"
|
||||
:closable="false"
|
||||
class="strategy-tip"
|
||||
/>
|
||||
|
||||
<div class="strategy-list">
|
||||
<ElCard
|
||||
v-for="item in items"
|
||||
:key="item.filePath"
|
||||
class="strategy-card"
|
||||
shadow="never"
|
||||
>
|
||||
<div class="strategy-card__header">
|
||||
<div>
|
||||
<div class="strategy-card__title">{{ item.fileName }}</div>
|
||||
<div class="strategy-card__meta">
|
||||
{{ item.analysis?.recommendedStructureType || '-' }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="strategy-card__badges">
|
||||
<ElTag type="success" effect="plain">
|
||||
{{
|
||||
item.analysis?.recommendedStrategyLabel ||
|
||||
$t('documentCollection.splitterDoc.autoStrategy')
|
||||
}}
|
||||
</ElTag>
|
||||
<ElTag effect="plain">
|
||||
{{ $t('documentCollection.importDoc.confidence') }}
|
||||
{{ item.analysis?.confidence ?? 0 }}
|
||||
</ElTag>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ElRow :gutter="16" class="strategy-card__content">
|
||||
<ElCol :span="12">
|
||||
<div class="strategy-block">
|
||||
<div class="strategy-block__label">
|
||||
{{ $t('documentCollection.importDoc.recommendReason') }}
|
||||
</div>
|
||||
<ul class="strategy-reason-list">
|
||||
<li
|
||||
v-for="reason in item.analysis?.reasons || []"
|
||||
:key="reason"
|
||||
class="strategy-reason-list__item"
|
||||
>
|
||||
{{ reason }}
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="strategy-block">
|
||||
<div class="strategy-block__label">
|
||||
{{ $t('documentCollection.importDoc.candidateStrategies') }}
|
||||
</div>
|
||||
<div class="strategy-candidate-list">
|
||||
<ElTag
|
||||
v-for="candidate in item.analysis?.candidateStrategies || []"
|
||||
:key="candidate.strategyCode"
|
||||
effect="plain"
|
||||
>
|
||||
{{ candidate.strategyLabel }} / {{ candidate.score }}
|
||||
</ElTag>
|
||||
</div>
|
||||
</div>
|
||||
</ElCol>
|
||||
|
||||
<ElCol :span="12">
|
||||
<ElForm
|
||||
ref="formRef"
|
||||
:model="form"
|
||||
:rules="rules"
|
||||
label-width="auto"
|
||||
class="custom-form"
|
||||
:model="formMap[item.filePath]"
|
||||
label-position="top"
|
||||
class="strategy-form"
|
||||
>
|
||||
<ElFormItem
|
||||
:label="$t('documentCollection.splitterDoc.fileType')"
|
||||
prop="fileType"
|
||||
:label="$t('documentCollection.importDoc.strategySelection')"
|
||||
>
|
||||
<ElSelect
|
||||
v-model="formMap[item.filePath].strategyCode"
|
||||
class="w-full"
|
||||
>
|
||||
<ElSelect v-model="form.fileType">
|
||||
<ElOption
|
||||
v-for="item in fileTypes"
|
||||
:key="item.value"
|
||||
v-bind="item"
|
||||
:label="item.label"
|
||||
/>
|
||||
</ElSelect>
|
||||
</ElFormItem>
|
||||
<ElFormItem
|
||||
:label="$t('documentCollection.splitterDoc.splitterName')"
|
||||
prop="splitterName"
|
||||
>
|
||||
<ElSelect v-model="form.splitterName">
|
||||
<ElOption
|
||||
v-for="item in splitterNames"
|
||||
:key="item.value"
|
||||
v-bind="item"
|
||||
:label="item.label"
|
||||
v-for="option in strategyOptions"
|
||||
:key="option.value"
|
||||
:label="option.label"
|
||||
:value="option.value"
|
||||
/>
|
||||
</ElSelect>
|
||||
</ElFormItem>
|
||||
|
||||
<ElFormItem
|
||||
v-if="showLengthSettings(formMap[item.filePath].strategyCode)"
|
||||
:label="$t('documentCollection.splitterDoc.chunkSize')"
|
||||
v-if="
|
||||
form.splitterName === 'SimpleDocumentSplitter' ||
|
||||
form.splitterName === 'SimpleTokenizeSplitter'
|
||||
"
|
||||
prop="chunkSize"
|
||||
>
|
||||
<ElSlider v-model="form.chunkSize" show-input :max="2048" />
|
||||
<ElSlider
|
||||
v-model="formMap[item.filePath].chunkSize"
|
||||
:max="2048"
|
||||
:min="128"
|
||||
show-input
|
||||
/>
|
||||
</ElFormItem>
|
||||
|
||||
<ElFormItem
|
||||
v-if="
|
||||
formMap[item.filePath].strategyCode === 'PARAGRAPH_LENGTH' ||
|
||||
formMap[item.filePath].strategyCode === 'AUTO'
|
||||
"
|
||||
:label="$t('documentCollection.splitterDoc.overlapSize')"
|
||||
>
|
||||
<ElSlider
|
||||
v-model="formMap[item.filePath].overlapSize"
|
||||
:max="512"
|
||||
:min="0"
|
||||
show-input
|
||||
/>
|
||||
</ElFormItem>
|
||||
|
||||
<ElFormItem
|
||||
v-if="
|
||||
form.splitterName === 'SimpleDocumentSplitter' ||
|
||||
form.splitterName === 'SimpleTokenizeSplitter'
|
||||
formMap[item.filePath].strategyCode === 'MARKDOWN_SECTION'
|
||||
"
|
||||
prop="overlapSize"
|
||||
>
|
||||
<ElSlider v-model="form.overlapSize" show-input :max="2048" />
|
||||
</ElFormItem>
|
||||
<ElFormItem
|
||||
:label="$t('documentCollection.splitterDoc.regex')"
|
||||
prop="regex"
|
||||
v-if="form.splitterName === 'RegexDocumentSplitter'"
|
||||
>
|
||||
<ElInput v-model="form.regex" />
|
||||
</ElFormItem>
|
||||
<ElFormItem
|
||||
v-if="form.splitterName === 'MarkdownHeaderSplitter'"
|
||||
:label="$t('documentCollection.splitterDoc.mdSplitterLevel')"
|
||||
prop="splitterName"
|
||||
>
|
||||
<ElSelect v-model="form.mdSplitterLevel">
|
||||
<ElSelect
|
||||
v-model="formMap[item.filePath].mdSplitterLevel"
|
||||
class="w-full"
|
||||
>
|
||||
<ElOption
|
||||
v-for="item in mdSplitterLevel"
|
||||
:key="item.value"
|
||||
:label="item.label"
|
||||
:value="item.value"
|
||||
v-for="level in mdLevels"
|
||||
:key="level"
|
||||
:label="'#'.repeat(level)"
|
||||
:value="level"
|
||||
/>
|
||||
</ElSelect>
|
||||
</ElFormItem>
|
||||
|
||||
<ElFormItem
|
||||
v-if="formMap[item.filePath].strategyCode === 'CUSTOM_REGEX'"
|
||||
:label="$t('documentCollection.splitterDoc.regex')"
|
||||
>
|
||||
<ElInput v-model="formMap[item.filePath].regex" />
|
||||
</ElFormItem>
|
||||
</ElForm>
|
||||
</ElCol>
|
||||
</ElRow>
|
||||
</ElCard>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.splitter-doc-container {
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
align-items: center;
|
||||
.strategy-container {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
.custom-form {
|
||||
width: 500px;
|
||||
|
||||
.strategy-tip {
|
||||
border-radius: 12px;
|
||||
}
|
||||
.custom-form :deep(.el-input),
|
||||
.custom-form :deep(.ElSelect) {
|
||||
width: 100%;
|
||||
|
||||
.strategy-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.strategy-card {
|
||||
border: 1px solid var(--el-border-color-light);
|
||||
border-radius: 16px;
|
||||
}
|
||||
|
||||
.strategy-card__header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
padding-bottom: 16px;
|
||||
border-bottom: 1px solid var(--el-border-color-lighter);
|
||||
}
|
||||
|
||||
.strategy-card__title {
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
color: var(--el-text-color-primary);
|
||||
}
|
||||
|
||||
.strategy-card__meta {
|
||||
margin-top: 4px;
|
||||
font-size: 13px;
|
||||
color: var(--el-text-color-secondary);
|
||||
}
|
||||
|
||||
.strategy-card__badges {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: flex-start;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.strategy-card__content {
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.strategy-block {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.strategy-block + .strategy-block {
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.strategy-block__label {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: var(--el-text-color-primary);
|
||||
}
|
||||
|
||||
.strategy-reason-list {
|
||||
margin: 0;
|
||||
padding-left: 18px;
|
||||
color: var(--el-text-color-regular);
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
.strategy-reason-list__item {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.strategy-candidate-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.strategy-form {
|
||||
padding: 16px;
|
||||
border-radius: 12px;
|
||||
background: var(--el-fill-color-light);
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,168 +1,286 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, ref, watch } from 'vue';
|
||||
import { useRoute } from 'vue-router';
|
||||
import { computed, ref, watch } from 'vue';
|
||||
|
||||
import { api } from '#/api/request';
|
||||
import CategoryPanel from '#/components/categoryPanel/CategoryPanel.vue';
|
||||
import PreviewSearchKnowledge from '#/views/ai/documentCollection/PreviewSearchKnowledge.vue';
|
||||
import { $t } from '@easyflow/locales';
|
||||
|
||||
export interface FileInfo {
|
||||
filePath: string;
|
||||
import {
|
||||
ElAlert,
|
||||
ElDescriptions,
|
||||
ElDescriptionsItem,
|
||||
ElEmpty,
|
||||
ElTabPane,
|
||||
ElTabs,
|
||||
ElTag,
|
||||
} from 'element-plus';
|
||||
|
||||
interface ChunkItem {
|
||||
answer?: string;
|
||||
charCount?: number;
|
||||
chunkId?: string;
|
||||
chunkType?: string;
|
||||
content?: string;
|
||||
headingPath?: string[];
|
||||
partNo?: number;
|
||||
partTotal?: number;
|
||||
question?: string;
|
||||
sourceLabel?: string;
|
||||
tokenEstimate?: number;
|
||||
warnings?: string[];
|
||||
}
|
||||
|
||||
interface PreviewItem {
|
||||
analysis?: {
|
||||
confidence?: number;
|
||||
recommendedStructureType?: string;
|
||||
};
|
||||
chunks?: ChunkItem[];
|
||||
fileName: string;
|
||||
previewSessionId: string;
|
||||
strategyLabel?: string;
|
||||
totalChunks?: number;
|
||||
totalWarnings?: number;
|
||||
}
|
||||
const props = defineProps({
|
||||
pageNumber: {
|
||||
default: 1,
|
||||
type: Number,
|
||||
},
|
||||
pageSize: {
|
||||
default: 10,
|
||||
type: Number,
|
||||
},
|
||||
knowledgeId: {
|
||||
default: '',
|
||||
type: String,
|
||||
},
|
||||
fliesList: {
|
||||
default: () => [],
|
||||
type: Array<FileInfo>,
|
||||
},
|
||||
splitterParams: {
|
||||
default: () => {},
|
||||
type: Object,
|
||||
},
|
||||
});
|
||||
const emit = defineEmits(['updateTotal']);
|
||||
const documentList = ref<any[]>([]);
|
||||
const route = useRoute();
|
||||
defineExpose({
|
||||
getFilesData() {
|
||||
return documentList.value.length;
|
||||
},
|
||||
});
|
||||
const knowledgeIdRef = ref<string>((route.query.id as string) || '');
|
||||
const selectedCategory = ref<any>();
|
||||
|
||||
watch(
|
||||
() => props.pageNumber,
|
||||
(newVal) => {
|
||||
if (selectedCategory.value) {
|
||||
splitterDocPreview(
|
||||
newVal,
|
||||
props.pageSize,
|
||||
selectedCategory.value.value,
|
||||
'textSplit',
|
||||
selectedCategory.value.label,
|
||||
);
|
||||
} else {
|
||||
splitterDocPreview(
|
||||
newVal,
|
||||
props.pageSize,
|
||||
props.fliesList[0]!.filePath,
|
||||
'textSplit',
|
||||
props.fliesList[0]!.fileName,
|
||||
);
|
||||
}
|
||||
},
|
||||
const props = defineProps<{
|
||||
previewItems?: PreviewItem[];
|
||||
}>();
|
||||
|
||||
const activeFile = ref('');
|
||||
|
||||
const previewItems = computed(() => props.previewItems ?? []);
|
||||
const currentPreview = computed(
|
||||
() =>
|
||||
previewItems.value.find(
|
||||
(item) => item.previewSessionId === activeFile.value,
|
||||
) || previewItems.value[0],
|
||||
);
|
||||
|
||||
watch(
|
||||
() => props.pageSize,
|
||||
(newVal) => {
|
||||
if (selectedCategory.value) {
|
||||
splitterDocPreview(
|
||||
props.pageNumber,
|
||||
newVal,
|
||||
selectedCategory.value.value,
|
||||
'textSplit',
|
||||
selectedCategory.value.label,
|
||||
);
|
||||
} else {
|
||||
splitterDocPreview(
|
||||
props.pageNumber,
|
||||
newVal,
|
||||
props.fliesList[0]!.filePath,
|
||||
'textSplit',
|
||||
props.fliesList[0]!.fileName,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
function splitterDocPreview(
|
||||
pageNumber: number,
|
||||
pageSize: number,
|
||||
filePath: string,
|
||||
operation: string,
|
||||
fileOriginName: string,
|
||||
) {
|
||||
api
|
||||
.post('/api/v1/document/textSplit', {
|
||||
pageNumber,
|
||||
pageSize,
|
||||
filePath,
|
||||
operation,
|
||||
knowledgeId: knowledgeIdRef.value,
|
||||
fileOriginName,
|
||||
...props.splitterParams,
|
||||
})
|
||||
.then((res) => {
|
||||
if (res.errorCode === 0) {
|
||||
documentList.value = res.data.previewData;
|
||||
emit('updateTotal', res.data.total);
|
||||
}
|
||||
});
|
||||
}
|
||||
onMounted(() => {
|
||||
if (props.fliesList.length === 0) {
|
||||
previewItems,
|
||||
(items) => {
|
||||
if (items.length === 0) {
|
||||
activeFile.value = '';
|
||||
return;
|
||||
}
|
||||
splitterDocPreview(
|
||||
props.pageNumber,
|
||||
props.pageSize,
|
||||
props.fliesList[0]!.filePath,
|
||||
'textSplit',
|
||||
props.fliesList[0]!.fileName,
|
||||
);
|
||||
});
|
||||
const changeCategory = (category: any) => {
|
||||
selectedCategory.value = category;
|
||||
splitterDocPreview(
|
||||
props.pageNumber,
|
||||
props.pageSize,
|
||||
category.value,
|
||||
'textSplit',
|
||||
category.label,
|
||||
);
|
||||
};
|
||||
if (!items.some((item) => item.previewSessionId === activeFile.value)) {
|
||||
activeFile.value = items[0]?.previewSessionId || '';
|
||||
}
|
||||
},
|
||||
{ immediate: true },
|
||||
);
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="splitter-doc-container">
|
||||
<div>
|
||||
<CategoryPanel
|
||||
:categories="fliesList"
|
||||
title-key="fileName"
|
||||
:need-hide-collapse="true"
|
||||
:expand-width="200"
|
||||
value-key="filePath"
|
||||
:default-selected-category="fliesList[0]!.filePath"
|
||||
@click="changeCategory"
|
||||
<div class="preview-shell">
|
||||
<ElAlert
|
||||
:title="$t('documentCollection.importDoc.previewTip')"
|
||||
type="info"
|
||||
:closable="false"
|
||||
class="preview-alert"
|
||||
/>
|
||||
|
||||
<ElEmpty
|
||||
v-if="previewItems.length === 0"
|
||||
:description="$t('documentCollection.importDoc.previewEmpty')"
|
||||
/>
|
||||
|
||||
<div v-else class="preview-panel">
|
||||
<ElTabs v-model="activeFile" class="preview-tabs">
|
||||
<ElTabPane
|
||||
v-for="item in previewItems"
|
||||
:key="item.previewSessionId"
|
||||
:label="item.fileName"
|
||||
:name="item.previewSessionId"
|
||||
/>
|
||||
</ElTabs>
|
||||
|
||||
<div v-if="currentPreview" class="preview-detail">
|
||||
<ElDescriptions :column="4" border class="preview-summary">
|
||||
<ElDescriptionsItem :label="$t('documentCollection.fileName')">
|
||||
{{ currentPreview.fileName }}
|
||||
</ElDescriptionsItem>
|
||||
<ElDescriptionsItem
|
||||
:label="$t('documentCollection.importDoc.strategySelection')"
|
||||
>
|
||||
{{ currentPreview.strategyLabel || '-' }}
|
||||
</ElDescriptionsItem>
|
||||
<ElDescriptionsItem :label="$t('documentCollection.total')">
|
||||
{{ currentPreview.totalChunks || 0 }}
|
||||
</ElDescriptionsItem>
|
||||
<ElDescriptionsItem
|
||||
:label="$t('documentCollection.importDoc.warningCount')"
|
||||
>
|
||||
{{ currentPreview.totalWarnings || 0 }}
|
||||
</ElDescriptionsItem>
|
||||
</ElDescriptions>
|
||||
|
||||
<div class="chunk-list">
|
||||
<div
|
||||
v-for="chunk in currentPreview.chunks || []"
|
||||
:key="chunk.chunkId"
|
||||
class="chunk-card"
|
||||
>
|
||||
<div class="chunk-card__header">
|
||||
<div>
|
||||
<div class="chunk-card__title">
|
||||
{{ chunk.sourceLabel || chunk.chunkId }}
|
||||
</div>
|
||||
<div
|
||||
v-if="chunk.headingPath && chunk.headingPath.length > 0"
|
||||
class="chunk-card__path"
|
||||
>
|
||||
{{ chunk.headingPath.join(' / ') }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="chunk-card__meta">
|
||||
<ElTag effect="plain">{{ chunk.chunkType || '-' }}</ElTag>
|
||||
<ElTag effect="plain">
|
||||
{{ chunk.charCount || 0 }} / {{ chunk.tokenEstimate || 0 }}
|
||||
</ElTag>
|
||||
<ElTag
|
||||
v-if="(chunk.partTotal || 1) > 1"
|
||||
type="warning"
|
||||
effect="plain"
|
||||
>
|
||||
{{ chunk.partNo }}/{{ chunk.partTotal }}
|
||||
</ElTag>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="preview-container">
|
||||
<PreviewSearchKnowledge :data="documentList" :hide-score="true" />
|
||||
<div v-if="chunk.chunkType === 'qa_pair'" class="qa-block">
|
||||
<div class="qa-block__item">
|
||||
<span class="qa-block__label">Q</span>
|
||||
<span>{{ chunk.question }}</span>
|
||||
</div>
|
||||
<div class="qa-block__item">
|
||||
<span class="qa-block__label">A</span>
|
||||
<span>{{ chunk.answer }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<pre class="chunk-card__content">{{ chunk.content }}</pre>
|
||||
|
||||
<div
|
||||
v-if="chunk.warnings && chunk.warnings.length > 0"
|
||||
class="chunk-card__warnings"
|
||||
>
|
||||
<ElTag
|
||||
v-for="warning in chunk.warnings"
|
||||
:key="warning"
|
||||
type="warning"
|
||||
effect="plain"
|
||||
>
|
||||
{{ warning }}
|
||||
</ElTag>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.splitter-doc-container {
|
||||
height: 100%;
|
||||
.preview-shell {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
.preview-container {
|
||||
flex: 1;
|
||||
overflow: scroll;
|
||||
|
||||
.preview-alert {
|
||||
border-radius: 12px;
|
||||
}
|
||||
|
||||
.preview-panel {
|
||||
padding: 20px;
|
||||
border: 1px solid var(--el-border-color-light);
|
||||
border-radius: 16px;
|
||||
background: var(--el-bg-color);
|
||||
}
|
||||
|
||||
.preview-summary {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.chunk-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
max-height: 560px;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.chunk-card {
|
||||
padding: 16px;
|
||||
border: 1px solid var(--el-border-color-lighter);
|
||||
border-radius: 14px;
|
||||
background: var(--el-fill-color-blank);
|
||||
}
|
||||
|
||||
.chunk-card__header {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.chunk-card__title {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
color: var(--el-text-color-primary);
|
||||
}
|
||||
|
||||
.chunk-card__path {
|
||||
margin-top: 6px;
|
||||
font-size: 12px;
|
||||
color: var(--el-text-color-secondary);
|
||||
}
|
||||
|
||||
.chunk-card__meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.chunk-card__content {
|
||||
margin: 16px 0 0;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-family: inherit;
|
||||
line-height: 1.7;
|
||||
color: var(--el-text-color-regular);
|
||||
}
|
||||
|
||||
.chunk-card__warnings {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
.qa-block {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
margin-top: 16px;
|
||||
padding: 12px;
|
||||
border-radius: 12px;
|
||||
background: var(--el-fill-color-light);
|
||||
}
|
||||
|
||||
.qa-block__item {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.qa-block__label {
|
||||
display: inline-flex;
|
||||
width: 22px;
|
||||
justify-content: center;
|
||||
border-radius: 999px;
|
||||
background: var(--el-color-primary-light-9);
|
||||
color: var(--el-color-primary);
|
||||
font-weight: 600;
|
||||
}
|
||||
</style>
|
||||
|
||||
Reference in New Issue
Block a user