发布 v1.10 #5

Merged
czm merged 147 commits from develop into main 2026-08-20 11:36:27 +08:00
32 changed files with 5672 additions and 156 deletions
Showing only changes of commit 0d14f1c165 - Show all commits

View File

@@ -4,6 +4,7 @@ import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Locale;
public class FileUtil { public class FileUtil {
@@ -33,23 +34,34 @@ public class FileUtil {
} }
/**
* 按文件名提取平台支持的文档扩展名。
*
* @param fileName 文件名或文件路径
* @return 小写扩展名;无法识别时返回 {@code null}
*/
public static String getFileTypeByExtension(String fileName) { public static String getFileTypeByExtension(String fileName) {
if (fileName.endsWith(".txt")) { if (fileName == null) {
return "txt"; return null;
} else if (fileName.endsWith(".pdf")) {
return "pdf";
} else if (fileName.endsWith(".md")) {
return "md";
} else if (fileName.endsWith(".docx")) {
return "docx";
} else if (fileName.endsWith(".xlsx")) {
return "xlsx";
} else if (fileName.endsWith(".ppt")) {
return "ppt";
} else if (fileName.endsWith(".pptx")) {
return "pptx";
} }
else { String normalizedFileName = fileName.toLowerCase(Locale.ROOT);
if (normalizedFileName.endsWith(".txt")) {
return "txt";
} else if (normalizedFileName.endsWith(".pdf")) {
return "pdf";
} else if (normalizedFileName.endsWith(".md")) {
return "md";
} else if (normalizedFileName.endsWith(".docx")) {
return "docx";
} else if (normalizedFileName.endsWith(".xlsx")) {
return "xlsx";
} else if (normalizedFileName.endsWith(".csv")) {
return "csv";
} else if (normalizedFileName.endsWith(".ppt")) {
return "ppt";
} else if (normalizedFileName.endsWith(".pptx")) {
return "pptx";
} else {
return null; return null;
} }
} }

View File

@@ -84,6 +84,10 @@
<groupId>org.jsoup</groupId> <groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId> <artifactId>jsoup</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.commonmark</groupId> <groupId>org.commonmark</groupId>
<artifactId>commonmark-ext-gfm-tables</artifactId> <artifactId>commonmark-ext-gfm-tables</artifactId>

View File

@@ -95,6 +95,9 @@ public final class DocumentImportDtos {
private BigInteger knowledgeId; private BigInteger knowledgeId;
private BigInteger documentId; private BigInteger documentId;
private List<PreviewFileRequest> files = new ArrayList<PreviewFileRequest>(); private List<PreviewFileRequest> files = new ArrayList<PreviewFileRequest>();
private String previewSessionId;
private Integer pageNo;
private Integer pageSize;
public BigInteger getKnowledgeId() { public BigInteger getKnowledgeId() {
return knowledgeId; return knowledgeId;
@@ -119,6 +122,60 @@ public final class DocumentImportDtos {
public void setFiles(List<PreviewFileRequest> files) { public void setFiles(List<PreviewFileRequest> files) {
this.files = files; this.files = files;
} }
/**
* 返回已有预览会话 ID。
*
* @return 预览会话 ID
*/
public String getPreviewSessionId() {
return previewSessionId;
}
/**
* 设置已有预览会话 ID用于只读取指定分页而不重新分块。
*
* @param previewSessionId 预览会话 ID
*/
public void setPreviewSessionId(String previewSessionId) {
this.previewSessionId = previewSessionId;
}
/**
* 返回预览页码。
*
* @return 从 1 开始的页码
*/
public Integer getPageNo() {
return pageNo;
}
/**
* 设置预览页码。
*
* @param pageNo 从 1 开始的页码
*/
public void setPageNo(Integer pageNo) {
this.pageNo = pageNo;
}
/**
* 返回单页分块数。
*
* @return 单页分块数
*/
public Integer getPageSize() {
return pageSize;
}
/**
* 设置单页分块数。
*
* @param pageSize 单页分块数
*/
public void setPageSize(Integer pageSize) {
this.pageSize = pageSize;
}
} }
public static class CommitRequest implements Serializable { public static class CommitRequest implements Serializable {
@@ -419,6 +476,8 @@ public final class DocumentImportDtos {
private AnalysisResult analysis; private AnalysisResult analysis;
private Integer totalChunks; private Integer totalChunks;
private Integer totalWarnings; private Integer totalWarnings;
private Integer pageNo;
private Integer pageSize;
private List<PreviewChunkResult> chunks = new ArrayList<PreviewChunkResult>(); private List<PreviewChunkResult> chunks = new ArrayList<PreviewChunkResult>();
public String getPreviewSessionId() { public String getPreviewSessionId() {
@@ -493,6 +552,42 @@ public final class DocumentImportDtos {
this.totalWarnings = totalWarnings; this.totalWarnings = totalWarnings;
} }
/**
* 返回当前预览页码。
*
* @return 从 1 开始的页码
*/
public Integer getPageNo() {
return pageNo;
}
/**
* 设置当前预览页码。
*
* @param pageNo 从 1 开始的页码
*/
public void setPageNo(Integer pageNo) {
this.pageNo = pageNo;
}
/**
* 返回当前预览页大小。
*
* @return 单页分块数
*/
public Integer getPageSize() {
return pageSize;
}
/**
* 设置当前预览页大小。
*
* @param pageSize 单页分块数
*/
public void setPageSize(Integer pageSize) {
this.pageSize = pageSize;
}
public List<PreviewChunkResult> getChunks() { public List<PreviewChunkResult> getChunks() {
return chunks; return chunks;
} }
@@ -640,6 +735,8 @@ public final class DocumentImportDtos {
private Document document; private Document document;
private List<DocumentChunk> documentChunks = new ArrayList<DocumentChunk>(); private List<DocumentChunk> documentChunks = new ArrayList<DocumentChunk>();
private List<RagChunk> previewChunks = new ArrayList<RagChunk>(); private List<RagChunk> previewChunks = new ArrayList<RagChunk>();
private String chunkSnapshotPath;
private Integer totalChunks;
private Date createdAt; private Date createdAt;
public String getSessionId() { public String getSessionId() {
@@ -730,6 +827,42 @@ public final class DocumentImportDtos {
this.previewChunks = previewChunks; this.previewChunks = previewChunks;
} }
/**
* 返回持久化分块快照清单路径。
*
* @return 快照清单路径
*/
public String getChunkSnapshotPath() {
return chunkSnapshotPath;
}
/**
* 设置持久化分块快照清单路径。
*
* @param chunkSnapshotPath 快照清单路径
*/
public void setChunkSnapshotPath(String chunkSnapshotPath) {
this.chunkSnapshotPath = chunkSnapshotPath;
}
/**
* 返回完整分块总数。
*
* @return 分块总数
*/
public Integer getTotalChunks() {
return totalChunks;
}
/**
* 设置完整分块总数。
*
* @param totalChunks 分块总数
*/
public void setTotalChunks(Integer totalChunks) {
this.totalChunks = totalChunks;
}
public Date getCreatedAt() { public Date getCreatedAt() {
return createdAt; return createdAt;
} }

View File

@@ -39,6 +39,10 @@ public final class DocumentImportKeys {
public static final String KEY_DOCUMENT_ROW_END = "rowEnd"; public static final String KEY_DOCUMENT_ROW_END = "rowEnd";
public static final String KEY_DOCUMENT_IMAGE_REFS = "imageRefs"; public static final String KEY_DOCUMENT_IMAGE_REFS = "imageRefs";
public static final String KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY = "parseArtifactSummary"; public static final String KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY = "parseArtifactSummary";
public static final String KEY_DOCUMENT_CSV_TABLE_SNAPSHOT_PATH = "parse.csvTableSnapshotPath";
public static final String KEY_DOCUMENT_CSV_ENCODING = "parse.csvEncoding";
public static final String KEY_DOCUMENT_CSV_ROW_COUNT = "parse.csvRowCount";
public static final String KEY_DOCUMENT_CSV_COLUMN_COUNT = "parse.csvColumnCount";
public static final String KEY_DOCUMENT_IMPORT_MODE = "import.mode"; public static final String KEY_DOCUMENT_IMPORT_MODE = "import.mode";
public static final String KEY_DOCUMENT_IMPORT_BATCH_ID = "import.batchId"; public static final String KEY_DOCUMENT_IMPORT_BATCH_ID = "import.batchId";
public static final String KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID = "import.batchItemId"; public static final String KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID = "import.batchItemId";

View File

@@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.common.web.exceptions.BusinessException;
import java.math.BigInteger;
import java.time.Duration; import java.time.Duration;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.UUID; import java.util.UUID;
@@ -27,6 +28,34 @@ public class DocumentImportPreviewService {
return sessionId; return sessionId;
} }
/**
* 为同一文档替换预览会话,返回被替换的旧会话供调用方清理对象存储工件。
*
* @param session 新预览会话
* @return 旧会话;不存在时返回 {@code null}
*/
public DocumentImportDtos.PreviewSession replaceForDocument(
DocumentImportDtos.PreviewSession session) {
String latestKey = buildLatestDocumentKey(
session.getKnowledgeId(), session.getDocumentId());
Object previousSessionId = defaultCache.get(latestKey);
DocumentImportDtos.PreviewSession previous = null;
if (previousSessionId instanceof String id) {
Object cached = defaultCache.get(buildKey(id));
if (cached instanceof DocumentImportDtos.PreviewSession oldSession) {
previous = oldSession;
}
}
String sessionId = put(session);
defaultCache.put(
latestKey, sessionId, SESSION_TTL.toMinutes(), TimeUnit.MINUTES);
if (previousSessionId instanceof String id
&& !id.equals(sessionId)) {
defaultCache.remove(buildKey(id));
}
return previous;
}
public DocumentImportDtos.PreviewSession getRequired(String sessionId) { public DocumentImportDtos.PreviewSession getRequired(String sessionId) {
Object cached = defaultCache.get(buildKey(sessionId)); Object cached = defaultCache.get(buildKey(sessionId));
if (!(cached instanceof DocumentImportDtos.PreviewSession)) { if (!(cached instanceof DocumentImportDtos.PreviewSession)) {
@@ -36,10 +65,33 @@ public class DocumentImportPreviewService {
} }
public void remove(String sessionId) { public void remove(String sessionId) {
Object cached = defaultCache.get(buildKey(sessionId));
defaultCache.remove(buildKey(sessionId)); defaultCache.remove(buildKey(sessionId));
if (cached instanceof DocumentImportDtos.PreviewSession session) {
String latestKey = buildLatestDocumentKey(
session.getKnowledgeId(), session.getDocumentId());
Object latestSessionId = defaultCache.get(latestKey);
if (sessionId.equals(latestSessionId)) {
defaultCache.remove(latestKey);
}
}
} }
private String buildKey(String sessionId) { private String buildKey(String sessionId) {
return DocumentImportKeys.CACHE_KEY_PREFIX + sessionId; return DocumentImportKeys.CACHE_KEY_PREFIX + sessionId;
} }
/**
* 构造文档当前预览会话索引键。
*
* @param knowledgeId 知识库 ID
* @param documentId 文档 ID
* @return 缓存键
*/
private String buildLatestDocumentKey(
BigInteger knowledgeId,
BigInteger documentId) {
return DocumentImportKeys.CACHE_KEY_PREFIX
+ "latest:" + knowledgeId + ":" + documentId;
}
} }

View File

@@ -0,0 +1,34 @@
package tech.easyflow.ai.documentimport.task;
import tech.easyflow.common.web.exceptions.BusinessException;
/**
* CSV 导入异常,携带可持久化的稳定失败码。
*
* @author Codex
* @since 2026-08-04
*/
public class CsvImportException extends BusinessException {
private final String failureCode;
/**
* 创建 CSV 导入异常。
*
* @param failureCode 稳定失败码
* @param message 用户可见错误信息
*/
public CsvImportException(String failureCode, String message) {
super(message);
this.failureCode = failureCode;
}
/**
* 返回稳定失败码。
*
* @return 失败码
*/
public String getFailureCode() {
return failureCode;
}
}

View File

@@ -57,7 +57,7 @@ public class DocumentImportBatchAppService {
private static final Logger LOG = LoggerFactory.getLogger(DocumentImportBatchAppService.class); private static final Logger LOG = LoggerFactory.getLogger(DocumentImportBatchAppService.class);
private static final Set<String> SUPPORTED_EXTENSIONS = private static final Set<String> SUPPORTED_EXTENSIONS =
Set.of("txt", "pdf", "docx", "md", "pptx", "xlsx"); DocumentImportFormatPolicy.supportedExtensions();
private static final Duration BATCH_MUTATION_LOCK_LEASE = Duration.ofMinutes(30); private static final Duration BATCH_MUTATION_LOCK_LEASE = Duration.ofMinutes(30);
private final DocumentImportBatchService batchService; private final DocumentImportBatchService batchService;

View File

@@ -1,10 +1,12 @@
package tech.easyflow.ai.documentimport.task; package tech.easyflow.ai.documentimport.task;
import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSON;
import com.easyagents.rag.ingestion.model.AnalysisResult;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import tech.easyflow.ai.document.support.DocumentInputStreamSupport; import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
import tech.easyflow.ai.documentimport.DocumentImportDtos; import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.easyagents.CustomMultipartFile; import tech.easyflow.ai.easyagents.CustomMultipartFile;
import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.util.StringUtil; import tech.easyflow.common.util.StringUtil;
import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.common.web.exceptions.BusinessException;
@@ -13,11 +15,20 @@ import javax.annotation.Resource;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HexFormat;
import java.util.List;
import java.util.UUID;
import java.util.function.Consumer;
/** /**
* 自动导入分块快照持久化服务。 * 自动导入分块快照持久化服务。
* *
* <p>快照写入对象存储,向量化任务只保存稳定路径,避免依赖短期预览缓存。</p> * <p>V2 快照使用“有界分片 + 最后写入清单”的提交协议,索引任务可以按批读取,
* 同时保留对历史 V1 单 JSON 快照的读取兼容。</p>
* *
* @author Codex * @author Codex
* @since 2026-07-31 * @since 2026-07-31
@@ -25,7 +36,11 @@ import java.nio.charset.StandardCharsets;
@Service @Service
public class DocumentImportChunkSnapshotService { public class DocumentImportChunkSnapshotService {
private static final long MAX_SNAPSHOT_BYTES = 256L * 1024L * 1024L; private static final String SNAPSHOT_VERSION_V2 = "v2";
private static final int DEFAULT_PART_CHUNK_COUNT = 128;
private static final long MAX_MANIFEST_BYTES = 4L * 1024L * 1024L;
private static final long MAX_PART_BYTES = 4L * 1024L * 1024L;
private static final long MAX_LEGACY_SNAPSHOT_BYTES = 256L * 1024L * 1024L;
@Resource(name = "default") @Resource(name = "default")
private FileStorageService storageService; private FileStorageService storageService;
@@ -34,61 +49,727 @@ public class DocumentImportChunkSnapshotService {
* 持久化预览会话及其最终分块。 * 持久化预览会话及其最终分块。
* *
* @param session 预览会话 * @param session 预览会话
* @return 快照存储路径 * @return V2 清单存储路径
*/ */
public String save(DocumentImportDtos.PreviewSession session) { public String save(DocumentImportDtos.PreviewSession session) {
if (session == null || session.getKnowledgeId() == null || session.getDocumentId() == null validateSession(session, true);
|| session.getDocumentChunks() == null || session.getDocumentChunks().isEmpty()) { try (SnapshotWriter writer = createWriter(session)) {
throw new BusinessException("分块快照内容不完整"); List<DocumentChunk> chunks = session.getDocumentChunks();
for (int start = 0; start < chunks.size(); start += DEFAULT_PART_CHUNK_COUNT) {
int end = Math.min(start + DEFAULT_PART_CHUNK_COUNT, chunks.size());
writer.append(new ArrayList<DocumentChunk>(chunks.subList(start, end)));
} }
String fileName = session.getDocumentId() + "-chunks.json"; return writer.finish();
byte[] payload = JSON.toJSONBytes(session);
if (payload.length > MAX_SNAPSHOT_BYTES) {
throw new BusinessException("分块快照过大,请调整文档后重试");
} }
CustomMultipartFile file = new CustomMultipartFile(
payload, fileName, fileName, "application/json");
String path = storageService.save(file,
"knowledge-import-snapshots/" + session.getKnowledgeId() + "/" + session.getDocumentId());
if (!StringUtil.hasText(path)) {
throw new BusinessException("分块快照保存失败");
}
return path;
} }
/** /**
* 从稳定存储恢复预览会话 * 创建流式快照写入器
*
* @param session 仅需包含归属、策略和预览元信息的会话
* @return 快照写入器
*/
public SnapshotWriter createWriter(DocumentImportDtos.PreviewSession session) {
validateSession(session, false);
return new SnapshotWriter(copyHeader(session));
}
/**
* 从稳定存储完整恢复预览会话。
*
* <p>该方法用于兼容现有调用和小型预览。大文件索引应使用
* {@link #forEachBatch(String, int, Consumer)}。</p>
* *
* @param path 快照路径 * @param path 快照路径
* @return 预览会话 * @return 预览会话
*/ */
public DocumentImportDtos.PreviewSession load(String path) { public DocumentImportDtos.PreviewSession load(String path) {
if (!StringUtil.hasText(path)) { SnapshotReadResult readResult = readManifestOrLegacy(path);
throw new BusinessException("分块快照不存在,请重试"); if (readResult.legacySession != null) {
return readResult.legacySession;
} }
try (InputStream inputStream = storageService.readStream(path)) { DocumentImportDtos.PreviewSession session = copyHeader(readResult.manifest.getSession());
byte[] payload = DocumentInputStreamSupport.readBytes(inputStream, MAX_SNAPSHOT_BYTES); List<DocumentChunk> chunks = new ArrayList<DocumentChunk>();
String json = new String(payload, StandardCharsets.UTF_8); long loadedBytes = 0L;
DocumentImportDtos.PreviewSession session = for (SnapshotPart part : readResult.manifest.getParts()) {
JSON.parseObject(json, DocumentImportDtos.PreviewSession.class); PartPayload payload = readPart(part);
if (session == null || session.getDocumentChunks() == null loadedBytes += payload.byteLength;
|| session.getDocumentChunks().isEmpty()) { if (loadedBytes > MAX_LEGACY_SNAPSHOT_BYTES) {
throw new BusinessException("分块快照无有效内容,请重试"); throw new BusinessException("分块快照过大,请使用分页或批量读取");
} }
chunks.addAll(payload.chunks);
}
session.setDocumentChunks(chunks);
session.setTotalChunks(chunks.size());
session.setChunkSnapshotPath(path);
return session; return session;
} catch (IOException error) { }
throw new BusinessException("分块快照读取失败,请重试");
/**
* 仅读取快照头部元信息。
*
* @param path 快照路径
* @return 不包含完整分块的预览会话
*/
public DocumentImportDtos.PreviewSession loadHeader(String path) {
SnapshotReadResult readResult = readManifestOrLegacy(path);
if (readResult.legacySession != null) {
DocumentImportDtos.PreviewSession header = copyHeader(readResult.legacySession);
int total = readResult.legacySession.getDocumentChunks() == null
? 0
: readResult.legacySession.getDocumentChunks().size();
header.setTotalChunks(total);
header.setChunkSnapshotPath(path);
return header;
}
DocumentImportDtos.PreviewSession header = copyHeader(readResult.manifest.getSession());
header.setTotalChunks(readResult.manifest.getTotalChunks());
header.setChunkSnapshotPath(path);
return header;
}
/**
* 分页读取快照分块。
*
* @param path 快照路径
* @param offset 零基偏移
* @param limit 最大返回数
* @return 指定页分块
*/
public List<DocumentChunk> loadPage(String path, int offset, int limit) {
if (offset < 0 || limit <= 0) {
throw new BusinessException("分块快照分页参数不合法");
}
SnapshotReadResult readResult = readManifestOrLegacy(path);
if (readResult.legacySession != null) {
List<DocumentChunk> chunks = readResult.legacySession.getDocumentChunks();
if (chunks == null || offset >= chunks.size()) {
return new ArrayList<DocumentChunk>();
}
int end = Math.min(offset + limit, chunks.size());
return new ArrayList<DocumentChunk>(chunks.subList(offset, end));
}
List<DocumentChunk> page = new ArrayList<DocumentChunk>(limit);
int skipped = 0;
for (SnapshotPart part : readResult.manifest.getParts()) {
if (page.size() >= limit) {
break;
}
if (skipped + part.getChunkCount() <= offset) {
skipped += part.getChunkCount();
continue;
}
List<DocumentChunk> chunks = readPart(part).chunks;
int localStart = Math.max(0, offset - skipped);
int localEnd = Math.min(chunks.size(), localStart + limit - page.size());
page.addAll(chunks.subList(localStart, localEnd));
skipped += chunks.size();
}
return page;
}
/**
* 按固定上限顺序消费快照分块。
*
* @param path 快照路径
* @param batchSize 单批最大分块数
* @param consumer 批处理函数
*/
public void forEachBatch(
String path,
int batchSize,
Consumer<List<DocumentChunk>> consumer) {
if (batchSize <= 0 || consumer == null) {
throw new IllegalArgumentException("快照批量消费参数不合法");
}
SnapshotReadResult readResult = readManifestOrLegacy(path);
if (readResult.legacySession != null) {
consumeInBatches(readResult.legacySession.getDocumentChunks(), batchSize, consumer);
return;
}
for (SnapshotPart part : readResult.manifest.getParts()) {
consumeInBatches(readPart(part).chunks, batchSize, consumer);
} }
} }
/** /**
* 删除已完成向量化的分块快照。 * 删除快照清单及其全部分片
* *
* @param path 快照路径 * @param path 快照清单或历史 V1 快照路径
*/ */
public void delete(String path) { public void delete(String path) {
if (!StringUtil.hasText(path)) {
return;
}
deleteParts(path);
deleteManifest(path);
}
/**
* 删除快照分片并保留清单。
*
* <p>任一分片删除失败时立即抛出异常,清单继续作为精确重试依据。</p>
*
* @param path 快照清单或历史 V1 快照路径
*/
public void deleteParts(String path) {
if (!StringUtil.hasText(path)) {
return;
}
SnapshotReadResult readResult = readManifestOrLegacy(path);
if (readResult.manifest == null) {
return;
}
for (SnapshotPart part : readResult.manifest.getParts()) {
if (StringUtil.hasText(part.getPath())) {
storageService.delete(part.getPath());
}
}
}
/**
* 删除快照清单或历史 V1 单对象。
*
* @param path 快照清单或历史 V1 快照路径
*/
public void deleteManifest(String path) {
if (StringUtil.hasText(path)) { if (StringUtil.hasText(path)) {
storageService.delete(path); storageService.delete(path);
} }
} }
/**
* 校验预览会话的最小完整性。
*
* @param session 预览会话
* @param requireChunks 是否要求已包含分块
*/
private void validateSession(
DocumentImportDtos.PreviewSession session,
boolean requireChunks) {
if (session == null || session.getKnowledgeId() == null || session.getDocumentId() == null) {
throw new BusinessException("分块快照归属信息不完整");
}
if (requireChunks
&& (session.getDocumentChunks() == null || session.getDocumentChunks().isEmpty())) {
throw new BusinessException("分块快照内容不完整");
}
}
/**
* 复制有界会话头,移除完整正文和分块数组。
*
* @param source 原始会话
* @return 会话头
*/
private DocumentImportDtos.PreviewSession copyHeader(
DocumentImportDtos.PreviewSession source) {
DocumentImportDtos.PreviewSession header = new DocumentImportDtos.PreviewSession();
header.setSessionId(source.getSessionId());
header.setKnowledgeId(source.getKnowledgeId());
header.setDocumentId(source.getDocumentId());
header.setFilePath(source.getFilePath());
header.setFileName(source.getFileName());
header.setSourceFormat(source.getSourceFormat());
header.setStrategyConfig(source.getStrategyConfig());
if (source.getAnalysis() != null) {
AnalysisResult analysis = JSON.parseObject(
JSON.toJSONBytes(source.getAnalysis()), AnalysisResult.class);
analysis.setNormalizedContent(null);
header.setAnalysis(analysis);
}
header.setDocument(null);
header.setDocumentChunks(new ArrayList<DocumentChunk>());
header.setPreviewChunks(new ArrayList<>());
header.setChunkSnapshotPath(source.getChunkSnapshotPath());
header.setTotalChunks(source.getTotalChunks());
header.setCreatedAt(source.getCreatedAt());
return header;
}
/**
* 读取 V2 清单或历史 V1 会话。
*
* @param path 快照路径
* @return 快照读取结果
*/
private SnapshotReadResult readManifestOrLegacy(String path) {
if (!StringUtil.hasText(path)) {
throw new BusinessException("分块快照不存在,请重试");
}
byte[] payload;
try (InputStream inputStream = storageService.readStream(path)) {
payload = DocumentInputStreamSupport.readBytes(
inputStream, MAX_LEGACY_SNAPSHOT_BYTES);
} catch (IOException error) {
throw new BusinessException("分块快照读取失败,请重试");
}
SnapshotManifest manifest = JSON.parseObject(payload, SnapshotManifest.class);
if (manifest != null && SNAPSHOT_VERSION_V2.equals(manifest.getVersion())) {
validateManifest(manifest);
return SnapshotReadResult.forManifest(manifest);
}
DocumentImportDtos.PreviewSession session = JSON.parseObject(
new String(payload, StandardCharsets.UTF_8),
DocumentImportDtos.PreviewSession.class);
if (session == null || session.getDocumentChunks() == null
|| session.getDocumentChunks().isEmpty()) {
throw new BusinessException("分块快照无有效内容,请重试");
}
return SnapshotReadResult.forLegacy(session);
}
/**
* 校验 V2 清单完整性。
*
* @param manifest 快照清单
*/
private void validateManifest(SnapshotManifest manifest) {
if (manifest.getSession() == null
|| manifest.getSession().getKnowledgeId() == null
|| manifest.getSession().getDocumentId() == null
|| manifest.getTotalChunks() <= 0
|| manifest.getParts() == null
|| manifest.getParts().isEmpty()) {
throw new BusinessException("分块快照清单损坏,请重试");
}
int total = 0;
for (SnapshotPart part : manifest.getParts()) {
if (part == null || !StringUtil.hasText(part.getPath())
|| part.getChunkCount() <= 0
|| part.getByteLength() <= 0
|| !StringUtil.hasText(part.getSha256())) {
throw new BusinessException("分块快照清单损坏,请重试");
}
total += part.getChunkCount();
}
if (total != manifest.getTotalChunks()) {
throw new BusinessException("分块快照清单计数不一致,请重试");
}
}
/**
* 读取并校验一个分块分片。
*
* @param part 分片清单
* @return 分片内容
*/
private PartPayload readPart(SnapshotPart part) {
try (InputStream inputStream = storageService.readStream(part.getPath())) {
byte[] payload = DocumentInputStreamSupport.readBytes(inputStream, MAX_PART_BYTES);
if (payload.length != part.getByteLength()
|| !sha256(payload).equals(part.getSha256())) {
throw new BusinessException("分块快照校验失败,请重试");
}
List<DocumentChunk> chunks = JSON.parseArray(
new String(payload, StandardCharsets.UTF_8), DocumentChunk.class);
if (chunks == null || chunks.size() != part.getChunkCount()) {
throw new BusinessException("分块快照分片损坏,请重试");
}
return new PartPayload(chunks, payload.length);
} catch (IOException error) {
throw new BusinessException("分块快照分片读取失败,请重试");
}
}
/**
* 将列表按固定大小交给消费者。
*
* @param chunks 分块
* @param batchSize 批大小
* @param consumer 消费者
*/
private void consumeInBatches(
List<DocumentChunk> chunks,
int batchSize,
Consumer<List<DocumentChunk>> consumer) {
if (chunks == null || chunks.isEmpty()) {
return;
}
for (int start = 0; start < chunks.size(); start += batchSize) {
int end = Math.min(start + batchSize, chunks.size());
consumer.accept(new ArrayList<DocumentChunk>(chunks.subList(start, end)));
}
}
/**
* 计算 SHA-256。
*
* @param payload 字节内容
* @return 十六进制摘要
*/
private String sha256(byte[] payload) {
try {
return HexFormat.of().formatHex(
MessageDigest.getInstance("SHA-256").digest(payload));
} catch (NoSuchAlgorithmException error) {
throw new IllegalStateException("当前运行环境不支持 SHA-256", error);
}
}
/**
* V2 快照流式写入器。
*/
public final class SnapshotWriter implements AutoCloseable {
private final SnapshotManifest manifest;
private final String storagePrefix;
private boolean committed;
/**
* 创建写入器。
*
* @param session 会话头
*/
private SnapshotWriter(DocumentImportDtos.PreviewSession session) {
this.manifest = new SnapshotManifest();
this.manifest.setVersion(SNAPSHOT_VERSION_V2);
this.manifest.setSession(session);
this.manifest.setCreatedAt(new Date());
this.storagePrefix = "knowledge-import-snapshots/"
+ session.getKnowledgeId() + "/"
+ session.getDocumentId() + "/"
+ UUID.randomUUID();
}
/**
* 追加一批分块,超出单分片上限时自动二分。
*
* @param chunks 分块
*/
public void append(List<DocumentChunk> chunks) {
if (committed) {
throw new IllegalStateException("分块快照已提交");
}
if (chunks == null || chunks.isEmpty()) {
return;
}
writeBoundedPart(new ArrayList<DocumentChunk>(chunks));
}
/**
* 最后写入清单并提交快照。
*
* @return 清单路径
*/
public String finish() {
if (committed) {
throw new IllegalStateException("分块快照已提交");
}
if (manifest.getTotalChunks() <= 0 || manifest.getParts().isEmpty()) {
throw new BusinessException("分块快照内容不完整");
}
byte[] payload = JSON.toJSONBytes(manifest);
if (payload.length > MAX_MANIFEST_BYTES) {
throw new BusinessException("分块快照清单过大,请调整文档后重试");
}
String fileName = manifest.getSession().getDocumentId() + "-manifest.json";
String path = storageService.save(
new CustomMultipartFile(payload, fileName, fileName, "application/json"),
storagePrefix);
if (!StringUtil.hasText(path)) {
throw new BusinessException("分块快照清单保存失败");
}
committed = true;
return path;
}
/**
* 未提交时删除已经写入的分片。
*/
@Override
public void close() {
if (committed) {
return;
}
for (SnapshotPart part : manifest.getParts()) {
if (StringUtil.hasText(part.getPath())) {
storageService.delete(part.getPath());
}
}
}
/**
* 写入一个有界分片。
*
* @param chunks 分块
*/
private void writeBoundedPart(List<DocumentChunk> chunks) {
byte[] payload = JSON.toJSONBytes(chunks);
if (payload.length > MAX_PART_BYTES && chunks.size() > 1) {
int midpoint = chunks.size() / 2;
writeBoundedPart(new ArrayList<DocumentChunk>(chunks.subList(0, midpoint)));
writeBoundedPart(new ArrayList<DocumentChunk>(chunks.subList(midpoint, chunks.size())));
return;
}
if (payload.length > MAX_PART_BYTES) {
throw new BusinessException("单个分块快照超过 4 MiB请缩小分块后重试");
}
int partNumber = manifest.getParts().size() + 1;
String fileName = String.format("part-%06d.json", partNumber);
String path = storageService.save(
new CustomMultipartFile(payload, fileName, fileName, "application/json"),
storagePrefix);
if (!StringUtil.hasText(path)) {
throw new BusinessException("分块快照分片保存失败");
}
SnapshotPart part = new SnapshotPart();
part.setPath(path);
part.setChunkCount(chunks.size());
part.setByteLength(payload.length);
part.setSha256(sha256(payload));
manifest.getParts().add(part);
manifest.setTotalChunks(manifest.getTotalChunks() + chunks.size());
}
}
/**
* V2 快照清单。
*/
public static class SnapshotManifest {
private String version;
private DocumentImportDtos.PreviewSession session;
private int totalChunks;
private List<SnapshotPart> parts = new ArrayList<SnapshotPart>();
private Date createdAt;
/**
* 返回版本。
*
* @return 版本
*/
public String getVersion() {
return version;
}
/**
* 设置版本。
*
* @param version 版本
*/
public void setVersion(String version) {
this.version = version;
}
/**
* 返回会话头。
*
* @return 会话头
*/
public DocumentImportDtos.PreviewSession getSession() {
return session;
}
/**
* 设置会话头。
*
* @param session 会话头
*/
public void setSession(DocumentImportDtos.PreviewSession session) {
this.session = session;
}
/**
* 返回总分块数。
*
* @return 总分块数
*/
public int getTotalChunks() {
return totalChunks;
}
/**
* 设置总分块数。
*
* @param totalChunks 总分块数
*/
public void setTotalChunks(int totalChunks) {
this.totalChunks = totalChunks;
}
/**
* 返回分片清单。
*
* @return 分片清单
*/
public List<SnapshotPart> getParts() {
return parts;
}
/**
* 设置分片清单。
*
* @param parts 分片清单
*/
public void setParts(List<SnapshotPart> parts) {
this.parts = parts == null
? new ArrayList<SnapshotPart>()
: parts;
}
/**
* 返回创建时间。
*
* @return 创建时间
*/
public Date getCreatedAt() {
return createdAt;
}
/**
* 设置创建时间。
*
* @param createdAt 创建时间
*/
public void setCreatedAt(Date createdAt) {
this.createdAt = createdAt;
}
}
/**
* V2 快照分片元信息。
*/
public static class SnapshotPart {
private String path;
private int chunkCount;
private long byteLength;
private String sha256;
/**
* 返回存储路径。
*
* @return 存储路径
*/
public String getPath() {
return path;
}
/**
* 设置存储路径。
*
* @param path 存储路径
*/
public void setPath(String path) {
this.path = path;
}
/**
* 返回分块数。
*
* @return 分块数
*/
public int getChunkCount() {
return chunkCount;
}
/**
* 设置分块数。
*
* @param chunkCount 分块数
*/
public void setChunkCount(int chunkCount) {
this.chunkCount = chunkCount;
}
/**
* 返回字节数。
*
* @return 字节数
*/
public long getByteLength() {
return byteLength;
}
/**
* 设置字节数。
*
* @param byteLength 字节数
*/
public void setByteLength(long byteLength) {
this.byteLength = byteLength;
}
/**
* 返回 SHA-256。
*
* @return SHA-256
*/
public String getSha256() {
return sha256;
}
/**
* 设置 SHA-256。
*
* @param sha256 SHA-256
*/
public void setSha256(String sha256) {
this.sha256 = sha256;
}
}
/**
* 内部读取结果。
*/
private static final class SnapshotReadResult {
private final SnapshotManifest manifest;
private final DocumentImportDtos.PreviewSession legacySession;
/**
* 创建读取结果。
*
* @param manifest V2 清单
* @param legacySession V1 会话
*/
private SnapshotReadResult(
SnapshotManifest manifest,
DocumentImportDtos.PreviewSession legacySession) {
this.manifest = manifest;
this.legacySession = legacySession;
}
/**
* 创建 V2 读取结果。
*
* @param manifest V2 清单
* @return 读取结果
*/
private static SnapshotReadResult forManifest(SnapshotManifest manifest) {
return new SnapshotReadResult(manifest, null);
}
/**
* 创建 V1 读取结果。
*
* @param session V1 会话
* @return 读取结果
*/
private static SnapshotReadResult forLegacy(
DocumentImportDtos.PreviewSession session) {
return new SnapshotReadResult(null, session);
}
}
/**
* 内部分片载荷。
*/
private static final class PartPayload {
private final List<DocumentChunk> chunks;
private final long byteLength;
/**
* 创建分片载荷。
*
* @param chunks 分块
* @param byteLength 字节数
*/
private PartPayload(List<DocumentChunk> chunks, long byteLength) {
this.chunks = chunks;
this.byteLength = byteLength;
}
}
} }

View File

@@ -0,0 +1,40 @@
package tech.easyflow.ai.documentimport.task;
import java.util.Set;
/**
* 知识库文档导入格式统一策略。
*
* @author Codex
* @since 2026-08-04
*/
public final class DocumentImportFormatPolicy {
private static final Set<String> SUPPORTED_EXTENSIONS =
Set.of("txt", "pdf", "docx", "md", "pptx", "xlsx", "csv");
/**
* 禁止实例化格式策略工具类。
*/
private DocumentImportFormatPolicy() {
}
/**
* 返回只读的支持格式集合。
*
* @return 支持的文件扩展名
*/
public static Set<String> supportedExtensions() {
return SUPPORTED_EXTENSIONS;
}
/**
* 判断扩展名是否属于知识库导入支持范围。
*
* @param extension 已转换为小写的文件扩展名
* @return 支持时返回 {@code true}
*/
public static boolean isSupported(String extension) {
return SUPPORTED_EXTENSIONS.contains(extension);
}
}

View File

@@ -0,0 +1,42 @@
package tech.easyflow.ai.documentimport.task;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tech.easyflow.common.cache.DistributedScheduledLock;
/**
* 文档导入中间快照清理调度器。
*
* @author Codex
* @since 2026-08-04
*/
@Component
public class DocumentImportSnapshotCleanupMonitor {
private final DocumentImportSnapshotCleanupService cleanupService;
/**
* 创建快照清理调度器。
*
* @param cleanupService 快照清理服务
*/
public DocumentImportSnapshotCleanupMonitor(
DocumentImportSnapshotCleanupService cleanupService) {
this.cleanupService = cleanupService;
}
/**
* 周期性重试到期的快照清理记录。
*/
@Scheduled(
fixedDelayString =
"${easyflow.ai.document-import.snapshot-cleanup-interval:60s}",
initialDelayString =
"${easyflow.ai.document-import.snapshot-cleanup-initial-delay:60s}")
@DistributedScheduledLock(
key = "easyflow:schedule:document-import:snapshot-cleanup",
leaseSeconds = 300L)
public void cleanupPendingSnapshots() {
cleanupService.processPendingCleanups();
}
}

View File

@@ -0,0 +1,352 @@
package tech.easyflow.ai.documentimport.task;
import com.mybatisflex.core.keygen.impl.FlexIDKeyGenerator;
import com.mybatisflex.core.query.QueryWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import tech.easyflow.ai.entity.DocumentImportSnapshotCleanup;
import tech.easyflow.ai.mapper.DocumentImportSnapshotCleanupMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.common.util.StringUtil;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.util.Date;
import java.util.HexFormat;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
/**
* 文档导入中间快照可靠清理服务。
*
* <p>先持久化精确清单路径,再按“分片、清单”两个阶段删除。阶段推进先于
* 清单删除落库,确保进程在任一位置中断后都能安全继续。</p>
*
* @author Codex
* @since 2026-08-04
*/
@Service
public class DocumentImportSnapshotCleanupService {
public static final String TYPE_CHUNK = "CHUNK";
public static final String TYPE_CSV_TABLE = "CSV_TABLE";
private static final Logger LOG =
LoggerFactory.getLogger(DocumentImportSnapshotCleanupService.class);
private static final String PHASE_PARTS_PENDING = "PARTS_PENDING";
private static final String PHASE_MANIFEST_PENDING = "MANIFEST_PENDING";
private static final int PROCESS_BATCH_SIZE = 50;
private static final int MAX_PATH_CHARS = 2_048;
private static final int MAX_ERROR_CHARS = 1_024;
private static final Duration CLAIM_LEASE = Duration.ofHours(1);
private static final Duration MIN_RETRY_DELAY = Duration.ofSeconds(30);
private static final Duration MAX_RETRY_DELAY = Duration.ofHours(1);
@Resource
private DocumentImportSnapshotCleanupMapper cleanupMapper;
@Resource
private DocumentImportChunkSnapshotService chunkSnapshotService;
@Resource
private CsvTableSnapshotService csvTableSnapshotService;
@Resource
private DocumentMapper documentMapper;
private final FlexIDKeyGenerator keyGenerator =
new FlexIDKeyGenerator();
/**
* 登记并尝试清理分块快照。
*
* @param manifestPath 分块快照清单路径
*/
public void scheduleChunkSnapshot(String manifestPath) {
schedule(null, null, TYPE_CHUNK, manifestPath);
}
/**
* 登记并尝试清理 CSV 表格快照。
*
* @param knowledgeId 知识库 ID
* @param documentId 文档 ID
* @param manifestPath CSV 表格清单路径
*/
public void scheduleCsvTableSnapshot(
BigInteger knowledgeId,
BigInteger documentId,
String manifestPath) {
schedule(
knowledgeId,
documentId,
TYPE_CSV_TABLE,
manifestPath);
}
/**
* 处理一批已到期的快照清理记录。
*/
public void processPendingCleanups() {
Date now = new Date();
List<DocumentImportSnapshotCleanup> records =
cleanupMapper.selectDueRecords(now, PROCESS_BATCH_SIZE);
for (DocumentImportSnapshotCleanup record : records) {
processRecord(record.getId());
}
}
/**
* 幂等登记清理记录,并在当前没有事务时立即尝试一次。
*
* @param knowledgeId 知识库 ID
* @param documentId 文档 ID
* @param snapshotType 快照类型
* @param manifestPath 清单路径
*/
private void schedule(
BigInteger knowledgeId,
BigInteger documentId,
String snapshotType,
String manifestPath) {
if (!StringUtil.hasText(manifestPath)) {
return;
}
if (manifestPath.length() > MAX_PATH_CHARS) {
throw new IllegalArgumentException("快照清单路径超过 2048 字符");
}
Date now = new Date();
String pathHash = sha256(manifestPath);
DocumentImportSnapshotCleanup record =
new DocumentImportSnapshotCleanup();
record.setId(generateId(record));
record.setKnowledgeId(knowledgeId);
record.setDocumentId(documentId);
record.setSnapshotType(snapshotType);
record.setManifestPath(manifestPath);
record.setPathHash(pathHash);
record.setPhase(PHASE_PARTS_PENDING);
record.setAttemptCount(0);
record.setNextRetryAt(now);
record.setCreated(now);
record.setModified(now);
cleanupMapper.insertIgnore(record);
DocumentImportSnapshotCleanup persisted =
cleanupMapper.selectOneByQuery(
QueryWrapper.create()
.eq(
DocumentImportSnapshotCleanup::getSnapshotType,
snapshotType)
.eq(
DocumentImportSnapshotCleanup::getPathHash,
pathHash));
if (persisted == null
|| !Objects.equals(
manifestPath, persisted.getManifestPath())) {
throw new IllegalStateException("快照清理记录登记失败");
}
// 事务中的登记必须随业务提交;后台调度器会在提交后处理。
if (!TransactionSynchronizationManager
.isActualTransactionActive()) {
processRecord(persisted.getId());
}
}
/**
* 领取并处理一条清理记录。
*
* @param recordId 清理记录 ID
*/
private void processRecord(BigInteger recordId) {
Date now = new Date();
String executionToken = UUID.randomUUID().toString();
Date leaseUntil = new Date(
now.getTime() + CLAIM_LEASE.toMillis());
if (cleanupMapper.claim(
recordId, executionToken, leaseUntil, now) <= 0) {
return;
}
DocumentImportSnapshotCleanup record =
cleanupMapper.selectOneByQuery(
QueryWrapper.create()
.eq(DocumentImportSnapshotCleanup::getId, recordId)
.eq(
DocumentImportSnapshotCleanup::getExecutionToken,
executionToken));
if (record == null) {
return;
}
try {
if (PHASE_PARTS_PENDING.equals(record.getPhase())) {
deleteParts(record);
if (cleanupMapper.advanceToManifest(
recordId, executionToken, new Date()) <= 0) {
throw new IllegalStateException(
"快照清理阶段推进失败");
}
record.setPhase(PHASE_MANIFEST_PENDING);
}
if (!PHASE_MANIFEST_PENDING.equals(record.getPhase())) {
throw new IllegalStateException(
"未知快照清理阶段: " + record.getPhase());
}
deleteManifest(record);
clearCsvSnapshotPointer(record);
if (cleanupMapper.deleteCompleted(
recordId, executionToken) <= 0) {
LOG.warn(
"快照对象已清理,但清理记录终态删除未命中: cleanupId={}, path={}",
recordId, record.getManifestPath());
}
} catch (RuntimeException error) {
releaseForRetry(record, executionToken, error);
}
}
/**
* 删除记录指向的全部快照分片。
*
* @param record 清理记录
*/
private void deleteParts(
DocumentImportSnapshotCleanup record) {
if (TYPE_CHUNK.equals(record.getSnapshotType())) {
chunkSnapshotService.deleteParts(
record.getManifestPath());
return;
}
if (TYPE_CSV_TABLE.equals(record.getSnapshotType())) {
csvTableSnapshotService.deleteParts(
record.getManifestPath());
return;
}
throw new IllegalStateException(
"未知快照类型: " + record.getSnapshotType());
}
/**
* 删除记录指向的快照清单。
*
* @param record 清理记录
*/
private void deleteManifest(
DocumentImportSnapshotCleanup record) {
if (TYPE_CHUNK.equals(record.getSnapshotType())) {
chunkSnapshotService.deleteManifest(
record.getManifestPath());
return;
}
if (TYPE_CSV_TABLE.equals(record.getSnapshotType())) {
csvTableSnapshotService.deleteManifest(
record.getManifestPath());
return;
}
throw new IllegalStateException(
"未知快照类型: " + record.getSnapshotType());
}
/**
* CSV 清单删除完成后原子移除仍指向该清单的文档选项。
*
* @param record 清理记录
*/
private void clearCsvSnapshotPointer(
DocumentImportSnapshotCleanup record) {
if (TYPE_CSV_TABLE.equals(record.getSnapshotType())
&& record.getDocumentId() != null) {
documentMapper.clearCsvSnapshotPath(
record.getDocumentId(),
record.getManifestPath(),
new Date());
}
}
/**
* 释放失败记录并按指数退避安排下一次重试。
*
* @param record 清理记录
* @param executionToken 执行令牌
* @param error 清理异常
*/
private void releaseForRetry(
DocumentImportSnapshotCleanup record,
String executionToken,
RuntimeException error) {
Date now = new Date();
int attempts = Math.max(
1,
record.getAttemptCount() == null
? 1
: record.getAttemptCount());
long multiplier = 1L << Math.min(7, attempts - 1);
long delayMillis = Math.min(
MAX_RETRY_DELAY.toMillis(),
MIN_RETRY_DELAY.toMillis() * multiplier);
String errorMessage = boundedError(error);
int released = cleanupMapper.releaseForRetry(
record.getId(),
executionToken,
new Date(now.getTime() + delayMillis),
errorMessage,
now);
LOG.error(
"文档导入快照清理失败,已安排重试: cleanupId={}, type={}, path={}, released={}",
record.getId(),
record.getSnapshotType(),
record.getManifestPath(),
released,
error);
}
/**
* 返回有界错误摘要。
*
* @param error 清理异常
* @return 有界摘要
*/
private String boundedError(RuntimeException error) {
String message = error.getMessage();
String summary = error.getClass().getSimpleName()
+ (StringUtil.hasText(message) ? ": " + message : "");
return summary.length() <= MAX_ERROR_CHARS
? summary
: summary.substring(0, MAX_ERROR_CHARS);
}
/**
* 计算清单路径 SHA-256。
*
* @param value 清单路径
* @return 十六进制摘要
*/
private String sha256(String value) {
try {
return HexFormat.of().formatHex(
MessageDigest.getInstance("SHA-256").digest(
value.getBytes(StandardCharsets.UTF_8)));
} catch (NoSuchAlgorithmException error) {
throw new IllegalStateException(
"当前运行环境不支持 SHA-256", error);
}
}
/**
* 生成清理记录主键。
*
* @param entity 清理记录
* @return 主键
*/
private BigInteger generateId(Object entity) {
return new BigInteger(
String.valueOf(keyGenerator.generate(entity, null)));
}
}

View File

@@ -119,6 +119,7 @@ public class KnowledgeDocumentImportTaskAppService {
private static final String KNOWLEDGE_PARSE_IMAGE_CATEGORY = "knowledge-parse"; private static final String KNOWLEDGE_PARSE_IMAGE_CATEGORY = "knowledge-parse";
private static final String OFFICE_PPTX_PAGE_STRATEGY = "OFFICE_PPTX_PAGE"; private static final String OFFICE_PPTX_PAGE_STRATEGY = "OFFICE_PPTX_PAGE";
private static final String OFFICE_XLSX_ROW_WINDOW_STRATEGY = "OFFICE_XLSX_ROW_WINDOW"; private static final String OFFICE_XLSX_ROW_WINDOW_STRATEGY = "OFFICE_XLSX_ROW_WINDOW";
private static final String TABLE_ROW_STRATEGY = "TABLE_ROW";
private static final String SEARCH_RENDER_MARKDOWN_METADATA_KEY = "renderMarkdown"; private static final String SEARCH_RENDER_MARKDOWN_METADATA_KEY = "renderMarkdown";
private static final String TASK_ERROR_PARSE_SERVICE_UNAVAILABLE = "parse_service_unavailable"; private static final String TASK_ERROR_PARSE_SERVICE_UNAVAILABLE = "parse_service_unavailable";
private static final String TASK_ERROR_PARSE_SERVICE_TIMEOUT = "parse_service_timeout"; private static final String TASK_ERROR_PARSE_SERVICE_TIMEOUT = "parse_service_timeout";
@@ -135,6 +136,8 @@ public class KnowledgeDocumentImportTaskAppService {
private static final Pattern HTTP_SERVER_ERROR_PATTERN = Pattern.compile("\\bstatus=5\\d{2}\\b"); private static final Pattern HTTP_SERVER_ERROR_PATTERN = Pattern.compile("\\bstatus=5\\d{2}\\b");
private static final Pattern MARKDOWN_IMAGE_PATTERN = Pattern.compile("!\\[(?:[^\\]]*)\\]\\(([^)]+)\\)"); private static final Pattern MARKDOWN_IMAGE_PATTERN = Pattern.compile("!\\[(?:[^\\]]*)\\]\\(([^)]+)\\)");
private final FlexIDKeyGenerator flexIdKeyGenerator = new FlexIDKeyGenerator(); private final FlexIDKeyGenerator flexIdKeyGenerator = new FlexIDKeyGenerator();
private final TabularRowWindowChunkBuilder tabularChunkBuilder =
new TabularRowWindowChunkBuilder();
@Resource @Resource
private DocumentMapper documentMapper; private DocumentMapper documentMapper;
@@ -182,6 +185,12 @@ public class KnowledgeDocumentImportTaskAppService {
@Resource @Resource
private DocumentImportChunkSnapshotService documentImportChunkSnapshotService; private DocumentImportChunkSnapshotService documentImportChunkSnapshotService;
@Resource
private CsvTableSnapshotService csvTableSnapshotService;
@Resource
private DocumentImportSnapshotCleanupService snapshotCleanupService;
@Resource @Resource
private RagIngestionService ragIngestionService; private RagIngestionService ragIngestionService;
@@ -713,20 +722,43 @@ public class KnowledgeDocumentImportTaskAppService {
tech.easyflow.ai.entity.Document document = requireDocumentForKnowledge(request.getDocumentId(), knowledge.getId()); tech.easyflow.ai.entity.Document document = requireDocumentForKnowledge(request.getDocumentId(), knowledge.getId());
ensurePreviewable(document); ensurePreviewable(document);
int pageNo = normalizePreviewPageNo(request.getPageNo());
int pageSize = normalizePreviewPageSize(request.getPageSize());
DocumentImportDtos.PreviewSession session;
String sessionId;
if (StringUtil.hasText(request.getPreviewSessionId())) {
sessionId = request.getPreviewSessionId();
session = documentImportPreviewService.getRequired(sessionId);
assertPreviewSessionOwner(knowledge, document, session);
} else {
StrategyConfig requestedStrategy = null; StrategyConfig requestedStrategy = null;
if (request.getFiles() != null && !request.getFiles().isEmpty()) { if (request.getFiles() != null && !request.getFiles().isEmpty()) {
requestedStrategy = request.getFiles().get(0).getStrategyConfig(); requestedStrategy = request.getFiles().get(0).getStrategyConfig();
} }
session = buildPreviewSessionForDocument(
knowledge, document, requestedStrategy);
DocumentImportDtos.PreviewSession previousSession =
documentImportPreviewService.replaceForDocument(session);
sessionId = session.getSessionId();
if (previousSession != null
&& StringUtil.hasText(previousSession.getChunkSnapshotPath())
&& !Objects.equals(
previousSession.getChunkSnapshotPath(),
session.getChunkSnapshotPath())) {
deleteChunkSnapshotAfterCompletion(
previousSession.getChunkSnapshotPath());
}
}
DocumentImportDtos.PreviewSession session = buildPreviewSessionForDocument(knowledge, document, requestedStrategy); List<DocumentChunk> pageChunks =
String sessionId = documentImportPreviewService.put(session); loadPreviewPage(session, pageNo, pageSize);
DocumentImportDtos.PreviewFileResult item = buildPreviewFileResult(
DocumentImportDtos.PreviewFileResult item = buildPreviewFileResult(document, session, sessionId); document, session, sessionId, pageNo, pageSize, pageChunks);
DocumentImportDtos.PreviewResponse response = new DocumentImportDtos.PreviewResponse(); DocumentImportDtos.PreviewResponse response = new DocumentImportDtos.PreviewResponse();
response.setItems(List.of(item)); response.setItems(List.of(item));
response.setTotalFiles(1); response.setTotalFiles(1);
response.setTotalChunks(session.getDocumentChunks().size()); response.setTotalChunks(resolveSessionTotalChunks(session));
return Result.ok(response); return Result.ok(response);
} }
@@ -755,16 +787,22 @@ public class KnowledgeDocumentImportTaskAppService {
request.getPreviewSessionId(), request.getPreviewSessionId(),
StringUtil.hasText(request.getPreviewSessionId()) ? null : existingSnapshotPath StringUtil.hasText(request.getPreviewSessionId()) ? null : existingSnapshotPath
); );
int totalChunks = session.getDocumentChunks().size(); int totalChunks = resolveSessionTotalChunks(session);
if (totalChunks <= 0) { if (totalChunks <= 0) {
throw new BusinessException("未生成有效分块,无法开始向量化"); throw new BusinessException("未生成有效分块,无法开始向量化");
} }
boolean createdSnapshot = StringUtil.hasText(request.getPreviewSessionId()) String preparedSnapshotPath = session.getChunkSnapshotPath();
|| !StringUtil.hasText(existingSnapshotPath); boolean createdSnapshot = !StringUtil.hasText(existingSnapshotPath)
String snapshotPath = createdSnapshot && !StringUtil.hasText(preparedSnapshotPath);
? documentImportChunkSnapshotService.save(session) String snapshotPath;
: existingSnapshotPath; if (StringUtil.hasText(preparedSnapshotPath)) {
snapshotPath = preparedSnapshotPath;
} else if (createdSnapshot) {
snapshotPath = documentImportChunkSnapshotService.save(session);
} else {
snapshotPath = existingSnapshotPath;
}
mergeDocumentPreviewOptions(document, session, snapshotPath); mergeDocumentPreviewOptions(document, session, snapshotPath);
if (!claimDocumentIndexing(document, totalChunks)) { if (!claimDocumentIndexing(document, totalChunks)) {
if (createdSnapshot) { if (createdSnapshot) {
@@ -993,11 +1031,13 @@ public class KnowledgeDocumentImportTaskAppService {
knowledge, document, requestedStrategy); knowledge, document, requestedStrategy);
previewSessionId = documentImportPreviewService.put(session); previewSessionId = documentImportPreviewService.put(session);
session.setSessionId(previewSessionId); session.setSessionId(previewSessionId);
int totalChunks = session.getDocumentChunks().size(); int totalChunks = resolveSessionTotalChunks(session);
if (totalChunks <= 0) { if (totalChunks <= 0) {
throw new BusinessException("未生成有效分块,无法开始向量化"); throw new BusinessException("未生成有效分块,无法开始向量化");
} }
snapshotPath = documentImportChunkSnapshotService.save(session); snapshotPath = StringUtil.hasText(session.getChunkSnapshotPath())
? session.getChunkSnapshotPath()
: documentImportChunkSnapshotService.save(session);
if (!touchRunningTask(task) || !executionLock.renew()) { if (!touchRunningTask(task) || !executionLock.renew()) {
throw new TaskOwnershipLostException(task.getId()); throw new TaskOwnershipLostException(task.getId());
} }
@@ -1015,8 +1055,11 @@ public class KnowledgeDocumentImportTaskAppService {
String errorMessage = truncateError(error.getMessage()); String errorMessage = truncateError(error.getMessage());
LOG.error("文档分块任务失败: taskId={}, documentId={}", LOG.error("文档分块任务失败: taskId={}, documentId={}",
taskId, task.getDocumentId(), error); taskId, task.getDocumentId(), error);
String failureCode = error instanceof CsvImportException csvError
? csvError.getFailureCode()
: TASK_ERROR_SPLIT_FAILED;
if (!selfProxy.failSplitTask( if (!selfProxy.failSplitTask(
task, task.getDocumentId(), errorMessage, TASK_ERROR_SPLIT_FAILED)) { task, task.getDocumentId(), errorMessage, failureCode)) {
LOG.warn("分块任务所有权已失效,忽略迟到失败: taskId={}", taskId); LOG.warn("分块任务所有权已失效,忽略迟到失败: taskId={}", taskId);
} }
} finally { } finally {
@@ -1182,37 +1225,57 @@ public class KnowledgeDocumentImportTaskAppService {
List<DocumentChunk> storedChunks = new ArrayList<DocumentChunk>(); List<DocumentChunk> storedChunks = new ArrayList<DocumentChunk>();
try { try {
DocumentCollection knowledge = assertDocumentCollection(task.getKnowledgeId()); DocumentCollection knowledge = assertDocumentCollection(task.getKnowledgeId());
DocumentImportDtos.PreviewSession session = resolveIndexPreviewSession(
knowledge,
document,
asString(task.getPayloadJson().get("previewSessionId")),
asString(task.getPayloadJson().get("chunkSnapshotPath"))
);
List<DocumentChunk> chunks = session.getDocumentChunks();
if (chunks == null || chunks.isEmpty()) {
throw new BusinessException("预览会话无有效分块");
}
assertUniqueChunkIds(chunks);
if (!touchRunningTask(task) || !executionLock.renew()) { if (!touchRunningTask(task) || !executionLock.renew()) {
throw new TaskOwnershipLostException(task.getId()); throw new TaskOwnershipLostException(task.getId());
} }
clearPersistedChunks(document.getId()); clearPersistedChunks(document.getId());
storeContext = prepareStoreContext(document); storeContext = prepareStoreContext(document);
int totalChunks = chunks.size(); String chunkSnapshotPath =
int completedChunks = 0; asString(task.getPayloadJson().get("chunkSnapshotPath"));
DocumentImportDtos.PreviewSession session =
resolveIndexPreviewSession(
knowledge,
document,
asString(task.getPayloadJson().get("previewSessionId")),
chunkSnapshotPath
);
int totalChunks = resolveSessionTotalChunks(session);
if (totalChunks <= 0) {
throw new BusinessException("预览会话无有效分块");
}
StoreExecutionContext activeStoreContext = storeContext;
DocumentImportTask activeTask = task;
Set<BigInteger> uniqueChunkIds =
new HashSet<BigInteger>(Math.min(totalChunks, 65_536));
int[] completedChunks = new int[]{0};
java.util.function.Consumer<List<DocumentChunk>> batchConsumer = batch -> {
assertUniqueChunkIds(batch, uniqueChunkIds);
LOG.info("文档向量化任务开始处理批次: taskId={}, documentId={}, batchStart={}, batchEnd={}, batchSize={}, totalChunks={}",
taskId, document.getId(), completedChunks[0],
completedChunks[0] + batch.size(), batch.size(), totalChunks);
// 提前登记当前批次,确保外部存储部分成功后仍能覆盖回滚范围。
storedChunks.addAll(toChunkIdMarkers(batch));
storeDocumentChunks(activeStoreContext, batch);
persistChunkBatch(document, batch);
completedChunks[0] += batch.size();
updateDocumentIndexProgress(
document.getId(), totalChunks, completedChunks[0]);
if (!touchRunningTask(activeTask) || !executionLock.renew()) {
throw new TaskOwnershipLostException(activeTask.getId());
}
};
if (StringUtil.hasText(chunkSnapshotPath)) {
documentImportChunkSnapshotService.forEachBatch(
chunkSnapshotPath, INDEX_BATCH_SIZE, batchConsumer);
} else {
List<DocumentChunk> chunks = session.getDocumentChunks();
if (chunks == null || chunks.isEmpty()) {
throw new BusinessException("预览会话无有效分块");
}
for (int start = 0; start < chunks.size(); start += INDEX_BATCH_SIZE) { for (int start = 0; start < chunks.size(); start += INDEX_BATCH_SIZE) {
int end = Math.min(start + INDEX_BATCH_SIZE, chunks.size()); int end = Math.min(start + INDEX_BATCH_SIZE, chunks.size());
List<DocumentChunk> batch = new ArrayList<DocumentChunk>(chunks.subList(start, end)); batchConsumer.accept(new ArrayList<DocumentChunk>(
LOG.info("文档向量化任务开始处理批次: taskId={}, documentId={}, batchStart={}, batchEnd={}, batchSize={}, totalChunks={}", chunks.subList(start, end)));
taskId, document.getId(), start, end, batch.size(), totalChunks);
storeDocumentChunks(storeContext, batch);
storedChunks.addAll(batch);
persistChunkBatch(document, batch);
completedChunks += batch.size();
updateDocumentIndexProgress(document.getId(), totalChunks, completedChunks);
if (!touchRunningTask(task) || !executionLock.renew()) {
throw new TaskOwnershipLostException(task.getId());
} }
} }
updateKnowledgeAfterStore(storeContext); updateKnowledgeAfterStore(storeContext);
@@ -1222,6 +1285,7 @@ public class KnowledgeDocumentImportTaskAppService {
} }
deleteChunkSnapshotAfterCompletion( deleteChunkSnapshotAfterCompletion(
asString(task.getPayloadJson().get("chunkSnapshotPath"))); asString(task.getPayloadJson().get("chunkSnapshotPath")));
cleanupCsvTableSnapshotAfterCompletion(document);
} catch (TaskOwnershipLostException ownershipLost) { } catch (TaskOwnershipLostException ownershipLost) {
// 旧执行失去 token 后不能清理可能已由新执行写入的索引或分块。 // 旧执行失去 token 后不能清理可能已由新执行写入的索引或分块。
LOG.warn("向量化任务执行令牌已失效,停止迟到回滚: taskId={}, documentId={}", LOG.warn("向量化任务执行令牌已失效,停止迟到回滚: taskId={}, documentId={}",
@@ -1239,9 +1303,13 @@ public class KnowledgeDocumentImportTaskAppService {
rollbackSucceeded = rollbackStoredChunks( rollbackSucceeded = rollbackStoredChunks(
taskId, document.getId(), storeContext, storedChunks); taskId, document.getId(), storeContext, storedChunks);
} }
String failureCode = e instanceof CsvImportException csvError
? csvError.getFailureCode()
: TASK_ERROR_INDEX_FAILED;
markIndexFailed( markIndexFailed(
task, document, task, document,
resolveIndexFailureMessage(e, rollbackSucceeded)); resolveIndexFailureMessage(e, rollbackSucceeded),
failureCode);
} finally { } finally {
closeStoreContext(storeContext); closeStoreContext(storeContext);
} }
@@ -1272,6 +1340,35 @@ public class KnowledgeDocumentImportTaskAppService {
String fileExt) { String fileExt) {
LOG.info("开始同步解析文档: taskId={}, documentId={}, fileName={}, fileExt={}", LOG.info("开始同步解析文档: taskId={}, documentId={}, fileName={}, fileExt={}",
task.getId(), document.getId(), document.getTitle(), fileExt); task.getId(), document.getId(), document.getTitle(), fileExt);
if ("csv".equals(fileExt)) {
CsvTableSnapshotService.CsvParseResult csvResult =
csvTableSnapshotService.parse(
document.getDocumentPath(),
document.getCollectionId(),
document.getId(),
task.getExecutionToken());
DocumentParsedResult parsedResult = new DocumentParsedResult();
parsedResult.setFileName(document.getTitle());
parsedResult.setPreferredText(csvResult.buildSummary());
parsedResult.setPlainText(csvResult.buildSummary());
parsedResult.setMetadata(csvResult.toMetadata());
LOG.info("CSV 流式解析完成: taskId={}, documentId={}, rows={}, columns={}, encoding={}",
task.getId(), document.getId(), csvResult.getRowCount(),
csvResult.getColumnCount(), csvResult.getEncoding());
boolean published = false;
try {
published = markParseSuccess(
task, document, parsedResult, fileExt, null);
} finally {
if (!published) {
snapshotCleanupService.scheduleCsvTableSnapshot(
document.getCollectionId(),
document.getId(),
csvResult.getManifestPath());
}
}
return;
}
String normalizedContent = readFileContent(document.getDocumentPath(), document.getTitle()); String normalizedContent = readFileContent(document.getDocumentPath(), document.getTitle());
DocumentParsedResult parsedResult = new DocumentParsedResult(); DocumentParsedResult parsedResult = new DocumentParsedResult();
parsedResult.setFileName(document.getTitle()); parsedResult.setFileName(document.getTitle());
@@ -1283,7 +1380,18 @@ public class KnowledgeDocumentImportTaskAppService {
markParseSuccess(task, document, parsedResult, fileExt, null); markParseSuccess(task, document, parsedResult, fileExt, null);
} }
private void markParseSuccess(DocumentImportTask task, /**
* 发布解析结果并原子推进文档和批次状态。
*
* @param task 解析任务
* @param document 文档实体
* @param parsedResult 解析结果
* @param sourceFormat 来源格式
* @param providerTaskId 外部解析任务 ID
* @return 结果是否由当前任务代次成功发布
*/
private boolean markParseSuccess(
DocumentImportTask task,
tech.easyflow.ai.entity.Document document, tech.easyflow.ai.entity.Document document,
DocumentParsedResult parsedResult, DocumentParsedResult parsedResult,
String sourceFormat, String sourceFormat,
@@ -1301,6 +1409,7 @@ public class KnowledgeDocumentImportTaskAppService {
} }
if (parsedResult.getMetadata() != null && !parsedResult.getMetadata().isEmpty()) { if (parsedResult.getMetadata() != null && !parsedResult.getMetadata().isEmpty()) {
options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_METADATA, new LinkedHashMap<String, Object>(parsedResult.getMetadata())); options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_METADATA, new LinkedHashMap<String, Object>(parsedResult.getMetadata()));
copyCsvParseMetadata(options, parsedResult.getMetadata());
} }
if (parsedResult.getWarnings() != null && !parsedResult.getWarnings().isEmpty()) { if (parsedResult.getWarnings() != null && !parsedResult.getWarnings().isEmpty()) {
options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_WARNINGS, new ArrayList<String>(parsedResult.getWarnings())); options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_WARNINGS, new ArrayList<String>(parsedResult.getWarnings()));
@@ -1337,11 +1446,12 @@ public class KnowledgeDocumentImportTaskAppService {
if (!selfProxy.completeParseTask(task, document, automaticBatch)) { if (!selfProxy.completeParseTask(task, document, automaticBatch)) {
LOG.warn("解析任务所有权已失效,忽略迟到结果: taskId={}, documentId={}", LOG.warn("解析任务所有权已失效,忽略迟到结果: taskId={}, documentId={}",
task.getId(), document.getId()); task.getId(), document.getId());
return; return false;
} }
LOG.info("文档解析任务完成: taskId={}, documentId={}, processStatus={}, providerTaskId={}, contentLength={}", LOG.info("文档解析任务完成: taskId={}, documentId={}, processStatus={}, providerTaskId={}, contentLength={}",
task.getId(), document.getId(), document.getProcessStatus(), providerTaskId, task.getId(), document.getId(), document.getProcessStatus(), providerTaskId,
parsedKnowledgeContent.documentLlmContent.length()); parsedKnowledgeContent.documentLlmContent.length());
return true;
} }
/** /**
@@ -1603,6 +1713,9 @@ public class KnowledgeDocumentImportTaskAppService {
Throwable current = error; Throwable current = error;
String fallbackCode = null; String fallbackCode = null;
while (current != null) { while (current != null) {
if (current instanceof CsvImportException csvError) {
return csvError.getFailureCode();
}
if (current instanceof DocumentParseBridgeException bridgeError) { if (current instanceof DocumentParseBridgeException bridgeError) {
String bridgeCode = bridgeError.getCode(); String bridgeCode = bridgeError.getCode();
if ("unsupported_source".equals(bridgeCode)) { if ("unsupported_source".equals(bridgeCode)) {
@@ -2628,6 +2741,20 @@ public class KnowledgeDocumentImportTaskAppService {
throw new BusinessException("文档尚未完成解析"); throw new BusinessException("文档尚未完成解析");
} }
String sourceFormat = normalizeSourceFormat(document); String sourceFormat = normalizeSourceFormat(document);
if ("csv".equals(sourceFormat)) {
StrategyConfig strategyConfig =
resolveOfficeStrategyConfig(document, requestedStrategy, sourceFormat);
String tableSnapshotPath = optionAsString(
document.getOptions(),
DocumentImportKeys.KEY_DOCUMENT_CSV_TABLE_SNAPSHOT_PATH);
if (!StringUtil.hasText(tableSnapshotPath)) {
throw new CsvImportException(
CsvTableSnapshotService.FAILURE_SNAPSHOT_CORRUPTED,
"CSV 表格快照不存在,请重新解析");
}
return csvTableSnapshotService.buildChunkSnapshot(
document, tableSnapshotPath, strategyConfig);
}
if (isOfficeDocument(sourceFormat)) { if (isOfficeDocument(sourceFormat)) {
return buildOfficePreviewSession(knowledge, document, requestedStrategy, sourceFormat); return buildOfficePreviewSession(knowledge, document, requestedStrategy, sourceFormat);
} }
@@ -2742,7 +2869,9 @@ public class KnowledgeDocumentImportTaskAppService {
if (rowsPerChunk == null || rowsPerChunk <= 0) { if (rowsPerChunk == null || rowsPerChunk <= 0) {
rowsPerChunk = 10; rowsPerChunk = 10;
} }
config.setStrategyCode(OFFICE_XLSX_ROW_WINDOW_STRATEGY); config.setStrategyCode("csv".equals(sourceFormat)
? TABLE_ROW_STRATEGY
: OFFICE_XLSX_ROW_WINDOW_STRATEGY);
config.setRowsPerChunk(rowsPerChunk); config.setRowsPerChunk(rowsPerChunk);
return config; return config;
} }
@@ -2906,14 +3035,21 @@ public class KnowledgeDocumentImportTaskAppService {
? new ArrayList<Map<String, Object>>(sheetRows.subList(1, sheetRows.size())) ? new ArrayList<Map<String, Object>>(sheetRows.subList(1, sheetRows.size()))
: new ArrayList<Map<String, Object>>(); : new ArrayList<Map<String, Object>>();
if (dataRows.isEmpty()) { if (dataRows.isEmpty()) {
chunks.add(buildXlsxWindowChunk(document, sorting++, sheetName, sheet, headerRow, List<DocumentChunk> windowChunks = buildXlsxWindowChunks(
new ArrayList<Map<String, Object>>(), sheetImages)); document, sorting, sheetName, sheet, headerRow,
new ArrayList<Map<String, Object>>(), sheetImages);
chunks.addAll(windowChunks);
sorting += windowChunks.size();
continue; continue;
} }
for (int start = 0; start < dataRows.size(); start += rowsPerChunk) { for (int start = 0; start < dataRows.size(); start += rowsPerChunk) {
int end = Math.min(start + rowsPerChunk, dataRows.size()); int end = Math.min(start + rowsPerChunk, dataRows.size());
List<Map<String, Object>> windowRows = new ArrayList<Map<String, Object>>(dataRows.subList(start, end)); List<Map<String, Object>> windowRows = new ArrayList<Map<String, Object>>(dataRows.subList(start, end));
chunks.add(buildXlsxWindowChunk(document, sorting++, sheetName, sheet, headerRow, windowRows, sheetImages)); List<DocumentChunk> windowChunks = buildXlsxWindowChunks(
document, sorting, sheetName, sheet, headerRow,
windowRows, sheetImages);
chunks.addAll(windowChunks);
sorting += windowChunks.size();
} }
} }
if (chunks.isEmpty()) { if (chunks.isEmpty()) {
@@ -2929,7 +3065,20 @@ public class KnowledgeDocumentImportTaskAppService {
return chunks; return chunks;
} }
private DocumentChunk buildXlsxWindowChunk(tech.easyflow.ai.entity.Document document, /**
* 构建 XLSX 行窗口分块。无图片窗口与 CSV 共用表格构建器,图片窗口保留现有 OCR 元信息。
*
* @param document 文档
* @param sorting 起始排序号
* @param sheetName Sheet 名称
* @param sheetSummary Sheet 摘要
* @param headerRow 表头行
* @param windowRows 数据行
* @param sheetImages Sheet 图片
* @return 一个或多个分块
*/
private List<DocumentChunk> buildXlsxWindowChunks(
tech.easyflow.ai.entity.Document document,
int sorting, int sorting,
String sheetName, String sheetName,
Map<String, Object> sheetSummary, Map<String, Object> sheetSummary,
@@ -2942,6 +3091,44 @@ public class KnowledgeDocumentImportTaskAppService {
? rowStart ? rowStart
: asInteger(windowRows.get(windowRows.size() - 1).get("rowIndex"), rowStart - 1) + 1; : asInteger(windowRows.get(windowRows.size() - 1).get("rowIndex"), rowStart - 1) + 1;
List<Map<String, Object>> selectedImages = selectWindowImages(sheetImages, rowStart, rowEnd, windowRows.isEmpty()); List<Map<String, Object>> selectedImages = selectWindowImages(sheetImages, rowStart, rowEnd, windowRows.isEmpty());
if (selectedImages.isEmpty()) {
List<Map<String, Object>> allRows =
new ArrayList<Map<String, Object>>();
allRows.add(headerRow);
allRows.addAll(windowRows);
int maxCol = resolveMaxColumnCount(allRows);
if (maxCol > 0) {
List<String> headers =
resolveTabularRowValues(headerRow, maxCol, true);
List<TabularRowWindowChunkBuilder.TabularRow> rows =
new ArrayList<TabularRowWindowChunkBuilder.TabularRow>();
for (Map<String, Object> row : windowRows) {
rows.add(new TabularRowWindowChunkBuilder.TabularRow(
asInteger(row.get("rowIndex"), 0) + 1,
resolveTabularRowValues(row, maxCol, false)));
}
List<DocumentChunk> chunks = tabularChunkBuilder.build(
document.getId(), document.getCollectionId(), sheetName,
headers, rows, sorting, RagChunkTypes.SECTION);
for (DocumentChunk chunk : chunks) {
Map<String, Object> options =
new LinkedHashMap<String, Object>(chunk.getOptions());
options.put(
DocumentImportKeys.KEY_DOCUMENT_ROW_START, rowStart);
options.put(
DocumentImportKeys.KEY_DOCUMENT_ROW_END, rowEnd);
options.put(
DocumentImportKeys.KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY,
buildXlsxChunkSummary(
sheetSummary, rowStart, rowEnd, selectedImages));
options.put(
DocumentImportKeys.KEY_DOCUMENT_IMAGE_REFS,
new ArrayList<String>());
chunk.setOptions(options);
}
return chunks;
}
}
String renderMarkdown = buildXlsxChunkRenderMarkdown(sheetName, headerRow, windowRows, selectedImages); String renderMarkdown = buildXlsxChunkRenderMarkdown(sheetName, headerRow, windowRows, selectedImages);
String llmContent = buildXlsxChunkLlmContent(sheetName, headerRow, windowRows, selectedImages); String llmContent = buildXlsxChunkLlmContent(sheetName, headerRow, windowRows, selectedImages);
Map<String, Object> options = new LinkedHashMap<String, Object>(); Map<String, Object> options = new LinkedHashMap<String, Object>();
@@ -2959,7 +3146,8 @@ public class KnowledgeDocumentImportTaskAppService {
options.put(DocumentImportKeys.KEY_DOCUMENT_ROW_END, rowEnd); options.put(DocumentImportKeys.KEY_DOCUMENT_ROW_END, rowEnd);
options.put(DocumentImportKeys.KEY_DOCUMENT_IMAGE_REFS, collectImageRefs(selectedImages)); options.put(DocumentImportKeys.KEY_DOCUMENT_IMAGE_REFS, collectImageRefs(selectedImages));
options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY, buildXlsxChunkSummary(sheetSummary, rowStart, rowEnd, selectedImages)); options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY, buildXlsxChunkSummary(sheetSummary, rowStart, rowEnd, selectedImages));
return buildOfficeChunk(document, sorting, llmContent, renderMarkdown, options); return Collections.singletonList(
buildOfficeChunk(document, sorting, llmContent, renderMarkdown, options));
} }
private DocumentChunk buildXlsxImageOnlyChunk(tech.easyflow.ai.entity.Document document, private DocumentChunk buildXlsxImageOnlyChunk(tech.easyflow.ai.entity.Document document,
@@ -3246,6 +3434,27 @@ public class KnowledgeDocumentImportTaskAppService {
private List<String> resolveMarkdownRowValues(Map<String, Object> row, private List<String> resolveMarkdownRowValues(Map<String, Object> row,
int maxCol, int maxCol,
boolean headerRow) { boolean headerRow) {
List<String> rawValues =
resolveTabularRowValues(row, maxCol, headerRow);
List<String> values = new ArrayList<String>(rawValues.size());
for (String value : rawValues) {
values.add(escapeMarkdownCell(value));
}
return values;
}
/**
* 将 XLSX 行转换为通用表格单元格值。
*
* @param row XLSX 行
* @param maxCol 最大列数
* @param headerRow 是否为表头
* @return 固定列数的原始值
*/
private List<String> resolveTabularRowValues(
Map<String, Object> row,
int maxCol,
boolean headerRow) {
List<String> values = new ArrayList<String>(Collections.nCopies(maxCol, "")); List<String> values = new ArrayList<String>(Collections.nCopies(maxCol, ""));
if (row == null) { if (row == null) {
return values; return values;
@@ -3260,7 +3469,7 @@ public class KnowledgeDocumentImportTaskAppService {
if (headerRow && !StringUtil.hasText(mergedText)) { if (headerRow && !StringUtil.hasText(mergedText)) {
mergedText = "" + (colIndex + 1); mergedText = "" + (colIndex + 1);
} }
values.set(colIndex, escapeMarkdownCell(mergedText)); values.set(colIndex, mergedText);
} }
return values; return values;
} }
@@ -3402,6 +3611,9 @@ public class KnowledgeDocumentImportTaskAppService {
if ("pptx".equals(sourceFormat) || OFFICE_PPTX_PAGE_STRATEGY.equals(strategyCode)) { if ("pptx".equals(sourceFormat) || OFFICE_PPTX_PAGE_STRATEGY.equals(strategyCode)) {
return "按页分块"; return "按页分块";
} }
if ("csv".equals(sourceFormat) || TABLE_ROW_STRATEGY.equals(strategyCode)) {
return "按表头 / 行窗口";
}
if ("xlsx".equals(sourceFormat) || OFFICE_XLSX_ROW_WINDOW_STRATEGY.equals(strategyCode)) { if ("xlsx".equals(sourceFormat) || OFFICE_XLSX_ROW_WINDOW_STRATEGY.equals(strategyCode)) {
return "按 Sheet / 行窗口"; return "按 Sheet / 行窗口";
} }
@@ -3414,11 +3626,18 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档实体 * @param document 文档实体
* @param session 预览会话 * @param session 预览会话
* @param sessionId 预览会话 ID * @param sessionId 预览会话 ID
* @param pageNo 当前页码
* @param pageSize 当前页大小
* @param pageChunks 当前页分块
* @return 预览文件结果 * @return 预览文件结果
*/ */
private DocumentImportDtos.PreviewFileResult buildPreviewFileResult(tech.easyflow.ai.entity.Document document, private DocumentImportDtos.PreviewFileResult buildPreviewFileResult(
tech.easyflow.ai.entity.Document document,
DocumentImportDtos.PreviewSession session, DocumentImportDtos.PreviewSession session,
String sessionId) { String sessionId,
int pageNo,
int pageSize,
List<DocumentChunk> pageChunks) {
DocumentImportDtos.PreviewFileResult item = new DocumentImportDtos.PreviewFileResult(); DocumentImportDtos.PreviewFileResult item = new DocumentImportDtos.PreviewFileResult();
item.setPreviewSessionId(sessionId); item.setPreviewSessionId(sessionId);
item.setFilePath(document.getDocumentPath()); item.setFilePath(document.getDocumentPath());
@@ -3427,12 +3646,77 @@ public class KnowledgeDocumentImportTaskAppService {
item.setStrategyCode(session.getStrategyConfig().getStrategyCode()); item.setStrategyCode(session.getStrategyConfig().getStrategyCode());
item.setStrategyLabel(resolveStrategyLabel(session.getSourceFormat(), session.getStrategyConfig())); item.setStrategyLabel(resolveStrategyLabel(session.getSourceFormat(), session.getStrategyConfig()));
item.setAnalysis(session.getAnalysis()); item.setAnalysis(session.getAnalysis());
item.setTotalChunks(session.getDocumentChunks().size()); item.setTotalChunks(resolveSessionTotalChunks(session));
item.setTotalWarnings(countChunkWarnings(session.getDocumentChunks())); item.setTotalWarnings(countChunkWarnings(pageChunks));
item.setChunks(toPreviewChunkResults(session.getDocumentChunks())); item.setPageNo(pageNo);
item.setPageSize(pageSize);
item.setChunks(toPreviewChunkResults(pageChunks));
return item; return item;
} }
/**
* 从内存会话或分片快照读取一个有界预览页。
*
* @param session 预览会话
* @param pageNo 从 1 开始的页码
* @param pageSize 单页分块数
* @return 当前页分块
*/
private List<DocumentChunk> loadPreviewPage(
DocumentImportDtos.PreviewSession session,
int pageNo,
int pageSize) {
long offsetValue = (long) (pageNo - 1) * pageSize;
if (offsetValue > Integer.MAX_VALUE) {
throw new BusinessException("预览页码超出范围");
}
int offset = (int) offsetValue;
if (StringUtil.hasText(session.getChunkSnapshotPath())) {
return documentImportChunkSnapshotService.loadPage(
session.getChunkSnapshotPath(), offset, pageSize);
}
List<DocumentChunk> chunks = session.getDocumentChunks();
if (chunks == null || offset >= chunks.size()) {
return new ArrayList<DocumentChunk>();
}
int end = Math.min(offset + pageSize, chunks.size());
return new ArrayList<DocumentChunk>(
chunks.subList(offset, end));
}
/**
* 规范化预览页码。
*
* @param pageNo 请求页码
* @return 至少为 1 的页码
*/
private int normalizePreviewPageNo(Integer pageNo) {
if (pageNo == null) {
return 1;
}
if (pageNo < 1) {
throw new BusinessException("预览页码必须大于 0");
}
return pageNo;
}
/**
* 规范化预览单页分块数。
*
* @param pageSize 请求单页分块数
* @return 1 到 50 之间的单页分块数
*/
private int normalizePreviewPageSize(Integer pageSize) {
if (pageSize == null) {
return 20;
}
if (pageSize < 1 || pageSize > 50) {
throw new BusinessException(
"预览每页分块数必须在 1 到 50 之间");
}
return pageSize;
}
private void mergeDocumentPreviewOptions(tech.easyflow.ai.entity.Document document, private void mergeDocumentPreviewOptions(tech.easyflow.ai.entity.Document document,
DocumentImportDtos.PreviewSession session, DocumentImportDtos.PreviewSession session,
String chunkSnapshotPath) { String chunkSnapshotPath) {
@@ -3460,7 +3744,7 @@ public class KnowledgeDocumentImportTaskAppService {
: optionAsString(document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH); : optionAsString(document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH);
if (StringUtil.hasText(persistedSnapshotPath)) { if (StringUtil.hasText(persistedSnapshotPath)) {
DocumentImportDtos.PreviewSession session = DocumentImportDtos.PreviewSession session =
documentImportChunkSnapshotService.load(persistedSnapshotPath); documentImportChunkSnapshotService.loadHeader(persistedSnapshotPath);
assertPreviewSessionOwner(knowledge, document, session); assertPreviewSessionOwner(knowledge, document, session);
return session; return session;
} }
@@ -4007,8 +4291,8 @@ public class KnowledgeDocumentImportTaskAppService {
} }
private void assertSupportedImportFile(String fileExt) { private void assertSupportedImportFile(String fileExt) {
if (!Arrays.asList("pdf", "docx", "txt", "md", "pptx", "xlsx").contains(fileExt)) { if (!DocumentImportFormatPolicy.isSupported(fileExt)) {
throw new BusinessException("当前仅支持 pdf/docx/txt/md/pptx/xlsx 文档导入"); throw new BusinessException("当前仅支持 pdf/docx/txt/md/pptx/xlsx/csv 文档导入");
} }
} }
@@ -4329,7 +4613,7 @@ public class KnowledgeDocumentImportTaskAppService {
} }
/** /**
* 向量化完成后尽力删除持久化分块快照。 * 登记并尝试删除持久化分块快照。
* *
* @param snapshotPath 快照路径 * @param snapshotPath 快照路径
*/ */
@@ -4338,9 +4622,12 @@ public class KnowledgeDocumentImportTaskAppService {
return; return;
} }
try { try {
documentImportChunkSnapshotService.delete(snapshotPath); snapshotCleanupService.scheduleChunkSnapshot(snapshotPath);
} catch (RuntimeException error) { } catch (RuntimeException error) {
LOG.error("删除已完成文档的分块快照失败: path={}", snapshotPath, error); LOG.error(
"登记文档分块快照清理失败: path={}",
snapshotPath,
error);
} }
} }
@@ -4356,6 +4643,36 @@ public class KnowledgeDocumentImportTaskAppService {
runAfterCommit(() -> deleteChunkSnapshotAfterCompletion(snapshotPath)); runAfterCommit(() -> deleteChunkSnapshotAfterCompletion(snapshotPath));
} }
/**
* CSV 向量化完成后登记中间表格快照清理。
*
* <p>清理记录持久化后由当前线程立即尝试,失败时由后台调度器继续重试。</p>
*
* @param document 已完成向量化的文档
*/
private void cleanupCsvTableSnapshotAfterCompletion(
tech.easyflow.ai.entity.Document document) {
if (document == null || document.getOptions() == null) {
return;
}
String tableSnapshotPath = optionAsString(
document.getOptions(),
DocumentImportKeys.KEY_DOCUMENT_CSV_TABLE_SNAPSHOT_PATH);
if (!StringUtil.hasText(tableSnapshotPath)) {
return;
}
try {
snapshotCleanupService.scheduleCsvTableSnapshot(
document.getCollectionId(),
document.getId(),
tableSnapshotPath);
} catch (RuntimeException error) {
LOG.error(
"登记已完成 CSV 文档的表格快照清理失败: documentId={}, path={}",
document.getId(), tableSnapshotPath, error);
}
}
private StrategyConfig resolveStrategyConfig(DocumentCollection knowledge, private StrategyConfig resolveStrategyConfig(DocumentCollection knowledge,
StrategyConfig requestConfig, StrategyConfig requestConfig,
AnalysisResult analysisResult) { AnalysisResult analysisResult) {
@@ -4653,6 +4970,63 @@ public class KnowledgeDocumentImportTaskAppService {
options.remove(DocumentImportKeys.KEY_DOCUMENT_PARSE_STATUS_MESSAGE); options.remove(DocumentImportKeys.KEY_DOCUMENT_PARSE_STATUS_MESSAGE);
} }
/**
* 将 CSV 解析元信息提升到文档有界选项中,供后续分块阶段直接定位表格快照。
*
* @param options 文档选项
* @param metadata 解析元信息
*/
private void copyCsvParseMetadata(
Map<String, Object> options,
Map<String, Object> metadata) {
if (options == null || metadata == null) {
return;
}
copyOptionIfPresent(
options, metadata, DocumentImportKeys.KEY_DOCUMENT_CSV_TABLE_SNAPSHOT_PATH);
copyOptionIfPresent(
options, metadata, DocumentImportKeys.KEY_DOCUMENT_CSV_ENCODING);
copyOptionIfPresent(
options, metadata, DocumentImportKeys.KEY_DOCUMENT_CSV_ROW_COUNT);
copyOptionIfPresent(
options, metadata, DocumentImportKeys.KEY_DOCUMENT_CSV_COLUMN_COUNT);
}
/**
* 复制一个存在的选项值。
*
* @param target 目标选项
* @param source 来源选项
* @param key 选项键
*/
private void copyOptionIfPresent(
Map<String, Object> target,
Map<String, Object> source,
String key) {
if (source.containsKey(key) && source.get(key) != null) {
target.put(key, source.get(key));
}
}
/**
* 返回会话的真实分块总数。
*
* @param session 预览会话
* @return 总分块数
*/
private int resolveSessionTotalChunks(
DocumentImportDtos.PreviewSession session) {
if (session == null) {
return 0;
}
if (session.getTotalChunks() != null) {
return session.getTotalChunks();
}
return session.getDocumentChunks() == null
? 0
: session.getDocumentChunks().size();
}
private BigInteger generateId(Object entity) { private BigInteger generateId(Object entity) {
return new BigInteger(String.valueOf(flexIdKeyGenerator.generate(entity, null))); return new BigInteger(String.valueOf(flexIdKeyGenerator.generate(entity, null)));
} }
@@ -4664,6 +5038,18 @@ public class KnowledgeDocumentImportTaskAppService {
*/ */
private void assertUniqueChunkIds(List<DocumentChunk> chunks) { private void assertUniqueChunkIds(List<DocumentChunk> chunks) {
Set<BigInteger> uniqueIds = new HashSet<BigInteger>(chunks.size()); Set<BigInteger> uniqueIds = new HashSet<BigInteger>(chunks.size());
assertUniqueChunkIds(chunks, uniqueIds);
}
/**
* 校验批次分块 ID并把已见 ID 保留到跨分片集合。
*
* @param chunks 当前批次
* @param uniqueIds 已见 ID
*/
private void assertUniqueChunkIds(
List<DocumentChunk> chunks,
Set<BigInteger> uniqueIds) {
for (DocumentChunk chunk : chunks) { for (DocumentChunk chunk : chunks) {
if (chunk == null || chunk.getId() == null) { if (chunk == null || chunk.getId() == null) {
throw new IllegalStateException("文档分块缺少 ID"); throw new IllegalStateException("文档分块缺少 ID");
@@ -4674,6 +5060,22 @@ public class KnowledgeDocumentImportTaskAppService {
} }
} }
/**
* 将已写入分块压缩为仅含主键的回滚标记,避免索引阶段长期持有全部正文。
*
* @param chunks 已写入分块
* @return 主键标记
*/
private List<DocumentChunk> toChunkIdMarkers(List<DocumentChunk> chunks) {
List<DocumentChunk> markers = new ArrayList<DocumentChunk>(chunks.size());
for (DocumentChunk chunk : chunks) {
DocumentChunk marker = new DocumentChunk();
marker.setId(chunk.getId());
markers.add(marker);
}
return markers;
}
private BigInteger resolveOperatorId() { private BigInteger resolveOperatorId() {
try { try {
return BigInteger.valueOf(StpUtil.getLoginIdAsLong()); return BigInteger.valueOf(StpUtil.getLoginIdAsLong());

View File

@@ -60,7 +60,7 @@ public class KnowledgeImportBatchFacade {
private static final Duration INCOMPLETE_SUBMISSION_TIMEOUT = private static final Duration INCOMPLETE_SUBMISSION_TIMEOUT =
Duration.ofMinutes(30); Duration.ofMinutes(30);
private static final Set<String> SUPPORTED_EXTENSIONS = private static final Set<String> SUPPORTED_EXTENSIONS =
Set.of("txt", "md", "pdf", "docx", "pptx", "xlsx"); DocumentImportFormatPolicy.supportedExtensions();
private final DocumentImportBatchAppService batchAppService; private final DocumentImportBatchAppService batchAppService;
private final DocumentImportBatchTracker batchTracker; private final DocumentImportBatchTracker batchTracker;

View File

@@ -0,0 +1,789 @@
package tech.easyflow.ai.documentimport.task;
import com.easyagents.rag.core.BgeM3ChunkSafety;
import com.easyagents.rag.core.RagDefaults;
import com.mybatisflex.core.keygen.impl.FlexIDKeyGenerator;
import tech.easyflow.ai.documentimport.DocumentImportKeys;
import tech.easyflow.ai.entity.DocumentChunk;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* 表格行窗口分块构建器。
*
* <p>CSV 与 XLSX 统一复用“表头 + 连续数据行”的语义,并对超长内容做有界续片,
* 保持单个分块满足下游模型的 Token 硬限制。</p>
*
* @author Codex
* @since 2026-08-04
*/
public class TabularRowWindowChunkBuilder {
/**
* 单个分块允许的最大 Java 字符数。
*/
public static final int MAX_CHUNK_CONTENT_CHARS = 60_000;
private final FlexIDKeyGenerator keyGenerator = new FlexIDKeyGenerator();
/**
* 构建一个表格行窗口对应的分块。
*
* @param documentId 文档 ID
* @param collectionId 知识库 ID
* @param sourceLabel 表格或 Sheet 名称
* @param headers 规范化表头
* @param rows 窗口数据行
* @param startingSorting 起始排序号
* @param chunkType 分块类型
* @return 一个或多个有界分块
*/
public List<DocumentChunk> build(
BigInteger documentId,
BigInteger collectionId,
String sourceLabel,
List<String> headers,
List<TabularRow> rows,
int startingSorting,
String chunkType) {
if (headers == null || headers.isEmpty()) {
throw new IllegalArgumentException("表格分块缺少表头");
}
List<TabularRow> safeRows = rows == null
? Collections.<TabularRow>emptyList()
: rows;
List<RenderedPart> parts =
renderParts(sourceLabel, headers, safeRows);
List<DocumentChunk> chunks =
new ArrayList<DocumentChunk>(parts.size());
for (int index = 0; index < parts.size(); index++) {
RenderedPart part = parts.get(index);
String content = part.content;
DocumentChunk chunk = new DocumentChunk();
chunk.setId(generateId(chunk));
chunk.setDocumentId(documentId);
chunk.setDocumentCollectionId(collectionId);
chunk.setSorting(startingSorting + index);
chunk.setContent(content);
Map<String, Object> options = new LinkedHashMap<String, Object>();
options.put("chunkType", chunkType);
options.put(
"sourceLabel",
sourceLabel + " · " + part.rowStart + "-" + part.rowEnd + "");
options.put("headingPath", Collections.singletonList(sourceLabel));
options.put("charCount", content.length());
options.put("tokenEstimate", Math.max(
1, BgeM3ChunkSafety.estimateContentTokens(content)));
options.put("partNo", index + 1);
options.put("partTotal", parts.size());
options.put("warnings", new ArrayList<String>());
options.put(DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN, content);
options.put(DocumentImportKeys.KEY_DOCUMENT_SHEET_NAME, sourceLabel);
options.put(
DocumentImportKeys.KEY_DOCUMENT_ROW_START, part.rowStart);
options.put(
DocumentImportKeys.KEY_DOCUMENT_ROW_END, part.rowEnd);
chunk.setOptions(options);
chunks.add(chunk);
}
return chunks;
}
/**
* 估算表格标题和表头渲染后的字符数。
*
* @param sourceLabel 表格名称
* @param headers 表头
* @return 渲染字符数
*/
public int estimateHeaderChars(
String sourceLabel,
List<String> headers) {
validateHeaders(headers);
long length = 6L + safeText(sourceLabel).length();
length += estimateCellsChars(headers);
length += 4L + 6L * headers.size();
return boundedInt(length);
}
/**
* 估算表格标题和表头渲染后的 Token 数。
*
* @param sourceLabel 表格名称
* @param headers 表头
* @return 保守 Token 数
*/
public int estimateHeaderTokens(
String sourceLabel,
List<String> headers) {
validateHeaders(headers);
long tokens = 6L
+ BgeM3ChunkSafety.estimateContentTokens(safeText(sourceLabel));
tokens += estimateCellsTokens(headers);
tokens += 4L + 6L * headers.size();
return boundedInt(tokens);
}
/**
* 估算单行追加到 Markdown 表格后的字符数。
*
* @param headers 表头
* @param row 表格数据行
* @return 渲染字符数
*/
public int estimateRowChars(
List<String> headers,
TabularRow row) {
validateRow(headers, row);
List<String> values =
normalizeValues(row.getValues(), headers.size());
return boundedInt(5L + estimateCellsChars(values));
}
/**
* 估算单行追加到 Markdown 表格后的 Token 数。
*
* @param headers 表头
* @param row 表格数据行
* @return 保守 Token 数
*/
public int estimateRowTokens(
List<String> headers,
TabularRow row) {
validateRow(headers, row);
List<String> values =
normalizeValues(row.getValues(), headers.size());
return boundedInt(5L + estimateCellsTokens(values));
}
/**
* 按表格语义生成有界 Markdown 续片。
*
* @param sourceLabel 表格名称
* @param headers 表头
* @param rows 数据行
* @return 有界续片
*/
private List<RenderedPart> renderParts(
String sourceLabel,
List<String> headers,
List<TabularRow> rows) {
int headerChars = estimateHeaderChars(sourceLabel, headers);
int headerTokens = estimateHeaderTokens(sourceLabel, headers);
if (!fitsLimits(headerChars, headerTokens)) {
throw new IllegalArgumentException("表格标题或表头超过分块上限");
}
if (rows.isEmpty()) {
return Collections.singletonList(new RenderedPart(
renderMarkdown(sourceLabel, headers, rows), 1, 1));
}
List<RenderedPart> parts = new ArrayList<RenderedPart>();
List<TabularRow> currentRows = new ArrayList<TabularRow>();
long currentChars = headerChars;
long currentTokens = headerTokens;
for (TabularRow row : rows) {
int rowChars = estimateRowChars(headers, row);
int rowTokens = estimateRowTokens(headers, row);
if (!fitsLimits(
(long) headerChars + rowChars,
(long) headerTokens + rowTokens)) {
flushRows(parts, sourceLabel, headers, currentRows);
currentRows.clear();
currentChars = headerChars;
currentTokens = headerTokens;
parts.addAll(renderOversizedRow(
sourceLabel, headers, row));
continue;
}
if (!currentRows.isEmpty()
&& !fitsLimits(
currentChars + rowChars,
currentTokens + rowTokens)) {
flushRows(parts, sourceLabel, headers, currentRows);
currentRows.clear();
currentChars = headerChars;
currentTokens = headerTokens;
}
currentRows.add(row);
currentChars += rowChars;
currentTokens += rowTokens;
}
flushRows(parts, sourceLabel, headers, currentRows);
return parts;
}
/**
* 输出一个完整行集合,续片始终包含标题和表头。
*
* @param parts 输出续片
* @param sourceLabel 表格名称
* @param headers 表头
* @param rows 数据行
*/
private void flushRows(
List<RenderedPart> parts,
String sourceLabel,
List<String> headers,
List<TabularRow> rows) {
if (rows.isEmpty()) {
return;
}
String content = renderMarkdown(sourceLabel, headers, rows);
assertWithinLimits(content);
parts.add(new RenderedPart(
content,
rows.get(0).getRowNumber(),
rows.get(rows.size() - 1).getRowNumber()));
}
/**
* 将单条超长记录按列边界生成连续续片。
*
* @param sourceLabel 表格名称
* @param headers 表头
* @param row 超长数据行
* @return 语义完整的续片
*/
private List<RenderedPart> renderOversizedRow(
String sourceLabel,
List<String> headers,
TabularRow row) {
List<String> values =
normalizeValues(row.getValues(), headers.size());
List<RenderedPart> parts = new ArrayList<RenderedPart>();
List<String> partHeaders = new ArrayList<String>();
List<String> partValues = new ArrayList<String>();
for (int index = 0; index < headers.size(); index++) {
String header = headers.get(index);
String value = values.get(index);
if (!fitsSingleRow(
sourceLabel,
Collections.singletonList(header),
Collections.singletonList(value),
row.getRowNumber())) {
flushColumnGroup(
parts, sourceLabel, partHeaders, partValues,
row.getRowNumber());
partHeaders.clear();
partValues.clear();
parts.addAll(renderOversizedCell(
sourceLabel, header, value, row.getRowNumber()));
continue;
}
List<String> candidateHeaders =
new ArrayList<String>(partHeaders);
candidateHeaders.add(header);
List<String> candidateValues =
new ArrayList<String>(partValues);
candidateValues.add(value);
if (!partHeaders.isEmpty()
&& !fitsLimits(renderSingleRow(
sourceLabel,
candidateHeaders,
candidateValues,
row.getRowNumber()))) {
flushColumnGroup(
parts, sourceLabel, partHeaders, partValues,
row.getRowNumber());
partHeaders.clear();
partValues.clear();
}
partHeaders.add(header);
partValues.add(value);
}
flushColumnGroup(
parts, sourceLabel, partHeaders, partValues,
row.getRowNumber());
return parts;
}
/**
* 通过有界估算判断单行表格是否满足字符与 Token 上限。
*
* @param sourceLabel 表格名称
* @param headers 表头
* @param values 行值
* @param rowNumber 逻辑行号
* @return 是否满足
*/
private boolean fitsSingleRow(
String sourceLabel,
List<String> headers,
List<String> values,
int rowNumber) {
TabularRow row = new TabularRow(rowNumber, values);
return fitsLimits(
(long) estimateHeaderChars(sourceLabel, headers)
+ estimateRowChars(headers, row),
(long) estimateHeaderTokens(sourceLabel, headers)
+ estimateRowTokens(headers, row));
}
/**
* 输出同一记录的一组完整列。
*
* @param parts 输出续片
* @param sourceLabel 表格名称
* @param headers 当前列头
* @param values 当前列值
* @param rowNumber 逻辑行号
*/
private void flushColumnGroup(
List<RenderedPart> parts,
String sourceLabel,
List<String> headers,
List<String> values,
int rowNumber) {
if (headers.isEmpty()) {
return;
}
String content = renderSingleRow(
sourceLabel, headers, values, rowNumber);
assertWithinLimits(content);
parts.add(new RenderedPart(content, rowNumber, rowNumber));
}
/**
* 对单个超长单元格做安全续片,每个续片重复当前列名。
*
* @param sourceLabel 表格名称
* @param header 当前列名
* @param value 单元格值
* @param rowNumber 逻辑行号
* @return 单列续片
*/
private List<RenderedPart> renderOversizedCell(
String sourceLabel,
String header,
String value,
int rowNumber) {
String normalizedValue = normalizeCellNewlines(value);
String emptyContent = renderSingleRow(
sourceLabel,
Collections.singletonList(header),
Collections.singletonList(""),
rowNumber);
if (!fitsLimits(emptyContent)) {
throw new IllegalArgumentException("表格列名超过分块上限");
}
int fixedChars = emptyContent.length();
int fixedTokens =
BgeM3ChunkSafety.estimateContentTokens(emptyContent);
int maxCandidateCodePoints = Math.max(
1,
Math.min(
MAX_CHUNK_CONTENT_CHARS - fixedChars,
RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT - fixedTokens));
List<RenderedPart> parts = new ArrayList<RenderedPart>();
int offset = 0;
while (offset < normalizedValue.length()) {
int remainingCodePoints =
normalizedValue.codePointCount(offset, normalizedValue.length());
int low = 1;
int high = Math.min(remainingCodePoints, maxCandidateCodePoints);
int acceptedEnd = -1;
String acceptedContent = null;
while (low <= high) {
int middle = low + (high - low) / 2;
int end = normalizedValue.offsetByCodePoints(offset, middle);
String fragment = normalizedValue.substring(offset, end);
String content = renderSingleRow(
sourceLabel,
Collections.singletonList(header),
Collections.singletonList(fragment),
rowNumber);
if (fitsLimits(content)) {
acceptedEnd = end;
acceptedContent = content;
low = middle + 1;
} else {
high = middle - 1;
}
}
if (acceptedEnd <= offset || acceptedContent == null) {
throw new IllegalArgumentException(
"表格单元格无法在分块预算内安全续片");
}
parts.add(new RenderedPart(
acceptedContent, rowNumber, rowNumber));
offset = acceptedEnd;
}
return parts;
}
/**
* 统一单元格换行,避免续片边界拆开 CRLF。
*
* @param value 原始单元格值
* @return 统一换行后的值
*/
private String normalizeCellNewlines(String value) {
return safeText(value)
.replace("\r\n", "\n")
.replace('\r', '\n');
}
/**
* 渲染单行 Markdown 表格。
*
* @param sourceLabel 表格名称
* @param headers 表头
* @param values 行值
* @param rowNumber 逻辑行号
* @return Markdown 内容
*/
private String renderSingleRow(
String sourceLabel,
List<String> headers,
List<String> values,
int rowNumber) {
return renderMarkdown(
sourceLabel,
headers,
Collections.singletonList(new TabularRow(rowNumber, values)));
}
/**
* 将表格渲染为 Markdown。
*
* @param sourceLabel 表格名称
* @param headers 表头
* @param rows 数据行
* @return Markdown 内容
*/
private String renderMarkdown(
String sourceLabel,
List<String> headers,
List<TabularRow> rows) {
StringBuilder builder = new StringBuilder();
builder.append("# ").append(safeText(sourceLabel)).append("\n\n| ");
appendCells(builder, headers);
builder.append(" |\n|");
for (int index = 0; index < headers.size(); index++) {
builder.append(" --- |");
}
for (TabularRow row : rows) {
builder.append("\n| ");
appendCells(builder, normalizeValues(row.getValues(), headers.size()));
builder.append(" |");
}
return builder.toString();
}
/**
* 追加经过 Markdown 转义的单元格。
*
* @param builder 输出缓冲区
* @param cells 单元格值
*/
private void appendCells(StringBuilder builder, List<String> cells) {
for (int index = 0; index < cells.size(); index++) {
if (index > 0) {
builder.append(" | ");
}
appendEscapedCell(builder, cells.get(index));
}
}
/**
* 将一行补齐到固定列数。
*
* @param values 原始值
* @param columnCount 列数
* @return 补齐后的值
*/
private List<String> normalizeValues(List<String> values, int columnCount) {
List<String> normalized = new ArrayList<String>(columnCount);
for (int index = 0; index < columnCount; index++) {
normalized.add(values != null && index < values.size() && values.get(index) != null
? values.get(index)
: "");
}
return normalized;
}
/**
* 转义 Markdown 表格单元格。
*
* @param value 原始值
* @return 转义后的值
*/
private void appendEscapedCell(
StringBuilder builder,
String value) {
String safeValue = safeText(value);
for (int index = 0; index < safeValue.length();) {
char current = safeValue.charAt(index);
if (current == '\r') {
builder.append("<br/>");
index += index + 1 < safeValue.length()
&& safeValue.charAt(index + 1) == '\n' ? 2 : 1;
continue;
}
if (current == '\n') {
builder.append("<br/>");
index++;
continue;
}
if (current == '\\' || current == '|') {
builder.append('\\');
}
int rawChars = rawUnitChars(safeValue, index);
builder.append(safeValue, index, index + rawChars);
index += rawChars;
}
}
/**
* 估算多个单元格完成 Markdown 转义后的字符数。
*
* @param cells 单元格
* @return 字符数
*/
private long estimateCellsChars(List<String> cells) {
long length = Math.max(0, cells.size() - 1) * 3L;
for (String cell : cells) {
String value = safeText(cell);
for (int index = 0; index < value.length();) {
char current = value.charAt(index);
if (current == '\r') {
length += 5L;
index += index + 1 < value.length()
&& value.charAt(index + 1) == '\n' ? 2 : 1;
continue;
}
if (current == '\n') {
length += 5L;
index++;
continue;
}
length += current == '\\' || current == '|'
? 2L
: rawUnitChars(value, index);
index += rawUnitChars(value, index);
if (length >= Integer.MAX_VALUE) {
return Integer.MAX_VALUE;
}
}
}
return length;
}
/**
* 估算多个单元格完成 Markdown 转义后的 Token 数。
*
* @param cells 单元格
* @return 保守 Token 数
*/
private long estimateCellsTokens(List<String> cells) {
long tokens = Math.max(0, cells.size() - 1) * 3L;
for (String cell : cells) {
String value = safeText(cell);
tokens += BgeM3ChunkSafety.estimateContentTokens(value);
for (int index = 0; index < value.length();) {
char current = value.charAt(index);
if (current == '\r') {
boolean crlf = index + 1 < value.length()
&& value.charAt(index + 1) == '\n';
tokens += crlf ? 3L : 4L;
index += crlf ? 2 : 1;
continue;
}
if (current == '\n') {
tokens += 4L;
} else if (current == '\\' || current == '|') {
tokens++;
}
index += rawUnitChars(value, index);
}
if (tokens >= Integer.MAX_VALUE) {
return Integer.MAX_VALUE;
}
}
return tokens;
}
/**
* 计算当前 Unicode 单元占用的 Java 字符数。
*
* @param value 原始值
* @param index 当前偏移
* @return Java 字符数
*/
private int rawUnitChars(String value, int index) {
char current = value.charAt(index);
if (Character.isHighSurrogate(current)
&& index + 1 < value.length()
&& Character.isLowSurrogate(value.charAt(index + 1))) {
return 2;
}
return 1;
}
/**
* 校验表头。
*
* @param headers 表头
*/
private void validateHeaders(List<String> headers) {
if (headers == null || headers.isEmpty()) {
throw new IllegalArgumentException("表格分块缺少表头");
}
}
/**
* 校验数据行。
*
* @param headers 表头
* @param row 数据行
*/
private void validateRow(
List<String> headers,
TabularRow row) {
validateHeaders(headers);
if (row == null) {
throw new IllegalArgumentException("表格分块缺少数据行");
}
}
/**
* 判断估算值是否同时满足字符与 Token 上限。
*
* @param chars 字符数
* @param tokens Token 数
* @return 是否满足
*/
private boolean fitsLimits(long chars, long tokens) {
return chars <= MAX_CHUNK_CONTENT_CHARS
&& tokens <= RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT;
}
/**
* 判断实际内容是否同时满足字符与 Token 上限。
*
* @param content 内容
* @return 是否满足
*/
private boolean fitsLimits(String content) {
return content.length() <= MAX_CHUNK_CONTENT_CHARS
&& BgeM3ChunkSafety.isWithinHardLimit(content);
}
/**
* 断言实际内容满足字符与 Token 上限。
*
* @param content 内容
*/
private void assertWithinLimits(String content) {
if (!fitsLimits(content)) {
throw new IllegalStateException("表格续片超过分块上限");
}
}
/**
* 将长度安全压缩到整数范围。
*
* @param value 长度
* @return 整数长度
*/
private int boundedInt(long value) {
return value >= Integer.MAX_VALUE
? Integer.MAX_VALUE
: (int) value;
}
/**
* 返回非空文本。
*
* @param value 原始文本
* @return 非空文本
*/
private String safeText(String value) {
return value == null ? "" : value;
}
/**
* 生成分块主键。
*
* @param entity 分块实体
* @return 主键
*/
private BigInteger generateId(Object entity) {
return new BigInteger(String.valueOf(keyGenerator.generate(entity, null)));
}
/**
* 已完成渲染的语义续片。
*/
private static final class RenderedPart {
private final String content;
private final int rowStart;
private final int rowEnd;
/**
* 创建语义续片。
*
* @param content Markdown 内容
* @param rowStart 起始行
* @param rowEnd 结束行
*/
private RenderedPart(
String content,
int rowStart,
int rowEnd) {
this.content = content;
this.rowStart = rowStart;
this.rowEnd = rowEnd;
}
}
/**
* 表格逻辑数据行。
*/
public static class TabularRow {
private final int rowNumber;
private final List<String> values;
/**
* 创建表格数据行。
*
* @param rowNumber 一基逻辑行号
* @param values 单元格值
*/
public TabularRow(int rowNumber, List<String> values) {
this.rowNumber = rowNumber;
this.values = values == null
? Collections.<String>emptyList()
: new ArrayList<String>(values);
}
/**
* 返回逻辑行号。
*
* @return 一基逻辑行号
*/
public int getRowNumber() {
return rowNumber;
}
/**
* 返回单元格值。
*
* @return 不可变语义的单元格值副本
*/
public List<String> getValues() {
return new ArrayList<String>(values);
}
}
}

View File

@@ -0,0 +1,325 @@
package tech.easyflow.ai.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
/**
* 文档导入中间快照清理记录。
*
* <p>记录精确清单路径、清理阶段和租约,使对象存储删除失败后可以跨进程重试。</p>
*
* @author Codex
* @since 2026-08-04
*/
@Table(
value = "tb_document_import_snapshot_cleanup",
comment = "文档导入中间快照清理记录")
public class DocumentImportSnapshotCleanup
extends DateEntity
implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "知识库ID")
private BigInteger knowledgeId;
@Column(comment = "文档ID")
private BigInteger documentId;
@Column(comment = "快照类型")
private String snapshotType;
@Column(comment = "快照清单路径")
private String manifestPath;
@Column(comment = "快照清单路径SHA-256")
private String pathHash;
@Column(comment = "清理阶段")
private String phase;
@Column(comment = "已尝试次数")
private Integer attemptCount;
@Column(comment = "下次重试时间")
private Date nextRetryAt;
@Column(comment = "执行令牌")
private String executionToken;
@Column(comment = "租约截止时间")
private Date leaseUntil;
@Column(comment = "最近错误")
private String lastError;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "修改时间")
private Date modified;
/**
* 返回主键。
*
* @return 主键
*/
public BigInteger getId() {
return id;
}
/**
* 设置主键。
*
* @param id 主键
*/
public void setId(BigInteger id) {
this.id = id;
}
/**
* 返回知识库 ID。
*
* @return 知识库 ID
*/
public BigInteger getKnowledgeId() {
return knowledgeId;
}
/**
* 设置知识库 ID。
*
* @param knowledgeId 知识库 ID
*/
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
/**
* 返回文档 ID。
*
* @return 文档 ID
*/
public BigInteger getDocumentId() {
return documentId;
}
/**
* 设置文档 ID。
*
* @param documentId 文档 ID
*/
public void setDocumentId(BigInteger documentId) {
this.documentId = documentId;
}
/**
* 返回快照类型。
*
* @return 快照类型
*/
public String getSnapshotType() {
return snapshotType;
}
/**
* 设置快照类型。
*
* @param snapshotType 快照类型
*/
public void setSnapshotType(String snapshotType) {
this.snapshotType = snapshotType;
}
/**
* 返回清单路径。
*
* @return 清单路径
*/
public String getManifestPath() {
return manifestPath;
}
/**
* 设置清单路径。
*
* @param manifestPath 清单路径
*/
public void setManifestPath(String manifestPath) {
this.manifestPath = manifestPath;
}
/**
* 返回路径摘要。
*
* @return 路径摘要
*/
public String getPathHash() {
return pathHash;
}
/**
* 设置路径摘要。
*
* @param pathHash 路径摘要
*/
public void setPathHash(String pathHash) {
this.pathHash = pathHash;
}
/**
* 返回清理阶段。
*
* @return 清理阶段
*/
public String getPhase() {
return phase;
}
/**
* 设置清理阶段。
*
* @param phase 清理阶段
*/
public void setPhase(String phase) {
this.phase = phase;
}
/**
* 返回已尝试次数。
*
* @return 已尝试次数
*/
public Integer getAttemptCount() {
return attemptCount;
}
/**
* 设置已尝试次数。
*
* @param attemptCount 已尝试次数
*/
public void setAttemptCount(Integer attemptCount) {
this.attemptCount = attemptCount;
}
/**
* 返回下次重试时间。
*
* @return 下次重试时间
*/
public Date getNextRetryAt() {
return nextRetryAt;
}
/**
* 设置下次重试时间。
*
* @param nextRetryAt 下次重试时间
*/
public void setNextRetryAt(Date nextRetryAt) {
this.nextRetryAt = nextRetryAt;
}
/**
* 返回执行令牌。
*
* @return 执行令牌
*/
public String getExecutionToken() {
return executionToken;
}
/**
* 设置执行令牌。
*
* @param executionToken 执行令牌
*/
public void setExecutionToken(String executionToken) {
this.executionToken = executionToken;
}
/**
* 返回租约截止时间。
*
* @return 租约截止时间
*/
public Date getLeaseUntil() {
return leaseUntil;
}
/**
* 设置租约截止时间。
*
* @param leaseUntil 租约截止时间
*/
public void setLeaseUntil(Date leaseUntil) {
this.leaseUntil = leaseUntil;
}
/**
* 返回最近错误。
*
* @return 最近错误
*/
public String getLastError() {
return lastError;
}
/**
* 设置最近错误。
*
* @param lastError 最近错误
*/
public void setLastError(String lastError) {
this.lastError = lastError;
}
/**
* 返回创建时间。
*
* @return 创建时间
*/
@Override
public Date getCreated() {
return created;
}
/**
* 设置创建时间。
*
* @param created 创建时间
*/
@Override
public void setCreated(Date created) {
this.created = created;
}
/**
* 返回修改时间。
*
* @return 修改时间
*/
@Override
public Date getModified() {
return modified;
}
/**
* 设置修改时间。
*
* @param modified 修改时间
*/
@Override
public void setModified(Date modified) {
this.modified = modified;
}
}

View File

@@ -0,0 +1,125 @@
package tech.easyflow.ai.mapper;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.ai.entity.DocumentImportSnapshotCleanup;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
/**
* 文档导入中间快照清理记录映射层。
*
* @author Codex
* @since 2026-08-04
*/
public interface DocumentImportSnapshotCleanupMapper
extends BaseMapper<DocumentImportSnapshotCleanup> {
/**
* 幂等登记一个快照清理请求。
*
* @param record 清理记录
* @return 新增行数
*/
@Insert("INSERT IGNORE INTO tb_document_import_snapshot_cleanup "
+ "(id, knowledge_id, document_id, snapshot_type, manifest_path, "
+ "path_hash, phase, attempt_count, next_retry_at, created, modified) "
+ "VALUES (#{id}, #{knowledgeId}, #{documentId}, #{snapshotType}, "
+ "#{manifestPath}, #{pathHash}, #{phase}, #{attemptCount}, "
+ "#{nextRetryAt}, #{created}, #{modified})")
int insertIgnore(DocumentImportSnapshotCleanup record);
/**
* 查询到期且未被有效租约持有的清理记录。
*
* @param now 当前时间
* @param limit 查询上限
* @return 待处理记录
*/
@Select("SELECT * FROM tb_document_import_snapshot_cleanup "
+ "WHERE next_retry_at <= #{now} "
+ "AND (execution_token IS NULL OR lease_until < #{now}) "
+ "ORDER BY next_retry_at, id LIMIT #{limit}")
List<DocumentImportSnapshotCleanup> selectDueRecords(
@Param("now") Date now,
@Param("limit") int limit);
/**
* 原子领取一条清理记录。
*
* @param id 记录 ID
* @param executionToken 执行令牌
* @param leaseUntil 租约截止时间
* @param now 当前时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_snapshot_cleanup SET "
+ "execution_token=#{executionToken}, lease_until=#{leaseUntil}, "
+ "attempt_count=attempt_count + 1, modified=#{now} "
+ "WHERE id=#{id} AND next_retry_at <= #{now} "
+ "AND (execution_token IS NULL OR lease_until < #{now})")
int claim(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@Param("leaseUntil") Date leaseUntil,
@Param("now") Date now);
/**
* 在分片全部删除后持久化推进到清单删除阶段。
*
* @param id 记录 ID
* @param executionToken 执行令牌
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_snapshot_cleanup SET "
+ "phase='MANIFEST_PENDING', last_error=NULL, modified=#{modified} "
+ "WHERE id=#{id} AND execution_token=#{executionToken} "
+ "AND phase='PARTS_PENDING'")
int advanceToManifest(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@Param("modified") Date modified);
/**
* 释放失败记录并安排下一次重试。
*
* @param id 记录 ID
* @param executionToken 执行令牌
* @param nextRetryAt 下次重试时间
* @param lastError 最近错误
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_snapshot_cleanup SET "
+ "execution_token=NULL, lease_until=NULL, "
+ "next_retry_at=#{nextRetryAt}, last_error=#{lastError}, "
+ "modified=#{modified} "
+ "WHERE id=#{id} AND execution_token=#{executionToken}")
int releaseForRetry(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@Param("nextRetryAt") Date nextRetryAt,
@Param("lastError") String lastError,
@Param("modified") Date modified);
/**
* 删除当前执行令牌已完成的清理记录。
*
* @param id 记录 ID
* @param executionToken 执行令牌
* @return 删除行数
*/
@Delete("DELETE FROM tb_document_import_snapshot_cleanup "
+ "WHERE id=#{id} AND execution_token=#{executionToken} "
+ "AND phase='MANIFEST_PENDING'")
int deleteCompleted(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken);
}

View File

@@ -1,7 +1,12 @@
package tech.easyflow.ai.mapper; package tech.easyflow.ai.mapper;
import tech.easyflow.ai.entity.Document;
import com.mybatisflex.core.BaseMapper; import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.ai.entity.Document;
import java.math.BigInteger;
import java.util.Date;
/** /**
* 映射层。 * 映射层。
@@ -11,4 +16,37 @@ import com.mybatisflex.core.BaseMapper;
*/ */
public interface DocumentMapper extends BaseMapper<Document> { public interface DocumentMapper extends BaseMapper<Document> {
/**
* 在路径仍指向已清理 CSV 快照时原子移除数据库指针。
*
* @param documentId 文档 ID
* @param manifestPath 已清理清单路径
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document SET options=CASE "
+ "WHEN JSON_UNQUOTE(JSON_EXTRACT(options, "
+ "'$.\"parse.csvTableSnapshotPath\"'))=#{manifestPath} "
+ "AND JSON_UNQUOTE(JSON_EXTRACT(options, "
+ "'$.\"parse.metadata\".\"parse.csvTableSnapshotPath\"'))="
+ "#{manifestPath} THEN JSON_REMOVE(options, "
+ "'$.\"parse.csvTableSnapshotPath\"', "
+ "'$.\"parse.metadata\".\"parse.csvTableSnapshotPath\"') "
+ "WHEN JSON_UNQUOTE(JSON_EXTRACT(options, "
+ "'$.\"parse.csvTableSnapshotPath\"'))=#{manifestPath} "
+ "THEN JSON_REMOVE(options, "
+ "'$.\"parse.csvTableSnapshotPath\"') "
+ "ELSE JSON_REMOVE(options, "
+ "'$.\"parse.metadata\".\"parse.csvTableSnapshotPath\"') END, "
+ "modified=#{modified} "
+ "WHERE id=#{documentId} AND JSON_VALID(options)=1 AND ("
+ "JSON_UNQUOTE(JSON_EXTRACT(options, "
+ "'$.\"parse.csvTableSnapshotPath\"'))=#{manifestPath} OR "
+ "JSON_UNQUOTE(JSON_EXTRACT(options, "
+ "'$.\"parse.metadata\".\"parse.csvTableSnapshotPath\"'))="
+ "#{manifestPath})")
int clearCsvSnapshotPath(
@Param("documentId") BigInteger documentId,
@Param("manifestPath") String manifestPath,
@Param("modified") Date modified);
} }

View File

@@ -34,6 +34,7 @@ import tech.easyflow.ai.config.SearcherFactory;
import tech.easyflow.ai.documentimport.DocumentImportDtos; import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.DocumentImportKeys; import tech.easyflow.ai.documentimport.DocumentImportKeys;
import tech.easyflow.ai.documentimport.DocumentImportPreviewService; import tech.easyflow.ai.documentimport.DocumentImportPreviewService;
import tech.easyflow.ai.documentimport.task.DocumentImportSnapshotCleanupService;
import tech.easyflow.ai.documentimport.task.KnowledgeDocumentImportTaskAppService; import tech.easyflow.ai.documentimport.task.KnowledgeDocumentImportTaskAppService;
import tech.easyflow.ai.entity.*; import tech.easyflow.ai.entity.*;
import tech.easyflow.ai.enums.DocumentProcessStatus; import tech.easyflow.ai.enums.DocumentProcessStatus;
@@ -101,6 +102,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
@Autowired @Autowired
private DocumentImportPreviewService documentImportPreviewService; private DocumentImportPreviewService documentImportPreviewService;
@Autowired
private DocumentImportSnapshotCleanupService snapshotCleanupService;
@Autowired @Autowired
private KnowledgeDocumentImportTaskAppService importTaskAppService; private KnowledgeDocumentImportTaskAppService importTaskAppService;
@@ -240,7 +244,18 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
: asString(oneByQuery.getOptions().get( : asString(oneByQuery.getOptions().get(
DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH)); DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH));
if (StringUtil.hasText(chunkSnapshotPath)) { if (StringUtil.hasText(chunkSnapshotPath)) {
storageService.delete(chunkSnapshotPath); snapshotCleanupService.scheduleChunkSnapshot(
chunkSnapshotPath);
}
String csvTableSnapshotPath = oneByQuery.getOptions() == null
? null
: asString(oneByQuery.getOptions().get(
DocumentImportKeys.KEY_DOCUMENT_CSV_TABLE_SNAPSHOT_PATH));
if (StringUtil.hasText(csvTableSnapshotPath)) {
snapshotCleanupService.scheduleCsvTableSnapshot(
oneByQuery.getCollectionId(),
oneByQuery.getId(),
csvTableSnapshotPath);
} }
storageService.delete(oneByQuery.getDocumentPath()); storageService.delete(oneByQuery.getDocumentPath());
// 主记录必须最后删除;否则接口返回成功后文档仍会出现在列表中。 // 主记录必须最后删除;否则接口返回成功后文档仍会出现在列表中。

View File

@@ -0,0 +1,67 @@
package tech.easyflow.ai.documentimport.task;
import org.junit.Assert;
import org.junit.Test;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.Set;
/**
* CSV 导入入口支持策略测试。
*/
public class CsvImportSupportPolicyTest {
/**
* 验证单文件、管理端批次和 Public API 批次三条入口都允许 CSV。
*
* @throws Exception 反射读取失败
*/
@Test
public void shouldAllowCsvAcrossAllKnowledgeImportEntrypoints()
throws Exception {
KnowledgeDocumentImportTaskAppService taskService =
new KnowledgeDocumentImportTaskAppService();
Method assertSupported = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("assertSupportedImportFile", String.class);
assertSupported.setAccessible(true);
Method normalizeExtension =
KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"normalizeFileExtension", String.class, String.class);
normalizeExtension.setAccessible(true);
String lowerCaseExtension = (String) normalizeExtension.invoke(
taskService, "results.csv", "/knowledge/results.csv");
String upperCaseExtension = (String) normalizeExtension.invoke(
taskService, "RESULTS.CSV", "/knowledge/RESULTS.CSV");
Assert.assertEquals("csv", lowerCaseExtension);
Assert.assertEquals("csv", upperCaseExtension);
assertSupported.invoke(taskService, lowerCaseExtension);
assertSupported.invoke(taskService, upperCaseExtension);
Assert.assertTrue(readSupportedExtensions(
DocumentImportBatchAppService.class).contains("csv"));
Assert.assertTrue(readSupportedExtensions(
KnowledgeImportBatchFacade.class).contains("csv"));
Assert.assertSame(
DocumentImportFormatPolicy.supportedExtensions(),
readSupportedExtensions(DocumentImportBatchAppService.class));
Assert.assertSame(
DocumentImportFormatPolicy.supportedExtensions(),
readSupportedExtensions(KnowledgeImportBatchFacade.class));
}
/**
* 读取入口类的支持扩展名集合。
*
* @param type 入口类
* @return 扩展名集合
* @throws Exception 字段读取失败
*/
@SuppressWarnings("unchecked")
private Set<String> readSupportedExtensions(Class<?> type)
throws Exception {
Field field = type.getDeclaredField("SUPPORTED_EXTENSIONS");
field.setAccessible(true);
return (Set<String>) field.get(null);
}
}

View File

@@ -0,0 +1,361 @@
package tech.easyflow.ai.documentimport.task;
import com.easyagents.rag.core.BgeM3ChunkSafety;
import com.easyagents.rag.ingestion.model.StrategyConfig;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.entity.DocumentChunk;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.List;
/**
* {@link CsvTableSnapshotService} 流式解析与分块测试。
*/
public class CsvTableSnapshotServiceTest {
/**
* 验证引号逗号、跨行字段、尾空列和重复表头都能稳定解析并按行窗口分块。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldParseRfc4180AndBuildRowWindowChunks() throws Exception {
InMemoryFileStorageService storage = new InMemoryFileStorageService();
String sourcePath = "source/demo.csv";
String csv = "姓名,备注,姓名,\r\n"
+ "张三,\"第一行\n第二行\",A,\r\n"
+ "李四,\"含,逗号\",B,\r\n";
storage.put(sourcePath, csv.getBytes(StandardCharsets.UTF_8));
CsvTableSnapshotService service = createService(storage);
CsvTableSnapshotService.CsvParseResult result = service.parse(
sourcePath, BigInteger.valueOf(7), BigInteger.valueOf(9), "token-1");
Assert.assertEquals("UTF-8", result.getEncoding());
Assert.assertEquals(2L, result.getRowCount());
Assert.assertEquals(4, result.getColumnCount());
Assert.assertEquals(
List.of("姓名", "备注", "姓名_2", "列_4"), result.getHeaders());
tech.easyflow.ai.entity.Document document =
new tech.easyflow.ai.entity.Document();
document.setId(BigInteger.valueOf(9));
document.setCollectionId(BigInteger.valueOf(7));
document.setTitle("demo.csv");
document.setDocumentPath(sourcePath);
StrategyConfig strategy = StrategyConfig.defaults();
strategy.setStrategyCode("TABLE_ROW");
strategy.setRowsPerChunk(1);
DocumentImportDtos.PreviewSession session =
service.buildChunkSnapshot(
document, result.getManifestPath(), strategy);
Assert.assertEquals(Integer.valueOf(2), session.getTotalChunks());
Assert.assertEquals(2, session.getDocumentChunks().size());
Assert.assertTrue(session.getDocumentChunks().get(0).getContent()
.contains("第一行<br/>第二行"));
Assert.assertTrue(session.getDocumentChunks().get(1).getContent()
.contains("含,逗号"));
Assert.assertTrue(session.getDocumentChunks().stream()
.allMatch(chunk -> "TABLE_ROW".equals(
chunk.getOptions().get("chunkType"))));
}
/**
* 验证无 BOM 且非 UTF-8 的 CSV 会重新打开源文件并回退到 GB18030。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldFallbackToGb18030AfterStrictUtf8Failure() throws Exception {
InMemoryFileStorageService storage = new InMemoryFileStorageService();
String sourcePath = "source/gb.csv";
storage.put(
sourcePath,
"名称,说明\r\n测试,中文内容\r\n".getBytes(Charset.forName("GB18030")));
CsvTableSnapshotService service = createService(storage);
CsvTableSnapshotService.CsvParseResult result = service.parse(
sourcePath, BigInteger.valueOf(7), BigInteger.valueOf(10), "token-2");
Assert.assertEquals("GB18030", result.getEncoding());
Assert.assertEquals(1L, result.getRowCount());
}
/**
* 验证数据行列数与表头不一致时返回稳定失败码。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldRejectColumnMismatchWithStableFailureCode() throws Exception {
InMemoryFileStorageService storage = new InMemoryFileStorageService();
String sourcePath = "source/broken.csv";
storage.put(
sourcePath,
"a,b\r\n1,2,3\r\n".getBytes(StandardCharsets.UTF_8));
CsvTableSnapshotService service = createService(storage);
try {
service.parse(
sourcePath, BigInteger.valueOf(7), BigInteger.valueOf(11), "token-3");
Assert.fail("应拒绝列数不一致的 CSV");
} catch (CsvImportException error) {
Assert.assertEquals(
CsvTableSnapshotService.FAILURE_COLUMN_MISMATCH,
error.getFailureCode());
Assert.assertTrue(error.getMessage().contains("列数"));
}
}
/**
* 验证二进制伪装 CSV 中的 NUL 字节会被明确拒绝。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldRejectNulByteAsMalformedCsv() throws Exception {
InMemoryFileStorageService storage = new InMemoryFileStorageService();
String sourcePath = "source/binary.csv";
storage.put(
sourcePath,
"name,value\nalice,\0binary\n".getBytes(StandardCharsets.UTF_8));
CsvTableSnapshotService service = createService(storage);
try {
service.parse(
sourcePath, BigInteger.valueOf(7), BigInteger.valueOf(12), "token-4");
Assert.fail("包含 NUL 字节的文件不应进入后续阶段");
} catch (CsvImportException error) {
Assert.assertEquals(
CsvTableSnapshotService.FAILURE_MALFORMED,
error.getFailureCode());
}
}
/**
* 验证超长表格正文会生成多个不超过数据库上限的续片。
*/
@Test
public void shouldSplitOversizedWindowWithoutSilentTruncation() {
TabularRowWindowChunkBuilder builder =
new TabularRowWindowChunkBuilder();
String longValue = "".repeat(
TabularRowWindowChunkBuilder.MAX_CHUNK_CONTENT_CHARS + 100);
List<DocumentChunk> chunks = builder.build(
BigInteger.ONE,
BigInteger.valueOf(2),
"大表",
List.of("正文"),
List.of(new TabularRowWindowChunkBuilder.TabularRow(
2, List.of(longValue))),
1,
"TABLE_ROW");
Assert.assertTrue(chunks.size() > 1);
Assert.assertTrue(chunks.stream().allMatch(
chunk -> chunk.getContent().length()
<= TabularRowWindowChunkBuilder.MAX_CHUNK_CONTENT_CHARS));
Assert.assertTrue(chunks.stream().allMatch(
chunk -> BgeM3ChunkSafety.isWithinHardLimit(chunk.getContent())));
Assert.assertTrue(chunks.stream().allMatch(
chunk -> chunk.getContent().startsWith("# 大表")
&& chunk.getContent().contains("| 正文 |")));
Assert.assertTrue(chunks.stream().allMatch(
chunk -> Integer.valueOf(2).equals(
chunk.getOptions().get("rowStart"))
&& Integer.valueOf(2).equals(
chunk.getOptions().get("rowEnd"))));
long restoredChars = chunks.stream()
.map(DocumentChunk::getContent)
.flatMapToInt(String::chars)
.filter(value -> value == '长')
.count();
Assert.assertEquals(longValue.length(), restoredChars);
Assert.assertEquals(
chunks.size(), chunks.get(0).getOptions().get("partTotal"));
}
/**
* 验证超长记录优先按列边界续片,且每个续片都保留标题和列名。
*/
@Test
public void shouldSplitOversizedRowAtColumnBoundaries() {
TabularRowWindowChunkBuilder builder =
new TabularRowWindowChunkBuilder();
String longValue = "".repeat(20_000);
List<DocumentChunk> chunks = builder.build(
BigInteger.ONE,
BigInteger.valueOf(2),
"列边界表",
List.of("左列", "正文", "右列"),
List.of(new TabularRowWindowChunkBuilder.TabularRow(
8, List.of("LEFT_MARKER", longValue, "RIGHT_MARKER"))),
1,
"TABLE_ROW");
Assert.assertTrue(chunks.size() > 2);
Assert.assertTrue(chunks.stream().allMatch(
chunk -> chunk.getContent().startsWith("# 列边界表")
&& chunk.getContent().contains("\n| ")));
String allContent = chunks.stream()
.map(DocumentChunk::getContent)
.reduce("", String::concat);
Assert.assertEquals(1, occurrences(allContent, "LEFT_MARKER"));
Assert.assertEquals(1, occurrences(allContent, "RIGHT_MARKER"));
Assert.assertEquals(
longValue.length(),
allContent.chars().filter(value -> value == '长').count());
Assert.assertTrue(chunks.stream().allMatch(
chunk -> Integer.valueOf(8).equals(
chunk.getOptions().get("rowStart"))
&& Integer.valueOf(8).equals(
chunk.getOptions().get("rowEnd"))));
}
/**
* 验证行数未满时仍会在加入下一行超过 Token 上限前闭合窗口。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldCloseWindowBeforeNextRowExceedsTokenLimit()
throws Exception {
InMemoryFileStorageService storage =
new InMemoryFileStorageService();
String sourcePath = "source/token-window.csv";
String value = "".repeat(5_000);
storage.put(
sourcePath,
("正文\n" + value + "\n" + value + "\n")
.getBytes(StandardCharsets.UTF_8));
CsvTableSnapshotService service = createService(storage);
CsvTableSnapshotService.CsvParseResult result = service.parse(
sourcePath,
BigInteger.valueOf(7),
BigInteger.valueOf(13),
"token-window");
tech.easyflow.ai.entity.Document document =
new tech.easyflow.ai.entity.Document();
document.setId(BigInteger.valueOf(13));
document.setCollectionId(BigInteger.valueOf(7));
document.setTitle("token-window.csv");
StrategyConfig strategy = StrategyConfig.defaults();
strategy.setStrategyCode("TABLE_ROW");
strategy.setRowsPerChunk(10);
DocumentImportDtos.PreviewSession session =
service.buildChunkSnapshot(
document, result.getManifestPath(), strategy);
Assert.assertEquals(Integer.valueOf(2), session.getTotalChunks());
Assert.assertEquals(
List.of(2, 3),
session.getDocumentChunks().stream()
.map(chunk -> (Integer) chunk.getOptions().get("rowStart"))
.toList());
}
/**
* 验证 CSV 行分片删除失败时清单保持可重试。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldKeepCsvManifestWhenPartDeletionFails()
throws Exception {
InMemoryFileStorageService storage =
new InMemoryFileStorageService();
String sourcePath = "source/cleanup.csv";
storage.put(
sourcePath,
"a,b\n1,2\n".getBytes(StandardCharsets.UTF_8));
CsvTableSnapshotService service = createService(storage);
CsvTableSnapshotService.CsvParseResult result = service.parse(
sourcePath,
BigInteger.valueOf(7),
BigInteger.valueOf(14),
"cleanup");
String manifestPath = result.getManifestPath();
String partPath = storage.paths().stream()
.filter(path -> !path.equals(sourcePath))
.filter(path -> !path.equals(manifestPath))
.findFirst()
.orElseThrow();
storage.failDelete(partPath, 1);
try {
service.delete(manifestPath);
Assert.fail("CSV 行分片删除失败时应抛出异常");
} catch (IllegalStateException expected) {
Assert.assertTrue(expected.getMessage().contains("模拟删除失败"));
}
Assert.assertTrue(storage.contains(manifestPath));
Assert.assertTrue(storage.contains(partPath));
service.delete(manifestPath);
Assert.assertEquals(1, storage.size());
Assert.assertTrue(storage.contains(sourcePath));
}
/**
* 统计文本出现次数。
*
* @param content 完整文本
* @param target 目标文本
* @return 出现次数
*/
private int occurrences(String content, String target) {
int count = 0;
int offset = 0;
while ((offset = content.indexOf(target, offset)) >= 0) {
count++;
offset += target.length();
}
return count;
}
/**
* 创建完成依赖注入的被测服务。
*
* @param storage 内存文件存储
* @return CSV 服务
* @throws Exception 反射注入失败
*/
private CsvTableSnapshotService createService(
InMemoryFileStorageService storage) throws Exception {
DocumentImportChunkSnapshotService chunkSnapshotService =
new DocumentImportChunkSnapshotService();
setField(chunkSnapshotService, "storageService", storage);
CsvTableSnapshotService service = new CsvTableSnapshotService();
setField(service, "storageService", storage);
setField(service, "chunkSnapshotService", chunkSnapshotService);
return service;
}
/**
* 反射设置测试字段。
*
* @param target 目标对象
* @param fieldName 字段名
* @param value 字段值
* @throws Exception 字段不存在或不可访问
*/
private void setField(
Object target,
String fieldName,
Object value) throws Exception {
Field field = target.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
field.set(target, value);
}
}

View File

@@ -2,17 +2,13 @@ package tech.easyflow.ai.documentimport.task;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.documentimport.DocumentImportDtos; import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.entity.DocumentChunk; import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.common.filestorage.FileStorageService;
import java.io.ByteArrayInputStream;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.math.BigInteger; import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
/** /**
* {@link DocumentImportChunkSnapshotService} 持久化恢复测试。 * {@link DocumentImportChunkSnapshotService} 持久化恢复测试。
@@ -26,19 +22,8 @@ public class DocumentImportChunkSnapshotServiceTest {
*/ */
@Test @Test
public void shouldPersistAndRestoreChunkSnapshot() throws Exception { public void shouldPersistAndRestoreChunkSnapshot() throws Exception {
String storedPath = "http://localhost/snapshots/9-chunks.json"; InMemoryFileStorageService storageService =
AtomicReference<byte[]> storedBytes = new AtomicReference<byte[]>(); new InMemoryFileStorageService();
FileStorageService storageService = Mockito.mock(FileStorageService.class);
Mockito.when(storageService.save(
Mockito.any(MultipartFile.class),
Mockito.anyString()
)).thenAnswer(invocation -> {
MultipartFile file = invocation.getArgument(0);
storedBytes.set(file.getBytes());
return storedPath;
});
Mockito.when(storageService.readStream(storedPath))
.thenAnswer(invocation -> new ByteArrayInputStream(storedBytes.get()));
DocumentImportChunkSnapshotService service = new DocumentImportChunkSnapshotService(); DocumentImportChunkSnapshotService service = new DocumentImportChunkSnapshotService();
Field storageField = DocumentImportChunkSnapshotService.class Field storageField = DocumentImportChunkSnapshotService.class
@@ -59,10 +44,108 @@ public class DocumentImportChunkSnapshotServiceTest {
String path = service.save(session); String path = service.save(session);
DocumentImportDtos.PreviewSession restored = service.load(path); DocumentImportDtos.PreviewSession restored = service.load(path);
Assert.assertEquals(storedPath, path); Assert.assertTrue(path.endsWith("-manifest.json"));
Assert.assertEquals(session.getKnowledgeId(), restored.getKnowledgeId()); Assert.assertEquals(session.getKnowledgeId(), restored.getKnowledgeId());
Assert.assertEquals(session.getDocumentId(), restored.getDocumentId()); Assert.assertEquals(session.getDocumentId(), restored.getDocumentId());
Assert.assertEquals(1, restored.getDocumentChunks().size()); Assert.assertEquals(1, restored.getDocumentChunks().size());
Assert.assertEquals("稳定分块", restored.getDocumentChunks().get(0).getContent()); Assert.assertEquals("稳定分块", restored.getDocumentChunks().get(0).getContent());
} }
/**
* 验证 V2 快照可分页并按小批顺序消费,删除时同时清理清单和分片。
*
* @throws Exception 反射注入异常
*/
@Test
public void shouldReadV2SnapshotByPageAndBatch() throws Exception {
InMemoryFileStorageService storageService =
new InMemoryFileStorageService();
DocumentImportChunkSnapshotService service =
new DocumentImportChunkSnapshotService();
Field storageField = DocumentImportChunkSnapshotService.class
.getDeclaredField("storageService");
storageField.setAccessible(true);
storageField.set(service, storageService);
DocumentImportDtos.PreviewSession session =
new DocumentImportDtos.PreviewSession();
session.setKnowledgeId(BigInteger.valueOf(7));
session.setDocumentId(BigInteger.valueOf(9));
List<DocumentChunk> chunks = new ArrayList<DocumentChunk>();
for (int index = 0; index < 5; index++) {
DocumentChunk chunk = new DocumentChunk();
chunk.setId(BigInteger.valueOf(100 + index));
chunk.setDocumentId(BigInteger.valueOf(9));
chunk.setDocumentCollectionId(BigInteger.valueOf(7));
chunk.setContent("chunk-" + index);
chunks.add(chunk);
}
session.setDocumentChunks(chunks);
String path = service.save(session);
List<DocumentChunk> page = service.loadPage(path, 2, 2);
List<String> consumed = new ArrayList<String>();
service.forEachBatch(path, 2, batch -> {
Assert.assertTrue(batch.size() <= 2);
for (DocumentChunk chunk : batch) {
consumed.add(chunk.getContent());
}
});
Assert.assertEquals(List.of("chunk-2", "chunk-3"),
page.stream().map(DocumentChunk::getContent).toList());
Assert.assertEquals(
List.of("chunk-0", "chunk-1", "chunk-2", "chunk-3", "chunk-4"),
consumed);
Assert.assertTrue(storageService.size() >= 2);
service.delete(path);
Assert.assertEquals(0, storageService.size());
}
/**
* 验证分片删除失败时保留清单,使后续重试仍能获取精确对象集合。
*
* @throws Exception 反射注入异常
*/
@Test
public void shouldKeepManifestWhenPartDeletionFails() throws Exception {
InMemoryFileStorageService storageService =
new InMemoryFileStorageService();
DocumentImportChunkSnapshotService service =
new DocumentImportChunkSnapshotService();
Field storageField = DocumentImportChunkSnapshotService.class
.getDeclaredField("storageService");
storageField.setAccessible(true);
storageField.set(service, storageService);
DocumentChunk chunk = new DocumentChunk();
chunk.setId(BigInteger.valueOf(21));
chunk.setDocumentId(BigInteger.valueOf(9));
chunk.setDocumentCollectionId(BigInteger.valueOf(7));
chunk.setContent("等待可靠清理");
DocumentImportDtos.PreviewSession session =
new DocumentImportDtos.PreviewSession();
session.setKnowledgeId(BigInteger.valueOf(7));
session.setDocumentId(BigInteger.valueOf(9));
session.setDocumentChunks(List.of(chunk));
String manifestPath = service.save(session);
String partPath = storageService.paths().stream()
.filter(path -> !path.equals(manifestPath))
.findFirst()
.orElseThrow();
storageService.failDelete(partPath, 1);
try {
service.delete(manifestPath);
Assert.fail("分片删除失败时应抛出异常");
} catch (IllegalStateException expected) {
Assert.assertTrue(expected.getMessage().contains("模拟删除失败"));
}
Assert.assertTrue(storageService.contains(manifestPath));
Assert.assertTrue(storageService.contains(partPath));
service.delete(manifestPath);
Assert.assertEquals(0, storageService.size());
}
} }

View File

@@ -0,0 +1,237 @@
package tech.easyflow.ai.documentimport.task;
import com.mybatisflex.core.query.QueryWrapper;
import org.junit.Test;
import org.mockito.Mockito;
import tech.easyflow.ai.entity.DocumentImportSnapshotCleanup;
import tech.easyflow.ai.mapper.DocumentImportSnapshotCleanupMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.Date;
import java.util.concurrent.atomic.AtomicReference;
/**
* {@link DocumentImportSnapshotCleanupService} 可靠清理测试。
*/
public class DocumentImportSnapshotCleanupServiceTest {
/**
* 验证分片和清单按持久化阶段顺序删除。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldPersistPhaseBeforeDeletingManifest()
throws Exception {
CleanupHarness harness = createHarness();
harness.service.scheduleChunkSnapshot(
"snapshot/chunk-manifest.json");
Mockito.verify(harness.chunkSnapshotService).deleteParts(
"snapshot/chunk-manifest.json");
Mockito.verify(harness.cleanupMapper).advanceToManifest(
Mockito.any(BigInteger.class),
Mockito.anyString(),
Mockito.any(Date.class));
Mockito.verify(harness.chunkSnapshotService).deleteManifest(
"snapshot/chunk-manifest.json");
Mockito.verify(harness.cleanupMapper).deleteCompleted(
Mockito.any(BigInteger.class),
Mockito.anyString());
Mockito.verify(harness.cleanupMapper, Mockito.never())
.releaseForRetry(
Mockito.any(BigInteger.class),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.anyString(),
Mockito.any(Date.class));
}
/**
* 验证分片删除失败后保留记录并安排重试。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldReleaseFailedCleanupForRetry()
throws Exception {
CleanupHarness harness = createHarness();
Mockito.doThrow(new IllegalStateException("对象存储暂不可用"))
.when(harness.chunkSnapshotService)
.deleteParts("snapshot/failing-manifest.json");
harness.service.scheduleChunkSnapshot(
"snapshot/failing-manifest.json");
Mockito.verify(harness.cleanupMapper).releaseForRetry(
Mockito.any(BigInteger.class),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.contains("对象存储暂不可用"),
Mockito.any(Date.class));
Mockito.verify(harness.chunkSnapshotService, Mockito.never())
.deleteManifest(Mockito.anyString());
Mockito.verify(harness.cleanupMapper, Mockito.never())
.deleteCompleted(
Mockito.any(BigInteger.class),
Mockito.anyString());
}
/**
* 验证 CSV 清单删除后原子清理文档中的旧快照指针。
*
* @throws Exception 反射注入失败
*/
@Test
public void shouldClearCsvPointerAfterManifestDeletion()
throws Exception {
CleanupHarness harness = createHarness();
BigInteger knowledgeId = BigInteger.valueOf(7);
BigInteger documentId = BigInteger.valueOf(9);
harness.service.scheduleCsvTableSnapshot(
knowledgeId,
documentId,
"snapshot/csv-manifest.json");
Mockito.verify(harness.csvTableSnapshotService).deleteParts(
"snapshot/csv-manifest.json");
Mockito.verify(harness.csvTableSnapshotService).deleteManifest(
"snapshot/csv-manifest.json");
Mockito.verify(harness.documentMapper).clearCsvSnapshotPath(
Mockito.eq(documentId),
Mockito.eq("snapshot/csv-manifest.json"),
Mockito.any(Date.class));
}
/**
* 创建带内存状态的清理服务测试夹具。
*
* @return 测试夹具
* @throws Exception 反射注入失败
*/
@SuppressWarnings("unchecked")
private CleanupHarness createHarness() throws Exception {
DocumentImportSnapshotCleanupService service =
new DocumentImportSnapshotCleanupService();
DocumentImportSnapshotCleanupMapper cleanupMapper =
Mockito.mock(DocumentImportSnapshotCleanupMapper.class);
DocumentImportChunkSnapshotService chunkSnapshotService =
Mockito.mock(DocumentImportChunkSnapshotService.class);
CsvTableSnapshotService csvTableSnapshotService =
Mockito.mock(CsvTableSnapshotService.class);
DocumentMapper documentMapper =
Mockito.mock(DocumentMapper.class);
AtomicReference<DocumentImportSnapshotCleanup> recordRef =
new AtomicReference<DocumentImportSnapshotCleanup>();
Mockito.when(cleanupMapper.insertIgnore(
Mockito.any(DocumentImportSnapshotCleanup.class)))
.thenAnswer(invocation -> {
recordRef.set(invocation.getArgument(0));
return 1;
});
Mockito.when(cleanupMapper.selectOneByQuery(
Mockito.any(QueryWrapper.class)))
.thenAnswer(invocation -> recordRef.get());
Mockito.when(cleanupMapper.claim(
Mockito.any(BigInteger.class),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.any(Date.class)))
.thenAnswer(invocation -> {
DocumentImportSnapshotCleanup record = recordRef.get();
record.setExecutionToken(invocation.getArgument(1));
record.setAttemptCount(
(record.getAttemptCount() == null
? 0
: record.getAttemptCount()) + 1);
return 1;
});
Mockito.when(cleanupMapper.advanceToManifest(
Mockito.any(BigInteger.class),
Mockito.anyString(),
Mockito.any(Date.class)))
.thenAnswer(invocation -> {
recordRef.get().setPhase("MANIFEST_PENDING");
return 1;
});
Mockito.when(cleanupMapper.deleteCompleted(
Mockito.any(BigInteger.class),
Mockito.anyString())).thenReturn(1);
Mockito.when(cleanupMapper.releaseForRetry(
Mockito.any(BigInteger.class),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.anyString(),
Mockito.any(Date.class))).thenReturn(1);
setField(service, "cleanupMapper", cleanupMapper);
setField(
service, "chunkSnapshotService", chunkSnapshotService);
setField(
service, "csvTableSnapshotService", csvTableSnapshotService);
setField(service, "documentMapper", documentMapper);
return new CleanupHarness(
service,
cleanupMapper,
chunkSnapshotService,
csvTableSnapshotService,
documentMapper);
}
/**
* 反射设置测试字段。
*
* @param target 目标对象
* @param fieldName 字段名
* @param value 字段值
* @throws Exception 字段不存在或不可访问
*/
private void setField(
Object target,
String fieldName,
Object value) throws Exception {
Field field = target.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
field.set(target, value);
}
/**
* 快照清理测试依赖集合。
*/
private static final class CleanupHarness {
private final DocumentImportSnapshotCleanupService service;
private final DocumentImportSnapshotCleanupMapper cleanupMapper;
private final DocumentImportChunkSnapshotService
chunkSnapshotService;
private final CsvTableSnapshotService csvTableSnapshotService;
private final DocumentMapper documentMapper;
/**
* 创建测试依赖集合。
*
* @param service 清理服务
* @param cleanupMapper 清理 Mapper
* @param chunkSnapshotService 分块快照服务
* @param csvTableSnapshotService CSV 快照服务
* @param documentMapper 文档 Mapper
*/
private CleanupHarness(
DocumentImportSnapshotCleanupService service,
DocumentImportSnapshotCleanupMapper cleanupMapper,
DocumentImportChunkSnapshotService chunkSnapshotService,
CsvTableSnapshotService csvTableSnapshotService,
DocumentMapper documentMapper) {
this.service = service;
this.cleanupMapper = cleanupMapper;
this.chunkSnapshotService = chunkSnapshotService;
this.csvTableSnapshotService = csvTableSnapshotService;
this.documentMapper = documentMapper;
}
}
}

View File

@@ -0,0 +1,148 @@
package tech.easyflow.ai.documentimport.task;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.common.filestorage.FileStorageService;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
/**
* 文档导入测试使用的内存文件存储。
*/
final class InMemoryFileStorageService implements FileStorageService {
private final AtomicInteger sequence = new AtomicInteger();
private final Map<String, byte[]> files = new LinkedHashMap<String, byte[]>();
private String failingDeletePath;
private int remainingDeleteFailures;
/**
* 保存到默认测试目录。
*
* @param file 上传文件
* @return 测试路径
*/
@Override
public String save(MultipartFile file) {
return save(file, "default");
}
/**
* 保存到指定测试目录。
*
* @param file 上传文件
* @param prePath 测试目录
* @return 测试路径
*/
@Override
public String save(MultipartFile file, String prePath) {
try {
String path = prePath + "/" + sequence.incrementAndGet()
+ "-" + file.getOriginalFilename();
files.put(path, file.getBytes());
return path;
} catch (IOException error) {
throw new IllegalStateException("测试文件保存失败", error);
}
}
/**
* 删除测试对象。
*
* @param path 测试路径
*/
@Override
public void delete(String path) {
if (remainingDeleteFailures > 0
&& path != null
&& path.equals(failingDeletePath)) {
remainingDeleteFailures--;
throw new IllegalStateException(
"模拟删除失败: " + path);
}
files.remove(path);
}
/**
* 打开测试对象。
*
* @param path 测试路径
* @return 输入流
* @throws IOException 对象不存在
*/
@Override
public InputStream readStream(String path) throws IOException {
byte[] bytes = files.get(path);
if (bytes == null) {
throw new IOException("测试对象不存在: " + path);
}
return new ByteArrayInputStream(bytes);
}
/**
* 返回测试对象大小。
*
* @param path 测试路径
* @return 字节数
*/
@Override
public long getFileSize(String path) {
byte[] bytes = files.get(path);
return bytes == null ? -1L : bytes.length;
}
/**
* 直接放入一个源文件。
*
* @param path 测试路径
* @param bytes 文件内容
*/
void put(String path, byte[] bytes) {
files.put(path, bytes);
}
/**
* 返回当前对象数量。
*
* @return 对象数量
*/
int size() {
return files.size();
}
/**
* 判断测试对象是否存在。
*
* @param path 测试路径
* @return 是否存在
*/
boolean contains(String path) {
return files.containsKey(path);
}
/**
* 返回当前全部测试路径快照。
*
* @return 路径集合
*/
Set<String> paths() {
return new LinkedHashSet<String>(files.keySet());
}
/**
* 配置指定路径接下来若干次删除失败。
*
* @param path 测试路径
* @param failureCount 失败次数
*/
void failDelete(String path, int failureCount) {
failingDeletePath = path;
remainingDeleteFailures = Math.max(0, failureCount);
}
}

View File

@@ -24,6 +24,7 @@ import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
import tech.easyflow.ai.document.model.DocumentParseArtifacts; import tech.easyflow.ai.document.model.DocumentParseArtifacts;
import tech.easyflow.ai.document.model.DocumentParsedResult; import tech.easyflow.ai.document.model.DocumentParsedResult;
import tech.easyflow.ai.document.model.DocumentSourceRef; import tech.easyflow.ai.document.model.DocumentSourceRef;
import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.DocumentImportKeys; import tech.easyflow.ai.documentimport.DocumentImportKeys;
import tech.easyflow.ai.entity.DocumentChunk; import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.DocumentCollection;
@@ -68,6 +69,46 @@ import java.util.concurrent.atomic.AtomicReference;
*/ */
public class KnowledgeDocumentImportTaskAppServiceTest { public class KnowledgeDocumentImportTaskAppServiceTest {
/**
* 验证预览翻页直接读取分片快照,避免恢复完整会话。
*
* @throws Exception 反射调用失败
*/
@Test
public void loadPreviewPageShouldReadOnlyRequestedSnapshotRange()
throws Exception {
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
DocumentImportChunkSnapshotService snapshotService =
Mockito.mock(DocumentImportChunkSnapshotService.class);
setField(
service,
"documentImportChunkSnapshotService",
snapshotService);
DocumentChunk chunk = new DocumentChunk();
chunk.setId(BigInteger.valueOf(9001));
Mockito.when(snapshotService.loadPage("snapshot.json", 20, 20))
.thenReturn(List.of(chunk));
DocumentImportDtos.PreviewSession session =
new DocumentImportDtos.PreviewSession();
session.setChunkSnapshotPath("snapshot.json");
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod(
"loadPreviewPage",
DocumentImportDtos.PreviewSession.class,
int.class,
int.class);
method.setAccessible(true);
@SuppressWarnings("unchecked")
List<DocumentChunk> result = (List<DocumentChunk>) method.invoke(
service, session, 2, 20);
Assert.assertEquals(List.of(chunk), result);
Mockito.verify(snapshotService).loadPage("snapshot.json", 20, 20);
}
/** /**
* 验证待处理任务重新投递只更新必要字段,避免自定义查询结果覆盖非空列。 * 验证待处理任务重新投递只更新必要字段,避免自定义查询结果覆盖非空列。
* *

View File

@@ -0,0 +1,30 @@
CREATE TABLE `tb_document_import_snapshot_cleanup`
(
`id` bigint UNSIGNED NOT NULL COMMENT '主键',
`knowledge_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '知识库ID',
`document_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '文档ID',
`snapshot_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '快照类型',
`manifest_path` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '快照清单路径',
`path_hash` char(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '快照清单路径SHA-256',
`phase` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '清理阶段',
`attempt_count` int NOT NULL DEFAULT 0 COMMENT '已尝试次数',
`next_retry_at` datetime NOT NULL COMMENT '下次重试时间',
`execution_token` varchar(64) CHARACTER SET ascii COLLATE ascii_bin NULL DEFAULT NULL COMMENT '执行令牌',
`lease_until` datetime NULL DEFAULT NULL COMMENT '租约截止时间',
`last_error` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '最近错误',
`created` datetime NULL DEFAULT NULL COMMENT '创建时间',
`created_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '创建人',
`modified` datetime NULL DEFAULT NULL COMMENT '修改时间',
`modified_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '修改人',
PRIMARY KEY (`id`) USING BTREE,
UNIQUE INDEX `uk_document_import_snapshot_cleanup_path`
(`snapshot_type`, `path_hash`) USING BTREE,
INDEX `idx_document_import_snapshot_cleanup_due`
(`next_retry_at`, `lease_until`, `id`) USING BTREE,
INDEX `idx_document_import_snapshot_cleanup_document`
(`document_id`, `snapshot_type`) USING BTREE
) ENGINE = InnoDB
CHARACTER SET = utf8mb4
COLLATE = utf8mb4_0900_ai_ci
COMMENT = '文档导入中间快照清理记录'
ROW_FORMAT = DYNAMIC;

View File

@@ -142,6 +142,7 @@ const isExcelChunk = (row: any) => {
return Boolean( return Boolean(
sourceFileExt === 'xlsx' || sourceFileExt === 'xlsx' ||
sourceFileExt === 'csv' ||
options?.sheetName || options?.sheetName ||
options?.rowStart || options?.rowStart ||
options?.rowEnd, options?.rowEnd,

View File

@@ -81,6 +81,7 @@ const MAX_FILE_COUNT = 2000;
const MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024; const MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024;
const MAX_TOTAL_SIZE_BYTES = 1024 * 1024 * 1024; const MAX_TOTAL_SIZE_BYTES = 1024 * 1024 * 1024;
const SUPPORTED_EXTENSIONS = new Set([ const SUPPORTED_EXTENSIONS = new Set([
'csv',
'docx', 'docx',
'md', 'md',
'pdf', 'pdf',
@@ -537,7 +538,7 @@ async function createClientFileKey(relativePath: string) {
class="native-file-input" class="native-file-input"
type="file" type="file"
multiple multiple
accept=".txt,.pdf,.docx,.md,.pptx,.xlsx" accept=".txt,.pdf,.docx,.md,.pptx,.xlsx,.csv"
@change="handleNativeSelection" @change="handleNativeSelection"
/> />
<input <input

View File

@@ -705,7 +705,7 @@ const endpointDocs = computed(() => {
'metadata 是 Content-Type 为 application/json 的 multipart Part不是普通表单字符串。', 'metadata 是 Content-Type 为 application/json 的 multipart Part不是普通表单字符串。',
'metadata.knowledgeId 填当前知识库 IDchunkStrategy 可省略duplicatePolicy 默认 SKIP。', 'metadata.knowledgeId 填当前知识库 IDchunkStrategy 可省略duplicatePolicy 默认 SKIP。',
'metadata.files 中每项填写 clientFileKey、fileName 和可选 relativePath文件大小由服务端实际读取并统计。', 'metadata.files 中每项填写 clientFileKey、fileName 和可选 relativePath文件大小由服务端实际读取并统计。',
'支持 txt、md、pdf、docx、pptx、xlsx单文件不超过 100 MiB单批不超过 200 MiB最多 200 个文件。', '支持 txt、md、pdf、docx、pptx、xlsx、csv;单文件不超过 100 MiB单批不超过 200 MiB最多 200 个文件。',
'metadata 最大 1 MiBmetadata.files 必须与 files Part 数量、顺序和文件名一致。', 'metadata 最大 1 MiBmetadata.files 必须与 files Part 数量、顺序和文件名一致。',
'chunkStrategy 可不传,默认 AUTOduplicatePolicy 支持 SKIP、OVERWRITE、REIMPORT。', 'chunkStrategy 可不传,默认 AUTOduplicatePolicy 支持 SKIP、OVERWRITE、REIMPORT。',
'服务端会按调用者、元数据和文件内容生成请求指纹,任务完成后 10 分钟内的重复提交会返回原 taskId。', '服务端会按调用者、元数据和文件内容生成请求指纹,任务完成后 10 分钟内的重复提交会返回原 taskId。',

View File

@@ -49,6 +49,8 @@ interface PreviewItem {
chunks?: ChunkItem[]; chunks?: ChunkItem[];
fileName: string; fileName: string;
normalizedContent?: string; normalizedContent?: string;
pageNo?: number;
pageSize?: number;
previewSessionId: string; previewSessionId: string;
strategyCode?: string; strategyCode?: string;
strategyLabel?: string; strategyLabel?: string;
@@ -97,6 +99,7 @@ const activeDocumentId = ref(props.documentId || '');
const previewError = ref(''); const previewError = ref('');
const previewLoading = ref(false); const previewLoading = ref(false);
const startLoading = ref(false); const startLoading = ref(false);
const previewPageSize = 20;
let previewDebounceTimer: null | ReturnType<typeof setTimeout> = null; let previewDebounceTimer: null | ReturnType<typeof setTimeout> = null;
let previewSequence = 0; let previewSequence = 0;
@@ -137,7 +140,9 @@ const fileExt = computed(
const isPptx = computed(() => fileExt.value === 'pptx'); const isPptx = computed(() => fileExt.value === 'pptx');
const isXlsx = computed(() => fileExt.value === 'xlsx'); const isXlsx = computed(() => fileExt.value === 'xlsx');
const showStrategySelector = computed(() => !isPptx.value && !isXlsx.value); const isCsv = computed(() => fileExt.value === 'csv');
const isTabular = computed(() => isXlsx.value || isCsv.value);
const showStrategySelector = computed(() => !isPptx.value && !isTabular.value);
const mdLevels = [1, 2, 3, 4, 5, 6]; const mdLevels = [1, 2, 3, 4, 5, 6];
@@ -169,10 +174,10 @@ const buildStrategyConfig = () => {
strategyCode: 'OFFICE_PPTX_PAGE', strategyCode: 'OFFICE_PPTX_PAGE',
}; };
} }
if (isXlsx.value) { if (isTabular.value) {
return { return {
rowsPerChunk: formState.rowsPerChunk, rowsPerChunk: formState.rowsPerChunk,
strategyCode: 'OFFICE_XLSX_ROW_WINDOW', strategyCode: isCsv.value ? 'TABLE_ROW' : 'OFFICE_XLSX_ROW_WINDOW',
}; };
} }
return { return {
@@ -233,6 +238,8 @@ const generatePreview = async () => {
}, },
], ],
knowledgeId: props.knowledgeId, knowledgeId: props.knowledgeId,
pageNo: 1,
pageSize: previewPageSize,
}, },
); );
if (requestSequence !== previewSequence) { if (requestSequence !== previewSequence) {
@@ -261,6 +268,52 @@ const generatePreview = async () => {
} }
}; };
const loadPreviewPage = async (pageNo: number) => {
if (
!activeDocumentId.value ||
!currentPreviewSessionId.value ||
previewLoading.value
) {
return;
}
const requestSequence = ++previewSequence;
previewLoading.value = true;
previewError.value = '';
try {
const res = await props.requestClient.post(
buildKnowledgePath(
props.endpointPrefix,
'/api/v1/document/import/task/preview',
),
{
documentId: activeDocumentId.value,
knowledgeId: props.knowledgeId,
pageNo,
pageSize: previewPageSize,
previewSessionId: currentPreviewSessionId.value,
},
);
if (requestSequence !== previewSequence) {
return;
}
previewItems.value = normalizePreviewItems(
(res.data?.items || []) as PreviewItem[],
);
} catch (error: any) {
if (requestSequence !== previewSequence) {
return;
}
const message =
error?.message || $t('documentCollection.importDoc.previewRequestFailed');
previewError.value = message;
ElMessage.error(message);
} finally {
if (requestSequence === previewSequence) {
previewLoading.value = false;
}
}
};
const schedulePreviewGeneration = () => { const schedulePreviewGeneration = () => {
if (!activeDocumentId.value) { if (!activeDocumentId.value) {
return; return;
@@ -321,8 +374,10 @@ watch(
if (isPptx.value) { if (isPptx.value) {
formState.strategyCode = 'OFFICE_PPTX_PAGE'; formState.strategyCode = 'OFFICE_PPTX_PAGE';
} }
if (isXlsx.value) { if (isTabular.value) {
formState.strategyCode = 'OFFICE_XLSX_ROW_WINDOW'; formState.strategyCode = isCsv.value
? 'TABLE_ROW'
: 'OFFICE_XLSX_ROW_WINDOW';
formState.rowsPerChunk = 10; formState.rowsPerChunk = 10;
} }
resetPreviewState(); resetPreviewState();
@@ -365,7 +420,7 @@ watch(
</ElFormItem> </ElFormItem>
<ElFormItem <ElFormItem
v-if="isXlsx" v-if="isTabular"
label="每多少行分一块" label="每多少行分一块"
class="workbench__form-full" class="workbench__form-full"
> >
@@ -450,6 +505,7 @@ watch(
<SplitterDocPreview <SplitterDocPreview
:loading="previewLoading" :loading="previewLoading"
:preview-items="previewItems" :preview-items="previewItems"
@page-change="loadPreviewPage"
@preview-session-change="handlePreviewSessionChange" @preview-session-change="handlePreviewSessionChange"
/> />
</section> </section>

View File

@@ -4,7 +4,7 @@ import ElXMarkdown from 'vue-element-plus-x/es/XMarkdown/index.js';
import { $t } from '@easyflow/locales'; import { $t } from '@easyflow/locales';
import { ElEmpty, ElSkeleton, ElTag } from 'element-plus'; import { ElEmpty, ElPagination, ElSkeleton, ElTag } from 'element-plus';
import { import {
markdownRenderProps, markdownRenderProps,
@@ -42,6 +42,8 @@ interface PreviewItem {
chunks?: ChunkItem[]; chunks?: ChunkItem[];
fileName: string; fileName: string;
normalizedContent?: string; normalizedContent?: string;
pageNo?: number;
pageSize?: number;
previewSessionId: string; previewSessionId: string;
strategyLabel?: string; strategyLabel?: string;
totalChunks?: number; totalChunks?: number;
@@ -63,7 +65,7 @@ const props = withDefaults(
}, },
); );
const emits = defineEmits(['previewSessionChange']); const emits = defineEmits(['pageChange', 'previewSessionChange']);
const activeFile = ref(''); const activeFile = ref('');
@@ -236,6 +238,23 @@ const isActiveChunk = (chunk: ChunkItem) =>
</div> </div>
</div> </div>
</div> </div>
<div
v-if="
Number(currentPreview?.totalChunks || 0) >
Number(currentPreview?.pageSize || 20)
"
class="preview-shell__pagination"
>
<ElPagination
:current-page="currentPreview?.pageNo || 1"
:disabled="loading"
:page-size="currentPreview?.pageSize || 20"
:total="currentPreview?.totalChunks || 0"
background
layout="prev, pager, next"
@current-change="emits('pageChange', $event)"
/>
</div>
</div> </div>
</div> </div>
</template> </template>
@@ -293,6 +312,13 @@ const isActiveChunk = (chunk: ChunkItem) =>
overflow: auto; overflow: auto;
} }
.preview-shell__pagination {
display: flex;
flex-shrink: 0;
justify-content: center;
padding-top: 16px;
}
.chunk-card { .chunk-card {
padding: 15px 16px 14px; padding: 15px 16px 14px;
cursor: pointer; cursor: pointer;

View File

@@ -38,6 +38,7 @@
<jsoup.version>1.16.1</jsoup.version> <jsoup.version>1.16.1</jsoup.version>
<openhtmltopdf.version>1.0.10</openhtmltopdf.version> <openhtmltopdf.version>1.0.10</openhtmltopdf.version>
<commons-io.version>2.18.0</commons-io.version> <commons-io.version>2.18.0</commons-io.version>
<commons-csv.version>1.14.1</commons-csv.version>
<commons-compress.version>1.28.0</commons-compress.version> <commons-compress.version>1.28.0</commons-compress.version>
<fastexcel.version>1.2.0</fastexcel.version> <fastexcel.version>1.2.0</fastexcel.version>
<hutool.version>5.8.36</hutool.version> <hutool.version>5.8.36</hutool.version>
@@ -373,6 +374,12 @@
<version>${commons-io.version}</version> <version>${commons-io.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>${commons-csv.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId> <artifactId>commons-compress</artifactId>