feat: 完善知识库批量导入与公共 API
- 新增批量异步导入、状态查询、失败重试与中断恢复链路 - 拆分知识库读取、导入、维护权限并完善 Public API 契约 - 补充数据库迁移、管理端交互、接口说明与相关测试
This commit is contained in:
@@ -5,11 +5,13 @@ import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import tech.easyflow.ai.documentimport.task.DocumentImportParseMonitorProperties;
|
||||
import tech.easyflow.ai.documentimport.task.DocumentImportBulkProperties;
|
||||
import tech.easyflow.ai.documentimport.task.DocumentImportStatusBroadcastProperties;
|
||||
|
||||
@MapperScan("tech.easyflow.ai.mapper")
|
||||
@ComponentScan("tech.easyflow.ai")
|
||||
@EnableConfigurationProperties({
|
||||
DocumentImportBulkProperties.class,
|
||||
DocumentImportParseMonitorProperties.class,
|
||||
DocumentImportStatusBroadcastProperties.class,
|
||||
RagHealthProperties.class
|
||||
|
||||
@@ -54,6 +54,19 @@ public interface DocumentParseBridgeService {
|
||||
*/
|
||||
DocumentParsedResult queryResult(String taskId);
|
||||
|
||||
/**
|
||||
* 按提交文档的源信息获取异步任务最终结果。
|
||||
*
|
||||
* <p>源信息仅用于精确选择提交任务时使用的解析服务,不会重新读取文档内容。</p>
|
||||
*
|
||||
* @param taskId 任务 ID
|
||||
* @param source 提交任务时的文档源信息
|
||||
* @return 标准化解析结果
|
||||
*/
|
||||
default DocumentParsedResult queryResult(String taskId, DocumentSourceRef source) {
|
||||
return queryResult(taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聚合查询异步任务信息。
|
||||
*
|
||||
@@ -64,4 +77,17 @@ public interface DocumentParseBridgeService {
|
||||
* @return 聚合任务信息
|
||||
*/
|
||||
DocumentParseTaskInfo queryTaskInfo(String taskId);
|
||||
|
||||
/**
|
||||
* 按提交文档的源信息聚合查询异步任务信息。
|
||||
*
|
||||
* <p>源信息仅用于精确选择提交任务时使用的解析服务,不会重新读取文档内容。</p>
|
||||
*
|
||||
* @param taskId 任务 ID
|
||||
* @param source 提交任务时的文档源信息
|
||||
* @return 聚合任务信息
|
||||
*/
|
||||
default DocumentParseTaskInfo queryTaskInfo(String taskId, DocumentSourceRef source) {
|
||||
return queryTaskInfo(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,12 +150,24 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
||||
*/
|
||||
@Override
|
||||
public DocumentParsedResult queryResult(String taskId) {
|
||||
return queryResult(taskId, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public DocumentParsedResult queryResult(String taskId, @Nullable DocumentSourceRef source) {
|
||||
if (!StringUtils.hasText(taskId)) {
|
||||
throw DocumentParseBridgeException.resultFetchFailed("taskId 不能为空");
|
||||
}
|
||||
try {
|
||||
LOG.info("桥接服务开始获取异步解析结果: providerTaskId={}", taskId);
|
||||
ParseResponse response = executeAgainstTaskService(taskId, service -> service.queryResult(taskId));
|
||||
ParseResponse response = executeAgainstTaskService(
|
||||
taskId,
|
||||
source,
|
||||
service -> service.queryResult(taskId)
|
||||
);
|
||||
DocumentParsedResult result = parseResultMapper.map(extractSingleResult(response, true));
|
||||
LOG.info("桥接服务获取异步解析结果完成: providerTaskId={}, preferredTextLength={}",
|
||||
taskId, resolveTextLength(result));
|
||||
@@ -174,11 +186,23 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
||||
*/
|
||||
@Override
|
||||
public DocumentParseTaskInfo queryTaskInfo(String taskId) {
|
||||
return queryTaskInfo(taskId, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public DocumentParseTaskInfo queryTaskInfo(String taskId, @Nullable DocumentSourceRef source) {
|
||||
if (!StringUtils.hasText(taskId)) {
|
||||
throw DocumentParseBridgeException.taskFailed("taskId 不能为空");
|
||||
}
|
||||
try {
|
||||
ParseTaskInfo taskInfo = executeAgainstTaskService(taskId, service -> service.queryTaskInfo(taskId));
|
||||
ParseTaskInfo taskInfo = executeAgainstTaskService(
|
||||
taskId,
|
||||
source,
|
||||
service -> service.queryTaskInfo(taskId)
|
||||
);
|
||||
DocumentParseTaskInfo mappedTaskInfo = parseResultMapper.map(taskInfo);
|
||||
LOG.info("桥接服务查询异步解析任务状态: providerTaskId={}, status={}, hasResult={}",
|
||||
taskId,
|
||||
@@ -223,6 +247,16 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
||||
|
||||
private DocumentParseService resolveService(LoadedDocumentSource loadedSource) {
|
||||
DocumentParseSourceType sourceType = DocumentParseSourceType.resolve(loadedSource.getFileName(), loadedSource.getContentType());
|
||||
return resolveService(sourceType);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按文档源类型选择解析服务。
|
||||
*
|
||||
* @param sourceType 文档源类型
|
||||
* @return 对应解析服务
|
||||
*/
|
||||
private DocumentParseService resolveService(DocumentParseSourceType sourceType) {
|
||||
switch (sourceType) {
|
||||
case PDF:
|
||||
return requireSpecificService(pdfDocumentParseService, defaultDocumentParseService, "PDF");
|
||||
@@ -249,6 +283,28 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
||||
throw DocumentParseBridgeException.serviceNotEnabled("未启用 " + sourceType + " 文档解析服务");
|
||||
}
|
||||
|
||||
/**
|
||||
* 在已知任务源信息时精确查询对应服务,缺少源信息时保留旧版兼容遍历。
|
||||
*
|
||||
* @param taskId 任务 ID
|
||||
* @param source 提交任务时的文档源信息
|
||||
* @param action 查询操作
|
||||
* @param <T> 查询结果类型
|
||||
* @return 查询结果
|
||||
*/
|
||||
private <T> T executeAgainstTaskService(String taskId,
|
||||
@Nullable DocumentSourceRef source,
|
||||
Function<DocumentParseService, T> action) {
|
||||
if (source == null) {
|
||||
return executeAgainstTaskService(taskId, action);
|
||||
}
|
||||
DocumentParseSourceType sourceType = DocumentParseSourceType.resolve(
|
||||
source.getFileName(),
|
||||
source.getContentType()
|
||||
);
|
||||
return action.apply(resolveService(sourceType));
|
||||
}
|
||||
|
||||
private <T> T executeAgainstTaskService(String taskId, Function<DocumentParseService, T> action) {
|
||||
List<DocumentParseService> services = availableServices();
|
||||
if (services.isEmpty()) {
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
/**
|
||||
* 批量导入建单上下文。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public final class DocumentImportBatchCreateContext {
|
||||
|
||||
private final ImportCallerContext caller;
|
||||
private final String idempotencyKeyHash;
|
||||
private final String requestDigest;
|
||||
private final String duplicatePolicy;
|
||||
private final String requestedStrategyJson;
|
||||
|
||||
/**
|
||||
* 创建批量导入建单上下文。
|
||||
*
|
||||
* @param caller 调用者上下文
|
||||
* @param idempotencyKeyHash 幂等键哈希
|
||||
* @param requestDigest 请求摘要
|
||||
* @param duplicatePolicy 重复文件策略
|
||||
* @param requestedStrategyJson 请求分块策略 JSON
|
||||
*/
|
||||
public DocumentImportBatchCreateContext(ImportCallerContext caller,
|
||||
String idempotencyKeyHash,
|
||||
String requestDigest,
|
||||
String duplicatePolicy,
|
||||
String requestedStrategyJson) {
|
||||
this.caller = caller;
|
||||
this.idempotencyKeyHash = idempotencyKeyHash;
|
||||
this.requestDigest = requestDigest;
|
||||
this.duplicatePolicy = duplicatePolicy;
|
||||
this.requestedStrategyJson = requestedStrategyJson;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取调用者上下文。
|
||||
*
|
||||
* @return 调用者上下文
|
||||
*/
|
||||
public ImportCallerContext getCaller() {
|
||||
return caller;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取幂等键哈希。
|
||||
*
|
||||
* @return 幂等键哈希
|
||||
*/
|
||||
public String getIdempotencyKeyHash() {
|
||||
return idempotencyKeyHash;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取请求摘要。
|
||||
*
|
||||
* @return 请求摘要
|
||||
*/
|
||||
public String getRequestDigest() {
|
||||
return requestDigest;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取重复文件策略。
|
||||
*
|
||||
* @return 重复文件策略
|
||||
*/
|
||||
public String getDuplicatePolicy() {
|
||||
return duplicatePolicy;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取请求分块策略 JSON。
|
||||
*
|
||||
* @return 分块策略 JSON
|
||||
*/
|
||||
public String getRequestedStrategyJson() {
|
||||
return requestedStrategyJson;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 文档批量导入接口数据对象。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public final class DocumentImportBatchDtos {
|
||||
|
||||
private DocumentImportBatchDtos() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 客户端文件清单项。
|
||||
*/
|
||||
public static class ManifestItem implements Serializable {
|
||||
private String clientFileKey;
|
||||
private String fileName;
|
||||
private String relativePath;
|
||||
private Long fileSize;
|
||||
|
||||
public String getClientFileKey() {
|
||||
return clientFileKey;
|
||||
}
|
||||
|
||||
public void setClientFileKey(String clientFileKey) {
|
||||
this.clientFileKey = clientFileKey;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
public String getRelativePath() {
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
public void setRelativePath(String relativePath) {
|
||||
this.relativePath = relativePath;
|
||||
}
|
||||
|
||||
public Long getFileSize() {
|
||||
return fileSize;
|
||||
}
|
||||
|
||||
public void setFileSize(Long fileSize) {
|
||||
this.fileSize = fileSize;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建批次请求。
|
||||
*/
|
||||
public static class CreateRequest implements Serializable {
|
||||
private BigInteger knowledgeId;
|
||||
private List<ManifestItem> files = new ArrayList<ManifestItem>();
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public List<ManifestItem> getFiles() {
|
||||
return files;
|
||||
}
|
||||
|
||||
public void setFiles(List<ManifestItem> files) {
|
||||
this.files = files;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 服务端文件项。
|
||||
*/
|
||||
public static class ItemResponse implements Serializable {
|
||||
private BigInteger itemId;
|
||||
private BigInteger documentId;
|
||||
private String clientFileKey;
|
||||
private String fileName;
|
||||
private String relativePath;
|
||||
private Long fileSize;
|
||||
private String stage;
|
||||
private String status;
|
||||
private String errorSummary;
|
||||
|
||||
public BigInteger getItemId() {
|
||||
return itemId;
|
||||
}
|
||||
|
||||
public void setItemId(BigInteger itemId) {
|
||||
this.itemId = itemId;
|
||||
}
|
||||
|
||||
public BigInteger getDocumentId() {
|
||||
return documentId;
|
||||
}
|
||||
|
||||
public void setDocumentId(BigInteger documentId) {
|
||||
this.documentId = documentId;
|
||||
}
|
||||
|
||||
public String getClientFileKey() {
|
||||
return clientFileKey;
|
||||
}
|
||||
|
||||
public void setClientFileKey(String clientFileKey) {
|
||||
this.clientFileKey = clientFileKey;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
public String getRelativePath() {
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
public void setRelativePath(String relativePath) {
|
||||
this.relativePath = relativePath;
|
||||
}
|
||||
|
||||
public Long getFileSize() {
|
||||
return fileSize;
|
||||
}
|
||||
|
||||
public void setFileSize(Long fileSize) {
|
||||
this.fileSize = fileSize;
|
||||
}
|
||||
|
||||
public String getStage() {
|
||||
return stage;
|
||||
}
|
||||
|
||||
public void setStage(String stage) {
|
||||
this.stage = stage;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public String getErrorSummary() {
|
||||
return errorSummary;
|
||||
}
|
||||
|
||||
public void setErrorSummary(String errorSummary) {
|
||||
this.errorSummary = errorSummary;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建批次响应。
|
||||
*/
|
||||
public static class CreateResponse implements Serializable {
|
||||
private BigInteger batchId;
|
||||
private Integer uploadConcurrency;
|
||||
private List<ItemResponse> items = new ArrayList<ItemResponse>();
|
||||
|
||||
public BigInteger getBatchId() {
|
||||
return batchId;
|
||||
}
|
||||
|
||||
public void setBatchId(BigInteger batchId) {
|
||||
this.batchId = batchId;
|
||||
}
|
||||
|
||||
public Integer getUploadConcurrency() {
|
||||
return uploadConcurrency;
|
||||
}
|
||||
|
||||
public void setUploadConcurrency(Integer uploadConcurrency) {
|
||||
this.uploadConcurrency = uploadConcurrency;
|
||||
}
|
||||
|
||||
public List<ItemResponse> getItems() {
|
||||
return items;
|
||||
}
|
||||
|
||||
public void setItems(List<ItemResponse> items) {
|
||||
this.items = items;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动批次请求。
|
||||
*/
|
||||
public static class StartRequest implements Serializable {
|
||||
private BigInteger knowledgeId;
|
||||
private BigInteger batchId;
|
||||
private String importMode;
|
||||
private String duplicatePolicy;
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public BigInteger getBatchId() {
|
||||
return batchId;
|
||||
}
|
||||
|
||||
public void setBatchId(BigInteger batchId) {
|
||||
this.batchId = batchId;
|
||||
}
|
||||
|
||||
public String getImportMode() {
|
||||
return importMode;
|
||||
}
|
||||
|
||||
public void setImportMode(String importMode) {
|
||||
this.importMode = importMode;
|
||||
}
|
||||
|
||||
public String getDuplicatePolicy() {
|
||||
return duplicatePolicy;
|
||||
}
|
||||
|
||||
public void setDuplicatePolicy(String duplicatePolicy) {
|
||||
this.duplicatePolicy = duplicatePolicy;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批次状态响应。
|
||||
*/
|
||||
public static class StatusResponse implements Serializable {
|
||||
private BigInteger batchId;
|
||||
private String importMode;
|
||||
private String status;
|
||||
private Integer totalCount;
|
||||
private Long totalBytes;
|
||||
private Integer completedCount;
|
||||
private Integer processingCount;
|
||||
private Integer failedCount;
|
||||
private Integer pendingCount;
|
||||
private Integer skippedCount;
|
||||
private Integer cancelledCount;
|
||||
private Integer retryableFailedCount;
|
||||
private Integer progressPercent;
|
||||
private Date startedAt;
|
||||
private Date finishedAt;
|
||||
|
||||
public BigInteger getBatchId() {
|
||||
return batchId;
|
||||
}
|
||||
|
||||
public void setBatchId(BigInteger batchId) {
|
||||
this.batchId = batchId;
|
||||
}
|
||||
|
||||
public String getImportMode() {
|
||||
return importMode;
|
||||
}
|
||||
|
||||
public void setImportMode(String importMode) {
|
||||
this.importMode = importMode;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Integer getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
public void setTotalCount(Integer totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
public Long getTotalBytes() {
|
||||
return totalBytes;
|
||||
}
|
||||
|
||||
public void setTotalBytes(Long totalBytes) {
|
||||
this.totalBytes = totalBytes;
|
||||
}
|
||||
|
||||
public Integer getCompletedCount() {
|
||||
return completedCount;
|
||||
}
|
||||
|
||||
public void setCompletedCount(Integer completedCount) {
|
||||
this.completedCount = completedCount;
|
||||
}
|
||||
|
||||
public Integer getProcessingCount() {
|
||||
return processingCount;
|
||||
}
|
||||
|
||||
public void setProcessingCount(Integer processingCount) {
|
||||
this.processingCount = processingCount;
|
||||
}
|
||||
|
||||
public Integer getFailedCount() {
|
||||
return failedCount;
|
||||
}
|
||||
|
||||
public void setFailedCount(Integer failedCount) {
|
||||
this.failedCount = failedCount;
|
||||
}
|
||||
|
||||
public Integer getPendingCount() {
|
||||
return pendingCount;
|
||||
}
|
||||
|
||||
public void setPendingCount(Integer pendingCount) {
|
||||
this.pendingCount = pendingCount;
|
||||
}
|
||||
|
||||
public Integer getSkippedCount() {
|
||||
return skippedCount;
|
||||
}
|
||||
|
||||
public void setSkippedCount(Integer skippedCount) {
|
||||
this.skippedCount = skippedCount;
|
||||
}
|
||||
|
||||
public Integer getCancelledCount() {
|
||||
return cancelledCount;
|
||||
}
|
||||
|
||||
public void setCancelledCount(Integer cancelledCount) {
|
||||
this.cancelledCount = cancelledCount;
|
||||
}
|
||||
|
||||
public Integer getRetryableFailedCount() {
|
||||
return retryableFailedCount;
|
||||
}
|
||||
|
||||
public void setRetryableFailedCount(Integer retryableFailedCount) {
|
||||
this.retryableFailedCount = retryableFailedCount;
|
||||
}
|
||||
|
||||
public Integer getProgressPercent() {
|
||||
return progressPercent;
|
||||
}
|
||||
|
||||
public void setProgressPercent(Integer progressPercent) {
|
||||
this.progressPercent = progressPercent;
|
||||
}
|
||||
|
||||
public Date getStartedAt() {
|
||||
return startedAt;
|
||||
}
|
||||
|
||||
public void setStartedAt(Date startedAt) {
|
||||
this.startedAt = startedAt;
|
||||
}
|
||||
|
||||
public Date getFinishedAt() {
|
||||
return finishedAt;
|
||||
}
|
||||
|
||||
public void setFinishedAt(Date finishedAt) {
|
||||
this.finishedAt = finishedAt;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* 文档批量导入重试的稳定响应快照。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public class DocumentImportBatchRetryResult {
|
||||
|
||||
private final BigInteger taskId;
|
||||
private final String status;
|
||||
private final Integer retryGeneration;
|
||||
private final Integer retriedCount;
|
||||
|
||||
/**
|
||||
* 创建重试响应快照。
|
||||
*
|
||||
* @param taskId 批次任务 ID
|
||||
* @param status 领取重试时的任务状态
|
||||
* @param retryGeneration 重试代次
|
||||
* @param retriedCount 本次领取的文件数
|
||||
*/
|
||||
public DocumentImportBatchRetryResult(BigInteger taskId,
|
||||
String status,
|
||||
Integer retryGeneration,
|
||||
Integer retriedCount) {
|
||||
this.taskId = taskId;
|
||||
this.status = status;
|
||||
this.retryGeneration = retryGeneration;
|
||||
this.retriedCount = retriedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取批次任务 ID。
|
||||
*
|
||||
* @return 批次任务 ID
|
||||
*/
|
||||
public BigInteger getTaskId() {
|
||||
return taskId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取领取重试时的任务状态。
|
||||
*
|
||||
* @return 任务状态
|
||||
*/
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取重试代次。
|
||||
*
|
||||
* @return 重试代次
|
||||
*/
|
||||
public Integer getRetryGeneration() {
|
||||
return retryGeneration;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取本次领取的文件数。
|
||||
*
|
||||
* @return 本次领取的文件数
|
||||
*/
|
||||
public Integer getRetriedCount() {
|
||||
return retriedCount;
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ public final class DocumentImportKeys {
|
||||
public static final String KEY_DOCUMENT_STRATEGY_CODE = "splitter.strategyCode";
|
||||
public static final String KEY_DOCUMENT_STRATEGY_LABEL = "splitter.strategyLabel";
|
||||
public static final String KEY_DOCUMENT_STRATEGY_SNAPSHOT = "splitter.strategySnapshot";
|
||||
public static final String KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH = "splitter.chunkSnapshotPath";
|
||||
public static final String KEY_DOCUMENT_ANALYSIS_SUMMARY = "splitter.analysisSummary";
|
||||
public static final String KEY_DOCUMENT_SOURCE_FILE_EXT = "splitter.sourceFileExt";
|
||||
public static final String KEY_DOCUMENT_PREVIEW_VERSION = "splitter.previewVersion";
|
||||
@@ -30,6 +31,7 @@ public final class DocumentImportKeys {
|
||||
public static final String KEY_DOCUMENT_PARSE_PROCESSED_ITEMS = "parse.processedItems";
|
||||
public static final String KEY_DOCUMENT_PARSE_TOTAL_ITEMS = "parse.totalItems";
|
||||
public static final String KEY_DOCUMENT_PARSE_STATUS_MESSAGE = "parse.statusMessage";
|
||||
public static final String KEY_DOCUMENT_TASK_ERROR_CODE = "task.errorCode";
|
||||
public static final String KEY_DOCUMENT_RENDER_MARKDOWN = "renderMarkdown";
|
||||
public static final String KEY_DOCUMENT_PAGE_INDEX = "pageIndex";
|
||||
public static final String KEY_DOCUMENT_SHEET_NAME = "sheetName";
|
||||
@@ -37,4 +39,8 @@ public final class DocumentImportKeys {
|
||||
public static final String KEY_DOCUMENT_ROW_END = "rowEnd";
|
||||
public static final String KEY_DOCUMENT_IMAGE_REFS = "imageRefs";
|
||||
public static final String KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY = "parseArtifactSummary";
|
||||
public static final String KEY_DOCUMENT_IMPORT_MODE = "import.mode";
|
||||
public static final String KEY_DOCUMENT_IMPORT_BATCH_ID = "import.batchId";
|
||||
public static final String KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID = "import.batchItemId";
|
||||
public static final String KEY_DOCUMENT_IMPORT_RELATIVE_PATH = "import.relativePath";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* 文档批量导入调用者上下文。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public final class ImportCallerContext {
|
||||
|
||||
private final ImportCallerType callerType;
|
||||
private final BigInteger callerId;
|
||||
|
||||
/**
|
||||
* 创建调用者上下文。
|
||||
*
|
||||
* @param callerType 调用者类型
|
||||
* @param callerId 调用者 ID
|
||||
* @throws BusinessException 调用者信息不完整时抛出
|
||||
*/
|
||||
public ImportCallerContext(ImportCallerType callerType, BigInteger callerId) {
|
||||
if (callerType == null || callerId == null) {
|
||||
throw new BusinessException("导入调用者信息不完整");
|
||||
}
|
||||
this.callerType = callerType;
|
||||
this.callerId = callerId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取调用者类型。
|
||||
*
|
||||
* @return 调用者类型
|
||||
*/
|
||||
public ImportCallerType getCallerType() {
|
||||
return callerType;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取调用者 ID。
|
||||
*
|
||||
* @return 调用者 ID
|
||||
*/
|
||||
public BigInteger getCallerId() {
|
||||
return callerId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
/**
|
||||
* 文档批量导入调用者类型。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public enum ImportCallerType {
|
||||
|
||||
/**
|
||||
* 管理端登录用户。
|
||||
*/
|
||||
ADMIN,
|
||||
|
||||
/**
|
||||
* Public API 访问令牌。
|
||||
*/
|
||||
PUBLIC_API
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
package tech.easyflow.ai.documentimport;
|
||||
|
||||
import com.easyagents.rag.ingestion.model.StrategyConfig;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 知识库 Public API 批量导入数据对象。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public final class PublicDocumentImportDtos {
|
||||
|
||||
private PublicDocumentImportDtos() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Multipart 元数据。
|
||||
*/
|
||||
public static class BatchMetadata implements Serializable {
|
||||
private BigInteger knowledgeId;
|
||||
private StrategyConfig chunkStrategy = StrategyConfig.defaults();
|
||||
private String duplicatePolicy = "SKIP";
|
||||
private List<DocumentImportBatchDtos.ManifestItem> files = new ArrayList<>();
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public StrategyConfig getChunkStrategy() {
|
||||
return chunkStrategy;
|
||||
}
|
||||
|
||||
public void setChunkStrategy(StrategyConfig chunkStrategy) {
|
||||
this.chunkStrategy = chunkStrategy == null
|
||||
? StrategyConfig.defaults()
|
||||
: chunkStrategy;
|
||||
}
|
||||
|
||||
public String getDuplicatePolicy() {
|
||||
return duplicatePolicy;
|
||||
}
|
||||
|
||||
public void setDuplicatePolicy(String duplicatePolicy) {
|
||||
this.duplicatePolicy = duplicatePolicy;
|
||||
}
|
||||
|
||||
public List<DocumentImportBatchDtos.ManifestItem> getFiles() {
|
||||
return files;
|
||||
}
|
||||
|
||||
public void setFiles(List<DocumentImportBatchDtos.ManifestItem> files) {
|
||||
this.files = files;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量提交响应。
|
||||
*/
|
||||
public static class SubmitResponse implements Serializable {
|
||||
private BigInteger taskId;
|
||||
private String status;
|
||||
private Integer totalCount;
|
||||
private Long totalBytes;
|
||||
private Date createdAt;
|
||||
|
||||
public BigInteger getTaskId() {
|
||||
return taskId;
|
||||
}
|
||||
|
||||
public void setTaskId(BigInteger taskId) {
|
||||
this.taskId = taskId;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Integer getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
public void setTotalCount(Integer totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
public Long getTotalBytes() {
|
||||
return totalBytes;
|
||||
}
|
||||
|
||||
public void setTotalBytes(Long totalBytes) {
|
||||
this.totalBytes = totalBytes;
|
||||
}
|
||||
|
||||
public Date getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
|
||||
public void setCreatedAt(Date createdAt) {
|
||||
this.createdAt = createdAt;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批次计数。
|
||||
*/
|
||||
public static class Counts implements Serializable {
|
||||
private Integer total;
|
||||
private Integer completed;
|
||||
private Integer processing;
|
||||
private Integer pending;
|
||||
private Integer failed;
|
||||
private Integer skipped;
|
||||
private Integer retryableFailed;
|
||||
|
||||
public Integer getTotal() {
|
||||
return total;
|
||||
}
|
||||
|
||||
public void setTotal(Integer total) {
|
||||
this.total = total;
|
||||
}
|
||||
|
||||
public Integer getCompleted() {
|
||||
return completed;
|
||||
}
|
||||
|
||||
public void setCompleted(Integer completed) {
|
||||
this.completed = completed;
|
||||
}
|
||||
|
||||
public Integer getProcessing() {
|
||||
return processing;
|
||||
}
|
||||
|
||||
public void setProcessing(Integer processing) {
|
||||
this.processing = processing;
|
||||
}
|
||||
|
||||
public Integer getPending() {
|
||||
return pending;
|
||||
}
|
||||
|
||||
public void setPending(Integer pending) {
|
||||
this.pending = pending;
|
||||
}
|
||||
|
||||
public Integer getFailed() {
|
||||
return failed;
|
||||
}
|
||||
|
||||
public void setFailed(Integer failed) {
|
||||
this.failed = failed;
|
||||
}
|
||||
|
||||
public Integer getSkipped() {
|
||||
return skipped;
|
||||
}
|
||||
|
||||
public void setSkipped(Integer skipped) {
|
||||
this.skipped = skipped;
|
||||
}
|
||||
|
||||
public Integer getRetryableFailed() {
|
||||
return retryableFailed;
|
||||
}
|
||||
|
||||
public void setRetryableFailed(Integer retryableFailed) {
|
||||
this.retryableFailed = retryableFailed;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件失败信息。
|
||||
*/
|
||||
public static class ItemError implements Serializable {
|
||||
private String code;
|
||||
private String message;
|
||||
|
||||
public String getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public void setCode(String code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批次文件状态记录。
|
||||
*/
|
||||
public static class ItemRecord implements Serializable {
|
||||
private String fileKey;
|
||||
private String relativePath;
|
||||
private BigInteger documentId;
|
||||
private String stage;
|
||||
private String status;
|
||||
private Integer attemptCount;
|
||||
private Boolean retryable;
|
||||
private ItemError error;
|
||||
|
||||
public String getFileKey() {
|
||||
return fileKey;
|
||||
}
|
||||
|
||||
public void setFileKey(String fileKey) {
|
||||
this.fileKey = fileKey;
|
||||
}
|
||||
|
||||
public String getRelativePath() {
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
public void setRelativePath(String relativePath) {
|
||||
this.relativePath = relativePath;
|
||||
}
|
||||
|
||||
public BigInteger getDocumentId() {
|
||||
return documentId;
|
||||
}
|
||||
|
||||
public void setDocumentId(BigInteger documentId) {
|
||||
this.documentId = documentId;
|
||||
}
|
||||
|
||||
public String getStage() {
|
||||
return stage;
|
||||
}
|
||||
|
||||
public void setStage(String stage) {
|
||||
this.stage = stage;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Integer getAttemptCount() {
|
||||
return attemptCount;
|
||||
}
|
||||
|
||||
public void setAttemptCount(Integer attemptCount) {
|
||||
this.attemptCount = attemptCount;
|
||||
}
|
||||
|
||||
public Boolean getRetryable() {
|
||||
return retryable;
|
||||
}
|
||||
|
||||
public void setRetryable(Boolean retryable) {
|
||||
this.retryable = retryable;
|
||||
}
|
||||
|
||||
public ItemError getError() {
|
||||
return error;
|
||||
}
|
||||
|
||||
public void setError(ItemError error) {
|
||||
this.error = error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批次文件状态分页。
|
||||
*/
|
||||
public static class ItemPage implements Serializable {
|
||||
private Long pageNumber;
|
||||
private Long pageSize;
|
||||
private Long total;
|
||||
private List<ItemRecord> records = new ArrayList<>();
|
||||
|
||||
public Long getPageNumber() {
|
||||
return pageNumber;
|
||||
}
|
||||
|
||||
public void setPageNumber(Long pageNumber) {
|
||||
this.pageNumber = pageNumber;
|
||||
}
|
||||
|
||||
public Long getPageSize() {
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
public void setPageSize(Long pageSize) {
|
||||
this.pageSize = pageSize;
|
||||
}
|
||||
|
||||
public Long getTotal() {
|
||||
return total;
|
||||
}
|
||||
|
||||
public void setTotal(Long total) {
|
||||
this.total = total;
|
||||
}
|
||||
|
||||
public List<ItemRecord> getRecords() {
|
||||
return records;
|
||||
}
|
||||
|
||||
public void setRecords(List<ItemRecord> records) {
|
||||
this.records = records;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批次状态响应。
|
||||
*/
|
||||
public static class StatusResponse implements Serializable {
|
||||
private BigInteger taskId;
|
||||
private BigInteger knowledgeId;
|
||||
private String status;
|
||||
private Integer progressPercent;
|
||||
private Counts counts;
|
||||
private Boolean canRetry;
|
||||
private ItemPage items;
|
||||
|
||||
public BigInteger getTaskId() {
|
||||
return taskId;
|
||||
}
|
||||
|
||||
public void setTaskId(BigInteger taskId) {
|
||||
this.taskId = taskId;
|
||||
}
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Integer getProgressPercent() {
|
||||
return progressPercent;
|
||||
}
|
||||
|
||||
public void setProgressPercent(Integer progressPercent) {
|
||||
this.progressPercent = progressPercent;
|
||||
}
|
||||
|
||||
public Counts getCounts() {
|
||||
return counts;
|
||||
}
|
||||
|
||||
public void setCounts(Counts counts) {
|
||||
this.counts = counts;
|
||||
}
|
||||
|
||||
public Boolean getCanRetry() {
|
||||
return canRetry;
|
||||
}
|
||||
|
||||
public void setCanRetry(Boolean canRetry) {
|
||||
this.canRetry = canRetry;
|
||||
}
|
||||
|
||||
public ItemPage getItems() {
|
||||
return items;
|
||||
}
|
||||
|
||||
public void setItems(ItemPage items) {
|
||||
this.items = items;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 异常任务重试请求。
|
||||
*/
|
||||
public static class RetryRequest implements Serializable {
|
||||
private BigInteger taskId;
|
||||
private List<String> fileKeys;
|
||||
|
||||
public BigInteger getTaskId() {
|
||||
return taskId;
|
||||
}
|
||||
|
||||
public void setTaskId(BigInteger taskId) {
|
||||
this.taskId = taskId;
|
||||
}
|
||||
|
||||
public List<String> getFileKeys() {
|
||||
return fileKeys;
|
||||
}
|
||||
|
||||
public void setFileKeys(List<String> fileKeys) {
|
||||
this.fileKeys = fileKeys;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 异常任务重试响应。
|
||||
*/
|
||||
public static class RetryResponse implements Serializable {
|
||||
private BigInteger taskId;
|
||||
private String status;
|
||||
private Integer retriedCount;
|
||||
|
||||
public BigInteger getTaskId() {
|
||||
return taskId;
|
||||
}
|
||||
|
||||
public void setTaskId(BigInteger taskId) {
|
||||
this.taskId = taskId;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Integer getRetriedCount() {
|
||||
return retriedCount;
|
||||
}
|
||||
|
||||
public void setRetriedCount(Integer retriedCount) {
|
||||
this.retriedCount = retriedCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,498 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportMode;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchService;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 批量导入文件项与批次汇总状态跟踪器。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
@Service
|
||||
public class DocumentImportBatchTracker {
|
||||
|
||||
private final DocumentImportBatchService batchService;
|
||||
private final DocumentImportBatchItemService itemService;
|
||||
private final DocumentImportBatchMapper batchMapper;
|
||||
private final DocumentImportBatchItemMapper itemMapper;
|
||||
|
||||
/**
|
||||
* 创建批次状态跟踪器。
|
||||
*
|
||||
* @param batchService 批次服务
|
||||
* @param itemService 批次项服务
|
||||
* @param batchMapper 批次 Mapper
|
||||
* @param itemMapper 批次项 Mapper
|
||||
*/
|
||||
public DocumentImportBatchTracker(DocumentImportBatchService batchService,
|
||||
DocumentImportBatchItemService itemService,
|
||||
DocumentImportBatchMapper batchMapper,
|
||||
DocumentImportBatchItemMapper itemMapper) {
|
||||
this.batchService = batchService;
|
||||
this.itemService = itemService;
|
||||
this.batchMapper = batchMapper;
|
||||
this.itemMapper = itemMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询批次。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @return 批次实体
|
||||
*/
|
||||
public DocumentImportBatch requireBatch(BigInteger batchId) {
|
||||
DocumentImportBatch batch = batchId == null ? null : batchService.getById(batchId);
|
||||
if (batch == null) {
|
||||
throw new BusinessException("导入批次不存在");
|
||||
}
|
||||
return batch;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询批次文件项。
|
||||
*
|
||||
* @param itemId 文件项 ID
|
||||
* @return 文件项实体
|
||||
*/
|
||||
public DocumentImportBatchItem requireItem(BigInteger itemId) {
|
||||
DocumentImportBatchItem item = itemId == null ? null : itemService.getById(itemId);
|
||||
if (item == null) {
|
||||
throw new BusinessException("导入文件不存在");
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断指定批次是否采用自动导入。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @return 是否自动导入
|
||||
*/
|
||||
public boolean isAutoBatch(BigInteger batchId) {
|
||||
if (batchId == null) {
|
||||
return false;
|
||||
}
|
||||
return DocumentImportMode.AUTO.name().equals(requireBatch(batchId).getImportMode());
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新文件项阶段与状态。
|
||||
*
|
||||
* @param itemId 文件项 ID
|
||||
* @param stage 处理阶段
|
||||
* @param status 处理状态
|
||||
* @param errorSummary 错误摘要
|
||||
*/
|
||||
@org.springframework.transaction.annotation.Transactional
|
||||
public boolean updateItem(BigInteger itemId,
|
||||
DocumentImportBatchItemStage stage,
|
||||
DocumentImportBatchItemStatus status,
|
||||
String errorSummary) {
|
||||
if (itemId == null) {
|
||||
return false;
|
||||
}
|
||||
DocumentImportBatchItem current = requireItem(itemId);
|
||||
int attemptDelta = (status == DocumentImportBatchItemStatus.PENDING
|
||||
|| status == DocumentImportBatchItemStatus.RUNNING)
|
||||
&& DocumentImportBatchItemStatus.FAILED.name().equals(current.getStatus())
|
||||
? 1
|
||||
: 0;
|
||||
return transitionItem(itemId, stage, status, errorSummary,
|
||||
status == DocumentImportBatchItemStatus.FAILED, attemptDelta, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按预期旧状态原子迁移文件项,并同步更新批次计数。
|
||||
*
|
||||
* @param itemId 文件项 ID
|
||||
* @param stage 新阶段
|
||||
* @param status 新状态
|
||||
* @param errorSummary 错误摘要
|
||||
* @param retryable 是否允许批量重试
|
||||
* @param attemptDelta 重试次数增量
|
||||
* @return 状态迁移或同状态刷新成功时返回 {@code true}
|
||||
*/
|
||||
@org.springframework.transaction.annotation.Transactional
|
||||
public boolean transitionItem(BigInteger itemId,
|
||||
DocumentImportBatchItemStage stage,
|
||||
DocumentImportBatchItemStatus status,
|
||||
String errorSummary,
|
||||
boolean retryable,
|
||||
int attemptDelta) {
|
||||
return transitionItem(itemId, stage, status, errorSummary,
|
||||
retryable, attemptDelta, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按预期旧状态原子迁移文件项,并记录稳定失败码。
|
||||
*
|
||||
* @param itemId 文件项 ID
|
||||
* @param stage 新阶段
|
||||
* @param status 新状态
|
||||
* @param errorSummary 错误摘要
|
||||
* @param retryable 是否允许批量重试
|
||||
* @param attemptDelta 重试次数增量
|
||||
* @param failureCode 稳定失败码
|
||||
* @return 状态迁移或同状态刷新成功时返回 {@code true}
|
||||
*/
|
||||
@org.springframework.transaction.annotation.Transactional
|
||||
public boolean transitionItem(BigInteger itemId,
|
||||
DocumentImportBatchItemStage stage,
|
||||
DocumentImportBatchItemStatus status,
|
||||
String errorSummary,
|
||||
boolean retryable,
|
||||
int attemptDelta,
|
||||
String failureCode) {
|
||||
if (itemId == null) {
|
||||
return false;
|
||||
}
|
||||
for (int attempt = 0; attempt < 3; attempt++) {
|
||||
DocumentImportBatchItem current = requireItem(itemId);
|
||||
DocumentImportBatchItemStatus currentStatus =
|
||||
DocumentImportBatchItemStatus.valueOf(current.getStatus());
|
||||
if (!isAllowedTransition(currentStatus, status)) {
|
||||
return false;
|
||||
}
|
||||
String expectedStatus = current.getStatus();
|
||||
Date now = new Date();
|
||||
int updated = itemMapper.transitionStatus(
|
||||
itemId,
|
||||
expectedStatus,
|
||||
stage.name(),
|
||||
status.name(),
|
||||
errorSummary,
|
||||
failureCode,
|
||||
retryable,
|
||||
Math.max(0, attemptDelta),
|
||||
now
|
||||
);
|
||||
if (updated <= 0) {
|
||||
continue;
|
||||
}
|
||||
CounterDelta delta = CounterDelta.between(current, status, retryable);
|
||||
if (!delta.isZero()) {
|
||||
batchMapper.adjustCounters(
|
||||
current.getBatchId(),
|
||||
delta.completed,
|
||||
delta.processing,
|
||||
delta.failed,
|
||||
delta.pending,
|
||||
delta.uploaded,
|
||||
delta.skipped,
|
||||
delta.cancelled,
|
||||
delta.retryableFailed,
|
||||
now
|
||||
);
|
||||
}
|
||||
refreshBatch(current.getBatchId());
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验文件项状态机,拒绝迟到任务覆盖终态。
|
||||
*
|
||||
* @param current 当前状态
|
||||
* @param next 目标状态
|
||||
* @return 是否允许迁移
|
||||
*/
|
||||
private boolean isAllowedTransition(DocumentImportBatchItemStatus current,
|
||||
DocumentImportBatchItemStatus next) {
|
||||
if (current == next) {
|
||||
return true;
|
||||
}
|
||||
return switch (next) {
|
||||
case UPLOADING -> current == DocumentImportBatchItemStatus.PENDING;
|
||||
case UPLOADED -> current == DocumentImportBatchItemStatus.UPLOADING;
|
||||
case RUNNING -> current == DocumentImportBatchItemStatus.PENDING
|
||||
|| current == DocumentImportBatchItemStatus.FAILED;
|
||||
case PENDING -> current == DocumentImportBatchItemStatus.RUNNING
|
||||
|| current == DocumentImportBatchItemStatus.FAILED;
|
||||
case FAILED, COMPLETED -> current == DocumentImportBatchItemStatus.RUNNING
|
||||
|| current == DocumentImportBatchItemStatus.PENDING;
|
||||
case SKIPPED -> current == DocumentImportBatchItemStatus.UPLOADED;
|
||||
case CANCELLED -> current == DocumentImportBatchItemStatus.PENDING
|
||||
|| current == DocumentImportBatchItemStatus.UPLOADING
|
||||
|| current == DocumentImportBatchItemStatus.UPLOADED;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 将批次文件项绑定到创建后的文档。
|
||||
*
|
||||
* @param itemId 文件项 ID
|
||||
* @param documentId 文档 ID
|
||||
*/
|
||||
@org.springframework.transaction.annotation.Transactional
|
||||
public void bindDocument(BigInteger itemId, BigInteger documentId) {
|
||||
DocumentImportBatchItem item = requireItem(itemId);
|
||||
if (documentId.equals(item.getDocumentId())
|
||||
&& DocumentImportBatchItemStatus.PENDING.name().equals(item.getStatus())) {
|
||||
return;
|
||||
}
|
||||
Date now = new Date();
|
||||
int updated = itemMapper.bindDocument(itemId, documentId, now);
|
||||
if (updated <= 0) {
|
||||
throw new BusinessException("导入文件状态已变化,请刷新后重试");
|
||||
}
|
||||
batchMapper.adjustCounters(item.getBatchId(),
|
||||
0, 0, 0, 0, -1, 0, 0, 0, now);
|
||||
refreshBatch(item.getBatchId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 原子完成文件上传并增量更新批次上传数。
|
||||
*
|
||||
* @param itemId 文件项 ID
|
||||
* @param filePath 存储路径
|
||||
* @param storageLocator 可恢复存储定位符
|
||||
* @return 文件项仍处于上传中且完成成功时返回 {@code true}
|
||||
*/
|
||||
@org.springframework.transaction.annotation.Transactional
|
||||
public boolean completeUpload(BigInteger itemId,
|
||||
String filePath,
|
||||
String storageLocator) {
|
||||
DocumentImportBatchItem item = requireItem(itemId);
|
||||
if (DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus())) {
|
||||
return filePath.equals(item.getFilePath())
|
||||
&& storageLocator.equals(item.getStorageLocator());
|
||||
}
|
||||
if (!DocumentImportBatchItemStatus.UPLOADING.name().equals(item.getStatus())) {
|
||||
return false;
|
||||
}
|
||||
Date now = new Date();
|
||||
int updated = itemMapper.completeUpload(
|
||||
itemId,
|
||||
filePath,
|
||||
storageLocator,
|
||||
now
|
||||
);
|
||||
if (updated <= 0) {
|
||||
return false;
|
||||
}
|
||||
batchMapper.adjustCounters(item.getBatchId(),
|
||||
0, 0, 0, 0, 1, 0, 0, 0, now);
|
||||
refreshBatch(item.getBatchId());
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录文件项成功后需要清理的历史文档。
|
||||
*
|
||||
* @param itemId 文件项 ID
|
||||
* @param replacedDocumentId 历史文档 ID
|
||||
*/
|
||||
@org.springframework.transaction.annotation.Transactional
|
||||
public void markReplacement(BigInteger itemId, BigInteger replacedDocumentId) {
|
||||
Date now = new Date();
|
||||
int updated = itemMapper.markReplacement(itemId, replacedDocumentId, now);
|
||||
if (updated <= 0) {
|
||||
throw new BusinessException("重复文件状态已变化,请刷新后重试");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清除已完成的历史文档覆盖标记。
|
||||
*
|
||||
* @param itemId 文件项 ID
|
||||
* @param replacedDocumentId 历史文档 ID
|
||||
*/
|
||||
@org.springframework.transaction.annotation.Transactional
|
||||
public void clearReplacement(BigInteger itemId, BigInteger replacedDocumentId) {
|
||||
itemMapper.clearReplacement(itemId, replacedDocumentId, new Date());
|
||||
}
|
||||
|
||||
/**
|
||||
* 汇总并持久化批次状态。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @return 最新状态
|
||||
*/
|
||||
public DocumentImportBatchDtos.StatusResponse refreshBatch(BigInteger batchId) {
|
||||
DocumentImportBatch batch = requireBatch(batchId);
|
||||
Date now = new Date();
|
||||
String nextStatus = batch.getStatus();
|
||||
Date nextFinishedAt = batch.getFinishedAt();
|
||||
if (batch.getImportMode() == null) {
|
||||
if (!DocumentImportBatchStatus.CANCELLED.name().equals(batch.getStatus())) {
|
||||
nextStatus = valueOrZero(batch.getUploadedCount()) == valueOrZero(batch.getTotalCount())
|
||||
? DocumentImportBatchStatus.READY.name()
|
||||
: DocumentImportBatchStatus.UPLOADING.name();
|
||||
}
|
||||
} else if (!DocumentImportBatchStatus.CANCELLED.name().equals(batch.getStatus())
|
||||
&& !DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())) {
|
||||
int terminalCount = valueOrZero(batch.getCompletedCount())
|
||||
+ valueOrZero(batch.getFailedCount())
|
||||
+ valueOrZero(batch.getSkippedCount())
|
||||
+ valueOrZero(batch.getCancelledCount());
|
||||
if (terminalCount >= valueOrZero(batch.getTotalCount())
|
||||
&& valueOrZero(batch.getProcessingCount()) == 0
|
||||
&& valueOrZero(batch.getPendingCount()) == 0) {
|
||||
nextStatus = valueOrZero(batch.getFailedCount()) > 0
|
||||
? DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name()
|
||||
: DocumentImportBatchStatus.COMPLETED.name();
|
||||
nextFinishedAt = now;
|
||||
} else {
|
||||
nextStatus = DocumentImportBatchStatus.RUNNING.name();
|
||||
}
|
||||
}
|
||||
if (!java.util.Objects.equals(nextStatus, batch.getStatus())
|
||||
|| !java.util.Objects.equals(nextFinishedAt, batch.getFinishedAt())) {
|
||||
batch.setStatus(nextStatus);
|
||||
batch.setFinishedAt(nextFinishedAt);
|
||||
batch.setModified(now);
|
||||
batchService.updateById(batch, false);
|
||||
}
|
||||
return toStatusResponse(batch);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将批次标记为已中断,保留已完成文件并允许批量继续。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
*/
|
||||
public void markInterrupted(BigInteger batchId) {
|
||||
if (batchId == null) {
|
||||
return;
|
||||
}
|
||||
Date now = new Date();
|
||||
DocumentImportBatch update = new DocumentImportBatch();
|
||||
update.setStatus(DocumentImportBatchStatus.INTERRUPTED.name());
|
||||
update.setModified(now);
|
||||
batchMapper.updateByQuery(update,
|
||||
QueryWrapper.create()
|
||||
.eq(DocumentImportBatch::getId, batchId)
|
||||
.eq(DocumentImportBatch::getStatus, DocumentImportBatchStatus.RUNNING.name()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 将批次转换为状态响应。
|
||||
*
|
||||
* @param batch 批次实体
|
||||
* @return 状态响应
|
||||
*/
|
||||
public DocumentImportBatchDtos.StatusResponse toStatusResponse(DocumentImportBatch batch) {
|
||||
DocumentImportBatchDtos.StatusResponse response = new DocumentImportBatchDtos.StatusResponse();
|
||||
response.setBatchId(batch.getId());
|
||||
response.setImportMode(batch.getImportMode());
|
||||
response.setStatus(batch.getStatus());
|
||||
response.setTotalCount(valueOrZero(batch.getTotalCount()));
|
||||
response.setTotalBytes(batch.getTotalBytes() == null ? 0L : batch.getTotalBytes());
|
||||
response.setCompletedCount(valueOrZero(batch.getCompletedCount()));
|
||||
response.setProcessingCount(valueOrZero(batch.getProcessingCount()));
|
||||
response.setFailedCount(valueOrZero(batch.getFailedCount()));
|
||||
response.setPendingCount(valueOrZero(batch.getPendingCount()));
|
||||
response.setSkippedCount(valueOrZero(batch.getSkippedCount()));
|
||||
response.setCancelledCount(valueOrZero(batch.getCancelledCount()));
|
||||
response.setRetryableFailedCount(valueOrZero(batch.getRetryableFailedCount()));
|
||||
int total = Math.max(1, valueOrZero(batch.getTotalCount()));
|
||||
int terminalCount = valueOrZero(batch.getCompletedCount())
|
||||
+ valueOrZero(batch.getFailedCount())
|
||||
+ valueOrZero(batch.getSkippedCount())
|
||||
+ valueOrZero(batch.getCancelledCount());
|
||||
response.setProgressPercent(Math.min(100, terminalCount * 100 / total));
|
||||
response.setStartedAt(batch.getStartedAt());
|
||||
response.setFinishedAt(batch.getFinishedAt());
|
||||
return response;
|
||||
}
|
||||
|
||||
private int valueOrZero(Integer value) {
|
||||
return value == null ? 0 : value;
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件项状态变化对应的批次计数增量。
|
||||
*/
|
||||
private static final class CounterDelta {
|
||||
private int completed;
|
||||
private int processing;
|
||||
private int failed;
|
||||
private int pending;
|
||||
private int uploaded;
|
||||
private int skipped;
|
||||
private int cancelled;
|
||||
private int retryableFailed;
|
||||
|
||||
/**
|
||||
* 计算文件项状态迁移前后的计数差值。
|
||||
*
|
||||
* @param current 当前文件项
|
||||
* @param nextStatus 新状态
|
||||
* @param nextRetryable 新状态是否允许重试
|
||||
* @return 计数差值
|
||||
*/
|
||||
private static CounterDelta between(DocumentImportBatchItem current,
|
||||
DocumentImportBatchItemStatus nextStatus,
|
||||
boolean nextRetryable) {
|
||||
CounterDelta delta = new CounterDelta();
|
||||
apply(delta, DocumentImportBatchItemStatus.valueOf(current.getStatus()),
|
||||
Boolean.TRUE.equals(current.getRetryable()), -1);
|
||||
apply(delta, nextStatus, nextRetryable, 1);
|
||||
return delta;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将一个状态映射到对应计数桶。
|
||||
*
|
||||
* @param delta 待修改的差值
|
||||
* @param status 文件项状态
|
||||
* @param retryable 是否允许重试
|
||||
* @param direction 增加或减少方向
|
||||
*/
|
||||
private static void apply(CounterDelta delta,
|
||||
DocumentImportBatchItemStatus status,
|
||||
boolean retryable,
|
||||
int direction) {
|
||||
switch (status) {
|
||||
case COMPLETED -> delta.completed += direction;
|
||||
case RUNNING -> delta.processing += direction;
|
||||
case FAILED -> {
|
||||
delta.failed += direction;
|
||||
if (retryable) {
|
||||
delta.retryableFailed += direction;
|
||||
}
|
||||
}
|
||||
case SKIPPED -> delta.skipped += direction;
|
||||
case CANCELLED -> delta.cancelled += direction;
|
||||
case PENDING, UPLOADING, UPLOADED -> delta.pending += direction;
|
||||
default -> throw new IllegalStateException("未知批次文件状态: " + status);
|
||||
}
|
||||
if (status == DocumentImportBatchItemStatus.UPLOADED) {
|
||||
delta.uploaded += direction;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断所有增量是否均为零。
|
||||
*
|
||||
* @return 是否没有计数变化
|
||||
*/
|
||||
private boolean isZero() {
|
||||
return completed == 0
|
||||
&& processing == 0
|
||||
&& failed == 0
|
||||
&& pending == 0
|
||||
&& uploaded == 0
|
||||
&& skipped == 0
|
||||
&& cancelled == 0
|
||||
&& retryableFailed == 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.util.unit.DataSize;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* 文档批量导入容量与并发缺省配置。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
@ConfigurationProperties(prefix = "easyflow.ai.document-import.bulk")
|
||||
public class DocumentImportBulkProperties {
|
||||
|
||||
private int maxFileCount = 2000;
|
||||
private DataSize maxTotalSize = DataSize.ofGigabytes(1);
|
||||
private DataSize maxFileSize = DataSize.ofMegabytes(100);
|
||||
private int uploadConcurrency = 3;
|
||||
private int parseMaxRunning = 2;
|
||||
private int splitMaxRunning = 2;
|
||||
private int indexMaxRunning = 2;
|
||||
private int perBatchParseMaxRunning = 2;
|
||||
private int pendingDispatchBatchSize = 100;
|
||||
private Duration pendingDispatchInterval = Duration.ofSeconds(2);
|
||||
private Duration pendingRedispatchDelay = Duration.ofSeconds(5);
|
||||
private Duration pendingTimeout = Duration.ofHours(24);
|
||||
private Duration parseSubmitTimeout = Duration.ofSeconds(120);
|
||||
private Duration interruptionTimeout = Duration.ofMinutes(10);
|
||||
private int maxTaskAttempts = 3;
|
||||
|
||||
public int getMaxFileCount() {
|
||||
return maxFileCount;
|
||||
}
|
||||
|
||||
public void setMaxFileCount(int maxFileCount) {
|
||||
this.maxFileCount = maxFileCount;
|
||||
}
|
||||
|
||||
public DataSize getMaxTotalSize() {
|
||||
return maxTotalSize;
|
||||
}
|
||||
|
||||
public void setMaxTotalSize(DataSize maxTotalSize) {
|
||||
this.maxTotalSize = maxTotalSize;
|
||||
}
|
||||
|
||||
public DataSize getMaxFileSize() {
|
||||
return maxFileSize;
|
||||
}
|
||||
|
||||
public void setMaxFileSize(DataSize maxFileSize) {
|
||||
this.maxFileSize = maxFileSize;
|
||||
}
|
||||
|
||||
public int getUploadConcurrency() {
|
||||
return uploadConcurrency;
|
||||
}
|
||||
|
||||
public void setUploadConcurrency(int uploadConcurrency) {
|
||||
this.uploadConcurrency = uploadConcurrency;
|
||||
}
|
||||
|
||||
public int getParseMaxRunning() {
|
||||
return parseMaxRunning;
|
||||
}
|
||||
|
||||
public void setParseMaxRunning(int parseMaxRunning) {
|
||||
this.parseMaxRunning = parseMaxRunning;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取全局分块任务并发上限。
|
||||
*
|
||||
* @return 分块任务并发上限
|
||||
*/
|
||||
public int getSplitMaxRunning() {
|
||||
return splitMaxRunning;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置全局分块任务并发上限。
|
||||
*
|
||||
* @param splitMaxRunning 分块任务并发上限
|
||||
*/
|
||||
public void setSplitMaxRunning(int splitMaxRunning) {
|
||||
this.splitMaxRunning = splitMaxRunning;
|
||||
}
|
||||
|
||||
public int getIndexMaxRunning() {
|
||||
return indexMaxRunning;
|
||||
}
|
||||
|
||||
public void setIndexMaxRunning(int indexMaxRunning) {
|
||||
this.indexMaxRunning = indexMaxRunning;
|
||||
}
|
||||
|
||||
public int getPerBatchParseMaxRunning() {
|
||||
return perBatchParseMaxRunning;
|
||||
}
|
||||
|
||||
public void setPerBatchParseMaxRunning(int perBatchParseMaxRunning) {
|
||||
this.perBatchParseMaxRunning = perBatchParseMaxRunning;
|
||||
}
|
||||
|
||||
public int getPendingDispatchBatchSize() {
|
||||
return pendingDispatchBatchSize;
|
||||
}
|
||||
|
||||
public void setPendingDispatchBatchSize(int pendingDispatchBatchSize) {
|
||||
this.pendingDispatchBatchSize = pendingDispatchBatchSize;
|
||||
}
|
||||
|
||||
public Duration getPendingDispatchInterval() {
|
||||
return pendingDispatchInterval;
|
||||
}
|
||||
|
||||
public void setPendingDispatchInterval(Duration pendingDispatchInterval) {
|
||||
this.pendingDispatchInterval = pendingDispatchInterval;
|
||||
}
|
||||
|
||||
public Duration getPendingRedispatchDelay() {
|
||||
return pendingRedispatchDelay;
|
||||
}
|
||||
|
||||
public void setPendingRedispatchDelay(Duration pendingRedispatchDelay) {
|
||||
this.pendingRedispatchDelay = pendingRedispatchDelay;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取待处理任务最长排队时间。
|
||||
*
|
||||
* @return 最长排队时间
|
||||
*/
|
||||
public Duration getPendingTimeout() {
|
||||
return pendingTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置待处理任务最长排队时间。
|
||||
*
|
||||
* @param pendingTimeout 最长排队时间
|
||||
*/
|
||||
public void setPendingTimeout(Duration pendingTimeout) {
|
||||
this.pendingTimeout = pendingTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取解析服务任务提交超时时间。
|
||||
*
|
||||
* @return 任务提交超时时间
|
||||
*/
|
||||
public Duration getParseSubmitTimeout() {
|
||||
return parseSubmitTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置解析服务任务提交超时时间。
|
||||
*
|
||||
* @param parseSubmitTimeout 任务提交超时时间
|
||||
*/
|
||||
public void setParseSubmitTimeout(Duration parseSubmitTimeout) {
|
||||
this.parseSubmitTimeout = parseSubmitTimeout;
|
||||
}
|
||||
|
||||
public Duration getInterruptionTimeout() {
|
||||
return interruptionTimeout;
|
||||
}
|
||||
|
||||
public void setInterruptionTimeout(Duration interruptionTimeout) {
|
||||
this.interruptionTimeout = interruptionTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取单文件每阶段最大执行次数。
|
||||
*
|
||||
* @return 最大执行次数
|
||||
*/
|
||||
public int getMaxTaskAttempts() {
|
||||
return maxTaskAttempts;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置单文件每阶段最大执行次数。
|
||||
*
|
||||
* @param maxTaskAttempts 最大执行次数
|
||||
*/
|
||||
public void setMaxTaskAttempts(int maxTaskAttempts) {
|
||||
this.maxTaskAttempts = maxTaskAttempts;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||
import tech.easyflow.ai.easyagents.CustomMultipartFile;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.util.StringUtil;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* 自动导入分块快照持久化服务。
|
||||
*
|
||||
* <p>快照写入对象存储,向量化任务只保存稳定路径,避免依赖短期预览缓存。</p>
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
@Service
|
||||
public class DocumentImportChunkSnapshotService {
|
||||
|
||||
private static final long MAX_SNAPSHOT_BYTES = 256L * 1024L * 1024L;
|
||||
|
||||
@Resource(name = "default")
|
||||
private FileStorageService storageService;
|
||||
|
||||
/**
|
||||
* 持久化预览会话及其最终分块。
|
||||
*
|
||||
* @param session 预览会话
|
||||
* @return 快照存储路径
|
||||
*/
|
||||
public String save(DocumentImportDtos.PreviewSession session) {
|
||||
if (session == null || session.getKnowledgeId() == null || session.getDocumentId() == null
|
||||
|| session.getDocumentChunks() == null || session.getDocumentChunks().isEmpty()) {
|
||||
throw new BusinessException("分块快照内容不完整");
|
||||
}
|
||||
String fileName = session.getDocumentId() + "-chunks.json";
|
||||
byte[] payload = JSON.toJSONBytes(session);
|
||||
if (payload.length > MAX_SNAPSHOT_BYTES) {
|
||||
throw new BusinessException("分块快照过大,请调整文档后重试");
|
||||
}
|
||||
CustomMultipartFile file = new CustomMultipartFile(
|
||||
payload, fileName, fileName, "application/json");
|
||||
String path = storageService.save(file,
|
||||
"knowledge-import-snapshots/" + session.getKnowledgeId() + "/" + session.getDocumentId());
|
||||
if (!StringUtil.hasText(path)) {
|
||||
throw new BusinessException("分块快照保存失败");
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从稳定存储恢复预览会话。
|
||||
*
|
||||
* @param path 快照路径
|
||||
* @return 预览会话
|
||||
*/
|
||||
public DocumentImportDtos.PreviewSession load(String path) {
|
||||
if (!StringUtil.hasText(path)) {
|
||||
throw new BusinessException("分块快照不存在,请重试");
|
||||
}
|
||||
try (InputStream inputStream = storageService.readStream(path)) {
|
||||
byte[] payload = DocumentInputStreamSupport.readBytes(inputStream, MAX_SNAPSHOT_BYTES);
|
||||
String json = new String(payload, StandardCharsets.UTF_8);
|
||||
DocumentImportDtos.PreviewSession session =
|
||||
JSON.parseObject(json, DocumentImportDtos.PreviewSession.class);
|
||||
if (session == null || session.getDocumentChunks() == null
|
||||
|| session.getDocumentChunks().isEmpty()) {
|
||||
throw new BusinessException("分块快照无有效内容,请重试");
|
||||
}
|
||||
return session;
|
||||
} catch (IOException error) {
|
||||
throw new BusinessException("分块快照读取失败,请重试");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除已完成向量化的分块快照。
|
||||
*
|
||||
* @param path 快照路径
|
||||
*/
|
||||
public void delete(String path) {
|
||||
if (StringUtil.hasText(path)) {
|
||||
storageService.delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.cache.DistributedScheduledLock;
|
||||
|
||||
/**
|
||||
* 文档导入待处理任务恢复调度器。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
@Component
|
||||
public class DocumentImportPendingTaskMonitor {
|
||||
|
||||
private final KnowledgeDocumentImportTaskAppService appService;
|
||||
|
||||
/**
|
||||
* 创建文档导入恢复调度器。
|
||||
*
|
||||
* @param appService 文档导入任务服务
|
||||
*/
|
||||
public DocumentImportPendingTaskMonitor(KnowledgeDocumentImportTaskAppService appService) {
|
||||
this.appService = appService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 周期性投递仍处于等待状态的解析与向量化任务。
|
||||
*/
|
||||
@Scheduled(
|
||||
fixedDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}",
|
||||
initialDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}"
|
||||
)
|
||||
@DistributedScheduledLock(
|
||||
key = "easyflow:schedule:document-import:pending-dispatch",
|
||||
leaseSeconds = 30L
|
||||
)
|
||||
public void dispatchPendingTasks() {
|
||||
appService.dispatchPendingTasks();
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测排队超时与长时间无心跳的任务。
|
||||
*/
|
||||
@Scheduled(
|
||||
fixedDelayString = "${easyflow.ai.document-import.bulk.interruption-scan-interval:60s}",
|
||||
initialDelayString = "${easyflow.ai.document-import.bulk.interruption-scan-interval:60s}"
|
||||
)
|
||||
@DistributedScheduledLock(
|
||||
key = "easyflow:schedule:document-import:interruption-scan",
|
||||
leaseSeconds = 30L
|
||||
)
|
||||
public void recoverTimedOutTasks() {
|
||||
appService.expireTimedOutParseSubmissions();
|
||||
appService.recoverInterruptedTasks();
|
||||
appService.expireTimedOutPendingTasks();
|
||||
appService.cleanupCompletedReplacements();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.mq.config.MQProperties;
|
||||
import tech.easyflow.common.mq.core.MQConsumerHandler;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
import tech.easyflow.common.mq.core.MQSubscription;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 文档分块任务消费者。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
@Component
|
||||
public class DocumentImportSplitTaskConsumer implements MQConsumerHandler {
|
||||
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(DocumentImportSplitTaskConsumer.class);
|
||||
|
||||
private final KnowledgeDocumentImportTaskAppService appService;
|
||||
private final MQProperties mqProperties;
|
||||
|
||||
/**
|
||||
* 创建分块任务消费者。
|
||||
*
|
||||
* @param appService 文档导入应用服务
|
||||
* @param mqProperties MQ 配置
|
||||
*/
|
||||
public DocumentImportSplitTaskConsumer(
|
||||
KnowledgeDocumentImportTaskAppService appService,
|
||||
MQProperties mqProperties) {
|
||||
this.appService = appService;
|
||||
this.mqProperties = mqProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取分块任务订阅。
|
||||
*
|
||||
* @return MQ 订阅信息
|
||||
*/
|
||||
@Override
|
||||
public MQSubscription subscription() {
|
||||
MQSubscription subscription = new MQSubscription();
|
||||
subscription.setTopic(DocumentImportTaskMqConstants.SPLIT_TOPIC);
|
||||
subscription.setConsumerGroup(DocumentImportTaskMqConstants.SPLIT_GROUP);
|
||||
subscription.setShardCount(resolveShardCount());
|
||||
return subscription;
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理一批分块任务消息。
|
||||
*
|
||||
* @param messages MQ 消息
|
||||
*/
|
||||
@Override
|
||||
public void handle(List<MQMessage> messages) {
|
||||
LOG.info("文档分块消费者收到消息批次: count={}",
|
||||
messages == null ? 0 : messages.size());
|
||||
for (MQMessage message : messages) {
|
||||
DocumentImportTaskMessage event =
|
||||
JSON.parseObject(message.getBody(), DocumentImportTaskMessage.class);
|
||||
if (event == null || event.getTaskId() == null) {
|
||||
LOG.warn("文档分块消费者跳过非法消息: streamMessageId={}, messageId={}",
|
||||
message == null ? null : message.getStreamMessageId(),
|
||||
message == null ? null : message.getMessageId());
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
appService.handleSplitTask(event.getTaskId());
|
||||
} catch (Exception error) {
|
||||
LOG.error("文档分块消费者处理失败: taskId={}, messageId={}, streamMessageId={}",
|
||||
event.getTaskId(), message.getMessageId(),
|
||||
message.getStreamMessageId(), error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前 Redis Stream 分片数。
|
||||
*
|
||||
* @return 分片数
|
||||
*/
|
||||
private int resolveShardCount() {
|
||||
return Math.max(
|
||||
mqProperties.getRedis().getChatPersistShardCount(), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
import tech.easyflow.common.mq.core.MQProducer;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 文档分块任务消息生产者。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
@Service
|
||||
public class DocumentImportSplitTaskProducer {
|
||||
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(DocumentImportSplitTaskProducer.class);
|
||||
|
||||
private final MQProducer mqProducer;
|
||||
|
||||
/**
|
||||
* 创建分块任务消息生产者。
|
||||
*
|
||||
* @param mqProducer MQ 生产者
|
||||
*/
|
||||
public DocumentImportSplitTaskProducer(MQProducer mqProducer) {
|
||||
this.mqProducer = mqProducer;
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送分块任务消息。
|
||||
*
|
||||
* @param taskId 任务 ID
|
||||
*/
|
||||
public void send(BigInteger taskId) {
|
||||
DocumentImportTaskMessage event = new DocumentImportTaskMessage();
|
||||
event.setTaskId(taskId);
|
||||
event.setOccurredAt(new Date());
|
||||
|
||||
MQMessage message = new MQMessage();
|
||||
message.setMessageId("split-" + taskId);
|
||||
message.setTopic(DocumentImportTaskMqConstants.SPLIT_TOPIC);
|
||||
message.setKey(String.valueOf(taskId));
|
||||
message.setCreatedAt(event.getOccurredAt());
|
||||
message.setBody(JSON.toJSONString(event));
|
||||
LOG.info("准备投递文档分块 MQ 消息: topic={}, taskId={}, messageId={}",
|
||||
message.getTopic(), taskId, message.getMessageId());
|
||||
String recordId = mqProducer.send(message);
|
||||
LOG.info("文档分块 MQ 消息投递完成: topic={}, taskId={}, messageId={}, recordId={}",
|
||||
message.getTopic(), taskId, message.getMessageId(), recordId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.ai.documentimport.ImportCallerContext;
|
||||
import tech.easyflow.ai.documentimport.ImportCallerType;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||
import tech.easyflow.common.cache.DistributedScheduledLock;
|
||||
import tech.easyflow.common.util.StringUtil;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 文档导入未完成批次与取消对象清理调度器。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
@Component
|
||||
public class DocumentImportStaleBatchMonitor {
|
||||
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(DocumentImportStaleBatchMonitor.class);
|
||||
private static final Duration INCOMPLETE_TIMEOUT = Duration.ofMinutes(30);
|
||||
private static final int BATCH_SIZE = 100;
|
||||
|
||||
private final DocumentImportBatchMapper batchMapper;
|
||||
private final DocumentImportBatchAppService batchAppService;
|
||||
|
||||
/**
|
||||
* 创建未完成批次清理调度器。
|
||||
*
|
||||
* @param batchMapper 批次 Mapper
|
||||
* @param batchAppService 批次应用服务
|
||||
*/
|
||||
public DocumentImportStaleBatchMonitor(
|
||||
DocumentImportBatchMapper batchMapper,
|
||||
DocumentImportBatchAppService batchAppService) {
|
||||
this.batchMapper = batchMapper;
|
||||
this.batchAppService = batchAppService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 分批回收长时间无进展的上传批次,并重试对象清理。
|
||||
*/
|
||||
@Scheduled(
|
||||
fixedDelayString =
|
||||
"${easyflow.ai.document-import.bulk.stale-batch-scan-interval:60s}",
|
||||
initialDelayString =
|
||||
"${easyflow.ai.document-import.bulk.stale-batch-scan-interval:60s}"
|
||||
)
|
||||
@DistributedScheduledLock(
|
||||
key = "easyflow:schedule:document-import:stale-batch-cleanup",
|
||||
leaseSeconds = 50L
|
||||
)
|
||||
public void cleanupStaleBatches() {
|
||||
Date cutoff = new Date(
|
||||
System.currentTimeMillis() - INCOMPLETE_TIMEOUT.toMillis()
|
||||
);
|
||||
List<DocumentImportBatch> candidates =
|
||||
batchMapper.selectStaleIncompleteBatches(cutoff, BATCH_SIZE);
|
||||
for (DocumentImportBatch batch : candidates) {
|
||||
cancelCandidate(batch, cutoff);
|
||||
}
|
||||
batchAppService.cleanupCancelledStoredObjects(BATCH_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过数据库条件更新尝试取消单个候选批次。
|
||||
*
|
||||
* @param batch 候选批次
|
||||
* @param cutoff 最后进展截止时间
|
||||
*/
|
||||
private void cancelCandidate(DocumentImportBatch batch, Date cutoff) {
|
||||
if (batch.getCallerType() == null || batch.getCallerId() == null) {
|
||||
LOG.warn("跳过调用者信息不完整的超时导入批次: batchId={}", batch.getId());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
ImportCallerContext caller = new ImportCallerContext(
|
||||
ImportCallerType.valueOf(batch.getCallerType()),
|
||||
batch.getCallerId()
|
||||
);
|
||||
boolean cancelled = batchAppService.cancelStaleBatch(
|
||||
batch.getKnowledgeId(),
|
||||
batch.getId(),
|
||||
caller,
|
||||
cutoff
|
||||
);
|
||||
if (cancelled && StringUtil.hasText(batch.getIdempotencyKeyHash())) {
|
||||
int released = batchMapper.releaseSubmissionFingerprint(
|
||||
batch.getId(),
|
||||
batch.getIdempotencyKeyHash(),
|
||||
cutoff,
|
||||
new Date()
|
||||
);
|
||||
if (released <= 0) {
|
||||
LOG.warn(
|
||||
"超时导入批次已取消但提交指纹未释放: batchId={}",
|
||||
batch.getId()
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (IllegalArgumentException error) {
|
||||
LOG.error(
|
||||
"超时导入批次调用者类型无效: batchId={}, callerType={}",
|
||||
batch.getId(),
|
||||
batch.getCallerType(),
|
||||
error
|
||||
);
|
||||
} catch (RuntimeException error) {
|
||||
LOG.error("回收超时导入批次失败: batchId={}", batch.getId(), error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,8 @@ public final class DocumentImportTaskMqConstants {
|
||||
|
||||
public static final String PARSE_TOPIC = "knowledge-document-parse";
|
||||
public static final String PARSE_GROUP = "knowledge-document-parse-group";
|
||||
public static final String SPLIT_TOPIC = "knowledge-document-split";
|
||||
public static final String SPLIT_GROUP = "knowledge-document-split-group";
|
||||
public static final String INDEX_TOPIC = "knowledge-document-index";
|
||||
public static final String INDEX_GROUP = "knowledge-document-index-group";
|
||||
}
|
||||
|
||||
@@ -147,6 +147,7 @@ public class DocumentImportTaskStatusStreamService {
|
||||
payload.put("parseCurrentStage", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_PARSE_CURRENT_STAGE));
|
||||
payload.put("parseStatusMessage", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_PARSE_STATUS_MESSAGE));
|
||||
payload.put("lastTaskError", document.getLastTaskError());
|
||||
payload.put("lastTaskErrorCode", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE));
|
||||
payload.put("taskModifiedAt", document.getTaskModifiedAt());
|
||||
return payload;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,333 @@
|
||||
package tech.easyflow.ai.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 知识库文档批量导入批次。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
@Table(value = "tb_document_import_batch", comment = "知识库文档批量导入批次")
|
||||
public class DocumentImportBatch extends DateEntity implements Serializable {
|
||||
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||
private BigInteger id;
|
||||
|
||||
@Column(comment = "知识库ID")
|
||||
private BigInteger knowledgeId;
|
||||
|
||||
@Column(comment = "调用者类型")
|
||||
private String callerType;
|
||||
|
||||
@Column(comment = "调用者ID")
|
||||
private BigInteger callerId;
|
||||
|
||||
@Column(comment = "幂等键哈希")
|
||||
private String idempotencyKeyHash;
|
||||
|
||||
@Column(comment = "请求摘要")
|
||||
private String requestDigest;
|
||||
|
||||
@Column(comment = "重复文件策略")
|
||||
private String duplicatePolicy;
|
||||
|
||||
@Column(comment = "请求分块策略")
|
||||
private String requestedStrategyJson;
|
||||
|
||||
@Column(comment = "重试代次")
|
||||
private Integer retryGeneration;
|
||||
|
||||
@Column(comment = "乐观锁版本")
|
||||
private Integer version;
|
||||
|
||||
@Column(comment = "导入模式")
|
||||
private String importMode;
|
||||
|
||||
@Column(comment = "批次状态")
|
||||
private String status;
|
||||
|
||||
@Column(comment = "文件总数")
|
||||
private Integer totalCount;
|
||||
|
||||
@Column(comment = "文件总字节数")
|
||||
private Long totalBytes;
|
||||
|
||||
@Column(comment = "完成数")
|
||||
private Integer completedCount;
|
||||
|
||||
@Column(comment = "处理中数量")
|
||||
private Integer processingCount;
|
||||
|
||||
@Column(comment = "失败数")
|
||||
private Integer failedCount;
|
||||
|
||||
@Column(comment = "等待数")
|
||||
private Integer pendingCount;
|
||||
|
||||
@Column(comment = "已上传数")
|
||||
private Integer uploadedCount;
|
||||
|
||||
@Column(comment = "跳过数")
|
||||
private Integer skippedCount;
|
||||
|
||||
@Column(comment = "取消数")
|
||||
private Integer cancelledCount;
|
||||
|
||||
@Column(comment = "可重试失败数")
|
||||
private Integer retryableFailedCount;
|
||||
|
||||
@Column(comment = "开始时间")
|
||||
private Date startedAt;
|
||||
|
||||
@Column(comment = "结束时间")
|
||||
private Date finishedAt;
|
||||
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(BigInteger id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public String getCallerType() {
|
||||
return callerType;
|
||||
}
|
||||
|
||||
public void setCallerType(String callerType) {
|
||||
this.callerType = callerType;
|
||||
}
|
||||
|
||||
public BigInteger getCallerId() {
|
||||
return callerId;
|
||||
}
|
||||
|
||||
public void setCallerId(BigInteger callerId) {
|
||||
this.callerId = callerId;
|
||||
}
|
||||
|
||||
public String getIdempotencyKeyHash() {
|
||||
return idempotencyKeyHash;
|
||||
}
|
||||
|
||||
public void setIdempotencyKeyHash(String idempotencyKeyHash) {
|
||||
this.idempotencyKeyHash = idempotencyKeyHash;
|
||||
}
|
||||
|
||||
public String getRequestDigest() {
|
||||
return requestDigest;
|
||||
}
|
||||
|
||||
public void setRequestDigest(String requestDigest) {
|
||||
this.requestDigest = requestDigest;
|
||||
}
|
||||
|
||||
public String getDuplicatePolicy() {
|
||||
return duplicatePolicy;
|
||||
}
|
||||
|
||||
public void setDuplicatePolicy(String duplicatePolicy) {
|
||||
this.duplicatePolicy = duplicatePolicy;
|
||||
}
|
||||
|
||||
public String getRequestedStrategyJson() {
|
||||
return requestedStrategyJson;
|
||||
}
|
||||
|
||||
public void setRequestedStrategyJson(String requestedStrategyJson) {
|
||||
this.requestedStrategyJson = requestedStrategyJson;
|
||||
}
|
||||
|
||||
public Integer getRetryGeneration() {
|
||||
return retryGeneration;
|
||||
}
|
||||
|
||||
public void setRetryGeneration(Integer retryGeneration) {
|
||||
this.retryGeneration = retryGeneration;
|
||||
}
|
||||
|
||||
public Integer getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
||||
public void setVersion(Integer version) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
public String getImportMode() {
|
||||
return importMode;
|
||||
}
|
||||
|
||||
public void setImportMode(String importMode) {
|
||||
this.importMode = importMode;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Integer getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
public void setTotalCount(Integer totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
public Long getTotalBytes() {
|
||||
return totalBytes;
|
||||
}
|
||||
|
||||
public void setTotalBytes(Long totalBytes) {
|
||||
this.totalBytes = totalBytes;
|
||||
}
|
||||
|
||||
public Integer getCompletedCount() {
|
||||
return completedCount;
|
||||
}
|
||||
|
||||
public void setCompletedCount(Integer completedCount) {
|
||||
this.completedCount = completedCount;
|
||||
}
|
||||
|
||||
public Integer getProcessingCount() {
|
||||
return processingCount;
|
||||
}
|
||||
|
||||
public void setProcessingCount(Integer processingCount) {
|
||||
this.processingCount = processingCount;
|
||||
}
|
||||
|
||||
public Integer getFailedCount() {
|
||||
return failedCount;
|
||||
}
|
||||
|
||||
public void setFailedCount(Integer failedCount) {
|
||||
this.failedCount = failedCount;
|
||||
}
|
||||
|
||||
public Integer getPendingCount() {
|
||||
return pendingCount;
|
||||
}
|
||||
|
||||
public void setPendingCount(Integer pendingCount) {
|
||||
this.pendingCount = pendingCount;
|
||||
}
|
||||
|
||||
public Integer getUploadedCount() {
|
||||
return uploadedCount;
|
||||
}
|
||||
|
||||
public void setUploadedCount(Integer uploadedCount) {
|
||||
this.uploadedCount = uploadedCount;
|
||||
}
|
||||
|
||||
public Integer getSkippedCount() {
|
||||
return skippedCount;
|
||||
}
|
||||
|
||||
public void setSkippedCount(Integer skippedCount) {
|
||||
this.skippedCount = skippedCount;
|
||||
}
|
||||
|
||||
public Integer getCancelledCount() {
|
||||
return cancelledCount;
|
||||
}
|
||||
|
||||
public void setCancelledCount(Integer cancelledCount) {
|
||||
this.cancelledCount = cancelledCount;
|
||||
}
|
||||
|
||||
public Integer getRetryableFailedCount() {
|
||||
return retryableFailedCount;
|
||||
}
|
||||
|
||||
public void setRetryableFailedCount(Integer retryableFailedCount) {
|
||||
this.retryableFailedCount = retryableFailedCount;
|
||||
}
|
||||
|
||||
public Date getStartedAt() {
|
||||
return startedAt;
|
||||
}
|
||||
|
||||
public void setStartedAt(Date startedAt) {
|
||||
this.startedAt = startedAt;
|
||||
}
|
||||
|
||||
public Date getFinishedAt() {
|
||||
return finishedAt;
|
||||
}
|
||||
|
||||
public void setFinishedAt(Date finishedAt) {
|
||||
this.finishedAt = finishedAt;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Date getCreated() {
|
||||
return created;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setCreated(Date created) {
|
||||
this.created = created;
|
||||
}
|
||||
|
||||
public BigInteger getCreatedBy() {
|
||||
return createdBy;
|
||||
}
|
||||
|
||||
public void setCreatedBy(BigInteger createdBy) {
|
||||
this.createdBy = createdBy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Date getModified() {
|
||||
return modified;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setModified(Date modified) {
|
||||
this.modified = modified;
|
||||
}
|
||||
|
||||
public BigInteger getModifiedBy() {
|
||||
return modifiedBy;
|
||||
}
|
||||
|
||||
public void setModifiedBy(BigInteger modifiedBy) {
|
||||
this.modifiedBy = modifiedBy;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package tech.easyflow.ai.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 知识库文档批量导入文件项。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
@Table(value = "tb_document_import_batch_item", comment = "知识库文档批量导入文件项")
|
||||
public class DocumentImportBatchItem extends DateEntity implements Serializable {
|
||||
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||
private BigInteger id;
|
||||
|
||||
@Column(comment = "批次ID")
|
||||
private BigInteger batchId;
|
||||
|
||||
@Column(comment = "知识库ID")
|
||||
private BigInteger knowledgeId;
|
||||
|
||||
@Column(comment = "文档ID")
|
||||
private BigInteger documentId;
|
||||
|
||||
@Column(comment = "待覆盖的历史文档ID")
|
||||
private BigInteger replacedDocumentId;
|
||||
|
||||
@Column(comment = "客户端文件键")
|
||||
private String clientFileKey;
|
||||
|
||||
@Column(comment = "文件名")
|
||||
private String fileName;
|
||||
|
||||
@Column(comment = "文件夹相对路径")
|
||||
private String relativePath;
|
||||
|
||||
@Column(comment = "文件大小")
|
||||
private Long fileSize;
|
||||
|
||||
@Column(comment = "存储路径")
|
||||
private String filePath;
|
||||
|
||||
@Column(comment = "可恢复存储定位符")
|
||||
private String storageLocator;
|
||||
|
||||
@Column(comment = "是否等待清理存储对象")
|
||||
private Boolean cleanupPending;
|
||||
|
||||
@Column(comment = "文件内容SHA-256")
|
||||
private String contentSha256;
|
||||
|
||||
@Column(comment = "当前阶段")
|
||||
private String stage;
|
||||
|
||||
@Column(comment = "当前状态")
|
||||
private String status;
|
||||
|
||||
@Column(comment = "错误摘要")
|
||||
private String errorSummary;
|
||||
|
||||
@Column(comment = "稳定失败码")
|
||||
private String failureCode;
|
||||
|
||||
@Column(comment = "实际分块策略编码")
|
||||
private String appliedStrategyCode;
|
||||
|
||||
@Column(comment = "分块策略快照")
|
||||
private String strategySnapshotJson;
|
||||
|
||||
@Column(comment = "是否允许批量重试")
|
||||
private Boolean retryable;
|
||||
|
||||
@Column(comment = "重试次数")
|
||||
private Integer attemptCount;
|
||||
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(BigInteger id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public BigInteger getBatchId() {
|
||||
return batchId;
|
||||
}
|
||||
|
||||
public void setBatchId(BigInteger batchId) {
|
||||
this.batchId = batchId;
|
||||
}
|
||||
|
||||
public BigInteger getKnowledgeId() {
|
||||
return knowledgeId;
|
||||
}
|
||||
|
||||
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public BigInteger getDocumentId() {
|
||||
return documentId;
|
||||
}
|
||||
|
||||
public void setDocumentId(BigInteger documentId) {
|
||||
this.documentId = documentId;
|
||||
}
|
||||
|
||||
public BigInteger getReplacedDocumentId() {
|
||||
return replacedDocumentId;
|
||||
}
|
||||
|
||||
public void setReplacedDocumentId(BigInteger replacedDocumentId) {
|
||||
this.replacedDocumentId = replacedDocumentId;
|
||||
}
|
||||
|
||||
public String getClientFileKey() {
|
||||
return clientFileKey;
|
||||
}
|
||||
|
||||
public void setClientFileKey(String clientFileKey) {
|
||||
this.clientFileKey = clientFileKey;
|
||||
}
|
||||
|
||||
public String getFileName() {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
public void setFileName(String fileName) {
|
||||
this.fileName = fileName;
|
||||
}
|
||||
|
||||
public String getRelativePath() {
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
public void setRelativePath(String relativePath) {
|
||||
this.relativePath = relativePath;
|
||||
}
|
||||
|
||||
public Long getFileSize() {
|
||||
return fileSize;
|
||||
}
|
||||
|
||||
public void setFileSize(Long fileSize) {
|
||||
this.fileSize = fileSize;
|
||||
}
|
||||
|
||||
public String getFilePath() {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
public void setFilePath(String filePath) {
|
||||
this.filePath = filePath;
|
||||
}
|
||||
|
||||
public String getStorageLocator() {
|
||||
return storageLocator;
|
||||
}
|
||||
|
||||
public void setStorageLocator(String storageLocator) {
|
||||
this.storageLocator = storageLocator;
|
||||
}
|
||||
|
||||
public Boolean getCleanupPending() {
|
||||
return cleanupPending;
|
||||
}
|
||||
|
||||
public void setCleanupPending(Boolean cleanupPending) {
|
||||
this.cleanupPending = cleanupPending;
|
||||
}
|
||||
|
||||
public String getContentSha256() {
|
||||
return contentSha256;
|
||||
}
|
||||
|
||||
public void setContentSha256(String contentSha256) {
|
||||
this.contentSha256 = contentSha256;
|
||||
}
|
||||
|
||||
public String getStage() {
|
||||
return stage;
|
||||
}
|
||||
|
||||
public void setStage(String stage) {
|
||||
this.stage = stage;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public String getErrorSummary() {
|
||||
return errorSummary;
|
||||
}
|
||||
|
||||
public void setErrorSummary(String errorSummary) {
|
||||
this.errorSummary = errorSummary;
|
||||
}
|
||||
|
||||
public String getFailureCode() {
|
||||
return failureCode;
|
||||
}
|
||||
|
||||
public void setFailureCode(String failureCode) {
|
||||
this.failureCode = failureCode;
|
||||
}
|
||||
|
||||
public String getAppliedStrategyCode() {
|
||||
return appliedStrategyCode;
|
||||
}
|
||||
|
||||
public void setAppliedStrategyCode(String appliedStrategyCode) {
|
||||
this.appliedStrategyCode = appliedStrategyCode;
|
||||
}
|
||||
|
||||
public String getStrategySnapshotJson() {
|
||||
return strategySnapshotJson;
|
||||
}
|
||||
|
||||
public void setStrategySnapshotJson(String strategySnapshotJson) {
|
||||
this.strategySnapshotJson = strategySnapshotJson;
|
||||
}
|
||||
|
||||
public Boolean getRetryable() {
|
||||
return retryable;
|
||||
}
|
||||
|
||||
public void setRetryable(Boolean retryable) {
|
||||
this.retryable = retryable;
|
||||
}
|
||||
|
||||
public Integer getAttemptCount() {
|
||||
return attemptCount;
|
||||
}
|
||||
|
||||
public void setAttemptCount(Integer attemptCount) {
|
||||
this.attemptCount = attemptCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Date getCreated() {
|
||||
return created;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setCreated(Date created) {
|
||||
this.created = created;
|
||||
}
|
||||
|
||||
public BigInteger getCreatedBy() {
|
||||
return createdBy;
|
||||
}
|
||||
|
||||
public void setCreatedBy(BigInteger createdBy) {
|
||||
this.createdBy = createdBy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Date getModified() {
|
||||
return modified;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setModified(Date modified) {
|
||||
this.modified = modified;
|
||||
}
|
||||
|
||||
public BigInteger getModifiedBy() {
|
||||
return modifiedBy;
|
||||
}
|
||||
|
||||
public void setModifiedBy(BigInteger modifiedBy) {
|
||||
this.modifiedBy = modifiedBy;
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,12 @@ public class DocumentImportTask extends DateEntity implements Serializable {
|
||||
@Column(comment = "知识库ID")
|
||||
private BigInteger knowledgeId;
|
||||
|
||||
@Column(comment = "批次ID")
|
||||
private BigInteger batchId;
|
||||
|
||||
@Column(comment = "批次文件项ID")
|
||||
private BigInteger batchItemId;
|
||||
|
||||
@Column(comment = "任务阶段")
|
||||
private String phase;
|
||||
|
||||
@@ -46,6 +52,21 @@ public class DocumentImportTask extends DateEntity implements Serializable {
|
||||
@Column(comment = "错误摘要")
|
||||
private String errorSummary;
|
||||
|
||||
@Column(comment = "稳定失败码")
|
||||
private String failureCode;
|
||||
|
||||
@Column(comment = "执行尝试次数")
|
||||
private Integer attemptNo;
|
||||
|
||||
@Column(comment = "执行令牌")
|
||||
private String executionToken;
|
||||
|
||||
@Column(comment = "租约到期时间")
|
||||
private Date leaseUntil;
|
||||
|
||||
@Column(comment = "乐观锁版本")
|
||||
private Integer version;
|
||||
|
||||
@Column(comment = "开始时间")
|
||||
private Date startedAt;
|
||||
|
||||
@@ -88,6 +109,22 @@ public class DocumentImportTask extends DateEntity implements Serializable {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
public BigInteger getBatchId() {
|
||||
return batchId;
|
||||
}
|
||||
|
||||
public void setBatchId(BigInteger batchId) {
|
||||
this.batchId = batchId;
|
||||
}
|
||||
|
||||
public BigInteger getBatchItemId() {
|
||||
return batchItemId;
|
||||
}
|
||||
|
||||
public void setBatchItemId(BigInteger batchItemId) {
|
||||
this.batchItemId = batchItemId;
|
||||
}
|
||||
|
||||
public String getPhase() {
|
||||
return phase;
|
||||
}
|
||||
@@ -128,6 +165,46 @@ public class DocumentImportTask extends DateEntity implements Serializable {
|
||||
this.errorSummary = errorSummary;
|
||||
}
|
||||
|
||||
public String getFailureCode() {
|
||||
return failureCode;
|
||||
}
|
||||
|
||||
public void setFailureCode(String failureCode) {
|
||||
this.failureCode = failureCode;
|
||||
}
|
||||
|
||||
public Integer getAttemptNo() {
|
||||
return attemptNo;
|
||||
}
|
||||
|
||||
public void setAttemptNo(Integer attemptNo) {
|
||||
this.attemptNo = attemptNo;
|
||||
}
|
||||
|
||||
public String getExecutionToken() {
|
||||
return executionToken;
|
||||
}
|
||||
|
||||
public void setExecutionToken(String executionToken) {
|
||||
this.executionToken = executionToken;
|
||||
}
|
||||
|
||||
public Date getLeaseUntil() {
|
||||
return leaseUntil;
|
||||
}
|
||||
|
||||
public void setLeaseUntil(Date leaseUntil) {
|
||||
this.leaseUntil = leaseUntil;
|
||||
}
|
||||
|
||||
public Integer getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
||||
public void setVersion(Integer version) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
public Date getStartedAt() {
|
||||
return startedAt;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package tech.easyflow.ai.enums;
|
||||
|
||||
/**
|
||||
* 文档批量导入项当前阶段。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public enum DocumentImportBatchItemStage {
|
||||
|
||||
/** 上传阶段。 */
|
||||
UPLOAD,
|
||||
|
||||
/** 解析阶段。 */
|
||||
PARSE,
|
||||
|
||||
/** 分块阶段。 */
|
||||
SPLIT,
|
||||
|
||||
/** 向量化阶段。 */
|
||||
INDEX,
|
||||
|
||||
/** 全流程结束。 */
|
||||
DONE
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package tech.easyflow.ai.enums;
|
||||
|
||||
/**
|
||||
* 文档批量导入项状态。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public enum DocumentImportBatchItemStatus {
|
||||
|
||||
/** 等待处理。 */
|
||||
PENDING,
|
||||
|
||||
/** 文件正在上传。 */
|
||||
UPLOADING,
|
||||
|
||||
/** 正在处理。 */
|
||||
RUNNING,
|
||||
|
||||
/** 文件已上传。 */
|
||||
UPLOADED,
|
||||
|
||||
/** 处理失败。 */
|
||||
FAILED,
|
||||
|
||||
/** 处理完成。 */
|
||||
COMPLETED,
|
||||
|
||||
/** 因重复而跳过。 */
|
||||
SKIPPED,
|
||||
|
||||
/** 文件随未启动批次一并取消。 */
|
||||
CANCELLED
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package tech.easyflow.ai.enums;
|
||||
|
||||
/**
|
||||
* 文档批量导入状态。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public enum DocumentImportBatchStatus {
|
||||
|
||||
/** 文件上传中。 */
|
||||
UPLOADING,
|
||||
|
||||
/** 文件已上传,等待选择导入方式。 */
|
||||
READY,
|
||||
|
||||
/** 批次处理中。 */
|
||||
RUNNING,
|
||||
|
||||
/** 批次处理已中断,可继续。 */
|
||||
INTERRUPTED,
|
||||
|
||||
/** 部分文件失败,可继续失败项。 */
|
||||
PARTIAL_SUCCEEDED,
|
||||
|
||||
/** 未启动的上传批次已取消。 */
|
||||
CANCELLED,
|
||||
|
||||
/** 批次全部完成。 */
|
||||
COMPLETED
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package tech.easyflow.ai.enums;
|
||||
|
||||
/**
|
||||
* 知识库文档导入模式。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public enum DocumentImportMode {
|
||||
|
||||
/**
|
||||
* 解析完成后由用户确认分块策略。
|
||||
*/
|
||||
MANUAL,
|
||||
|
||||
/**
|
||||
* 自动完成解析、分块、向量化和入库。
|
||||
*/
|
||||
AUTO
|
||||
}
|
||||
@@ -13,6 +13,11 @@ public enum DocumentImportTaskPhase {
|
||||
*/
|
||||
PARSE,
|
||||
|
||||
/**
|
||||
* 文档分块阶段。
|
||||
*/
|
||||
SPLIT,
|
||||
|
||||
/**
|
||||
* 向量化阶段。
|
||||
*/
|
||||
|
||||
@@ -28,6 +28,16 @@ public enum DocumentProcessStatus {
|
||||
*/
|
||||
READY_FOR_SEGMENT,
|
||||
|
||||
/**
|
||||
* 自动分块处理中。
|
||||
*/
|
||||
SPLITTING,
|
||||
|
||||
/**
|
||||
* 自动分块失败。
|
||||
*/
|
||||
SPLIT_FAILED,
|
||||
|
||||
/**
|
||||
* 已确认分块,可开始向量化。
|
||||
*/
|
||||
@@ -54,6 +64,18 @@ public enum DocumentProcessStatus {
|
||||
* @return 是否运行中
|
||||
*/
|
||||
public boolean isProcessing() {
|
||||
return this == PARSING || this == INDEXING;
|
||||
return this == PARSING || this == SPLITTING || this == INDEXING;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断状态名称是否属于运行中状态。
|
||||
*
|
||||
* @param status 状态名称
|
||||
* @return 是否运行中
|
||||
*/
|
||||
public static boolean isProcessing(String status) {
|
||||
return PARSING.name().equals(status)
|
||||
|| SPLITTING.name().equals(status)
|
||||
|| INDEXING.name().equals(status);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package tech.easyflow.ai.enums;
|
||||
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 知识库 Public API 产品权限范围。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public enum KnowledgeApiPermissionScope {
|
||||
|
||||
/**
|
||||
* 知识库读取权限。
|
||||
*/
|
||||
KNOWLEDGE_READ,
|
||||
|
||||
/**
|
||||
* 知识库导入权限。
|
||||
*/
|
||||
KNOWLEDGE_IMPORT,
|
||||
|
||||
/**
|
||||
* 知识库维护权限。
|
||||
*/
|
||||
KNOWLEDGE_MAINTENANCE;
|
||||
|
||||
/**
|
||||
* 根据三个权限开关构造稳定 Scope 集合。
|
||||
*
|
||||
* @param readEnabled 是否开启读取
|
||||
* @param importEnabled 是否开启导入
|
||||
* @param maintenanceEnabled 是否开启维护
|
||||
* @return 已开启的权限 Scope
|
||||
*/
|
||||
public static Set<String> enabledScopes(boolean readEnabled,
|
||||
boolean importEnabled,
|
||||
boolean maintenanceEnabled) {
|
||||
Set<String> scopes = new LinkedHashSet<>();
|
||||
if (readEnabled) {
|
||||
scopes.add(KNOWLEDGE_READ.name());
|
||||
}
|
||||
if (importEnabled) {
|
||||
scopes.add(KNOWLEDGE_IMPORT.name());
|
||||
}
|
||||
if (maintenanceEnabled) {
|
||||
scopes.add(KNOWLEDGE_MAINTENANCE.name());
|
||||
}
|
||||
return scopes;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
package tech.easyflow.ai.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 文档批量导入文件项映射层。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImportBatchItem> {
|
||||
|
||||
/**
|
||||
* 原子领取文件上传权并刷新批次进度时间。
|
||||
*
|
||||
* <p>文件项与批次在同一条 MySQL 多表更新中加锁,确保上传领取和
|
||||
* 超时取消之间不存在旧快照窗口。</p>
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @param itemId 文件项 ID
|
||||
* @param knowledgeId 知识库 ID
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch batch "
|
||||
+ "INNER JOIN tb_document_import_batch_item item "
|
||||
+ "ON item.batch_id=batch.id SET "
|
||||
+ "item.stage='UPLOAD', item.status='UPLOADING', "
|
||||
+ "item.error_summary=NULL, item.failure_code=NULL, "
|
||||
+ "item.retryable=0, item.modified=#{modified}, "
|
||||
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
|
||||
+ "WHERE batch.id=#{batchId} AND batch.knowledge_id=#{knowledgeId} "
|
||||
+ "AND batch.status IN ('UPLOADING','READY') "
|
||||
+ "AND item.id=#{itemId} AND item.status='PENDING' "
|
||||
+ "AND item.cleanup_pending=0 AND item.storage_locator IS NULL")
|
||||
int claimUpload(
|
||||
@Param("batchId") BigInteger batchId,
|
||||
@Param("itemId") BigInteger itemId,
|
||||
@Param("knowledgeId") BigInteger knowledgeId,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 按预期状态原子迁移文件项。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param expectedStatus 预期状态
|
||||
* @param stage 新阶段
|
||||
* @param status 新状态
|
||||
* @param errorSummary 错误摘要
|
||||
* @param failureCode 稳定失败码
|
||||
* @param retryable 是否允许批量重试
|
||||
* @param attemptDelta 重试次数增量
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch_item SET "
|
||||
+ "stage=#{stage}, status=#{status}, error_summary=#{errorSummary}, "
|
||||
+ "failure_code=#{failureCode}, "
|
||||
+ "retryable=#{retryable}, attempt_count=attempt_count + #{attemptDelta}, "
|
||||
+ "modified=#{modified} WHERE id=#{id} AND status=#{expectedStatus}")
|
||||
int transitionStatus(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("expectedStatus") String expectedStatus,
|
||||
@Param("stage") String stage,
|
||||
@Param("status") String status,
|
||||
@Param("errorSummary") String errorSummary,
|
||||
@Param("failureCode") String failureCode,
|
||||
@Param("retryable") boolean retryable,
|
||||
@Param("attemptDelta") int attemptDelta,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 将上传项原子绑定到创建后的文档。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param documentId 文档 ID
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch_item SET "
|
||||
+ "document_id=#{documentId}, stage='PARSE', status='PENDING', "
|
||||
+ "error_summary=NULL, retryable=0, modified=#{modified} "
|
||||
+ "WHERE id=#{id} AND status='UPLOADED' AND document_id IS NULL")
|
||||
int bindDocument(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("documentId") BigInteger documentId,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 在物理写入前持久化可恢复存储定位符。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param storageLocator 可恢复存储定位符
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch_item SET "
|
||||
+ "storage_locator=#{storageLocator}, cleanup_pending=0, "
|
||||
+ "modified=#{modified} "
|
||||
+ "WHERE id=#{id} AND status='UPLOADING' "
|
||||
+ "AND storage_locator IS NULL AND cleanup_pending=0")
|
||||
int registerUploadWriteIntent(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("storageLocator") String storageLocator,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 在物理写入前原子撤销上传领取与写意图登记。
|
||||
*
|
||||
* <p>同时兼容登记未提交和提交结果未知两种情况;仅清除空定位符或
|
||||
* 本次预期定位符,避免覆盖其他请求的新写入意图。</p>
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param storageLocator 本次预期存储定位符,可为空
|
||||
* @param errorSummary 上传准备失败摘要
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch batch "
|
||||
+ "INNER JOIN tb_document_import_batch_item item "
|
||||
+ "ON item.batch_id=batch.id SET "
|
||||
+ "item.status='PENDING', item.storage_locator=NULL, "
|
||||
+ "item.file_path=NULL, item.cleanup_pending=0, "
|
||||
+ "item.error_summary=#{errorSummary}, item.failure_code=NULL, "
|
||||
+ "item.retryable=0, item.modified=#{modified}, "
|
||||
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
|
||||
+ "WHERE item.id=#{id} AND item.status='UPLOADING' "
|
||||
+ "AND item.cleanup_pending=0 "
|
||||
+ "AND (item.storage_locator IS NULL "
|
||||
+ "OR item.storage_locator=#{storageLocator})")
|
||||
int abortUploadBeforeWrite(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("storageLocator") String storageLocator,
|
||||
@Param("errorSummary") String errorSummary,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 在文件仍处于上传中时原子绑定存储路径并完成上传。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param filePath 存储路径
|
||||
* @param storageLocator 可恢复存储定位符
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch_item SET "
|
||||
+ "file_path=#{filePath}, status='UPLOADED', error_summary=NULL, "
|
||||
+ "cleanup_pending=0, retryable=0, modified=#{modified} "
|
||||
+ "WHERE id=#{id} AND status='UPLOADING' "
|
||||
+ "AND storage_locator=#{storageLocator}")
|
||||
int completeUpload(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("filePath") String filePath,
|
||||
@Param("storageLocator") String storageLocator,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 将指定可恢复写意图标记为等待清理。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param storageLocator 可恢复存储定位符
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch_item SET "
|
||||
+ "cleanup_pending=1, modified=#{modified} "
|
||||
+ "WHERE id=#{id} AND status IN ('UPLOADING','CANCELLED') "
|
||||
+ "AND storage_locator=#{storageLocator}")
|
||||
int markUploadCleanupPending(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("storageLocator") String storageLocator,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 将已取消文件项的现有存储引用标记为等待清理。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch_item SET "
|
||||
+ "cleanup_pending=1, modified=#{modified} "
|
||||
+ "WHERE id=#{id} AND status='CANCELLED' "
|
||||
+ "AND (storage_locator IS NOT NULL OR file_path IS NOT NULL)")
|
||||
int markCancelledCleanupPending(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 为新导入项记录待覆盖的历史文档。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param replacedDocumentId 历史文档 ID
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch_item SET "
|
||||
+ "replaced_document_id=#{replacedDocumentId}, modified=#{modified} "
|
||||
+ "WHERE id=#{id} AND status='UPLOADED' AND replaced_document_id IS NULL")
|
||||
int markReplacement(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("replacedDocumentId") BigInteger replacedDocumentId,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 完成历史文档清理后原子清除覆盖标记。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param replacedDocumentId 历史文档 ID
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch_item SET "
|
||||
+ "replaced_document_id=NULL, modified=#{modified} "
|
||||
+ "WHERE id=#{id} AND replaced_document_id=#{replacedDocumentId}")
|
||||
int clearReplacement(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("replacedDocumentId") BigInteger replacedDocumentId,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 对象删除成功后原子清理上传中项并恢复待上传状态。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param storageLocator 已删除对象的恢复定位符
|
||||
* @param filePath 已删除对象的兼容存储路径
|
||||
* @param recoveryMessage 恢复为待上传时的提示
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch batch "
|
||||
+ "INNER JOIN tb_document_import_batch_item item "
|
||||
+ "ON item.batch_id=batch.id SET "
|
||||
+ "item.stage='UPLOAD', item.error_summary=#{recoveryMessage}, "
|
||||
+ "item.failure_code=NULL, item.retryable=0, item.status='PENDING', "
|
||||
+ "item.cleanup_pending=0, item.storage_locator=NULL, "
|
||||
+ "item.file_path=NULL, item.modified=#{modified}, "
|
||||
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
|
||||
+ "WHERE item.id=#{id} AND item.cleanup_pending=1 "
|
||||
+ "AND item.status='UPLOADING' "
|
||||
+ "AND item.storage_locator <=> #{storageLocator} "
|
||||
+ "AND item.file_path <=> #{filePath}")
|
||||
int completeUploadingStorageCleanup(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("storageLocator") String storageLocator,
|
||||
@Param("filePath") String filePath,
|
||||
@Param("recoveryMessage") String recoveryMessage,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 对象删除成功后原子清理已取消项并保持取消状态。
|
||||
*
|
||||
* @param id 文件项 ID
|
||||
* @param storageLocator 已删除对象的恢复定位符
|
||||
* @param filePath 已删除对象的兼容存储路径
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch batch "
|
||||
+ "INNER JOIN tb_document_import_batch_item item "
|
||||
+ "ON item.batch_id=batch.id SET "
|
||||
+ "item.cleanup_pending=0, item.storage_locator=NULL, "
|
||||
+ "item.file_path=NULL, item.modified=#{modified}, "
|
||||
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
|
||||
+ "WHERE item.id=#{id} AND item.cleanup_pending=1 "
|
||||
+ "AND item.status='CANCELLED' "
|
||||
+ "AND item.storage_locator <=> #{storageLocator} "
|
||||
+ "AND item.file_path <=> #{filePath}")
|
||||
int completeCancelledStorageCleanup(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("storageLocator") String storageLocator,
|
||||
@Param("filePath") String filePath,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package tech.easyflow.ai.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 文档批量导入批次映射层。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatch> {
|
||||
|
||||
/**
|
||||
* 锁定并读取调用者拥有的批次,保证后续重试基于最新状态。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @param callerType 调用者类型
|
||||
* @param callerId 调用者 ID
|
||||
* @return 批次;不存在时返回 null
|
||||
*/
|
||||
@Select("SELECT * FROM tb_document_import_batch "
|
||||
+ "WHERE id=#{batchId} AND caller_type=#{callerType} "
|
||||
+ "AND caller_id=#{callerId} FOR UPDATE")
|
||||
DocumentImportBatch selectOwnedForUpdate(
|
||||
@Param("batchId") BigInteger batchId,
|
||||
@Param("callerType") String callerType,
|
||||
@Param("callerId") BigInteger callerId
|
||||
);
|
||||
|
||||
/**
|
||||
* 按文件项状态迁移增量更新批次计数。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @param completedDelta 完成数增量
|
||||
* @param processingDelta 处理中数量增量
|
||||
* @param failedDelta 失败数增量
|
||||
* @param pendingDelta 等待数增量
|
||||
* @param uploadedDelta 已上传数增量
|
||||
* @param skippedDelta 跳过数增量
|
||||
* @param cancelledDelta 取消数增量
|
||||
* @param retryableFailedDelta 可重试失败数增量
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch SET "
|
||||
+ "completed_count=GREATEST(0, completed_count + #{completedDelta}), "
|
||||
+ "processing_count=GREATEST(0, processing_count + #{processingDelta}), "
|
||||
+ "failed_count=GREATEST(0, failed_count + #{failedDelta}), "
|
||||
+ "pending_count=GREATEST(0, pending_count + #{pendingDelta}), "
|
||||
+ "uploaded_count=GREATEST(0, uploaded_count + #{uploadedDelta}), "
|
||||
+ "skipped_count=GREATEST(0, skipped_count + #{skippedDelta}), "
|
||||
+ "cancelled_count=GREATEST(0, cancelled_count + #{cancelledDelta}), "
|
||||
+ "retryable_failed_count=GREATEST(0, retryable_failed_count + #{retryableFailedDelta}), "
|
||||
+ "modified=#{modified} WHERE id=#{batchId}")
|
||||
int adjustCounters(
|
||||
@Param("batchId") BigInteger batchId,
|
||||
@Param("completedDelta") int completedDelta,
|
||||
@Param("processingDelta") int processingDelta,
|
||||
@Param("failedDelta") int failedDelta,
|
||||
@Param("pendingDelta") int pendingDelta,
|
||||
@Param("uploadedDelta") int uploadedDelta,
|
||||
@Param("skippedDelta") int skippedDelta,
|
||||
@Param("cancelledDelta") int cancelledDelta,
|
||||
@Param("retryableFailedDelta") int retryableFailedDelta,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 原子领取 Public API 重试代次。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @param callerType 调用者类型
|
||||
* @param callerId 调用者 ID
|
||||
* @param expectedGeneration 预期重试代次
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch SET "
|
||||
+ "status='RUNNING', finished_at=NULL, "
|
||||
+ "retry_generation=retry_generation + 1, "
|
||||
+ "version=version + 1, modified=#{modified} "
|
||||
+ "WHERE id=#{batchId} AND caller_type=#{callerType} AND caller_id=#{callerId} "
|
||||
+ "AND retry_generation=#{expectedGeneration} "
|
||||
+ "AND status IN ('PARTIAL_SUCCEEDED','INTERRUPTED')")
|
||||
int claimRetry(
|
||||
@Param("batchId") BigInteger batchId,
|
||||
@Param("callerType") String callerType,
|
||||
@Param("callerId") BigInteger callerId,
|
||||
@Param("expectedGeneration") int expectedGeneration,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 原子领取无进展的未完成批次取消权。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @param knowledgeId 知识库 ID
|
||||
* @param callerType 调用者类型
|
||||
* @param callerId 调用者 ID
|
||||
* @param incompleteCutoff 最后进展截止时间
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch SET "
|
||||
+ "status='CANCELLED', finished_at=#{modified}, "
|
||||
+ "version=version + 1, modified=#{modified} "
|
||||
+ "WHERE id=#{batchId} AND knowledge_id=#{knowledgeId} "
|
||||
+ "AND caller_type=#{callerType} AND caller_id=#{callerId} "
|
||||
+ "AND status IN ('UPLOADING','READY') "
|
||||
+ "AND ((modified IS NOT NULL AND modified < #{incompleteCutoff}) "
|
||||
+ "OR (modified IS NULL AND created < #{incompleteCutoff}))")
|
||||
int claimStaleCancellation(
|
||||
@Param("batchId") BigInteger batchId,
|
||||
@Param("knowledgeId") BigInteger knowledgeId,
|
||||
@Param("callerType") String callerType,
|
||||
@Param("callerId") BigInteger callerId,
|
||||
@Param("incompleteCutoff") Date incompleteCutoff,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
|
||||
/**
|
||||
* 分批查询无进展的未完成批次。
|
||||
*
|
||||
* @param incompleteCutoff 最后进展截止时间
|
||||
* @param limit 最大返回数量
|
||||
* @return 待回收批次
|
||||
*/
|
||||
@Select("SELECT * FROM tb_document_import_batch "
|
||||
+ "WHERE status IN ('UPLOADING','READY') "
|
||||
+ "AND ((modified IS NOT NULL AND modified < #{incompleteCutoff}) "
|
||||
+ "OR (modified IS NULL AND created < #{incompleteCutoff})) "
|
||||
+ "ORDER BY COALESCE(modified, created), id LIMIT #{limit}")
|
||||
List<DocumentImportBatch> selectStaleIncompleteBatches(
|
||||
@Param("incompleteCutoff") Date incompleteCutoff,
|
||||
@Param("limit") int limit
|
||||
);
|
||||
|
||||
/**
|
||||
* 释放已取消或超过去重窗口的服务端提交指纹。
|
||||
*
|
||||
* @param batchId 批次 ID
|
||||
* @param submissionFingerprint 当前服务端提交指纹
|
||||
* @param deduplicationCutoff 去重窗口起点
|
||||
* @param modified 修改时间
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_batch SET "
|
||||
+ "idempotency_key_hash=NULL, modified=#{modified}, version=version + 1 "
|
||||
+ "WHERE id=#{batchId} AND idempotency_key_hash=#{submissionFingerprint} "
|
||||
+ "AND (status='CANCELLED' OR ("
|
||||
+ "COALESCE(finished_at, modified, created) < #{deduplicationCutoff} "
|
||||
+ "AND status IN ('INTERRUPTED','PARTIAL_SUCCEEDED','COMPLETED')))")
|
||||
int releaseSubmissionFingerprint(
|
||||
@Param("batchId") BigInteger batchId,
|
||||
@Param("submissionFingerprint") String submissionFingerprint,
|
||||
@Param("deduplicationCutoff") Date deduplicationCutoff,
|
||||
@Param("modified") Date modified
|
||||
);
|
||||
}
|
||||
@@ -1,8 +1,15 @@
|
||||
package tech.easyflow.ai.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
import tech.easyflow.ai.entity.DocumentImportTask;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 文档导入任务映射层。
|
||||
*
|
||||
@@ -10,4 +17,169 @@ import tech.easyflow.ai.entity.DocumentImportTask;
|
||||
* @since 2026-04-14
|
||||
*/
|
||||
public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask> {
|
||||
|
||||
/**
|
||||
* 按任务阶段和批次轮转查询待投递任务,避免大批次独占投递窗口。
|
||||
*
|
||||
* @param redispatchBefore 允许重新投递的修改时间边界
|
||||
* @param limit 最大任务数
|
||||
* @return 公平排序后的待投递任务
|
||||
*/
|
||||
@Select("SELECT task.* FROM tb_document_import_task task JOIN ("
|
||||
+ "SELECT task.id, ROW_NUMBER() OVER ("
|
||||
+ "PARTITION BY task.phase, COALESCE(task.batch_id, task.id) "
|
||||
+ "ORDER BY task.created, task.id) AS lane_row "
|
||||
+ "FROM tb_document_import_task task "
|
||||
+ "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore}"
|
||||
+ ") ranked ON ranked.id=task.id "
|
||||
+ "ORDER BY ranked.lane_row, task.created, task.id LIMIT #{limit}")
|
||||
List<DocumentImportTask> selectPendingFairly(
|
||||
@Param("redispatchBefore") Date redispatchBefore,
|
||||
@Param("limit") int limit
|
||||
);
|
||||
|
||||
/**
|
||||
* 仅在任务仍待处理时原子更新时间戳,取得本轮重新投递资格。
|
||||
*
|
||||
* @param id 任务 ID
|
||||
* @param redispatchBefore 允许重新投递的修改时间边界
|
||||
* @param now 当前时间
|
||||
* @param operatorId 操作人 ID
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_task SET modified=#{now}, "
|
||||
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
||||
+ "WHERE id=#{id} AND status='PENDING' "
|
||||
+ "AND modified <= #{redispatchBefore}")
|
||||
int touchPendingForDispatch(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("redispatchBefore") Date redispatchBefore,
|
||||
@Param("now") Date now,
|
||||
@Param("operatorId") BigInteger operatorId
|
||||
);
|
||||
|
||||
/**
|
||||
* 查询租约已过期的运行任务,并兼容迁移前没有租约的历史任务。
|
||||
*
|
||||
* @param now 当前时间
|
||||
* @param legacyCutoff 历史任务失联时间边界
|
||||
* @param limit 最大任务数
|
||||
* @return 已失去执行租约的运行任务
|
||||
*/
|
||||
@Select("SELECT * FROM tb_document_import_task "
|
||||
+ "WHERE status='RUNNING' AND ("
|
||||
+ "(lease_until IS NOT NULL AND lease_until <= #{now}) OR "
|
||||
+ "(lease_until IS NULL AND modified <= #{legacyCutoff})) "
|
||||
+ "ORDER BY COALESCE(lease_until, modified), id LIMIT #{limit}")
|
||||
List<DocumentImportTask> selectExpiredRunningTasks(
|
||||
@Param("now") Date now,
|
||||
@Param("legacyCutoff") Date legacyCutoff,
|
||||
@Param("limit") int limit
|
||||
);
|
||||
|
||||
/**
|
||||
* 使用执行令牌原子领取一个待处理任务。
|
||||
*
|
||||
* @param id 任务 ID
|
||||
* @param executionToken 本轮执行令牌
|
||||
* @param leaseUntil 租约到期时间
|
||||
* @param now 当前时间
|
||||
* @param operatorId 操作人 ID
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_task SET status='RUNNING', "
|
||||
+ "attempt_no=COALESCE(attempt_no, 0) + 1, "
|
||||
+ "execution_token=#{executionToken}, lease_until=#{leaseUntil}, "
|
||||
+ "started_at=COALESCE(started_at, #{now}), finished_at=NULL, "
|
||||
+ "error_summary=NULL, failure_code=NULL, modified=#{now}, "
|
||||
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
||||
+ "WHERE id=#{id} AND status='PENDING'")
|
||||
int claimPending(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("executionToken") String executionToken,
|
||||
@Param("leaseUntil") Date leaseUntil,
|
||||
@Param("now") Date now,
|
||||
@Param("operatorId") BigInteger operatorId
|
||||
);
|
||||
|
||||
/**
|
||||
* 在仍持有执行令牌时续租任务。
|
||||
*
|
||||
* @param id 任务 ID
|
||||
* @param executionToken 本轮执行令牌
|
||||
* @param leaseUntil 新租约到期时间
|
||||
* @param now 当前时间
|
||||
* @param operatorId 操作人 ID
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_task SET lease_until=#{leaseUntil}, "
|
||||
+ "modified=#{now}, modified_by=#{operatorId}, "
|
||||
+ "version=COALESCE(version, 0) + 1 "
|
||||
+ "WHERE id=#{id} AND status='RUNNING' "
|
||||
+ "AND execution_token=#{executionToken}")
|
||||
int renewLease(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("executionToken") String executionToken,
|
||||
@Param("leaseUntil") Date leaseUntil,
|
||||
@Param("now") Date now,
|
||||
@Param("operatorId") BigInteger operatorId
|
||||
);
|
||||
|
||||
/**
|
||||
* 在仍持有执行令牌时写入任务终态。
|
||||
*
|
||||
* @param id 任务 ID
|
||||
* @param executionToken 本轮执行令牌
|
||||
* @param status 任务终态
|
||||
* @param errorSummary 错误摘要
|
||||
* @param failureCode 稳定失败码
|
||||
* @param now 当前时间
|
||||
* @param operatorId 操作人 ID
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_task SET status=#{status}, "
|
||||
+ "error_summary=#{errorSummary}, failure_code=#{failureCode}, "
|
||||
+ "lease_until=NULL, finished_at=#{now}, modified=#{now}, "
|
||||
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
||||
+ "WHERE id=#{id} AND status='RUNNING' "
|
||||
+ "AND execution_token=#{executionToken}")
|
||||
int finishOwned(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("executionToken") String executionToken,
|
||||
@Param("status") String status,
|
||||
@Param("errorSummary") String errorSummary,
|
||||
@Param("failureCode") String failureCode,
|
||||
@Param("now") Date now,
|
||||
@Param("operatorId") BigInteger operatorId
|
||||
);
|
||||
|
||||
/**
|
||||
* 仅在当前执行令牌仍持有已过期租约时将任务标记为失败。
|
||||
*
|
||||
* @param id 任务 ID
|
||||
* @param executionToken 本轮执行令牌
|
||||
* @param errorSummary 错误摘要
|
||||
* @param failureCode 稳定失败码
|
||||
* @param now 当前时间
|
||||
* @param legacyCutoff 历史任务失联时间边界
|
||||
* @param operatorId 操作人 ID
|
||||
* @return 更新行数
|
||||
*/
|
||||
@Update("UPDATE tb_document_import_task SET status='FAILED', "
|
||||
+ "error_summary=#{errorSummary}, failure_code=#{failureCode}, "
|
||||
+ "lease_until=NULL, finished_at=#{now}, modified=#{now}, "
|
||||
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
||||
+ "WHERE id=#{id} AND status='RUNNING' "
|
||||
+ "AND execution_token <=> #{executionToken} AND ("
|
||||
+ "(lease_until IS NOT NULL AND lease_until <= #{now}) OR "
|
||||
+ "(lease_until IS NULL AND modified <= #{legacyCutoff}))")
|
||||
int failExpiredOwned(
|
||||
@Param("id") BigInteger id,
|
||||
@Param("executionToken") String executionToken,
|
||||
@Param("errorSummary") String errorSummary,
|
||||
@Param("failureCode") String failureCode,
|
||||
@Param("now") Date now,
|
||||
@Param("legacyCutoff") Date legacyCutoff,
|
||||
@Param("operatorId") BigInteger operatorId
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package tech.easyflow.ai.service;
|
||||
|
||||
import com.mybatisflex.core.service.IService;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||
|
||||
/**
|
||||
* 文档批量导入文件项服务。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public interface DocumentImportBatchItemService extends IService<DocumentImportBatchItem> {
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package tech.easyflow.ai.service;
|
||||
|
||||
import com.mybatisflex.core.service.IService;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||
|
||||
/**
|
||||
* 文档批量导入批次服务。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public interface DocumentImportBatchService extends IService<DocumentImportBatch> {
|
||||
}
|
||||
@@ -19,7 +19,37 @@ import java.util.List;
|
||||
*/
|
||||
public interface DocumentService extends IService<Document> {
|
||||
|
||||
Page<Document> getDocumentList(String knowledgeId , int pageSize, int pageNum, String fileName);
|
||||
/**
|
||||
* 按知识库和文件标题查询文档分页。
|
||||
*
|
||||
* @param knowledgeId 知识库 ID
|
||||
* @param pageSize 每页条数
|
||||
* @param pageNum 页码
|
||||
* @param fileName 可选的文件标题筛选
|
||||
* @return 文档分页
|
||||
*/
|
||||
Page<Document> getDocumentList(
|
||||
String knowledgeId,
|
||||
int pageSize,
|
||||
int pageNum,
|
||||
String fileName
|
||||
);
|
||||
|
||||
/**
|
||||
* 按知识库和文档 ID 查询文档分页。
|
||||
*
|
||||
* @param knowledgeId 知识库 ID
|
||||
* @param pageSize 每页条数
|
||||
* @param pageNum 页码
|
||||
* @param documentId 可选的文档 ID
|
||||
* @return 文档分页
|
||||
*/
|
||||
Page<Document> getDocumentListById(
|
||||
String knowledgeId,
|
||||
int pageSize,
|
||||
int pageNum,
|
||||
BigInteger documentId
|
||||
);
|
||||
|
||||
boolean removeDoc(String id);
|
||||
|
||||
@@ -44,4 +74,6 @@ public interface DocumentService extends IService<Document> {
|
||||
Result<DocumentImportDtos.TaskStartIndexResponse> retryParseTask(DocumentImportDtos.TaskRetryRequest request);
|
||||
|
||||
Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(DocumentImportDtos.TaskRetryRequest request);
|
||||
|
||||
Result<DocumentImportDtos.TaskStartIndexResponse> retryFailedTask(DocumentImportDtos.TaskRetryRequest request);
|
||||
}
|
||||
|
||||
@@ -27,6 +27,27 @@ public interface KnowledgeSharePermissionService {
|
||||
*/
|
||||
void replaceApiShareEnabled(BigInteger apiKeyId, boolean enabled);
|
||||
|
||||
/**
|
||||
* 替换访问令牌的三类知识库 Public API 权限。
|
||||
*
|
||||
* @param apiKeyId 系统访问令牌 ID
|
||||
* @param readEnabled 是否开启读取
|
||||
* @param importEnabled 是否开启导入
|
||||
* @param maintenanceEnabled 是否开启维护
|
||||
*/
|
||||
void replaceApiPermissions(BigInteger apiKeyId,
|
||||
boolean readEnabled,
|
||||
boolean importEnabled,
|
||||
boolean maintenanceEnabled);
|
||||
|
||||
/**
|
||||
* 查询访问令牌已开启的知识库 Public API 权限。
|
||||
*
|
||||
* @param apiKeyId 系统访问令牌 ID
|
||||
* @return 权限 Scope 集合
|
||||
*/
|
||||
Set<String> getApiPermissionScopes(BigInteger apiKeyId);
|
||||
|
||||
/**
|
||||
* 断言当前令牌具备知识库分享权限。
|
||||
*
|
||||
|
||||
@@ -486,6 +486,10 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
||||
}
|
||||
item.setContent(content);
|
||||
item.addMetadata("chunkId", chunkId);
|
||||
item.addMetadata(
|
||||
"resultType",
|
||||
DocumentCollection.TYPE_DOCUMENT
|
||||
);
|
||||
Object sourceDocumentId = hitSnapshot.findSourceDocumentId(item.getId());
|
||||
if (sourceDocumentId != null) {
|
||||
item.addMetadata("documentId", sourceDocumentId);
|
||||
@@ -605,6 +609,11 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
||||
.collect(Collectors.toList());
|
||||
metadataMap.put("chunkId", item.getId());
|
||||
metadataMap.put("documentId", item.getId());
|
||||
metadataMap.put("resultType", DocumentCollection.TYPE_FAQ);
|
||||
metadataMap.put("faqId", faqItem.getId());
|
||||
metadataMap.put("question", faqItem.getQuestion());
|
||||
metadataMap.put("answerText", faqItem.getAnswerText());
|
||||
metadataMap.put("categoryId", faqItem.getCategoryId());
|
||||
metadataMap.put("imageUrls", imageUrls);
|
||||
item.setMetadataMap(metadataMap);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package tech.easyflow.ai.service.impl;
|
||||
|
||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
||||
|
||||
/**
|
||||
* 文档批量导入文件项服务实现。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
@Service
|
||||
public class DocumentImportBatchItemServiceImpl
|
||||
extends ServiceImpl<DocumentImportBatchItemMapper, DocumentImportBatchItem>
|
||||
implements DocumentImportBatchItemService {
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package tech.easyflow.ai.service.impl;
|
||||
|
||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchService;
|
||||
|
||||
/**
|
||||
* 文档批量导入批次服务实现。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
@Service
|
||||
public class DocumentImportBatchServiceImpl
|
||||
extends ServiceImpl<DocumentImportBatchMapper, DocumentImportBatch>
|
||||
implements DocumentImportBatchService {
|
||||
}
|
||||
@@ -106,6 +106,45 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
|
||||
@Override
|
||||
public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) {
|
||||
return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按知识库和文档 ID 查询文档分页。
|
||||
*
|
||||
* @param knowledgeId 知识库 ID
|
||||
* @param pageSize 每页条数
|
||||
* @param pageNum 页码
|
||||
* @param documentId 可选的文档 ID
|
||||
* @return 文档分页
|
||||
*/
|
||||
@Override
|
||||
public Page<Document> getDocumentListById(
|
||||
String knowledgeId,
|
||||
int pageSize,
|
||||
int pageNum,
|
||||
BigInteger documentId
|
||||
) {
|
||||
return queryDocumentList(knowledgeId, pageSize, pageNum, null, documentId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行文档分页查询。
|
||||
*
|
||||
* @param knowledgeId 知识库 ID
|
||||
* @param pageSize 每页条数
|
||||
* @param pageNum 页码
|
||||
* @param fileName 可选的文件标题筛选
|
||||
* @param documentId 可选的文档 ID
|
||||
* @return 文档分页
|
||||
*/
|
||||
private Page<Document> queryDocumentList(
|
||||
String knowledgeId,
|
||||
int pageSize,
|
||||
int pageNum,
|
||||
String fileName,
|
||||
BigInteger documentId
|
||||
) {
|
||||
QueryWrapper queryWrapper=QueryWrapper.create()
|
||||
.select(
|
||||
DOCUMENT.ALL_COLUMNS,
|
||||
@@ -120,19 +159,23 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
if (fileName != null && !fileName.trim().isEmpty()) {
|
||||
queryWrapper.and(DOCUMENT.TITLE.like(fileName));
|
||||
}
|
||||
if (documentId != null) {
|
||||
queryWrapper.and(DOCUMENT.ID.eq(documentId));
|
||||
}
|
||||
// 分组
|
||||
queryWrapper.groupBy(DOCUMENT.ID);
|
||||
Page<Document> documentVoPage = documentMapper.paginateAs(pageNum, pageSize, queryWrapper, Document.class);
|
||||
return documentVoPage;
|
||||
return documentMapper.paginateAs(pageNum, pageSize, queryWrapper, Document.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据文档id删除文件
|
||||
* 删除文档的向量、搜索索引、分块、存储文件和主记录。
|
||||
*
|
||||
* @param id 文档id
|
||||
* @return
|
||||
* @param id 文档 ID
|
||||
* @return 全部数据库清理成功时返回 true
|
||||
* @throws BusinessException 文档仍在处理中时抛出
|
||||
*/
|
||||
@Override
|
||||
@Transactional
|
||||
public boolean removeDoc(String id) {
|
||||
// 查询该文档对应哪些分割的字段,先删除
|
||||
QueryWrapper queryWrapperDocument = QueryWrapper.create().eq(Document::getId, id);
|
||||
@@ -140,8 +183,7 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
if (oneByQuery == null) {
|
||||
return false;
|
||||
}
|
||||
if (DocumentProcessStatus.PARSING.name().equals(oneByQuery.getProcessStatus())
|
||||
|| DocumentProcessStatus.INDEXING.name().equals(oneByQuery.getProcessStatus())) {
|
||||
if (DocumentProcessStatus.isProcessing(oneByQuery.getProcessStatus())) {
|
||||
throw new BusinessException("文档处理中,暂不允许删除");
|
||||
}
|
||||
DocumentCollection knowledge = knowledgeService.getById(oneByQuery.getCollectionId());
|
||||
@@ -149,28 +191,40 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
return false;
|
||||
}
|
||||
|
||||
// 存储到知识库
|
||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
||||
if (documentStore == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
QueryWrapper queryWrapper = QueryWrapper.create()
|
||||
.select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id);
|
||||
List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(
|
||||
queryWrapper,
|
||||
BigInteger.class
|
||||
);
|
||||
DocumentStore documentStore = null;
|
||||
try {
|
||||
Model model = modelService.getById(knowledge.getVectorEmbedModelId());
|
||||
if (model == null) {
|
||||
return false;
|
||||
if (!chunkIds.isEmpty()) {
|
||||
documentStore = knowledge.toDocumentStore();
|
||||
if (documentStore == null) {
|
||||
return false;
|
||||
}
|
||||
Model model = modelService.getById(
|
||||
knowledge.getVectorEmbedModelId()
|
||||
);
|
||||
if (model == null) {
|
||||
return false;
|
||||
}
|
||||
StoreOptions options = StoreOptions.ofCollectionName(
|
||||
knowledge.getVectorStoreCollection()
|
||||
);
|
||||
EmbeddingOptions embeddingOptions = new EmbeddingOptions();
|
||||
embeddingOptions.setModel(model.getModelName());
|
||||
options.setEmbeddingOptions(embeddingOptions);
|
||||
StoreResult deleteResult = documentStore.delete(chunkIds, options);
|
||||
if (deleteResult == null || !deleteResult.isSuccess()) {
|
||||
String failReason = deleteResult == null
|
||||
? "未返回结果"
|
||||
: deleteResult.getFailReason();
|
||||
Log.error("删除文档向量失败: documentId={}, reason={}", id, failReason);
|
||||
throw new BusinessException("文档向量删除失败");
|
||||
}
|
||||
}
|
||||
// 设置向量模型
|
||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
||||
EmbeddingOptions embeddingOptions = new EmbeddingOptions();
|
||||
embeddingOptions.setModel(model.getModelName());
|
||||
options.setEmbeddingOptions(embeddingOptions);
|
||||
options.setCollectionName(knowledge.getVectorStoreCollection());
|
||||
// 查询文本分割表tb_document_chunk中对应的有哪些数据,找出来删除
|
||||
QueryWrapper queryWrapper = QueryWrapper.create()
|
||||
.select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id);
|
||||
List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class);
|
||||
documentStore.delete(chunkIds, options);
|
||||
// 删除搜索引擎中的数据
|
||||
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||
if (searcher != null) {
|
||||
@@ -181,9 +235,16 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
return false;
|
||||
}
|
||||
// 再删除指定路径下的文件
|
||||
Document document = documentMapper.selectOneByQuery(queryWrapperDocument);
|
||||
storageService.delete(document.getDocumentPath());
|
||||
return true;
|
||||
String chunkSnapshotPath = oneByQuery.getOptions() == null
|
||||
? null
|
||||
: asString(oneByQuery.getOptions().get(
|
||||
DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH));
|
||||
if (StringUtil.hasText(chunkSnapshotPath)) {
|
||||
storageService.delete(chunkSnapshotPath);
|
||||
}
|
||||
storageService.delete(oneByQuery.getDocumentPath());
|
||||
// 主记录必须最后删除;否则接口返回成功后文档仍会出现在列表中。
|
||||
return documentMapper.deleteById(oneByQuery.getId()) > 0;
|
||||
} finally {
|
||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
||||
}
|
||||
@@ -1012,4 +1073,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
public Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(DocumentImportDtos.TaskRetryRequest request) {
|
||||
return importTaskAppService.retryIndexTask(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<DocumentImportDtos.TaskStartIndexResponse> retryFailedTask(DocumentImportDtos.TaskRetryRequest request) {
|
||||
return importTaskAppService.retryFailedTask(request);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,13 @@ package tech.easyflow.ai.service.impl;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
|
||||
import tech.easyflow.ai.enums.KnowledgeShareActionScope;
|
||||
import tech.easyflow.ai.service.KnowledgeSharePermissionService;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor.LockHandle;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.system.entity.SysApiKey;
|
||||
import tech.easyflow.system.entity.SysApiKeyResource;
|
||||
@@ -15,9 +20,9 @@ import tech.easyflow.system.service.SysApiKeyService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
@@ -30,10 +35,51 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
|
||||
|
||||
public static final String RESOURCE_TYPE_KNOWLEDGE = "KNOWLEDGE";
|
||||
|
||||
private static final Map<String, List<String>> URI_SCOPE_MAPPING = new LinkedHashMap<>();
|
||||
private static final String PERMISSION_LOCK_PREFIX =
|
||||
"easyflow:lock:knowledge-api-permission:";
|
||||
private static final Duration PERMISSION_LOCK_WAIT = Duration.ofSeconds(2);
|
||||
private static final Duration PERMISSION_LOCK_LEASE = Duration.ofSeconds(15);
|
||||
|
||||
private static final Map<String, List<String>> API_SCOPE_URI_MAPPING = new LinkedHashMap<>();
|
||||
private static final Map<String, List<String>> LEGACY_ACTION_URI_MAPPING = new LinkedHashMap<>();
|
||||
|
||||
static {
|
||||
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.VIEW.name(), List.of(
|
||||
API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_READ.name(), List.of(
|
||||
"/public-api/knowledge-share/detail",
|
||||
"/public-api/knowledge-share/search",
|
||||
"/public-api/knowledge-share/document/page",
|
||||
"/public-api/knowledge-share/document/download",
|
||||
"/public-api/knowledge-share/documentChunk/page",
|
||||
"/public-api/knowledge-share/faq/page",
|
||||
"/public-api/knowledge-share/faq/detail",
|
||||
"/public-api/knowledge-share/faq/exportExcel"
|
||||
));
|
||||
API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name(), List.of(
|
||||
"/public-api/knowledge-share/document/import/batch",
|
||||
"/public-api/knowledge-share/document/import/batch/status",
|
||||
"/public-api/knowledge-share/document/import/batch/retry",
|
||||
"/public-api/knowledge-share/document/import/analyze",
|
||||
"/public-api/knowledge-share/document/import/preview",
|
||||
"/public-api/knowledge-share/document/import/commit",
|
||||
"/public-api/knowledge-share/document/import/task/create",
|
||||
"/public-api/knowledge-share/document/import/task/detail",
|
||||
"/public-api/knowledge-share/document/import/task/preview",
|
||||
"/public-api/knowledge-share/document/import/task/startIndex",
|
||||
"/public-api/knowledge-share/document/import/task/retryParse",
|
||||
"/public-api/knowledge-share/document/import/task/retryIndex",
|
||||
"/public-api/knowledge-share/faq/importExcel",
|
||||
"/public-api/knowledge-share/faq/downloadImportTemplate"
|
||||
));
|
||||
API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name(), List.of(
|
||||
"/public-api/knowledge-share/document/remove",
|
||||
"/public-api/knowledge-share/documentChunk/update",
|
||||
"/public-api/knowledge-share/documentChunk/remove",
|
||||
"/public-api/knowledge-share/faq/save",
|
||||
"/public-api/knowledge-share/faq/update",
|
||||
"/public-api/knowledge-share/faq/remove"
|
||||
));
|
||||
|
||||
LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.VIEW.name(), List.of(
|
||||
"/public-api/knowledge-share/detail",
|
||||
"/public-api/knowledge-share/document/page",
|
||||
"/public-api/knowledge-share/document/download",
|
||||
@@ -42,10 +88,10 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
|
||||
"/public-api/knowledge-share/faq/page",
|
||||
"/public-api/knowledge-share/faq/detail"
|
||||
));
|
||||
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.SEARCH.name(), List.of(
|
||||
LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.SEARCH.name(), List.of(
|
||||
"/public-api/knowledge-share/search"
|
||||
));
|
||||
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_CREATE.name(), List.of(
|
||||
LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_CREATE.name(), List.of(
|
||||
"/public-api/knowledge-share/document/import/analyze",
|
||||
"/public-api/knowledge-share/document/import/preview",
|
||||
"/public-api/knowledge-share/document/import/commit",
|
||||
@@ -54,18 +100,21 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
|
||||
"/public-api/knowledge-share/document/import/task/startIndex",
|
||||
"/public-api/knowledge-share/document/import/task/retryParse",
|
||||
"/public-api/knowledge-share/document/import/task/retryIndex",
|
||||
"/public-api/knowledge-share/document/import/batch",
|
||||
"/public-api/knowledge-share/document/import/batch/status",
|
||||
"/public-api/knowledge-share/document/import/batch/retry",
|
||||
"/public-api/knowledge-share/faq/save"
|
||||
));
|
||||
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_UPDATE.name(), List.of(
|
||||
LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_UPDATE.name(), List.of(
|
||||
"/public-api/knowledge-share/documentChunk/update",
|
||||
"/public-api/knowledge-share/faq/update"
|
||||
));
|
||||
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_DELETE.name(), List.of(
|
||||
LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_DELETE.name(), List.of(
|
||||
"/public-api/knowledge-share/document/remove",
|
||||
"/public-api/knowledge-share/documentChunk/remove",
|
||||
"/public-api/knowledge-share/faq/remove"
|
||||
));
|
||||
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.IMPORT_EXPORT.name(), List.of(
|
||||
LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.IMPORT_EXPORT.name(), List.of(
|
||||
"/public-api/knowledge-share/faq/importExcel",
|
||||
"/public-api/knowledge-share/faq/exportExcel",
|
||||
"/public-api/knowledge-share/faq/downloadImportTemplate"
|
||||
@@ -78,6 +127,8 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
|
||||
private SysApiKeyResourceService resourceService;
|
||||
@Resource
|
||||
private SysApiKeyResourceMappingService mappingService;
|
||||
@Resource
|
||||
private RedisLockExecutor redisLockExecutor;
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
@@ -97,32 +148,48 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
|
||||
throw new BusinessException("动作范围不能为空");
|
||||
}
|
||||
|
||||
mappingService.removeScopedMappings(apiKeyId, RESOURCE_TYPE_KNOWLEDGE, knowledgeId);
|
||||
List<SysApiKeyResourceMapping> rows = new ArrayList<>();
|
||||
for (String scope : normalizedScopes) {
|
||||
List<String> uris = URI_SCOPE_MAPPING.get(scope);
|
||||
if (uris == null || uris.isEmpty()) {
|
||||
continue;
|
||||
Runnable releaseLock = acquirePermissionMutationLock(apiKeyId);
|
||||
try {
|
||||
mappingService.removeScopedMappings(
|
||||
apiKeyId,
|
||||
RESOURCE_TYPE_KNOWLEDGE,
|
||||
knowledgeId
|
||||
);
|
||||
List<SysApiKeyResourceMapping> rows = new ArrayList<>();
|
||||
for (String legacyScope : normalizedScopes) {
|
||||
List<String> uris = LEGACY_ACTION_URI_MAPPING.get(legacyScope);
|
||||
if (uris == null || uris.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
for (String uri : uris) {
|
||||
rows.add(buildMapping(
|
||||
apiKeyId,
|
||||
knowledgeId,
|
||||
uri,
|
||||
requireApiScope(uri)
|
||||
));
|
||||
}
|
||||
}
|
||||
for (String uri : uris) {
|
||||
SysApiKeyResource resource = ensureResource(uri);
|
||||
SysApiKeyResourceMapping row = new SysApiKeyResourceMapping();
|
||||
row.setApiKeyId(apiKeyId);
|
||||
row.setApiKeyResourceId(resource.getId());
|
||||
row.setResourceType(RESOURCE_TYPE_KNOWLEDGE);
|
||||
row.setResourceTargetId(knowledgeId);
|
||||
row.setActionScope(scope);
|
||||
rows.add(row);
|
||||
if (!rows.isEmpty()) {
|
||||
mappingService.saveBatch(rows);
|
||||
}
|
||||
}
|
||||
if (!rows.isEmpty()) {
|
||||
mappingService.saveBatch(rows);
|
||||
} finally {
|
||||
releaseLock.run();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void replaceApiShareEnabled(BigInteger apiKeyId, boolean enabled) {
|
||||
replaceApiPermissions(apiKeyId, enabled, enabled, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void replaceApiPermissions(BigInteger apiKeyId,
|
||||
boolean readEnabled,
|
||||
boolean importEnabled,
|
||||
boolean maintenanceEnabled) {
|
||||
if (apiKeyId == null) {
|
||||
throw new BusinessException("系统访问令牌不能为空");
|
||||
}
|
||||
@@ -130,30 +197,73 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
|
||||
if (apiKey == null) {
|
||||
throw new BusinessException("系统访问令牌不存在");
|
||||
}
|
||||
mappingService.removeScopedMappings(apiKeyId, RESOURCE_TYPE_KNOWLEDGE);
|
||||
if (!enabled) {
|
||||
return;
|
||||
}
|
||||
Runnable releaseLock = acquirePermissionMutationLock(apiKeyId);
|
||||
try {
|
||||
// 全局开关只替换全局授权,保留分享页配置的指定知识库权限。
|
||||
mappingService.remove(
|
||||
QueryWrapper.create()
|
||||
.eq(SysApiKeyResourceMapping::getApiKeyId, apiKeyId)
|
||||
.eq(
|
||||
SysApiKeyResourceMapping::getResourceType,
|
||||
RESOURCE_TYPE_KNOWLEDGE
|
||||
)
|
||||
.isNull(
|
||||
SysApiKeyResourceMapping::getResourceTargetId
|
||||
)
|
||||
);
|
||||
Set<String> enabledScopes = KnowledgeApiPermissionScope.enabledScopes(
|
||||
readEnabled,
|
||||
importEnabled,
|
||||
maintenanceEnabled
|
||||
);
|
||||
if (enabledScopes.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<SysApiKeyResourceMapping> rows = new ArrayList<>();
|
||||
for (String scope : KnowledgeShareActionScope.defaultApiScopes()) {
|
||||
List<String> uris = URI_SCOPE_MAPPING.get(scope);
|
||||
if (uris == null || uris.isEmpty()) {
|
||||
continue;
|
||||
List<SysApiKeyResourceMapping> rows = new ArrayList<>();
|
||||
for (String scope : enabledScopes) {
|
||||
List<String> uris = API_SCOPE_URI_MAPPING.get(scope);
|
||||
if (uris == null || uris.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
for (String uri : uris) {
|
||||
rows.add(buildMapping(apiKeyId, null, uri, scope));
|
||||
}
|
||||
}
|
||||
for (String uri : uris) {
|
||||
SysApiKeyResource resource = ensureResource(uri);
|
||||
SysApiKeyResourceMapping row = new SysApiKeyResourceMapping();
|
||||
row.setApiKeyId(apiKeyId);
|
||||
row.setApiKeyResourceId(resource.getId());
|
||||
row.setResourceType(RESOURCE_TYPE_KNOWLEDGE);
|
||||
row.setActionScope(scope);
|
||||
rows.add(row);
|
||||
if (!rows.isEmpty()) {
|
||||
mappingService.saveBatch(rows);
|
||||
}
|
||||
} finally {
|
||||
releaseLock.run();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getApiPermissionScopes(BigInteger apiKeyId) {
|
||||
if (apiKeyId == null) {
|
||||
return Set.of();
|
||||
}
|
||||
List<SysApiKeyResourceMapping> mappings = mappingService.list(
|
||||
QueryWrapper.create()
|
||||
.select(SysApiKeyResourceMapping::getActionScope)
|
||||
.eq(SysApiKeyResourceMapping::getApiKeyId, apiKeyId)
|
||||
.eq(SysApiKeyResourceMapping::getResourceType, RESOURCE_TYPE_KNOWLEDGE)
|
||||
.isNull(SysApiKeyResourceMapping::getResourceTargetId)
|
||||
.in(
|
||||
SysApiKeyResourceMapping::getActionScope,
|
||||
API_SCOPE_URI_MAPPING.keySet()
|
||||
)
|
||||
);
|
||||
if (mappings == null || mappings.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<String> scopes = new java.util.LinkedHashSet<>();
|
||||
for (SysApiKeyResourceMapping mapping : mappings) {
|
||||
if (mapping.getActionScope() != null) {
|
||||
scopes.add(mapping.getActionScope());
|
||||
}
|
||||
}
|
||||
if (!rows.isEmpty()) {
|
||||
mappingService.saveBatch(rows);
|
||||
}
|
||||
return scopes;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -161,9 +271,56 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
|
||||
if (apiKeyId == null || knowledgeId == null) {
|
||||
throw new BusinessException("API 分享鉴权参数不完整");
|
||||
}
|
||||
if (!API_SCOPE_URI_MAPPING.containsKey(actionScope)) {
|
||||
throw new IllegalArgumentException("未知的知识库 API 权限范围");
|
||||
}
|
||||
sysApiKeyService.checkResourceScope(apiKeyId, requestUri, RESOURCE_TYPE_KNOWLEDGE, knowledgeId, actionScope);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造单条知识库 API 权限映射。
|
||||
*
|
||||
* @param apiKeyId 访问令牌 ID
|
||||
* @param knowledgeId 知识库 ID;为空表示全局授权
|
||||
* @param uri 请求 URI
|
||||
* @param scope 产品权限 Scope
|
||||
* @return 权限映射
|
||||
*/
|
||||
private SysApiKeyResourceMapping buildMapping(BigInteger apiKeyId,
|
||||
BigInteger knowledgeId,
|
||||
String uri,
|
||||
String scope) {
|
||||
SysApiKeyResource resource = ensureResource(uri);
|
||||
SysApiKeyResourceMapping row = new SysApiKeyResourceMapping();
|
||||
row.setApiKeyId(apiKeyId);
|
||||
row.setApiKeyResourceId(resource.getId());
|
||||
row.setResourceType(RESOURCE_TYPE_KNOWLEDGE);
|
||||
row.setResourceTargetId(knowledgeId);
|
||||
row.setActionScope(scope);
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 URI 获取唯一的产品权限 Scope。
|
||||
*
|
||||
* @param uri 请求 URI
|
||||
* @return 产品权限 Scope
|
||||
*/
|
||||
private String requireApiScope(String uri) {
|
||||
for (Map.Entry<String, List<String>> entry : API_SCOPE_URI_MAPPING.entrySet()) {
|
||||
if (entry.getValue().contains(uri)) {
|
||||
return entry.getKey();
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("知识库接口未归类: " + uri);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取或创建固定 URI 对应的 API 资源。
|
||||
*
|
||||
* @param requestInterface 请求 URI
|
||||
* @return API 资源
|
||||
*/
|
||||
private SysApiKeyResource ensureResource(String requestInterface) {
|
||||
QueryWrapper wrapper = QueryWrapper.create()
|
||||
.eq(SysApiKeyResource::getRequestInterface, requestInterface);
|
||||
@@ -177,4 +334,35 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
|
||||
resourceService.save(resource);
|
||||
return resource;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取访问令牌知识库权限变更锁,并在事务完成后释放。
|
||||
*
|
||||
* @param apiKeyId 访问令牌 ID
|
||||
* @return 非事务直接调用时使用的释放动作
|
||||
*/
|
||||
private Runnable acquirePermissionMutationLock(BigInteger apiKeyId) {
|
||||
LockHandle handle = redisLockExecutor.tryAcquire(
|
||||
PERMISSION_LOCK_PREFIX + apiKeyId,
|
||||
PERMISSION_LOCK_WAIT,
|
||||
PERMISSION_LOCK_LEASE
|
||||
);
|
||||
if (handle == null) {
|
||||
throw new BusinessException("访问令牌权限正在更新,请稍后重试");
|
||||
}
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()
|
||||
&& TransactionSynchronizationManager.isActualTransactionActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(
|
||||
new TransactionSynchronization() {
|
||||
@Override
|
||||
public void afterCompletion(int status) {
|
||||
handle.release();
|
||||
}
|
||||
}
|
||||
);
|
||||
return () -> {
|
||||
};
|
||||
}
|
||||
return handle::release;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +108,26 @@ public class DocumentParseBridgeServiceImplTest {
|
||||
Assert.assertEquals("# demo", taskInfo.getResult().getPreferredText());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证带 PDF 源信息的任务查询会直达 PDF 服务,不试探无关的 PPTX 服务。
|
||||
*/
|
||||
@Test
|
||||
public void shouldQueryPdfTaskAgainstResolvedService() {
|
||||
FakePdfDocumentParseService pdfService = new FakePdfDocumentParseService();
|
||||
pdfService.taskStatusValue = "completed";
|
||||
FakePptxDocumentParseService pptxService = new FakePptxDocumentParseService();
|
||||
DocumentParseBridgeServiceImpl bridgeService =
|
||||
buildBridgeService(pdfService, pptxService, null, pdfService);
|
||||
|
||||
DocumentParseTaskInfo taskInfo = bridgeService.queryTaskInfo("task-1", buildSource());
|
||||
DocumentParsedResult result = bridgeService.queryResult("task-1", buildSource());
|
||||
|
||||
Assert.assertEquals("completed", taskInfo.getStatus());
|
||||
Assert.assertEquals("# demo", result.getPreferredText());
|
||||
Assert.assertEquals(0, pptxService.queryTaskInfoCallCount);
|
||||
Assert.assertEquals(0, pptxService.queryResultCallCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证缺少底层服务时抛出稳定错误码。
|
||||
*/
|
||||
@@ -390,6 +410,8 @@ public class DocumentParseBridgeServiceImplTest {
|
||||
private static class FakePptxDocumentParseService implements PptxDocumentParseService {
|
||||
|
||||
private int parseCallCount;
|
||||
private int queryTaskInfoCallCount;
|
||||
private int queryResultCallCount;
|
||||
|
||||
@Override
|
||||
public ParseResponse parse(ParseRequest request) {
|
||||
@@ -415,6 +437,13 @@ public class DocumentParseBridgeServiceImplTest {
|
||||
|
||||
@Override
|
||||
public ParseResponse queryResult(String taskId) {
|
||||
queryResultCallCount++;
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ParseTaskInfo queryTaskInfo(String taskId) {
|
||||
queryTaskInfoCallCount++;
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,913 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult;
|
||||
import tech.easyflow.ai.documentimport.ImportCallerContext;
|
||||
import tech.easyflow.ai.documentimport.ImportCallerType;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchService;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.filestorage.FileStorageWriteHandle;
|
||||
import tech.easyflow.common.filestorage.FileStorageWriteResult;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* {@link DocumentImportBatchAppService} 批次启动与重复策略回归测试。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-07-31
|
||||
*/
|
||||
public class DocumentImportBatchAppServiceTest {
|
||||
|
||||
/**
|
||||
* 验证上传领取通过单条多表更新同步刷新批次进度时间。
|
||||
*/
|
||||
@Test
|
||||
public void uploadShouldAtomicallyClaimItemAndTouchBatch() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
|
||||
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(41));
|
||||
DocumentImportBatchItem uploaded = pendingUploadItem(BigInteger.valueOf(41));
|
||||
uploaded.setStatus(DocumentImportBatchItemStatus.UPLOADED.name());
|
||||
uploaded.setFilePath("/stored/demo.docx");
|
||||
FileStorageWriteHandle handle = writeHandle(pending);
|
||||
String locator = handle.encodeLocator();
|
||||
uploaded.setStorageLocator(locator);
|
||||
uploaded.setCleanupPending(false);
|
||||
Mockito.when(context.batchTracker.requireItem(pending.getId()))
|
||||
.thenReturn(pending, uploaded);
|
||||
Mockito.when(context.itemMapper.claimUpload(
|
||||
Mockito.eq(batch.getId()),
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(batch.getKnowledgeId()),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.itemMapper.registerUploadWriteIntent(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.batchTracker.completeUpload(
|
||||
pending.getId(),
|
||||
"/stored/demo.docx",
|
||||
locator
|
||||
)).thenReturn(true);
|
||||
FileStorageService storage = Mockito.mock(FileStorageService.class);
|
||||
Mockito.when(storage.prepareRecoverableWrite(
|
||||
Mockito.anyString(),
|
||||
Mockito.anyString()
|
||||
)).thenReturn(handle);
|
||||
Mockito.when(storage.saveRecoverable(
|
||||
Mockito.any(MultipartFile.class),
|
||||
Mockito.eq(handle)
|
||||
)).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator));
|
||||
setStorageService(context.service, storage);
|
||||
MultipartFile file = uploadFile("demo.docx", pending.getFileSize());
|
||||
|
||||
DocumentImportBatchDtos.ItemResponse response = context.service.uploadItem(
|
||||
batch.getKnowledgeId(),
|
||||
batch.getId(),
|
||||
pending.getId(),
|
||||
file,
|
||||
publicCaller()
|
||||
);
|
||||
|
||||
Assert.assertEquals(
|
||||
DocumentImportBatchItemStatus.UPLOADED.name(),
|
||||
response.getStatus()
|
||||
);
|
||||
org.mockito.InOrder order = Mockito.inOrder(context.itemMapper, storage);
|
||||
order.verify(context.itemMapper).claimUpload(
|
||||
Mockito.eq(batch.getId()),
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(batch.getKnowledgeId()),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
order.verify(context.itemMapper).registerUploadWriteIntent(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
order.verify(storage).saveRecoverable(file, handle);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证对象绑定失败后先清理,再恢复为可重新上传状态。
|
||||
*/
|
||||
@Test
|
||||
public void uploadBindFailureShouldCleanupAndAllowSameItemRetry() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
|
||||
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(42));
|
||||
FileStorageWriteHandle handle = writeHandle(pending);
|
||||
String locator = handle.encodeLocator();
|
||||
DocumentImportBatchItem cleanup = pendingUploadItem(pending.getId());
|
||||
cleanup.setStatus(DocumentImportBatchItemStatus.UPLOADING.name());
|
||||
cleanup.setStorageLocator(locator);
|
||||
cleanup.setFilePath("/stored/demo.docx");
|
||||
cleanup.setCleanupPending(true);
|
||||
DocumentImportBatchItem retryPending = pendingUploadItem(pending.getId());
|
||||
DocumentImportBatchItem uploaded = pendingUploadItem(pending.getId());
|
||||
uploaded.setStatus(DocumentImportBatchItemStatus.UPLOADED.name());
|
||||
uploaded.setStorageLocator(locator);
|
||||
uploaded.setFilePath("/stored/demo.docx");
|
||||
Mockito.when(context.batchTracker.requireItem(pending.getId()))
|
||||
.thenReturn(pending, pending, cleanup, retryPending, uploaded);
|
||||
Mockito.when(context.itemMapper.claimUpload(
|
||||
Mockito.eq(batch.getId()),
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(batch.getKnowledgeId()),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1, 1);
|
||||
Mockito.when(context.itemMapper.registerUploadWriteIntent(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1, 1);
|
||||
Mockito.when(context.batchTracker.completeUpload(
|
||||
pending.getId(),
|
||||
"/stored/demo.docx",
|
||||
locator
|
||||
)).thenReturn(false, true);
|
||||
Mockito.when(context.itemMapper.markUploadCleanupPending(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.itemMapper.completeUploadingStorageCleanup(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.eq("/stored/demo.docx"),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
FileStorageService storage = Mockito.mock(FileStorageService.class);
|
||||
Mockito.when(storage.prepareRecoverableWrite(
|
||||
Mockito.anyString(),
|
||||
Mockito.anyString()
|
||||
)).thenReturn(handle);
|
||||
Mockito.when(storage.saveRecoverable(
|
||||
Mockito.any(MultipartFile.class),
|
||||
Mockito.eq(handle)
|
||||
)).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator));
|
||||
setStorageService(context.service, storage);
|
||||
MultipartFile file = uploadFile("demo.docx", pending.getFileSize());
|
||||
|
||||
try {
|
||||
context.service.uploadItem(
|
||||
batch.getKnowledgeId(),
|
||||
batch.getId(),
|
||||
pending.getId(),
|
||||
file,
|
||||
publicCaller()
|
||||
);
|
||||
Assert.fail("Expected cancelled upload rejection");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("上传批次已取消"));
|
||||
}
|
||||
|
||||
DocumentImportBatchDtos.ItemResponse retried = context.service.uploadItem(
|
||||
batch.getKnowledgeId(),
|
||||
batch.getId(),
|
||||
pending.getId(),
|
||||
file,
|
||||
publicCaller()
|
||||
);
|
||||
|
||||
Assert.assertEquals(
|
||||
DocumentImportBatchItemStatus.UPLOADED.name(),
|
||||
retried.getStatus()
|
||||
);
|
||||
org.mockito.InOrder order = Mockito.inOrder(context.itemMapper, storage);
|
||||
order.verify(context.itemMapper).registerUploadWriteIntent(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
order.verify(storage).saveRecoverable(file, handle);
|
||||
order.verify(context.itemMapper).markUploadCleanupPending(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
order.verify(storage).deleteRecoverable(handle);
|
||||
order.verify(context.itemMapper).completeUploadingStorageCleanup(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.eq("/stored/demo.docx"),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
Mockito.verify(context.itemMapper, Mockito.times(2)).claimUpload(
|
||||
Mockito.eq(batch.getId()),
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(batch.getKnowledgeId()),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
Mockito.verify(context.batchTracker, Mockito.never()).transitionItem(
|
||||
Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(),
|
||||
Mockito.anyBoolean(), Mockito.anyInt()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证批次并发取消后,对象清理不会把文件项恢复为待上传。
|
||||
*/
|
||||
@Test
|
||||
public void uploadCleanupShouldPreserveConcurrentCancellation() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
|
||||
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(44));
|
||||
FileStorageWriteHandle handle = writeHandle(pending);
|
||||
String locator = handle.encodeLocator();
|
||||
DocumentImportBatchItem cancelled = pendingUploadItem(pending.getId());
|
||||
cancelled.setStatus(DocumentImportBatchItemStatus.CANCELLED.name());
|
||||
cancelled.setStorageLocator(locator);
|
||||
cancelled.setFilePath("/stored/demo.docx");
|
||||
cancelled.setCleanupPending(true);
|
||||
Mockito.when(context.batchTracker.requireItem(pending.getId()))
|
||||
.thenReturn(pending, pending, cancelled);
|
||||
Mockito.when(context.itemMapper.claimUpload(
|
||||
Mockito.eq(batch.getId()),
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(batch.getKnowledgeId()),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.itemMapper.registerUploadWriteIntent(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.batchTracker.completeUpload(
|
||||
pending.getId(),
|
||||
"/stored/demo.docx",
|
||||
locator
|
||||
)).thenReturn(false);
|
||||
Mockito.when(context.itemMapper.markUploadCleanupPending(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.itemMapper.completeCancelledStorageCleanup(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.eq("/stored/demo.docx"),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
FileStorageService storage = Mockito.mock(FileStorageService.class);
|
||||
Mockito.when(storage.prepareRecoverableWrite(
|
||||
Mockito.anyString(),
|
||||
Mockito.anyString()
|
||||
)).thenReturn(handle);
|
||||
Mockito.when(storage.saveRecoverable(
|
||||
Mockito.any(MultipartFile.class),
|
||||
Mockito.eq(handle)
|
||||
)).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator));
|
||||
setStorageService(context.service, storage);
|
||||
|
||||
try {
|
||||
context.service.uploadItem(
|
||||
batch.getKnowledgeId(),
|
||||
batch.getId(),
|
||||
pending.getId(),
|
||||
uploadFile("demo.docx", pending.getFileSize()),
|
||||
publicCaller()
|
||||
);
|
||||
Assert.fail("Expected cancelled upload rejection");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("上传批次已取消"));
|
||||
}
|
||||
|
||||
Mockito.verify(storage).deleteRecoverable(handle);
|
||||
Mockito.verify(context.itemMapper).completeCancelledStorageCleanup(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.eq("/stored/demo.docx"),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证对象删除后的数据库恢复失败会保留待办,并由下一轮幂等完成。
|
||||
*/
|
||||
@Test
|
||||
public void cleanupRecoveryFailureShouldRemainRetryable() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatchItem cleanup = pendingUploadItem(BigInteger.valueOf(45));
|
||||
FileStorageWriteHandle handle = writeHandle(cleanup);
|
||||
String locator = handle.encodeLocator();
|
||||
cleanup.setStatus(DocumentImportBatchItemStatus.UPLOADING.name());
|
||||
cleanup.setStorageLocator(locator);
|
||||
cleanup.setFilePath("/stored/demo.docx");
|
||||
cleanup.setCleanupPending(true);
|
||||
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
||||
.thenReturn(List.of(cleanup));
|
||||
Mockito.when(context.itemMapper.completeUploadingStorageCleanup(
|
||||
Mockito.eq(cleanup.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.eq("/stored/demo.docx"),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class)
|
||||
)).thenThrow(new RuntimeException("database unavailable"))
|
||||
.thenReturn(1);
|
||||
FileStorageService storage = Mockito.mock(FileStorageService.class);
|
||||
setStorageService(context.service, storage);
|
||||
|
||||
Assert.assertEquals(0, context.service.cleanupCancelledStoredObjects(10));
|
||||
Assert.assertEquals(1, context.service.cleanupCancelledStoredObjects(10));
|
||||
|
||||
Mockito.verify(storage, Mockito.times(2)).deleteRecoverable(handle);
|
||||
Mockito.verify(context.itemMapper, Mockito.times(2))
|
||||
.completeUploadingStorageCleanup(
|
||||
Mockito.eq(cleanup.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.eq("/stored/demo.docx"),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证写意图登记已提交但调用抛异常时,写入前撤销会清理定位符。
|
||||
*/
|
||||
@Test
|
||||
public void uploadIntentUnknownCommitShouldAbortBeforePhysicalWrite() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
|
||||
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(46));
|
||||
FileStorageWriteHandle handle = writeHandle(pending);
|
||||
String locator = handle.encodeLocator();
|
||||
Mockito.when(context.batchTracker.requireItem(pending.getId()))
|
||||
.thenReturn(pending);
|
||||
Mockito.when(context.itemMapper.claimUpload(
|
||||
Mockito.eq(batch.getId()),
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(batch.getKnowledgeId()),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.itemMapper.registerUploadWriteIntent(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
)).thenThrow(new RuntimeException("commit result unknown"));
|
||||
Mockito.when(context.itemMapper.abortUploadBeforeWrite(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
FileStorageService storage = Mockito.mock(FileStorageService.class);
|
||||
Mockito.when(storage.prepareRecoverableWrite(
|
||||
Mockito.anyString(),
|
||||
Mockito.anyString()
|
||||
)).thenReturn(handle);
|
||||
setStorageService(context.service, storage);
|
||||
|
||||
try {
|
||||
context.service.uploadItem(
|
||||
batch.getKnowledgeId(),
|
||||
batch.getId(),
|
||||
pending.getId(),
|
||||
uploadFile("demo.docx", pending.getFileSize()),
|
||||
publicCaller()
|
||||
);
|
||||
Assert.fail("Expected unknown commit exception");
|
||||
} catch (RuntimeException expected) {
|
||||
Assert.assertEquals("commit result unknown", expected.getMessage());
|
||||
}
|
||||
|
||||
Mockito.verify(context.itemMapper).abortUploadBeforeWrite(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
Mockito.verify(storage, Mockito.never()).saveRecoverable(
|
||||
Mockito.any(MultipartFile.class),
|
||||
Mockito.any(FileStorageWriteHandle.class)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证提交响应异常但数据库已完成上传时不会误删有效对象。
|
||||
*/
|
||||
@Test
|
||||
public void uploadCommitUnknownShouldKeepConfirmedUploadedObject() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
|
||||
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(43));
|
||||
FileStorageWriteHandle handle = writeHandle(pending);
|
||||
String locator = handle.encodeLocator();
|
||||
DocumentImportBatchItem committed = pendingUploadItem(pending.getId());
|
||||
committed.setStatus(DocumentImportBatchItemStatus.UPLOADED.name());
|
||||
committed.setFilePath("/stored/demo.docx");
|
||||
committed.setStorageLocator(locator);
|
||||
committed.setCleanupPending(false);
|
||||
Mockito.when(context.batchTracker.requireItem(pending.getId()))
|
||||
.thenReturn(pending, committed);
|
||||
Mockito.when(context.itemMapper.claimUpload(
|
||||
Mockito.eq(batch.getId()),
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(batch.getKnowledgeId()),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.itemMapper.registerUploadWriteIntent(
|
||||
Mockito.eq(pending.getId()),
|
||||
Mockito.eq(locator),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.batchTracker.completeUpload(
|
||||
pending.getId(),
|
||||
"/stored/demo.docx",
|
||||
locator
|
||||
)).thenThrow(new RuntimeException("commit result unknown"));
|
||||
FileStorageService storage = Mockito.mock(FileStorageService.class);
|
||||
Mockito.when(storage.prepareRecoverableWrite(
|
||||
Mockito.anyString(),
|
||||
Mockito.anyString()
|
||||
)).thenReturn(handle);
|
||||
Mockito.when(storage.saveRecoverable(
|
||||
Mockito.any(MultipartFile.class),
|
||||
Mockito.eq(handle)
|
||||
)).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator));
|
||||
setStorageService(context.service, storage);
|
||||
|
||||
DocumentImportBatchDtos.ItemResponse response = context.service.uploadItem(
|
||||
batch.getKnowledgeId(),
|
||||
batch.getId(),
|
||||
pending.getId(),
|
||||
uploadFile("demo.docx", pending.getFileSize()),
|
||||
publicCaller()
|
||||
);
|
||||
|
||||
Assert.assertEquals(
|
||||
DocumentImportBatchItemStatus.UPLOADED.name(),
|
||||
response.getStatus()
|
||||
);
|
||||
Mockito.verify(context.itemMapper, Mockito.never())
|
||||
.markUploadCleanupPending(
|
||||
Mockito.any(),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
Mockito.verify(storage, Mockito.never())
|
||||
.deleteRecoverable(Mockito.any(FileStorageWriteHandle.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证同一知识库已有运行中的自动批次时拒绝再次启动。
|
||||
*/
|
||||
@Test
|
||||
public void startAutoShouldRejectOtherRunningBatch() {
|
||||
TestContext context = createContext();
|
||||
Mockito.when(context.batchService.count(Mockito.any(QueryWrapper.class))).thenReturn(1L);
|
||||
beginTransactionSynchronization();
|
||||
try {
|
||||
context.service.startBatch(startRequest("AUTO", "SKIP"));
|
||||
Assert.fail("Expected active automatic batch rejection");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("已有自动导入批次"));
|
||||
} finally {
|
||||
completeTransactionSynchronization(TransactionSynchronization.STATUS_ROLLED_BACK);
|
||||
}
|
||||
Mockito.verify(context.batchMapper, Mockito.never())
|
||||
.updateByQuery(Mockito.any(DocumentImportBatch.class), Mockito.any());
|
||||
Mockito.verify(context.lockHandle).release();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证覆盖策略会持久化历史文档 ID,并继续创建新文档任务。
|
||||
*/
|
||||
@Test
|
||||
public void startOverwriteShouldRecordHistoricalDocument() {
|
||||
TestContext context = createContext();
|
||||
Mockito.when(context.batchService.count(Mockito.any(QueryWrapper.class))).thenReturn(0L);
|
||||
DocumentImportBatchItem uploaded = uploadedItem(BigInteger.valueOf(11), BigInteger.ONE);
|
||||
DocumentImportBatchItem historical = uploadedItem(BigInteger.valueOf(21), BigInteger.valueOf(22));
|
||||
historical.setBatchId(BigInteger.valueOf(99));
|
||||
historical.setDocumentId(BigInteger.valueOf(23));
|
||||
historical.setStatus(DocumentImportBatchItemStatus.COMPLETED.name());
|
||||
historical.setCreated(new Date());
|
||||
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
||||
.thenReturn(List.of(uploaded), List.of(historical));
|
||||
tech.easyflow.ai.entity.Document historicalDocument = new tech.easyflow.ai.entity.Document();
|
||||
historicalDocument.setId(historical.getDocumentId());
|
||||
historicalDocument.setCollectionId(BigInteger.TWO);
|
||||
Mockito.when(context.documentMapper.selectListByQuery(Mockito.any()))
|
||||
.thenReturn(List.of(historicalDocument));
|
||||
Mockito.when(context.batchMapper.updateByQuery(
|
||||
Mockito.any(DocumentImportBatch.class), Mockito.any()
|
||||
)).thenReturn(1);
|
||||
Mockito.when(context.batchTracker.refreshBatch(BigInteger.ONE))
|
||||
.thenReturn(new DocumentImportBatchDtos.StatusResponse());
|
||||
|
||||
beginTransactionSynchronization();
|
||||
try {
|
||||
context.service.startBatch(startRequest("AUTO", "OVERWRITE"));
|
||||
} finally {
|
||||
completeTransactionSynchronization(TransactionSynchronization.STATUS_COMMITTED);
|
||||
}
|
||||
|
||||
Mockito.verify(context.batchTracker)
|
||||
.markReplacement(uploaded.getId(), historical.getDocumentId());
|
||||
Mockito.verify(context.taskAppService)
|
||||
.createBatchImportTasks(Mockito.any(DocumentImportBatch.class), Mockito.anyList());
|
||||
Mockito.verify(context.lockHandle).release();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证重试代次由服务端读取并原子领取。
|
||||
*/
|
||||
@Test
|
||||
public void retryShouldClaimCurrentGeneration() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
|
||||
batch.setRetryGeneration(0);
|
||||
DocumentImportBatchItem failed = uploadedItem(
|
||||
BigInteger.valueOf(31),
|
||||
batch.getId()
|
||||
);
|
||||
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||
failed.setRetryable(true);
|
||||
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
||||
.thenReturn(List.of(failed));
|
||||
Mockito.when(context.batchMapper.claimRetry(
|
||||
Mockito.any(),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(),
|
||||
Mockito.anyInt(),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
|
||||
beginTransactionSynchronization();
|
||||
DocumentImportBatchRetryResult first;
|
||||
try {
|
||||
first = context.service.retryOwnedBatch(
|
||||
batch.getId(),
|
||||
publicCaller(),
|
||||
Set.of("file-key")
|
||||
);
|
||||
} finally {
|
||||
completeTransactionSynchronization(
|
||||
TransactionSynchronization.STATUS_ROLLED_BACK
|
||||
);
|
||||
}
|
||||
|
||||
Assert.assertEquals(Integer.valueOf(1), first.getRetryGeneration());
|
||||
Assert.assertEquals(Integer.valueOf(1), first.getRetriedCount());
|
||||
Mockito.verify(context.batchMapper).claimRetry(
|
||||
Mockito.any(),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(),
|
||||
Mockito.eq(0),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
Mockito.verify(context.lockHandle).release();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证修复前的可恢复失败项会在重试领取事务中恢复重试资格和批次计数。
|
||||
*/
|
||||
@Test
|
||||
public void retryShouldRestoreLegacyRecoverableItem() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
|
||||
batch.setRetryGeneration(0);
|
||||
DocumentImportBatchItem failed = uploadedItem(
|
||||
BigInteger.valueOf(32),
|
||||
batch.getId()
|
||||
);
|
||||
failed.setStage(DocumentImportBatchItemStage.PARSE.name());
|
||||
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||
failed.setRetryable(false);
|
||||
failed.setFailureCode("parse_failed");
|
||||
failed.setErrorSummary("历史代码异常");
|
||||
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
||||
.thenReturn(List.of(failed));
|
||||
Mockito.when(context.taskAppService.isRecoverableBatchFailure(failed))
|
||||
.thenReturn(true);
|
||||
Mockito.when(context.batchTracker.transitionItem(
|
||||
Mockito.eq(failed.getId()),
|
||||
Mockito.eq(DocumentImportBatchItemStage.PARSE),
|
||||
Mockito.eq(DocumentImportBatchItemStatus.FAILED),
|
||||
Mockito.eq("历史代码异常"),
|
||||
Mockito.eq(true),
|
||||
Mockito.eq(0),
|
||||
Mockito.eq("parse_failed")
|
||||
)).thenReturn(true);
|
||||
Mockito.when(context.batchMapper.claimRetry(
|
||||
Mockito.any(),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(),
|
||||
Mockito.anyInt(),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
|
||||
beginTransactionSynchronization();
|
||||
DocumentImportBatchRetryResult result;
|
||||
try {
|
||||
result = context.service.retryOwnedBatch(
|
||||
batch.getId(),
|
||||
publicCaller(),
|
||||
Set.of()
|
||||
);
|
||||
} finally {
|
||||
completeTransactionSynchronization(
|
||||
TransactionSynchronization.STATUS_ROLLED_BACK
|
||||
);
|
||||
}
|
||||
|
||||
Assert.assertEquals(Integer.valueOf(1), result.getRetriedCount());
|
||||
Assert.assertTrue(failed.getRetryable());
|
||||
Mockito.verify(context.batchTracker).transitionItem(
|
||||
failed.getId(),
|
||||
DocumentImportBatchItemStage.PARSE,
|
||||
DocumentImportBatchItemStatus.FAILED,
|
||||
"历史代码异常",
|
||||
true,
|
||||
0,
|
||||
"parse_failed"
|
||||
);
|
||||
Mockito.verify(context.batchMapper).claimRetry(
|
||||
Mockito.eq(batch.getId()),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(),
|
||||
Mockito.eq(0),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
Mockito.verify(context.lockHandle).release();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证运行中的任务拒绝再次领取,调用方可继续查询原 taskId。
|
||||
*/
|
||||
@Test
|
||||
public void retryShouldRejectTaskThatIsAlreadyRunning() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
|
||||
|
||||
beginTransactionSynchronization();
|
||||
try {
|
||||
context.service.retryOwnedBatch(
|
||||
BigInteger.ONE,
|
||||
publicCaller(),
|
||||
Set.of()
|
||||
);
|
||||
Assert.fail("Expected running task rejection");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(
|
||||
expected.getMessage().contains("当前任务状态不允许重试")
|
||||
);
|
||||
} finally {
|
||||
completeTransactionSynchronization(
|
||||
TransactionSynchronization.STATUS_ROLLED_BACK
|
||||
);
|
||||
}
|
||||
Mockito.verify(context.batchMapper, Mockito.never()).claimRetry(
|
||||
Mockito.any(),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(),
|
||||
Mockito.anyInt(),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
Mockito.verify(context.lockHandle).release();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证候选快照过期后若任务已恢复进展,不会继续取消或清理文件项。
|
||||
*/
|
||||
@Test
|
||||
public void staleCancellationShouldStopWhenConditionalClaimMisses() {
|
||||
TestContext context = createContext();
|
||||
Mockito.when(context.batchMapper.claimStaleCancellation(
|
||||
Mockito.any(),
|
||||
Mockito.any(),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(0);
|
||||
|
||||
beginTransactionSynchronization();
|
||||
boolean cancelled;
|
||||
try {
|
||||
cancelled = context.service.cancelStaleBatch(
|
||||
BigInteger.TWO,
|
||||
BigInteger.ONE,
|
||||
publicCaller(),
|
||||
new Date(System.currentTimeMillis() - 30L * 60L * 1000L)
|
||||
);
|
||||
} finally {
|
||||
completeTransactionSynchronization(
|
||||
TransactionSynchronization.STATUS_ROLLED_BACK
|
||||
);
|
||||
}
|
||||
|
||||
Assert.assertFalse(cancelled);
|
||||
Mockito.verify(context.itemService, Mockito.never())
|
||||
.list(Mockito.any(QueryWrapper.class));
|
||||
Mockito.verifyNoInteractions(context.itemMapper);
|
||||
Mockito.verify(context.lockHandle).release();
|
||||
}
|
||||
|
||||
private TestContext createContext() {
|
||||
DocumentImportBatchService batchService = Mockito.mock(DocumentImportBatchService.class);
|
||||
DocumentImportBatchItemService itemService = Mockito.mock(DocumentImportBatchItemService.class);
|
||||
DocumentImportBatchTracker batchTracker = Mockito.mock(DocumentImportBatchTracker.class);
|
||||
KnowledgeDocumentImportTaskAppService taskAppService =
|
||||
Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
|
||||
DocumentImportBatchMapper batchMapper = Mockito.mock(DocumentImportBatchMapper.class);
|
||||
DocumentImportBatchItemMapper itemMapper =
|
||||
Mockito.mock(DocumentImportBatchItemMapper.class);
|
||||
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
|
||||
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
|
||||
Mockito.when(redisLockExecutor.tryAcquire(
|
||||
Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||
)).thenReturn(lockHandle);
|
||||
|
||||
DocumentImportBatch batch = new DocumentImportBatch();
|
||||
batch.setId(BigInteger.ONE);
|
||||
batch.setKnowledgeId(BigInteger.TWO);
|
||||
batch.setStatus(DocumentImportBatchStatus.READY.name());
|
||||
Mockito.when(batchTracker.requireBatch(BigInteger.ONE)).thenReturn(batch);
|
||||
Mockito.when(batchService.getOne(Mockito.any(QueryWrapper.class)))
|
||||
.thenReturn(batch);
|
||||
Mockito.when(batchMapper.selectOwnedForUpdate(
|
||||
Mockito.any(), Mockito.anyString(), Mockito.any()
|
||||
)).thenReturn(batch);
|
||||
|
||||
DocumentImportBatchAppService service = new DocumentImportBatchAppService(
|
||||
batchService,
|
||||
itemService,
|
||||
batchTracker,
|
||||
new DocumentImportBulkProperties(),
|
||||
taskAppService,
|
||||
batchMapper,
|
||||
itemMapper,
|
||||
documentMapper,
|
||||
redisLockExecutor
|
||||
);
|
||||
return new TestContext(
|
||||
service, batchService, itemService, batchTracker,
|
||||
taskAppService, batchMapper, itemMapper, documentMapper, lockHandle
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 Public API 测试调用者。
|
||||
*
|
||||
* @return Public API 调用者
|
||||
*/
|
||||
private ImportCallerContext publicCaller() {
|
||||
return new ImportCallerContext(
|
||||
ImportCallerType.PUBLIC_API,
|
||||
BigInteger.valueOf(77)
|
||||
);
|
||||
}
|
||||
|
||||
private DocumentImportBatchDtos.StartRequest startRequest(String mode, String duplicatePolicy) {
|
||||
DocumentImportBatchDtos.StartRequest request = new DocumentImportBatchDtos.StartRequest();
|
||||
request.setKnowledgeId(BigInteger.TWO);
|
||||
request.setBatchId(BigInteger.ONE);
|
||||
request.setImportMode(mode);
|
||||
request.setDuplicatePolicy(duplicatePolicy);
|
||||
return request;
|
||||
}
|
||||
|
||||
private DocumentImportBatchItem uploadedItem(BigInteger itemId, BigInteger batchId) {
|
||||
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||
item.setId(itemId);
|
||||
item.setBatchId(batchId);
|
||||
item.setKnowledgeId(BigInteger.TWO);
|
||||
item.setClientFileKey("file-key");
|
||||
item.setFileName("demo.docx");
|
||||
item.setFilePath("/demo.docx");
|
||||
item.setStatus(DocumentImportBatchItemStatus.UPLOADED.name());
|
||||
return item;
|
||||
}
|
||||
|
||||
private DocumentImportBatchItem pendingUploadItem(BigInteger itemId) {
|
||||
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||
item.setId(itemId);
|
||||
item.setBatchId(BigInteger.ONE);
|
||||
item.setKnowledgeId(BigInteger.TWO);
|
||||
item.setClientFileKey("file-key-" + itemId);
|
||||
item.setFileName("demo.docx");
|
||||
item.setRelativePath("demo.docx");
|
||||
item.setFileSize(4L);
|
||||
item.setStage("UPLOAD");
|
||||
item.setStatus(DocumentImportBatchItemStatus.PENDING.name());
|
||||
item.setCleanupPending(false);
|
||||
return item;
|
||||
}
|
||||
|
||||
private FileStorageWriteHandle writeHandle(DocumentImportBatchItem item) {
|
||||
return new FileStorageWriteHandle(
|
||||
"mock-storage",
|
||||
"",
|
||||
"/tmp/easyflow-test",
|
||||
"knowledge-import/" + item.getBatchId() + "/" + item.getId(),
|
||||
item.getId() + ".docx"
|
||||
);
|
||||
}
|
||||
|
||||
private MultipartFile uploadFile(String fileName, long size) {
|
||||
MultipartFile file = Mockito.mock(MultipartFile.class);
|
||||
Mockito.when(file.getOriginalFilename()).thenReturn(fileName);
|
||||
Mockito.when(file.getSize()).thenReturn(size);
|
||||
Mockito.when(file.isEmpty()).thenReturn(false);
|
||||
return file;
|
||||
}
|
||||
|
||||
private void setStorageService(DocumentImportBatchAppService service,
|
||||
FileStorageService storageService) {
|
||||
try {
|
||||
Field field = DocumentImportBatchAppService.class.getDeclaredField(
|
||||
"storageService"
|
||||
);
|
||||
field.setAccessible(true);
|
||||
field.set(service, storageService);
|
||||
} catch (ReflectiveOperationException error) {
|
||||
throw new AssertionError("Failed to inject storage service", error);
|
||||
}
|
||||
}
|
||||
|
||||
private void beginTransactionSynchronization() {
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
}
|
||||
|
||||
private void completeTransactionSynchronization(int status) {
|
||||
List<TransactionSynchronization> synchronizations =
|
||||
TransactionSynchronizationManager.getSynchronizations();
|
||||
for (TransactionSynchronization synchronization : synchronizations) {
|
||||
if (status == TransactionSynchronization.STATUS_COMMITTED) {
|
||||
synchronization.afterCommit();
|
||||
}
|
||||
synchronization.afterCompletion(status);
|
||||
}
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试依赖集合。
|
||||
*/
|
||||
private record TestContext(
|
||||
DocumentImportBatchAppService service,
|
||||
DocumentImportBatchService batchService,
|
||||
DocumentImportBatchItemService itemService,
|
||||
DocumentImportBatchTracker batchTracker,
|
||||
KnowledgeDocumentImportTaskAppService taskAppService,
|
||||
DocumentImportBatchMapper batchMapper,
|
||||
DocumentImportBatchItemMapper itemMapper,
|
||||
DocumentMapper documentMapper,
|
||||
RedisLockExecutor.LockHandle lockHandle
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportMode;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 文档批量导入状态汇总测试。
|
||||
*/
|
||||
public class DocumentImportBatchTrackerTest {
|
||||
|
||||
/**
|
||||
* 验证运行批次的完成、处理、失败与等待数量。
|
||||
*/
|
||||
@Test
|
||||
public void shouldAggregateRunningBatchProgress() {
|
||||
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
|
||||
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
|
||||
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
|
||||
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
|
||||
DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 4);
|
||||
batch.setCompletedCount(1);
|
||||
batch.setProcessingCount(1);
|
||||
batch.setFailedCount(1);
|
||||
batch.setPendingCount(1);
|
||||
when(batchService.getById(batch.getId())).thenReturn(batch);
|
||||
|
||||
DocumentImportBatchTracker tracker =
|
||||
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
|
||||
DocumentImportBatchDtos.StatusResponse response = tracker.refreshBatch(batch.getId());
|
||||
|
||||
assertEquals(1, response.getCompletedCount().intValue());
|
||||
assertEquals(1, response.getProcessingCount().intValue());
|
||||
assertEquals(1, response.getFailedCount().intValue());
|
||||
assertEquals(1, response.getPendingCount().intValue());
|
||||
assertEquals(50, response.getProgressPercent().intValue());
|
||||
assertEquals(DocumentImportBatchStatus.RUNNING.name(), response.getStatus());
|
||||
verify(batchService, never()).updateById(batch, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证中断批次在汇总失败项后仍保留可继续状态。
|
||||
*/
|
||||
@Test
|
||||
public void shouldPreserveInterruptedStatusUntilUserContinues() {
|
||||
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
|
||||
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
|
||||
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
|
||||
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
|
||||
DocumentImportBatch batch = batch(DocumentImportBatchStatus.INTERRUPTED, 2);
|
||||
batch.setCompletedCount(1);
|
||||
batch.setFailedCount(1);
|
||||
batch.setRetryableFailedCount(1);
|
||||
when(batchService.getById(batch.getId())).thenReturn(batch);
|
||||
|
||||
DocumentImportBatchTracker tracker =
|
||||
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
|
||||
DocumentImportBatchDtos.StatusResponse response = tracker.refreshBatch(batch.getId());
|
||||
|
||||
assertEquals(DocumentImportBatchStatus.INTERRUPTED.name(), response.getStatus());
|
||||
assertEquals(100, response.getProgressPercent().intValue());
|
||||
assertEquals(1, response.getRetryableFailedCount().intValue());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证失败项重试通过状态 CAS 增量迁移计数。
|
||||
*/
|
||||
@Test
|
||||
public void shouldMoveRetryableFailureBackToPendingAtomically() {
|
||||
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
|
||||
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
|
||||
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
|
||||
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
|
||||
DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1);
|
||||
batch.setFailedCount(1);
|
||||
batch.setRetryableFailedCount(1);
|
||||
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||
item.setId(BigInteger.TEN);
|
||||
item.setBatchId(batch.getId());
|
||||
item.setStage(DocumentImportBatchItemStage.INDEX.name());
|
||||
item.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||
item.setRetryable(true);
|
||||
when(itemService.getById(item.getId())).thenReturn(item);
|
||||
when(itemMapper.transitionStatus(
|
||||
eq(item.getId()),
|
||||
eq(DocumentImportBatchItemStatus.FAILED.name()),
|
||||
eq(DocumentImportBatchItemStage.INDEX.name()),
|
||||
eq(DocumentImportBatchItemStatus.PENDING.name()),
|
||||
eq(null),
|
||||
eq(null),
|
||||
eq(false),
|
||||
eq(1),
|
||||
any(Date.class)
|
||||
)).thenReturn(1);
|
||||
when(batchService.getById(batch.getId())).thenReturn(batch);
|
||||
|
||||
DocumentImportBatchTracker tracker =
|
||||
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
|
||||
|
||||
assertTrue(tracker.updateItem(item.getId(),
|
||||
DocumentImportBatchItemStage.INDEX,
|
||||
DocumentImportBatchItemStatus.PENDING,
|
||||
null));
|
||||
verify(batchMapper).adjustCounters(
|
||||
eq(batch.getId()),
|
||||
eq(0),
|
||||
eq(0),
|
||||
eq(-1),
|
||||
eq(1),
|
||||
eq(0),
|
||||
eq(0),
|
||||
eq(0),
|
||||
eq(-1),
|
||||
any(Date.class)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证迟到任务不能把已完成文件重新改为处理中。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRejectLateTransitionFromCompletedToRunning() {
|
||||
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
|
||||
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
|
||||
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
|
||||
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
|
||||
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||
item.setId(BigInteger.TEN);
|
||||
item.setBatchId(BigInteger.ONE);
|
||||
item.setStatus(DocumentImportBatchItemStatus.COMPLETED.name());
|
||||
when(itemService.getById(item.getId())).thenReturn(item);
|
||||
|
||||
DocumentImportBatchTracker tracker =
|
||||
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
|
||||
|
||||
assertFalse(tracker.updateItem(item.getId(),
|
||||
DocumentImportBatchItemStage.INDEX,
|
||||
DocumentImportBatchItemStatus.RUNNING,
|
||||
null));
|
||||
verify(itemMapper, never()).transitionStatus(
|
||||
any(), any(), any(), any(), any(), any(), anyBoolean(), anyInt(), any(Date.class));
|
||||
}
|
||||
|
||||
private DocumentImportBatch batch(DocumentImportBatchStatus status, int totalCount) {
|
||||
DocumentImportBatch batch = new DocumentImportBatch();
|
||||
batch.setId(BigInteger.ONE);
|
||||
batch.setKnowledgeId(BigInteger.TWO);
|
||||
batch.setImportMode(DocumentImportMode.AUTO.name());
|
||||
batch.setStatus(status.name());
|
||||
batch.setTotalCount(totalCount);
|
||||
batch.setTotalBytes(100L);
|
||||
batch.setCompletedCount(0);
|
||||
batch.setProcessingCount(0);
|
||||
batch.setFailedCount(0);
|
||||
batch.setPendingCount(0);
|
||||
batch.setUploadedCount(0);
|
||||
batch.setSkippedCount(0);
|
||||
batch.setCancelledCount(0);
|
||||
batch.setRetryableFailedCount(0);
|
||||
return batch;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||
import tech.easyflow.ai.entity.DocumentChunk;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.lang.reflect.Field;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
* {@link DocumentImportChunkSnapshotService} 持久化恢复测试。
|
||||
*/
|
||||
public class DocumentImportChunkSnapshotServiceTest {
|
||||
|
||||
/**
|
||||
* 验证分块快照可跨缓存写入并从文件存储完整恢复。
|
||||
*
|
||||
* @throws Exception 反射注入或文件读取异常
|
||||
*/
|
||||
@Test
|
||||
public void shouldPersistAndRestoreChunkSnapshot() throws Exception {
|
||||
String storedPath = "http://localhost/snapshots/9-chunks.json";
|
||||
AtomicReference<byte[]> storedBytes = new AtomicReference<byte[]>();
|
||||
FileStorageService storageService = Mockito.mock(FileStorageService.class);
|
||||
Mockito.when(storageService.save(
|
||||
Mockito.any(MultipartFile.class),
|
||||
Mockito.anyString()
|
||||
)).thenAnswer(invocation -> {
|
||||
MultipartFile file = invocation.getArgument(0);
|
||||
storedBytes.set(file.getBytes());
|
||||
return storedPath;
|
||||
});
|
||||
Mockito.when(storageService.readStream(storedPath))
|
||||
.thenAnswer(invocation -> new ByteArrayInputStream(storedBytes.get()));
|
||||
|
||||
DocumentImportChunkSnapshotService service = new DocumentImportChunkSnapshotService();
|
||||
Field storageField = DocumentImportChunkSnapshotService.class
|
||||
.getDeclaredField("storageService");
|
||||
storageField.setAccessible(true);
|
||||
storageField.set(service, storageService);
|
||||
|
||||
DocumentChunk chunk = new DocumentChunk();
|
||||
chunk.setId(BigInteger.valueOf(11));
|
||||
chunk.setDocumentId(BigInteger.valueOf(9));
|
||||
chunk.setDocumentCollectionId(BigInteger.valueOf(7));
|
||||
chunk.setContent("稳定分块");
|
||||
DocumentImportDtos.PreviewSession session = new DocumentImportDtos.PreviewSession();
|
||||
session.setKnowledgeId(BigInteger.valueOf(7));
|
||||
session.setDocumentId(BigInteger.valueOf(9));
|
||||
session.setDocumentChunks(List.of(chunk));
|
||||
|
||||
String path = service.save(session);
|
||||
DocumentImportDtos.PreviewSession restored = service.load(path);
|
||||
|
||||
Assert.assertEquals(storedPath, path);
|
||||
Assert.assertEquals(session.getKnowledgeId(), restored.getKnowledgeId());
|
||||
Assert.assertEquals(session.getDocumentId(), restored.getDocumentId());
|
||||
Assert.assertEquals(1, restored.getDocumentChunks().size());
|
||||
Assert.assertEquals("稳定分块", restored.getDocumentChunks().get(0).getContent());
|
||||
}
|
||||
}
|
||||
@@ -6,11 +6,15 @@ import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
||||
import tech.easyflow.ai.entity.Document;
|
||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.math.BigInteger;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
@@ -18,6 +22,33 @@ import java.util.concurrent.atomic.AtomicReference;
|
||||
*/
|
||||
public class DocumentImportTaskStatusStreamServiceTest {
|
||||
|
||||
/**
|
||||
* 验证状态流会携带稳定错误码,供前端进行本地化展示。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void buildDocumentPayloadShouldIncludeTaskErrorCode() throws Exception {
|
||||
Document document = new Document();
|
||||
document.setId(BigInteger.valueOf(88));
|
||||
document.setCollectionId(BigInteger.valueOf(99));
|
||||
document.setOptions(new LinkedHashMap<String, Object>(Map.of(
|
||||
DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE,
|
||||
"parse_service_unavailable"
|
||||
)));
|
||||
DocumentImportTaskStatusStreamService service = new DocumentImportTaskStatusStreamService();
|
||||
Method method = DocumentImportTaskStatusStreamService.class.getDeclaredMethod(
|
||||
"buildDocumentPayload",
|
||||
Document.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> payload = (Map<String, Object>) method.invoke(service, document);
|
||||
|
||||
Assert.assertEquals("parse_service_unavailable", payload.get("lastTaskErrorCode"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文档状态变更会向 Redis 广播文档 ID。
|
||||
*
|
||||
|
||||
@@ -3,28 +3,53 @@ package tech.easyflow.ai.documentimport.task;
|
||||
import com.easyagents.document.core.entity.DocumentBlock;
|
||||
import com.easyagents.document.core.entity.DocumentImage;
|
||||
import com.easyagents.document.core.entity.DocumentTable;
|
||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.rag.ingestion.model.StrategyConfig;
|
||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||
import tech.easyflow.ai.document.model.DocumentParseArtifacts;
|
||||
import tech.easyflow.ai.document.model.DocumentParsedResult;
|
||||
import tech.easyflow.ai.document.model.DocumentSourceRef;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
||||
import tech.easyflow.ai.entity.DocumentChunk;
|
||||
import tech.easyflow.ai.entity.DocumentCollection;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||
import tech.easyflow.ai.entity.DocumentImportTask;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportTaskStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportTaskPhase;
|
||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||
import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
||||
import tech.easyflow.ai.service.DocumentImportTaskService;
|
||||
import tech.easyflow.ai.service.DocumentService;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Proxy;
|
||||
import java.math.BigInteger;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -38,6 +63,666 @@ import java.util.concurrent.atomic.AtomicReference;
|
||||
*/
|
||||
public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
|
||||
/**
|
||||
* 验证待处理任务重新投递只更新必要字段,避免自定义查询结果覆盖非空列。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void dispatchPendingTasksShouldTouchPendingWithCas()
|
||||
throws Exception {
|
||||
BigInteger taskId = BigInteger.valueOf(29);
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setId(taskId);
|
||||
task.setPhase(DocumentImportTaskPhase.PARSE.name());
|
||||
task.setStatus(DocumentImportTaskStatus.PENDING.name());
|
||||
|
||||
DocumentImportTaskMapper taskMapper =
|
||||
Mockito.mock(DocumentImportTaskMapper.class);
|
||||
Mockito.when(taskMapper.selectPendingFairly(
|
||||
Mockito.any(Date.class), Mockito.anyInt()))
|
||||
.thenReturn(List.of(task));
|
||||
Mockito.when(taskMapper.touchPendingForDispatch(
|
||||
Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
|
||||
Mockito.any(BigInteger.class)))
|
||||
.thenReturn(1);
|
||||
DocumentImportParseTaskProducer producer =
|
||||
Mockito.mock(DocumentImportParseTaskProducer.class);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportTaskMapper", taskMapper);
|
||||
setField(service, "parseTaskProducer", producer);
|
||||
setField(service, "bulkProperties", new DocumentImportBulkProperties());
|
||||
|
||||
service.dispatchPendingTasks();
|
||||
|
||||
ArgumentCaptor<Date> selectCutoffCaptor =
|
||||
ArgumentCaptor.forClass(Date.class);
|
||||
Mockito.verify(taskMapper).selectPendingFairly(
|
||||
selectCutoffCaptor.capture(), Mockito.anyInt());
|
||||
ArgumentCaptor<Date> touchCutoffCaptor =
|
||||
ArgumentCaptor.forClass(Date.class);
|
||||
Mockito.verify(taskMapper).touchPendingForDispatch(
|
||||
Mockito.eq(taskId), touchCutoffCaptor.capture(), Mockito.any(Date.class),
|
||||
Mockito.any(BigInteger.class));
|
||||
Assert.assertEquals(selectCutoffCaptor.getValue(), touchCutoffCaptor.getValue());
|
||||
Mockito.verify(producer).send(taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证任务已离开待处理状态时不再发送重复消息。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void dispatchPendingTasksShouldSkipChangedTask()
|
||||
throws Exception {
|
||||
BigInteger taskId = BigInteger.valueOf(30);
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setId(taskId);
|
||||
task.setPhase(DocumentImportTaskPhase.PARSE.name());
|
||||
|
||||
DocumentImportTaskMapper taskMapper =
|
||||
Mockito.mock(DocumentImportTaskMapper.class);
|
||||
Mockito.when(taskMapper.selectPendingFairly(
|
||||
Mockito.any(Date.class), Mockito.anyInt()))
|
||||
.thenReturn(List.of(task));
|
||||
Mockito.when(taskMapper.touchPendingForDispatch(
|
||||
Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
|
||||
Mockito.any(BigInteger.class)))
|
||||
.thenReturn(0);
|
||||
DocumentImportParseTaskProducer producer =
|
||||
Mockito.mock(DocumentImportParseTaskProducer.class);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportTaskMapper", taskMapper);
|
||||
setField(service, "parseTaskProducer", producer);
|
||||
setField(service, "bulkProperties", new DocumentImportBulkProperties());
|
||||
|
||||
service.dispatchPendingTasks();
|
||||
|
||||
Mockito.verify(producer, Mockito.never()).send(Mockito.any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证重新投递资格更新包含待处理状态和过期时间边界,确保竞争更新仅一个成功。
|
||||
*
|
||||
* @throws Exception 映射方法不存在时抛出
|
||||
*/
|
||||
@Test
|
||||
public void touchPendingForDispatchSqlShouldFenceConcurrentDispatch()
|
||||
throws Exception {
|
||||
Method method = DocumentImportTaskMapper.class.getMethod(
|
||||
"touchPendingForDispatch",
|
||||
BigInteger.class,
|
||||
Date.class,
|
||||
Date.class,
|
||||
BigInteger.class
|
||||
);
|
||||
Update update = method.getAnnotation(Update.class);
|
||||
String sql = String.join(" ", update.value());
|
||||
|
||||
Assert.assertTrue(sql.contains("status='PENDING'"));
|
||||
Assert.assertTrue(sql.contains("modified <= #{redispatchBefore}"));
|
||||
Assert.assertTrue(sql.contains("modified=#{now}"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证分块策略快照使用忽略 null 的部分实体更新。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void persistAppliedStrategyShouldKeepRequiredBatchItemFields()
|
||||
throws Exception {
|
||||
BigInteger itemId = BigInteger.valueOf(31);
|
||||
DocumentImportBatchItemService itemService =
|
||||
Mockito.mock(DocumentImportBatchItemService.class);
|
||||
Mockito.when(itemService.updateById(Mockito.any()))
|
||||
.thenReturn(true);
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportBatchItemService", itemService);
|
||||
|
||||
Method method =
|
||||
KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"persistAppliedStrategy",
|
||||
BigInteger.class,
|
||||
StrategyConfig.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
method.invoke(service, itemId, StrategyConfig.defaults());
|
||||
|
||||
ArgumentCaptor<DocumentImportBatchItem> updateCaptor =
|
||||
ArgumentCaptor.forClass(DocumentImportBatchItem.class);
|
||||
Mockito.verify(itemService).updateById(updateCaptor.capture());
|
||||
Mockito.verify(itemService, Mockito.never()).updateById(
|
||||
Mockito.any(), Mockito.anyBoolean());
|
||||
Assert.assertEquals(itemId, updateCaptor.getValue().getId());
|
||||
Assert.assertNotNull(
|
||||
updateCaptor.getValue().getStrategySnapshotJson());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 SPLIT 待处理任务使用 execution token 和租约原子领取。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void tryMarkSplitTaskRunningShouldCreateExecutionFence()
|
||||
throws Exception {
|
||||
BigInteger taskId = BigInteger.valueOf(35);
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setId(taskId);
|
||||
task.setPhase(DocumentImportTaskPhase.SPLIT.name());
|
||||
task.setStatus(DocumentImportTaskStatus.PENDING.name());
|
||||
|
||||
DocumentImportTaskService taskService =
|
||||
Mockito.mock(DocumentImportTaskService.class);
|
||||
Mockito.when(taskService.getById(taskId)).thenReturn(task);
|
||||
Mockito.when(taskService.count(
|
||||
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
|
||||
.thenReturn(0L);
|
||||
DocumentImportTaskMapper taskMapper =
|
||||
Mockito.mock(DocumentImportTaskMapper.class);
|
||||
Mockito.when(taskMapper.claimPending(
|
||||
Mockito.eq(taskId),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.nullable(BigInteger.class)
|
||||
)).thenReturn(1);
|
||||
RedisLockExecutor lockExecutor =
|
||||
Mockito.mock(RedisLockExecutor.class);
|
||||
RedisLockExecutor.LockHandle lockHandle =
|
||||
Mockito.mock(RedisLockExecutor.LockHandle.class);
|
||||
Mockito.when(lockExecutor.tryAcquire(
|
||||
Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||
)).thenReturn(lockHandle);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportTaskService", taskService);
|
||||
setField(service, "documentImportTaskMapper", taskMapper);
|
||||
setField(service, "redisLockExecutor", lockExecutor);
|
||||
setField(service, "bulkProperties", new DocumentImportBulkProperties());
|
||||
|
||||
Assert.assertTrue(service.tryMarkTaskRunning(taskId));
|
||||
|
||||
ArgumentCaptor<String> token = ArgumentCaptor.forClass(String.class);
|
||||
ArgumentCaptor<Date> lease = ArgumentCaptor.forClass(Date.class);
|
||||
Mockito.verify(taskMapper).claimPending(
|
||||
Mockito.eq(taskId),
|
||||
token.capture(),
|
||||
lease.capture(),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.nullable(BigInteger.class)
|
||||
);
|
||||
Assert.assertFalse(token.getValue().isBlank());
|
||||
Assert.assertTrue(lease.getValue().after(new Date()));
|
||||
Mockito.verify(lockHandle).release();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证新文档完成后会清理待覆盖的历史文档并清除持久化待办。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void cleanupCompletedReplacementShouldDeleteHistoricalDocument() throws Exception {
|
||||
BigInteger itemId = BigInteger.valueOf(901);
|
||||
BigInteger documentId = BigInteger.valueOf(902);
|
||||
BigInteger replacedDocumentId = BigInteger.valueOf(903);
|
||||
BigInteger knowledgeId = BigInteger.valueOf(904);
|
||||
|
||||
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||
item.setId(itemId);
|
||||
item.setDocumentId(documentId);
|
||||
item.setReplacedDocumentId(replacedDocumentId);
|
||||
tech.easyflow.ai.entity.Document replacement = new tech.easyflow.ai.entity.Document();
|
||||
replacement.setId(documentId);
|
||||
replacement.setCollectionId(knowledgeId);
|
||||
replacement.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||
tech.easyflow.ai.entity.Document historical = new tech.easyflow.ai.entity.Document();
|
||||
historical.setId(replacedDocumentId);
|
||||
historical.setCollectionId(knowledgeId);
|
||||
historical.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||
|
||||
DocumentImportBatchTracker tracker = Mockito.mock(DocumentImportBatchTracker.class);
|
||||
Mockito.when(tracker.requireItem(itemId)).thenReturn(item);
|
||||
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||
Mockito.when(documentMapper.selectOneById(documentId)).thenReturn(replacement);
|
||||
Mockito.when(documentMapper.selectOneById(replacedDocumentId)).thenReturn(historical);
|
||||
DocumentService documentService = Mockito.mock(DocumentService.class);
|
||||
Mockito.when(documentService.removeDoc(replacedDocumentId.toString())).thenReturn(true);
|
||||
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
|
||||
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
|
||||
Mockito.when(redisLockExecutor.tryAcquire(
|
||||
Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||
)).thenReturn(lockHandle);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportBatchTracker", tracker);
|
||||
setField(service, "documentMapper", documentMapper);
|
||||
setField(service, "documentService", documentService);
|
||||
setField(service, "redisLockExecutor", redisLockExecutor);
|
||||
|
||||
service.cleanupCompletedReplacement(itemId);
|
||||
|
||||
Mockito.verify(documentService).removeDoc(replacedDocumentId.toString());
|
||||
Mockito.verify(documentMapper).deleteById(replacedDocumentId);
|
||||
Mockito.verify(tracker).clearReplacement(itemId, replacedDocumentId);
|
||||
Mockito.verify(lockHandle).release();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证失联且关联文档已删除的运行任务会被收口,避免持续占用并发名额。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void recoverInterruptedTaskShouldFailOrphanWithoutDocument() throws Exception {
|
||||
BigInteger taskId = BigInteger.valueOf(41);
|
||||
Date cutoff = new Date(System.currentTimeMillis() - 60_000L);
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setId(taskId);
|
||||
task.setDocumentId(BigInteger.valueOf(42));
|
||||
task.setPhase(DocumentImportTaskPhase.PARSE.name());
|
||||
task.setStatus(DocumentImportTaskStatus.RUNNING.name());
|
||||
task.setModified(new Date(cutoff.getTime() - 1_000L));
|
||||
|
||||
DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class);
|
||||
Mockito.when(taskService.getById(taskId)).thenReturn(task);
|
||||
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
|
||||
Mockito.when(taskMapper.failExpiredOwned(
|
||||
Mockito.eq(taskId),
|
||||
Mockito.isNull(),
|
||||
Mockito.anyString(),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.eq(cutoff),
|
||||
Mockito.any()
|
||||
)).thenReturn(1);
|
||||
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||
Mockito.when(documentMapper.selectOneById(task.getDocumentId())).thenReturn(null);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportTaskService", taskService);
|
||||
setField(service, "documentImportTaskMapper", taskMapper);
|
||||
setField(service, "documentMapper", documentMapper);
|
||||
|
||||
service.recoverInterruptedTask(taskId, cutoff);
|
||||
|
||||
Mockito.verify(taskMapper).failExpiredOwned(
|
||||
Mockito.eq(taskId),
|
||||
Mockito.isNull(),
|
||||
Mockito.eq("任务执行中断,请重试"),
|
||||
Mockito.eq("execution_interrupted"),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.eq(cutoff),
|
||||
Mockito.any()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证运行中断收口后仍保留批次文件的人工重试资格。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void recoveredBatchStateShouldRemainRetryableBeforeAttemptLimit()
|
||||
throws Exception {
|
||||
BigInteger batchId = BigInteger.valueOf(61);
|
||||
BigInteger itemId = BigInteger.valueOf(62);
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setBatchId(batchId);
|
||||
task.setBatchItemId(itemId);
|
||||
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||
item.setId(itemId);
|
||||
item.setAttemptCount(0);
|
||||
|
||||
DocumentImportBatchTracker tracker =
|
||||
Mockito.mock(DocumentImportBatchTracker.class);
|
||||
Mockito.when(tracker.requireItem(itemId)).thenReturn(item);
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportBatchTracker", tracker);
|
||||
setField(service, "bulkProperties", new DocumentImportBulkProperties());
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class
|
||||
.getDeclaredMethod(
|
||||
"finishRecoveredBatchState",
|
||||
DocumentImportTask.class,
|
||||
DocumentImportBatchItemStage.class,
|
||||
String.class,
|
||||
String.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
method.invoke(
|
||||
service,
|
||||
task,
|
||||
DocumentImportBatchItemStage.PARSE,
|
||||
"任务执行中断,请继续批次",
|
||||
"execution_interrupted"
|
||||
);
|
||||
|
||||
Mockito.verify(tracker).transitionItem(
|
||||
itemId,
|
||||
DocumentImportBatchItemStage.PARSE,
|
||||
DocumentImportBatchItemStatus.FAILED,
|
||||
"任务执行中断,请继续批次",
|
||||
true,
|
||||
0,
|
||||
"execution_interrupted"
|
||||
);
|
||||
Mockito.verify(tracker).markInterrupted(batchId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证待处理超时按创建时间判定,即使重新投递刷新了修改时间也会正常收口。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void expireTimedOutPendingTaskShouldUseCreatedTime() throws Exception {
|
||||
BigInteger taskId = BigInteger.valueOf(51);
|
||||
BigInteger documentId = BigInteger.valueOf(52);
|
||||
Date cutoff = new Date(System.currentTimeMillis() - 60_000L);
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setId(taskId);
|
||||
task.setDocumentId(documentId);
|
||||
task.setPhase(DocumentImportTaskPhase.PARSE.name());
|
||||
task.setStatus(DocumentImportTaskStatus.PENDING.name());
|
||||
task.setCreated(new Date(cutoff.getTime() - 1_000L));
|
||||
task.setModified(new Date());
|
||||
|
||||
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
|
||||
document.setId(documentId);
|
||||
document.setCollectionId(BigInteger.valueOf(53));
|
||||
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef =
|
||||
new AtomicReference<tech.easyflow.ai.entity.Document>();
|
||||
|
||||
DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class);
|
||||
Mockito.when(taskService.getById(taskId)).thenReturn(task);
|
||||
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
|
||||
Mockito.when(taskMapper.updateByQuery(
|
||||
Mockito.any(DocumentImportTask.class),
|
||||
Mockito.any()
|
||||
)).thenReturn(1);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportTaskService", taskService);
|
||||
setField(service, "documentImportTaskMapper", taskMapper);
|
||||
setField(service, "documentMapper", mockDocumentMapper(document, updatedDocumentRef));
|
||||
setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService());
|
||||
|
||||
service.expireTimedOutPendingTask(taskId, cutoff);
|
||||
|
||||
tech.easyflow.ai.entity.Document updatedDocument = updatedDocumentRef.get();
|
||||
Assert.assertNotNull(updatedDocument);
|
||||
Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(), updatedDocument.getProcessStatus());
|
||||
Assert.assertEquals("任务排队超时,请重试", updatedDocument.getLastTaskError());
|
||||
Assert.assertEquals("pending_timeout",
|
||||
updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证未取得 MinerU 任务 ID 的超时提交会失败并释放运行状态。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void expireTimedOutParseSubmissionShouldExposeFriendlyTimeout() throws Exception {
|
||||
BigInteger taskId = BigInteger.valueOf(61);
|
||||
BigInteger documentId = BigInteger.valueOf(62);
|
||||
Date cutoff = new Date(System.currentTimeMillis() - 60_000L);
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setId(taskId);
|
||||
task.setDocumentId(documentId);
|
||||
task.setPhase(DocumentImportTaskPhase.PARSE.name());
|
||||
task.setStatus(DocumentImportTaskStatus.RUNNING.name());
|
||||
task.setStartedAt(new Date(cutoff.getTime() - 1_000L));
|
||||
|
||||
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
|
||||
document.setId(documentId);
|
||||
document.setCollectionId(BigInteger.valueOf(63));
|
||||
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef =
|
||||
new AtomicReference<tech.easyflow.ai.entity.Document>();
|
||||
|
||||
DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class);
|
||||
Mockito.when(taskService.getById(taskId)).thenReturn(task);
|
||||
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
|
||||
Mockito.when(taskMapper.updateByQuery(
|
||||
Mockito.any(DocumentImportTask.class),
|
||||
Mockito.any()
|
||||
)).thenReturn(1);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentImportTaskService", taskService);
|
||||
setField(service, "documentImportTaskMapper", taskMapper);
|
||||
setField(service, "documentMapper", mockDocumentMapper(document, updatedDocumentRef));
|
||||
setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService());
|
||||
|
||||
service.expireTimedOutParseSubmission(taskId, cutoff);
|
||||
|
||||
ArgumentCaptor<DocumentImportTask> updateCaptor = ArgumentCaptor.forClass(DocumentImportTask.class);
|
||||
Mockito.verify(taskMapper).updateByQuery(updateCaptor.capture(), Mockito.any());
|
||||
Assert.assertEquals(DocumentImportTaskStatus.FAILED.name(), updateCaptor.getValue().getStatus());
|
||||
Assert.assertEquals("文档解析服务响应超时,请重试", updateCaptor.getValue().getErrorSummary());
|
||||
|
||||
tech.easyflow.ai.entity.Document updatedDocument = updatedDocumentRef.get();
|
||||
Assert.assertNotNull(updatedDocument);
|
||||
Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(), updatedDocument.getProcessStatus());
|
||||
Assert.assertEquals("文档解析服务响应超时,请重试", updatedDocument.getLastTaskError());
|
||||
Assert.assertEquals("parse_service_timeout",
|
||||
updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 MinerU 任意 5xx 会归一化为服务暂不可用错误。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void resolveParseFailureCodeShouldRecognizeMineruServerErrors() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"resolveParseFailureCode",
|
||||
Throwable.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
for (int statusCode : new int[] {500, 502, 503, 504, 599}) {
|
||||
Object code = method.invoke(service,
|
||||
new RuntimeException("MinerU request failed: path=/tasks, status=" + statusCode + ", body="));
|
||||
Assert.assertEquals("status=" + statusCode, "parse_service_unavailable", code);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证底层读取超时会归一化为解析服务超时错误。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void resolveParseFailureCodeShouldRecognizeSocketTimeout() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"resolveParseFailureCode",
|
||||
Throwable.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
Object code = method.invoke(service,
|
||||
new RuntimeException("wrapped", new SocketTimeoutException("timeout")));
|
||||
|
||||
Assert.assertEquals("parse_service_timeout", code);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文档源读取失败优先于底层 UnknownHostException 分类,避免误报 MinerU 不可用。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void resolveParseFailureCodeShouldPreferDocumentSourceFailure() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"resolveParseFailureCode",
|
||||
Throwable.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
DocumentParseBridgeException sourceError = DocumentParseBridgeException.sourceLoadFailed(
|
||||
"下载文档 URL 失败",
|
||||
new UnknownHostException("远端文档地址不允许访问非公网目标")
|
||||
);
|
||||
Object code = method.invoke(service, new RuntimeException("wrapped", sourceError));
|
||||
|
||||
Assert.assertEquals("document_source_unavailable", code);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证解析桥接层的确定性输入错误会转换为不可重试错误码。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void resolveParseFailureCodeShouldClassifyPermanentBridgeFailures()
|
||||
throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class
|
||||
.getDeclaredMethod("resolveParseFailureCode", Throwable.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
Object unsupported = method.invoke(
|
||||
service,
|
||||
DocumentParseBridgeException.unsupportedSource("不支持的文件类型")
|
||||
);
|
||||
Object invalidRequest = method.invoke(
|
||||
service,
|
||||
DocumentParseBridgeException.requestBuildFailed("解析请求无效")
|
||||
);
|
||||
|
||||
Assert.assertEquals("unsupported_document_source", unsupported);
|
||||
Assert.assertEquals("invalid_parse_request", invalidRequest);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证未识别的系统异常仍会获得稳定错误码并允许后续人工重试。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void resolveParseFailureCodeShouldStabilizeUnknownFailures()
|
||||
throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class
|
||||
.getDeclaredMethod("resolveParseFailureCode", Throwable.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
Object code = method.invoke(service, new IllegalStateException("代码异常"));
|
||||
|
||||
Assert.assertEquals("parse_failed", code);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证未知系统异常和临时源读取失败允许人工重试,确定性输入错误除外。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void parseRetryabilityShouldDefaultToRecoverable()
|
||||
throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class
|
||||
.getDeclaredMethod("isRetryableParseFailure", String.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
Assert.assertTrue((Boolean) method.invoke(service, new Object[] {null}));
|
||||
Assert.assertTrue((Boolean) method.invoke(
|
||||
service, "document_source_unavailable"));
|
||||
Assert.assertTrue((Boolean) method.invoke(service, "parse_failed"));
|
||||
Assert.assertFalse((Boolean) method.invoke(
|
||||
service, "unsupported_document_source"));
|
||||
Assert.assertFalse((Boolean) method.invoke(
|
||||
service, "invalid_parse_request"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证批次上传文件通过受信任存储服务读取为字节,避免内部附件 URL 进入公网 URL 校验。
|
||||
*
|
||||
* @throws Exception 反射调用或存储桩异常
|
||||
*/
|
||||
@Test
|
||||
public void buildBridgeSourceRefShouldReadBatchFileFromStorage() throws Exception {
|
||||
byte[] content = "batch-document".getBytes(StandardCharsets.UTF_8);
|
||||
String storedUrl = "http://127.0.0.1:39000/easyflow/attachment/test.docx";
|
||||
FileStorageService storageService = Mockito.mock(FileStorageService.class);
|
||||
Mockito.when(storageService.readStream(storedUrl))
|
||||
.thenReturn(new ByteArrayInputStream(content));
|
||||
|
||||
DocumentImportBulkProperties properties = new DocumentImportBulkProperties();
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "storageService", storageService);
|
||||
setField(service, "bulkProperties", properties);
|
||||
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setBatchId(BigInteger.valueOf(61));
|
||||
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
|
||||
document.setTitle("test.docx");
|
||||
document.setDocumentPath(storedUrl);
|
||||
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"buildBridgeSourceRef",
|
||||
DocumentImportTask.class,
|
||||
tech.easyflow.ai.entity.Document.class,
|
||||
String.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
DocumentSourceRef sourceRef = (DocumentSourceRef) method.invoke(service, task, document, "docx");
|
||||
|
||||
Assert.assertArrayEquals(content, sourceRef.getContentBytes());
|
||||
Assert.assertNull(sourceRef.getFilePath());
|
||||
Assert.assertEquals(Long.valueOf(content.length), sourceRef.getSize());
|
||||
Mockito.verify(storageService).readStream(storedUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证非批次远程文档仍保留 URL,由文档源加载器继续执行 SSRF 防护。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void buildBridgeSourceRefShouldKeepExternalUrlForSsrfValidation() throws Exception {
|
||||
FileStorageService storageService = Mockito.mock(FileStorageService.class);
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "storageService", storageService);
|
||||
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
|
||||
document.setTitle("external.pdf");
|
||||
document.setDocumentPath("https://example.com/external.pdf");
|
||||
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"buildBridgeSourceRef",
|
||||
DocumentImportTask.class,
|
||||
tech.easyflow.ai.entity.Document.class,
|
||||
String.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
DocumentSourceRef sourceRef = (DocumentSourceRef) method.invoke(service, task, document, "pdf");
|
||||
|
||||
Assert.assertEquals("https://example.com/external.pdf", sourceRef.getFilePath());
|
||||
Assert.assertNull(sourceRef.getContentBytes());
|
||||
Mockito.verifyNoInteractions(storageService);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证向量化失败会按整文档失败语义重置进度,并刷新任务错误信息。
|
||||
*
|
||||
@@ -59,11 +744,16 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
persistedDocument.setLastTaskError("旧错误");
|
||||
|
||||
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef = new AtomicReference<tech.easyflow.ai.entity.Document>();
|
||||
AtomicReference<DocumentImportTask> updatedTaskRef = new AtomicReference<DocumentImportTask>();
|
||||
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
|
||||
Mockito.when(taskMapper.finishOwned(
|
||||
Mockito.any(), Mockito.anyString(), Mockito.anyString(),
|
||||
Mockito.any(), Mockito.anyString(), Mockito.any(),
|
||||
Mockito.any()
|
||||
)).thenReturn(1);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentMapper", mockDocumentMapper(persistedDocument, updatedDocumentRef));
|
||||
setField(service, "documentImportTaskService", mockDocumentImportTaskService(updatedTaskRef));
|
||||
setField(service, "documentImportTaskMapper", taskMapper);
|
||||
setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService());
|
||||
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
@@ -72,6 +762,7 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
task.setKnowledgeId(knowledgeId);
|
||||
task.setStatus(DocumentImportTaskStatus.RUNNING.name());
|
||||
task.setErrorSummary("旧错误");
|
||||
task.setExecutionToken("attempt-token");
|
||||
|
||||
tech.easyflow.ai.entity.Document inputDocument = new tech.easyflow.ai.entity.Document();
|
||||
inputDocument.setId(documentId);
|
||||
@@ -93,11 +784,62 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
Assert.assertEquals(Integer.valueOf(8), updatedDocument.getFailedChunks());
|
||||
Assert.assertEquals(Integer.valueOf(0), updatedDocument.getProgressPercent());
|
||||
Assert.assertEquals("新错误", updatedDocument.getLastTaskError());
|
||||
Assert.assertEquals("index_failed",
|
||||
updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE));
|
||||
|
||||
DocumentImportTask updatedTask = updatedTaskRef.get();
|
||||
Assert.assertNotNull(updatedTask);
|
||||
Assert.assertEquals(DocumentImportTaskStatus.FAILED.name(), updatedTask.getStatus());
|
||||
Assert.assertEquals("新错误", updatedTask.getErrorSummary());
|
||||
Mockito.verify(taskMapper).finishOwned(
|
||||
Mockito.eq(task.getId()),
|
||||
Mockito.eq("attempt-token"),
|
||||
Mockito.eq(DocumentImportTaskStatus.FAILED.name()),
|
||||
Mockito.eq("新错误"),
|
||||
Mockito.eq("index_failed"),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.any()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证看门狗已经收口任务后,迟到执行者不能覆盖文档终态。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void markIndexFailedShouldIgnoreLostTaskOwnership() throws Exception {
|
||||
BigInteger documentId = BigInteger.valueOf(81);
|
||||
tech.easyflow.ai.entity.Document persistedDocument = new tech.easyflow.ai.entity.Document();
|
||||
persistedDocument.setId(documentId);
|
||||
persistedDocument.setProcessStatus(DocumentProcessStatus.PARSE_FAILED.name());
|
||||
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef =
|
||||
new AtomicReference<tech.easyflow.ai.entity.Document>();
|
||||
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
|
||||
Mockito.when(taskMapper.updateByQuery(
|
||||
Mockito.any(DocumentImportTask.class),
|
||||
Mockito.any()
|
||||
)).thenReturn(0);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "documentMapper", mockDocumentMapper(persistedDocument, updatedDocumentRef));
|
||||
setField(service, "documentImportTaskMapper", taskMapper);
|
||||
|
||||
DocumentImportTask task = new DocumentImportTask();
|
||||
task.setId(BigInteger.valueOf(82));
|
||||
task.setDocumentId(documentId);
|
||||
task.setStatus(DocumentImportTaskStatus.RUNNING.name());
|
||||
tech.easyflow.ai.entity.Document inputDocument = new tech.easyflow.ai.entity.Document();
|
||||
inputDocument.setId(documentId);
|
||||
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"markIndexFailed",
|
||||
DocumentImportTask.class,
|
||||
tech.easyflow.ai.entity.Document.class,
|
||||
String.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
method.invoke(service, task, inputDocument, "迟到错误");
|
||||
|
||||
Assert.assertNull(updatedDocumentRef.get());
|
||||
Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(),
|
||||
persistedDocument.getProcessStatus());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -232,6 +974,8 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
|
||||
Assert.assertEquals(2, chunks.size());
|
||||
DocumentChunk firstChunk = chunks.get(0);
|
||||
Assert.assertNotNull(firstChunk.getId());
|
||||
Assert.assertNotEquals(firstChunk.getId(), chunks.get(1).getId());
|
||||
Assert.assertTrue(firstChunk.getContent().contains("Slide 1"));
|
||||
Assert.assertTrue(firstChunk.getContent().contains("本页介绍季度目标"));
|
||||
Assert.assertEquals("https://example.com/slides/slide-001.png",
|
||||
@@ -352,6 +1096,61 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
Assert.assertTrue(chunks.isEmpty());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证向量存储回滚会去重分块 ID,并在 Milvus 删除失败后停止清理搜索索引。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void rollbackStoredChunksShouldValidateVectorDeleteResult() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
|
||||
DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class);
|
||||
StoreOptions storeOptions = StoreOptions.ofCollectionName("knowledge-test");
|
||||
Mockito.when(documentStore.delete(Mockito.anyCollection(), Mockito.same(storeOptions)))
|
||||
.thenReturn(StoreResult.fail("milvus unavailable"));
|
||||
|
||||
DocumentCollection knowledge = new DocumentCollection();
|
||||
knowledge.setId(BigInteger.valueOf(501));
|
||||
Class<?> contextClass = Class.forName(
|
||||
KnowledgeDocumentImportTaskAppService.class.getName() + "$StoreExecutionContext"
|
||||
);
|
||||
Constructor<?> constructor = contextClass.getDeclaredConstructor(
|
||||
DocumentCollection.class,
|
||||
EmbeddingModel.class,
|
||||
DocumentStore.class,
|
||||
StoreOptions.class,
|
||||
DocumentSearcher.class
|
||||
);
|
||||
constructor.setAccessible(true);
|
||||
Object context = constructor.newInstance(knowledge, null, documentStore, storeOptions, searcher);
|
||||
|
||||
DocumentChunk first = new DocumentChunk();
|
||||
first.setId(BigInteger.valueOf(601));
|
||||
DocumentChunk duplicate = new DocumentChunk();
|
||||
duplicate.setId(first.getId());
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"rollbackStoredChunks",
|
||||
BigInteger.class,
|
||||
BigInteger.class,
|
||||
contextClass,
|
||||
List.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
method.invoke(service,
|
||||
BigInteger.valueOf(701),
|
||||
BigInteger.valueOf(702),
|
||||
context,
|
||||
List.of(first, duplicate));
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
ArgumentCaptor<Collection> idsCaptor = ArgumentCaptor.forClass(Collection.class);
|
||||
Mockito.verify(documentStore).delete(idsCaptor.capture(), Mockito.same(storeOptions));
|
||||
Assert.assertEquals(1, idsCaptor.getValue().size());
|
||||
Assert.assertTrue(idsCaptor.getValue().contains(first.getId()));
|
||||
Mockito.verifyNoInteractions(searcher);
|
||||
}
|
||||
|
||||
private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument,
|
||||
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef) {
|
||||
return (DocumentMapper) Proxy.newProxyInstance(
|
||||
@@ -370,20 +1169,6 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
);
|
||||
}
|
||||
|
||||
private static DocumentImportTaskService mockDocumentImportTaskService(AtomicReference<DocumentImportTask> updatedTaskRef) {
|
||||
return (DocumentImportTaskService) Proxy.newProxyInstance(
|
||||
DocumentImportTaskService.class.getClassLoader(),
|
||||
new Class<?>[]{DocumentImportTaskService.class},
|
||||
(proxy, method, args) -> {
|
||||
if ("updateById".equals(method.getName())) {
|
||||
updatedTaskRef.set((DocumentImportTask) args[0]);
|
||||
return true;
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private static FileStorageService mockFileStorageService(AtomicReference<String> savedPrePathRef,
|
||||
AtomicReference<String> savedFilenameRef) {
|
||||
return (FileStorageService) Proxy.newProxyInstance(
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext;
|
||||
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
||||
import tech.easyflow.ai.documentimport.ImportCallerContext;
|
||||
import tech.easyflow.ai.documentimport.ImportCallerType;
|
||||
import tech.easyflow.ai.documentimport.PublicDocumentImportDtos;
|
||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
||||
import tech.easyflow.ai.service.DocumentImportBatchService;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Public API 批量导入门面边界与服务端去重测试。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public class KnowledgeImportBatchFacadeTest {
|
||||
|
||||
/**
|
||||
* 验证单次文件数超过 200 时在创建批次前拒绝。
|
||||
*/
|
||||
@Test
|
||||
public void submitShouldRejectMoreThanTwoHundredFiles() {
|
||||
TestContext context = createContext();
|
||||
PublicDocumentImportDtos.BatchMetadata metadata =
|
||||
new PublicDocumentImportDtos.BatchMetadata();
|
||||
metadata.setKnowledgeId(BigInteger.ONE);
|
||||
List<DocumentImportBatchDtos.ManifestItem> manifest = new ArrayList<>();
|
||||
List<MultipartFile> files = new ArrayList<>();
|
||||
for (int index = 0; index < 201; index++) {
|
||||
String name = "file-" + index + ".txt";
|
||||
manifest.add(manifest(name, name, 1L));
|
||||
files.add(file(name, new byte[]{'a'}));
|
||||
}
|
||||
metadata.setFiles(manifest);
|
||||
|
||||
try {
|
||||
context.facade.submit(context.caller, metadata, files);
|
||||
Assert.fail("Expected file count rejection");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("最多上传200个文件"));
|
||||
}
|
||||
Mockito.verifyNoInteractions(context.batchAppService);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文件夹清单拒绝父目录穿越。
|
||||
*/
|
||||
@Test
|
||||
public void submitShouldRejectRelativePathTraversal() {
|
||||
TestContext context = createContext();
|
||||
PublicDocumentImportDtos.BatchMetadata metadata =
|
||||
metadata("demo.txt", "../demo.txt", 4L);
|
||||
|
||||
try {
|
||||
context.facade.submit(
|
||||
context.caller,
|
||||
metadata,
|
||||
List.of(file(
|
||||
"demo.txt",
|
||||
"demo".getBytes(StandardCharsets.UTF_8)
|
||||
))
|
||||
);
|
||||
Assert.fail("Expected relative path rejection");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("相对路径无效"));
|
||||
}
|
||||
Mockito.verifyNoInteractions(context.batchAppService);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 Public API 可省略 fileSize,内部批次使用服务端实测大小。
|
||||
*/
|
||||
@Test
|
||||
public void submitShouldUseMeasuredSizeWhenMetadataOmitsFileSize() {
|
||||
TestContext context = createContext();
|
||||
PublicDocumentImportDtos.BatchMetadata metadata =
|
||||
metadata("demo.txt", "folder/demo.txt", 4L);
|
||||
metadata.getFiles().get(0).setFileSize(null);
|
||||
byte[] content = "demo".getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
DocumentImportBatchDtos.CreateResponse created =
|
||||
new DocumentImportBatchDtos.CreateResponse();
|
||||
created.setBatchId(BigInteger.valueOf(91));
|
||||
DocumentImportBatchDtos.ItemResponse createdItem =
|
||||
new DocumentImportBatchDtos.ItemResponse();
|
||||
createdItem.setItemId(BigInteger.valueOf(92));
|
||||
created.setItems(List.of(createdItem));
|
||||
Mockito.when(context.batchAppService.createBatch(
|
||||
Mockito.any(), Mockito.any()
|
||||
)).thenReturn(created);
|
||||
Mockito.when(context.itemService.updateById(Mockito.any()))
|
||||
.thenReturn(true);
|
||||
DocumentImportBatch batch = new DocumentImportBatch();
|
||||
batch.setId(created.getBatchId());
|
||||
batch.setKnowledgeId(BigInteger.ONE);
|
||||
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
|
||||
batch.setTotalCount(1);
|
||||
batch.setTotalBytes((long) content.length);
|
||||
batch.setCreated(new Date());
|
||||
Mockito.when(context.batchAppService.requireOwnedBatch(
|
||||
BigInteger.ONE, created.getBatchId(), context.caller
|
||||
)).thenReturn(batch);
|
||||
|
||||
context.facade.submit(
|
||||
context.caller,
|
||||
metadata,
|
||||
List.of(file("demo.txt", content))
|
||||
);
|
||||
|
||||
ArgumentCaptor<DocumentImportBatchDtos.CreateRequest> request =
|
||||
ArgumentCaptor.forClass(DocumentImportBatchDtos.CreateRequest.class);
|
||||
Mockito.verify(context.batchAppService).createBatch(
|
||||
request.capture(), Mockito.any());
|
||||
Assert.assertEquals(
|
||||
Long.valueOf(content.length),
|
||||
request.getValue().getFiles().get(0).getFileSize()
|
||||
);
|
||||
Assert.assertNull(metadata.getFiles().get(0).getFileSize());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证相同提交由服务端指纹复用,包含仍在上传的并发请求。
|
||||
*/
|
||||
@Test
|
||||
public void submitShouldReuseTaskByServerFingerprint() {
|
||||
TestContext context = createContext();
|
||||
PublicDocumentImportDtos.BatchMetadata metadata =
|
||||
metadata("demo.txt", "folder/demo.txt", 4L);
|
||||
List<MultipartFile> files = List.of(file(
|
||||
"demo.txt",
|
||||
"demo".getBytes(StandardCharsets.UTF_8)
|
||||
));
|
||||
|
||||
DocumentImportBatchDtos.CreateResponse created =
|
||||
new DocumentImportBatchDtos.CreateResponse();
|
||||
created.setBatchId(BigInteger.valueOf(91));
|
||||
DocumentImportBatchDtos.ItemResponse createdItem =
|
||||
new DocumentImportBatchDtos.ItemResponse();
|
||||
createdItem.setItemId(BigInteger.valueOf(92));
|
||||
created.setItems(List.of(createdItem));
|
||||
Mockito.when(context.batchAppService.createBatch(
|
||||
Mockito.any(), Mockito.any()
|
||||
)).thenReturn(created);
|
||||
Mockito.when(context.itemService.updateById(Mockito.any()))
|
||||
.thenReturn(true);
|
||||
DocumentImportBatch batch = new DocumentImportBatch();
|
||||
batch.setId(created.getBatchId());
|
||||
batch.setKnowledgeId(BigInteger.ONE);
|
||||
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
|
||||
batch.setTotalCount(1);
|
||||
batch.setTotalBytes(4L);
|
||||
batch.setCreated(new Date());
|
||||
Mockito.when(context.batchAppService.requireOwnedBatch(
|
||||
BigInteger.ONE, created.getBatchId(), context.caller
|
||||
)).thenReturn(batch);
|
||||
|
||||
context.facade.submit(context.caller, metadata, files);
|
||||
|
||||
ArgumentCaptor<tech.easyflow.ai.entity.DocumentImportBatchItem> hashUpdate =
|
||||
ArgumentCaptor.forClass(tech.easyflow.ai.entity.DocumentImportBatchItem.class);
|
||||
Mockito.verify(context.itemService).updateById(hashUpdate.capture());
|
||||
Assert.assertNotNull(hashUpdate.getValue().getContentSha256());
|
||||
Assert.assertEquals(createdItem.getItemId(), hashUpdate.getValue().getId());
|
||||
ArgumentCaptor<DocumentImportBatchCreateContext> createContext =
|
||||
ArgumentCaptor.forClass(DocumentImportBatchCreateContext.class);
|
||||
Mockito.verify(context.batchAppService).createBatch(
|
||||
Mockito.any(), createContext.capture());
|
||||
batch.setRequestDigest(createContext.getValue().getRequestDigest());
|
||||
Mockito.when(context.batchService.getOne(
|
||||
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
|
||||
.thenReturn(batch);
|
||||
|
||||
PublicDocumentImportDtos.SubmitResponse repeated =
|
||||
context.facade.submit(
|
||||
context.caller, metadata, files);
|
||||
|
||||
Assert.assertEquals(batch.getId(), repeated.getTaskId());
|
||||
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
|
||||
PublicDocumentImportDtos.SubmitResponse uploading =
|
||||
context.facade.submit(context.caller, metadata, files);
|
||||
|
||||
Assert.assertEquals(batch.getId(), uploading.getTaskId());
|
||||
Mockito.verify(context.batchAppService, Mockito.times(1))
|
||||
.createBatch(Mockito.any(), Mockito.any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证长时间无进展的上传任务会先取消并释放指纹,再创建新任务。
|
||||
*/
|
||||
@Test
|
||||
public void submitShouldReplaceStaleIncompleteTask() {
|
||||
TestContext context = createContext();
|
||||
PublicDocumentImportDtos.BatchMetadata metadata =
|
||||
metadata("demo.txt", "folder/demo.txt", 4L);
|
||||
List<MultipartFile> files = List.of(file(
|
||||
"demo.txt",
|
||||
"demo".getBytes(StandardCharsets.UTF_8)
|
||||
));
|
||||
DocumentImportBatch stale = new DocumentImportBatch();
|
||||
stale.setId(BigInteger.valueOf(81));
|
||||
stale.setKnowledgeId(BigInteger.ONE);
|
||||
stale.setStatus(DocumentImportBatchStatus.UPLOADING.name());
|
||||
stale.setModified(new Date(
|
||||
System.currentTimeMillis() - 31L * 60L * 1000L
|
||||
));
|
||||
Mockito.when(context.batchService.getOne(
|
||||
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
|
||||
.thenReturn(stale)
|
||||
.thenReturn(null);
|
||||
Mockito.when(context.batchAppService.cancelStaleBatch(
|
||||
Mockito.eq(BigInteger.ONE),
|
||||
Mockito.eq(stale.getId()),
|
||||
Mockito.eq(context.caller),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(true);
|
||||
Mockito.when(context.batchMapper.releaseSubmissionFingerprint(
|
||||
Mockito.eq(stale.getId()),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.any(Date.class)
|
||||
)).thenReturn(1);
|
||||
|
||||
DocumentImportBatchDtos.CreateResponse created =
|
||||
new DocumentImportBatchDtos.CreateResponse();
|
||||
created.setBatchId(BigInteger.valueOf(91));
|
||||
DocumentImportBatchDtos.ItemResponse item =
|
||||
new DocumentImportBatchDtos.ItemResponse();
|
||||
item.setItemId(BigInteger.valueOf(92));
|
||||
created.setItems(List.of(item));
|
||||
Mockito.when(context.batchAppService.createBatch(
|
||||
Mockito.any(), Mockito.any()
|
||||
)).thenReturn(created);
|
||||
Mockito.when(context.itemService.updateById(Mockito.any()))
|
||||
.thenReturn(true);
|
||||
DocumentImportBatch replacement = new DocumentImportBatch();
|
||||
replacement.setId(created.getBatchId());
|
||||
replacement.setKnowledgeId(BigInteger.ONE);
|
||||
replacement.setStatus(DocumentImportBatchStatus.RUNNING.name());
|
||||
replacement.setTotalCount(1);
|
||||
replacement.setTotalBytes(4L);
|
||||
replacement.setCreated(new Date());
|
||||
Mockito.when(context.batchAppService.requireOwnedBatch(
|
||||
BigInteger.ONE, created.getBatchId(), context.caller
|
||||
)).thenReturn(replacement);
|
||||
|
||||
PublicDocumentImportDtos.SubmitResponse response =
|
||||
context.facade.submit(context.caller, metadata, files);
|
||||
|
||||
Assert.assertEquals(replacement.getId(), response.getTaskId());
|
||||
Mockito.verify(context.batchAppService).cancelStaleBatch(
|
||||
Mockito.eq(BigInteger.ONE),
|
||||
Mockito.eq(stale.getId()),
|
||||
Mockito.eq(context.caller),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
Mockito.verify(context.batchMapper).releaseSubmissionFingerprint(
|
||||
Mockito.eq(stale.getId()),
|
||||
Mockito.anyString(),
|
||||
Mockito.any(Date.class),
|
||||
Mockito.any(Date.class)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证显式空文件键数组会被拒绝,省略时由服务端重试全部异常文件。
|
||||
*/
|
||||
@Test
|
||||
public void retryShouldDistinguishOmittedAndEmptyFileKeys() {
|
||||
TestContext context = createContext();
|
||||
PublicDocumentImportDtos.RetryRequest empty =
|
||||
new PublicDocumentImportDtos.RetryRequest();
|
||||
empty.setTaskId(BigInteger.ONE);
|
||||
empty.setFileKeys(List.of());
|
||||
|
||||
try {
|
||||
context.facade.retry(context.caller, empty);
|
||||
Assert.fail("Expected empty fileKeys rejection");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("不能为空数组"));
|
||||
}
|
||||
|
||||
PublicDocumentImportDtos.RetryRequest omitted =
|
||||
new PublicDocumentImportDtos.RetryRequest();
|
||||
omitted.setTaskId(BigInteger.ONE);
|
||||
tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult result =
|
||||
new tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult(
|
||||
BigInteger.ONE,
|
||||
DocumentImportBatchStatus.RUNNING.name(),
|
||||
1,
|
||||
2
|
||||
);
|
||||
Mockito.when(context.batchAppService.retryOwnedBatch(
|
||||
BigInteger.ONE,
|
||||
context.caller,
|
||||
java.util.Set.of()
|
||||
)).thenReturn(result);
|
||||
|
||||
PublicDocumentImportDtos.RetryResponse response =
|
||||
context.facade.retry(context.caller, omitted);
|
||||
|
||||
Assert.assertEquals(Integer.valueOf(2), response.getRetriedCount());
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建单文件元数据。
|
||||
*
|
||||
* @param fileName 文件名
|
||||
* @param relativePath 相对路径
|
||||
* @param fileSize 文件大小
|
||||
* @return 元数据
|
||||
*/
|
||||
private PublicDocumentImportDtos.BatchMetadata metadata(
|
||||
String fileName,
|
||||
String relativePath,
|
||||
long fileSize) {
|
||||
PublicDocumentImportDtos.BatchMetadata metadata =
|
||||
new PublicDocumentImportDtos.BatchMetadata();
|
||||
metadata.setKnowledgeId(BigInteger.ONE);
|
||||
metadata.setFiles(List.of(
|
||||
manifest(fileName, relativePath, fileSize)));
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建清单项。
|
||||
*
|
||||
* @param fileName 文件名
|
||||
* @param relativePath 相对路径
|
||||
* @param fileSize 文件大小
|
||||
* @return 清单项
|
||||
*/
|
||||
private DocumentImportBatchDtos.ManifestItem manifest(
|
||||
String fileName,
|
||||
String relativePath,
|
||||
long fileSize) {
|
||||
DocumentImportBatchDtos.ManifestItem item =
|
||||
new DocumentImportBatchDtos.ManifestItem();
|
||||
item.setClientFileKey(relativePath);
|
||||
item.setFileName(fileName);
|
||||
item.setRelativePath(relativePath);
|
||||
item.setFileSize(fileSize);
|
||||
return item;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建可重复读取的 MultipartFile。
|
||||
*
|
||||
* @param fileName 文件名
|
||||
* @param bytes 文件内容
|
||||
* @return Multipart 文件
|
||||
*/
|
||||
private MultipartFile file(String fileName, byte[] bytes) {
|
||||
MultipartFile file = Mockito.mock(MultipartFile.class);
|
||||
try {
|
||||
Mockito.when(file.getInputStream())
|
||||
.thenAnswer(invocation -> new ByteArrayInputStream(bytes));
|
||||
} catch (java.io.IOException error) {
|
||||
throw new IllegalStateException(error);
|
||||
}
|
||||
Mockito.when(file.getOriginalFilename()).thenReturn(fileName);
|
||||
Mockito.when(file.getSize()).thenReturn((long) bytes.length);
|
||||
Mockito.when(file.isEmpty()).thenReturn(bytes.length == 0);
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建测试依赖。
|
||||
*
|
||||
* @return 测试上下文
|
||||
*/
|
||||
private TestContext createContext() {
|
||||
DocumentImportBatchAppService batchAppService =
|
||||
Mockito.mock(DocumentImportBatchAppService.class);
|
||||
DocumentImportBatchTracker batchTracker =
|
||||
Mockito.mock(DocumentImportBatchTracker.class);
|
||||
DocumentImportBatchService batchService =
|
||||
Mockito.mock(DocumentImportBatchService.class);
|
||||
DocumentImportBatchItemService itemService =
|
||||
Mockito.mock(DocumentImportBatchItemService.class);
|
||||
DocumentImportBatchMapper batchMapper =
|
||||
Mockito.mock(DocumentImportBatchMapper.class);
|
||||
KnowledgeImportBatchFacade facade = new KnowledgeImportBatchFacade(
|
||||
batchAppService,
|
||||
batchTracker,
|
||||
batchService,
|
||||
itemService,
|
||||
batchMapper
|
||||
);
|
||||
return new TestContext(
|
||||
facade,
|
||||
batchAppService,
|
||||
batchService,
|
||||
itemService,
|
||||
batchMapper,
|
||||
new ImportCallerContext(
|
||||
ImportCallerType.PUBLIC_API,
|
||||
BigInteger.valueOf(77)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试依赖集合。
|
||||
*/
|
||||
private record TestContext(
|
||||
KnowledgeImportBatchFacade facade,
|
||||
DocumentImportBatchAppService batchAppService,
|
||||
DocumentImportBatchService batchService,
|
||||
DocumentImportBatchItemService itemService,
|
||||
DocumentImportBatchMapper batchMapper,
|
||||
ImportCallerContext caller
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import tech.easyflow.ai.config.SearcherFactory;
|
||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||
import tech.easyflow.ai.mapper.FaqItemMapper;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.lang.reflect.Field;
|
||||
@@ -92,6 +93,89 @@ public class DocumentCollectionServiceImplTest {
|
||||
Assert.assertEquals(completedChunkId, result.get(0).getId());
|
||||
Assert.assertEquals("completed chunk", result.get(0).getContent());
|
||||
Assert.assertEquals(String.valueOf(knowledgeId), searcher.lastKnowledgeId);
|
||||
Assert.assertEquals(
|
||||
tech.easyflow.ai.entity.DocumentCollection.TYPE_DOCUMENT,
|
||||
result.get(0).getMetadata("resultType")
|
||||
);
|
||||
Assert.assertEquals(
|
||||
completedDocumentId,
|
||||
result.get(0).getMetadata("documentId")
|
||||
);
|
||||
Assert.assertEquals(
|
||||
completedDocument.getTitle(),
|
||||
result.get(0).getMetadata("sourceFileName")
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 FAQ 检索使用当前数据库记录回填稳定的 FAQ 来源信息。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void searchShouldFillFaqSourceMetadataFromDatabase() throws Exception {
|
||||
BigInteger knowledgeId = BigInteger.ONE;
|
||||
BigInteger faqId = BigInteger.valueOf(2001);
|
||||
BigInteger categoryId = BigInteger.valueOf(3001);
|
||||
tech.easyflow.ai.entity.DocumentCollection collection =
|
||||
new tech.easyflow.ai.entity.DocumentCollection();
|
||||
collection.setId(knowledgeId);
|
||||
collection.setCollectionType(
|
||||
tech.easyflow.ai.entity.DocumentCollection.TYPE_FAQ
|
||||
);
|
||||
collection.setOptions(new HashMap<String, Object>() {{
|
||||
put(KEY_DOC_RECALL_MAX_NUM, 5);
|
||||
put(KEY_SIMILARITY_THRESHOLD, BigDecimal.ZERO);
|
||||
}});
|
||||
|
||||
tech.easyflow.ai.entity.FaqItem faqItem =
|
||||
new tech.easyflow.ai.entity.FaqItem();
|
||||
faqItem.setId(faqId);
|
||||
faqItem.setCollectionId(knowledgeId);
|
||||
faqItem.setCategoryId(categoryId);
|
||||
faqItem.setQuestion("如何申请账号?");
|
||||
faqItem.setAnswerText("请联系管理员。");
|
||||
TestKeywordSearcher searcher = new TestKeywordSearcher(
|
||||
List.of(buildHit(faqId, 0.85D))
|
||||
);
|
||||
|
||||
DocumentCollectionServiceImpl service =
|
||||
new TestDocumentCollectionService(collection);
|
||||
setField(
|
||||
service,
|
||||
"searcherFactory",
|
||||
new SearcherFactory(
|
||||
new StaticObjectProvider<DocumentSearcher>(searcher)
|
||||
)
|
||||
);
|
||||
setField(service, "faqItemMapper", mockFaqItemMapper(faqItem));
|
||||
|
||||
tech.easyflow.ai.rag.KnowledgeRetrievalRequest request =
|
||||
new tech.easyflow.ai.rag.KnowledgeRetrievalRequest();
|
||||
request.setKnowledgeId(knowledgeId);
|
||||
request.setQuery("账号");
|
||||
request.setRetrievalMode(
|
||||
com.easyagents.rag.retrieval.RetrievalMode.KEYWORD
|
||||
);
|
||||
|
||||
List<Document> result = service.search(request);
|
||||
|
||||
Assert.assertEquals(1, result.size());
|
||||
Document item = result.get(0);
|
||||
Assert.assertEquals(
|
||||
tech.easyflow.ai.entity.DocumentCollection.TYPE_FAQ,
|
||||
item.getMetadata("resultType")
|
||||
);
|
||||
Assert.assertEquals(faqId, item.getMetadata("faqId"));
|
||||
Assert.assertEquals(
|
||||
faqItem.getQuestion(),
|
||||
item.getMetadata("question")
|
||||
);
|
||||
Assert.assertEquals(
|
||||
faqItem.getAnswerText(),
|
||||
item.getMetadata("answerText")
|
||||
);
|
||||
Assert.assertEquals(categoryId, item.getMetadata("categoryId"));
|
||||
}
|
||||
|
||||
private static Document buildHit(BigInteger id, double score) {
|
||||
@@ -132,6 +216,27 @@ public class DocumentCollectionServiceImplTest {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建返回固定 FAQ 的 Mapper 桩。
|
||||
*
|
||||
* @param faqItem FAQ 数据
|
||||
* @return Mapper 桩
|
||||
*/
|
||||
private static FaqItemMapper mockFaqItemMapper(
|
||||
tech.easyflow.ai.entity.FaqItem faqItem
|
||||
) {
|
||||
return (FaqItemMapper) Proxy.newProxyInstance(
|
||||
FaqItemMapper.class.getClassLoader(),
|
||||
new Class<?>[]{FaqItemMapper.class},
|
||||
(proxy, method, args) -> {
|
||||
if ("selectListByQuery".equals(method.getName())) {
|
||||
return List.of(faqItem);
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private static void setField(Object target, String fieldName, Object value) throws Exception {
|
||||
Field field = DocumentCollectionServiceImpl.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
package tech.easyflow.ai.service.impl;
|
||||
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.ai.config.SearcherFactory;
|
||||
import tech.easyflow.ai.entity.Document;
|
||||
import tech.easyflow.ai.entity.DocumentCollection;
|
||||
import tech.easyflow.ai.entity.Model;
|
||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||
import tech.easyflow.ai.service.ModelService;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* {@link DocumentServiceImpl} 文档维护回归测试。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public class DocumentServiceImplTest {
|
||||
|
||||
/**
|
||||
* 验证删除链路在外部索引和分块清理后删除文档主记录。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void removeDocShouldDeleteDocumentRecord() throws Exception {
|
||||
BigInteger documentId = BigInteger.valueOf(101);
|
||||
BigInteger knowledgeId = BigInteger.valueOf(102);
|
||||
BigInteger modelId = BigInteger.valueOf(103);
|
||||
Document document = new Document();
|
||||
document.setId(documentId);
|
||||
document.setCollectionId(knowledgeId);
|
||||
document.setDocumentPath("storage://document.txt");
|
||||
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||
|
||||
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
|
||||
Mockito.when(documentStore.delete(
|
||||
Mockito.anyList(), Mockito.any())).thenReturn(StoreResult.success());
|
||||
DocumentCollection knowledge = Mockito.mock(DocumentCollection.class);
|
||||
Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId);
|
||||
Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test");
|
||||
Mockito.when(knowledge.toDocumentStore()).thenReturn(documentStore);
|
||||
Model model = new Model();
|
||||
model.setModelName("embedding-test");
|
||||
|
||||
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||
Mockito.when(documentMapper.selectOneByQuery(
|
||||
Mockito.any(QueryWrapper.class))).thenReturn(document);
|
||||
Mockito.when(documentMapper.deleteById(documentId)).thenReturn(1);
|
||||
DocumentChunkMapper chunkMapper =
|
||||
Mockito.mock(DocumentChunkMapper.class);
|
||||
Mockito.when(chunkMapper.selectListByQueryAs(
|
||||
Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class)))
|
||||
.thenReturn(List.of(BigInteger.valueOf(201)));
|
||||
Mockito.when(chunkMapper.deleteByQuery(
|
||||
Mockito.any(QueryWrapper.class))).thenReturn(1);
|
||||
DocumentCollectionService knowledgeService =
|
||||
Mockito.mock(DocumentCollectionService.class);
|
||||
Mockito.when(knowledgeService.getById(knowledgeId))
|
||||
.thenReturn(knowledge);
|
||||
ModelService modelService = Mockito.mock(ModelService.class);
|
||||
Mockito.when(modelService.getById(modelId)).thenReturn(model);
|
||||
FileStorageService storageService =
|
||||
Mockito.mock(FileStorageService.class);
|
||||
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
|
||||
|
||||
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||
setField(service, "documentMapper", documentMapper);
|
||||
setField(service, "documentChunkMapper", chunkMapper);
|
||||
setField(service, "knowledgeService", knowledgeService);
|
||||
setField(service, "modelService", modelService);
|
||||
setField(service, "storageService", storageService);
|
||||
setField(service, "searcherFactory", searcherFactory);
|
||||
|
||||
Assert.assertTrue(service.removeDoc(documentId.toString()));
|
||||
|
||||
Mockito.verify(documentMapper).deleteById(documentId);
|
||||
Mockito.verify(storageService).delete(document.getDocumentPath());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证最后一个分块已单独删除时,文档删除跳过空向量请求并完成清理。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void removeDocShouldHandleDocumentWithoutChunks() throws Exception {
|
||||
BigInteger documentId = BigInteger.valueOf(202);
|
||||
BigInteger knowledgeId = BigInteger.valueOf(203);
|
||||
BigInteger modelId = BigInteger.valueOf(204);
|
||||
Document document = new Document();
|
||||
document.setId(documentId);
|
||||
document.setCollectionId(knowledgeId);
|
||||
document.setDocumentPath("storage://empty-document.txt");
|
||||
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||
|
||||
DocumentCollection knowledge = Mockito.mock(DocumentCollection.class);
|
||||
Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId);
|
||||
Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test");
|
||||
Mockito.when(knowledge.toDocumentStore()).thenReturn(null);
|
||||
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||
Mockito.when(documentMapper.selectOneByQuery(
|
||||
Mockito.any(QueryWrapper.class))).thenReturn(document);
|
||||
Mockito.when(documentMapper.deleteById(documentId)).thenReturn(1);
|
||||
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||
Mockito.when(chunkMapper.selectListByQueryAs(
|
||||
Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class)))
|
||||
.thenReturn(List.of());
|
||||
Mockito.when(chunkMapper.deleteByQuery(
|
||||
Mockito.any(QueryWrapper.class))).thenReturn(0);
|
||||
DocumentCollectionService knowledgeService =
|
||||
Mockito.mock(DocumentCollectionService.class);
|
||||
Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge);
|
||||
ModelService modelService = Mockito.mock(ModelService.class);
|
||||
FileStorageService storageService = Mockito.mock(FileStorageService.class);
|
||||
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
|
||||
|
||||
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||
setField(service, "documentMapper", documentMapper);
|
||||
setField(service, "documentChunkMapper", chunkMapper);
|
||||
setField(service, "knowledgeService", knowledgeService);
|
||||
setField(service, "modelService", modelService);
|
||||
setField(service, "storageService", storageService);
|
||||
setField(service, "searcherFactory", searcherFactory);
|
||||
|
||||
Assert.assertTrue(service.removeDoc(documentId.toString()));
|
||||
|
||||
Mockito.verify(knowledge, Mockito.never()).toDocumentStore();
|
||||
Mockito.verifyNoInteractions(modelService);
|
||||
Mockito.verify(documentMapper).deleteById(documentId);
|
||||
Mockito.verify(storageService).delete(document.getDocumentPath());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证分块中的文档禁止删除,且不会触发任何外部或数据库清理。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void removeDocShouldRejectSplittingDocumentWithoutSideEffects()
|
||||
throws Exception {
|
||||
BigInteger documentId = BigInteger.valueOf(301);
|
||||
Document document = new Document();
|
||||
document.setId(documentId);
|
||||
document.setProcessStatus(DocumentProcessStatus.SPLITTING.name());
|
||||
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||
Mockito.when(documentMapper.selectOneByQuery(
|
||||
Mockito.any(QueryWrapper.class))).thenReturn(document);
|
||||
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||
DocumentCollectionService knowledgeService =
|
||||
Mockito.mock(DocumentCollectionService.class);
|
||||
|
||||
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||
setField(service, "documentMapper", documentMapper);
|
||||
setField(service, "documentChunkMapper", chunkMapper);
|
||||
setField(service, "knowledgeService", knowledgeService);
|
||||
|
||||
try {
|
||||
service.removeDoc(documentId.toString());
|
||||
Assert.fail("分块中的文档应拒绝删除");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertEquals("文档处理中,暂不允许删除", expected.getMessage());
|
||||
}
|
||||
|
||||
Mockito.verifyNoInteractions(chunkMapper, knowledgeService);
|
||||
Mockito.verify(documentMapper, Mockito.never()).deleteById(Mockito.any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证向量删除失败时停止后续清理,避免接口返回虚假成功。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void removeDocShouldStopWhenVectorDeleteFails() throws Exception {
|
||||
BigInteger documentId = BigInteger.valueOf(401);
|
||||
BigInteger knowledgeId = BigInteger.valueOf(402);
|
||||
BigInteger modelId = BigInteger.valueOf(403);
|
||||
Document document = new Document();
|
||||
document.setId(documentId);
|
||||
document.setCollectionId(knowledgeId);
|
||||
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
|
||||
Mockito.when(documentStore.delete(
|
||||
Mockito.anyList(), Mockito.any())).thenReturn(StoreResult.fail("测试失败"));
|
||||
DocumentCollection knowledge = Mockito.mock(DocumentCollection.class);
|
||||
Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId);
|
||||
Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test");
|
||||
Mockito.when(knowledge.toDocumentStore()).thenReturn(documentStore);
|
||||
Model model = new Model();
|
||||
model.setModelName("embedding-test");
|
||||
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||
Mockito.when(documentMapper.selectOneByQuery(
|
||||
Mockito.any(QueryWrapper.class))).thenReturn(document);
|
||||
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||
Mockito.when(chunkMapper.selectListByQueryAs(
|
||||
Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class)))
|
||||
.thenReturn(List.of(BigInteger.valueOf(404)));
|
||||
DocumentCollectionService knowledgeService =
|
||||
Mockito.mock(DocumentCollectionService.class);
|
||||
Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge);
|
||||
ModelService modelService = Mockito.mock(ModelService.class);
|
||||
Mockito.when(modelService.getById(modelId)).thenReturn(model);
|
||||
FileStorageService storageService = Mockito.mock(FileStorageService.class);
|
||||
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
|
||||
|
||||
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||
setField(service, "documentMapper", documentMapper);
|
||||
setField(service, "documentChunkMapper", chunkMapper);
|
||||
setField(service, "knowledgeService", knowledgeService);
|
||||
setField(service, "modelService", modelService);
|
||||
setField(service, "storageService", storageService);
|
||||
setField(service, "searcherFactory", searcherFactory);
|
||||
|
||||
try {
|
||||
service.removeDoc(documentId.toString());
|
||||
Assert.fail("向量删除失败时应停止文档删除");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertEquals("文档向量删除失败", expected.getMessage());
|
||||
}
|
||||
|
||||
Mockito.verify(chunkMapper, Mockito.never()).deleteByQuery(Mockito.any());
|
||||
Mockito.verify(documentMapper, Mockito.never()).deleteById(Mockito.any());
|
||||
Mockito.verifyNoInteractions(storageService);
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过反射注入测试依赖。
|
||||
*
|
||||
* @param target 目标对象
|
||||
* @param fieldName 字段名
|
||||
* @param value 字段值
|
||||
* @throws Exception 字段不存在或无法访问时抛出
|
||||
*/
|
||||
private static void setField(Object target,
|
||||
String fieldName,
|
||||
Object value) throws Exception {
|
||||
Field field = DocumentServiceImpl.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package tech.easyflow.ai.service.impl;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.system.entity.SysApiKey;
|
||||
import tech.easyflow.system.entity.SysApiKeyResource;
|
||||
import tech.easyflow.system.entity.SysApiKeyResourceMapping;
|
||||
import tech.easyflow.system.service.SysApiKeyResourceMappingService;
|
||||
import tech.easyflow.system.service.SysApiKeyResourceService;
|
||||
import tech.easyflow.system.service.SysApiKeyService;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* 知识库 Public API 产品权限映射测试。
|
||||
*
|
||||
* @author Codex
|
||||
* @since 2026-08-02
|
||||
*/
|
||||
public class KnowledgeSharePermissionServiceImplTest {
|
||||
|
||||
/**
|
||||
* 验证读取、导入和维护权限分别只生成各自接口映射。
|
||||
*
|
||||
* @throws Exception 依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void replacePermissionsShouldKeepReadImportMaintenanceSeparated()
|
||||
throws Exception {
|
||||
BigInteger apiKeyId = BigInteger.valueOf(701);
|
||||
SysApiKeyService apiKeyService = Mockito.mock(SysApiKeyService.class);
|
||||
SysApiKeyResourceService resourceService =
|
||||
Mockito.mock(SysApiKeyResourceService.class);
|
||||
SysApiKeyResourceMappingService mappingService =
|
||||
Mockito.mock(SysApiKeyResourceMappingService.class);
|
||||
RedisLockExecutor redisLockExecutor =
|
||||
Mockito.mock(RedisLockExecutor.class);
|
||||
RedisLockExecutor.LockHandle lockHandle =
|
||||
Mockito.mock(RedisLockExecutor.LockHandle.class);
|
||||
Mockito.when(redisLockExecutor.tryAcquire(
|
||||
Mockito.anyString(),
|
||||
Mockito.any(),
|
||||
Mockito.any()
|
||||
)).thenReturn(lockHandle);
|
||||
Mockito.when(apiKeyService.getById(apiKeyId)).thenReturn(new SysApiKey());
|
||||
AtomicLong resourceId = new AtomicLong(800);
|
||||
Mockito.when(resourceService.getOne(
|
||||
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
SysApiKeyResource resource = new SysApiKeyResource();
|
||||
resource.setId(BigInteger.valueOf(resourceId.incrementAndGet()));
|
||||
return resource;
|
||||
});
|
||||
|
||||
KnowledgeSharePermissionServiceImpl service =
|
||||
new KnowledgeSharePermissionServiceImpl();
|
||||
setField(service, "sysApiKeyService", apiKeyService);
|
||||
setField(service, "resourceService", resourceService);
|
||||
setField(service, "mappingService", mappingService);
|
||||
setField(service, "redisLockExecutor", redisLockExecutor);
|
||||
|
||||
service.replaceApiPermissions(apiKeyId, true, false, false);
|
||||
service.replaceApiPermissions(apiKeyId, false, true, false);
|
||||
service.replaceApiPermissions(apiKeyId, false, false, true);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<List<SysApiKeyResourceMapping>> mappings =
|
||||
ArgumentCaptor.forClass(List.class);
|
||||
Mockito.verify(mappingService, Mockito.times(3)).remove(
|
||||
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)
|
||||
);
|
||||
Mockito.verify(mappingService, Mockito.times(3))
|
||||
.saveBatch(mappings.capture());
|
||||
Mockito.verify(lockHandle, Mockito.times(3)).release();
|
||||
Assert.assertEquals(3, mappings.getAllValues().size());
|
||||
assertScopeMappings(
|
||||
mappings.getAllValues().get(0),
|
||||
KnowledgeApiPermissionScope.KNOWLEDGE_READ,
|
||||
8
|
||||
);
|
||||
assertScopeMappings(
|
||||
mappings.getAllValues().get(1),
|
||||
KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT,
|
||||
14
|
||||
);
|
||||
assertScopeMappings(
|
||||
mappings.getAllValues().get(2),
|
||||
KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE,
|
||||
6
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 断言一组全局知识库接口映射只包含指定权限。
|
||||
*
|
||||
* @param mappings 接口映射
|
||||
* @param expectedScope 预期权限
|
||||
* @param expectedSize 预期接口数量
|
||||
*/
|
||||
private void assertScopeMappings(
|
||||
List<SysApiKeyResourceMapping> mappings,
|
||||
KnowledgeApiPermissionScope expectedScope,
|
||||
int expectedSize) {
|
||||
Assert.assertEquals(expectedSize, mappings.size());
|
||||
for (SysApiKeyResourceMapping mapping : mappings) {
|
||||
Assert.assertEquals(
|
||||
expectedScope.name(),
|
||||
mapping.getActionScope()
|
||||
);
|
||||
Assert.assertNull(mapping.getResourceTargetId());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 反射注入测试依赖。
|
||||
*
|
||||
* @param target 目标对象
|
||||
* @param fieldName 字段名
|
||||
* @param value 字段值
|
||||
* @throws Exception 字段不存在或不可访问
|
||||
*/
|
||||
private static void setField(
|
||||
Object target,
|
||||
String fieldName,
|
||||
Object value) throws Exception {
|
||||
Field field = target.getClass().getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user