From 6cbc330f55b57c148a57ed6f7c8280d8af9c9f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Fri, 4 Sep 2026 11:29:06 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=81=A2=E5=A4=8D=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E9=87=8D=E5=90=AF=E5=90=8E=E7=9A=84=E8=87=AA=E5=8A=A8=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DocumentParseBridgeException.java | 4 + .../impl/DocumentParseBridgeServiceImpl.java | 9 + .../task/DocumentImportBatchAppService.java | 25 +- .../DocumentImportBatchCircuitBreaker.java | 84 +- .../task/DocumentImportBatchTracker.java | 253 +++- .../DocumentImportRecoveryCoordinator.java | 75 ++ .../task/DocumentImportRecoveryException.java | 71 ++ ...KnowledgeDocumentImportTaskAppService.java | 680 +++++++++-- .../mapper/DocumentImportBatchItemMapper.java | 112 +- .../ai/mapper/DocumentImportBatchMapper.java | 40 +- .../ai/mapper/DocumentImportTaskMapper.java | 52 +- .../DocumentParseBridgeServiceImplTest.java | 32 + .../DocumentImportBatchAppServiceTest.java | 61 +- ...DocumentImportBatchCircuitBreakerTest.java | 169 ++- .../task/DocumentImportBatchTrackerTest.java | 162 +++ ...DocumentImportRecoveryCoordinatorTest.java | 74 ++ ...ledgeDocumentImportTaskAppServiceTest.java | 1073 ++++++++++++++++- 17 files changed, 2676 insertions(+), 300 deletions(-) create mode 100644 easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryCoordinator.java create mode 100644 easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryException.java create mode 100644 easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryCoordinatorTest.java diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/exception/DocumentParseBridgeException.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/exception/DocumentParseBridgeException.java index fa321214..75f3da2b 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/exception/DocumentParseBridgeException.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/exception/DocumentParseBridgeException.java @@ -75,6 +75,10 @@ public class DocumentParseBridgeException extends RuntimeException { return new DocumentParseBridgeException("task_failed", message); } + public static DocumentParseBridgeException taskNotFound(String message, Throwable cause) { + return new DocumentParseBridgeException("task_not_found", message, cause); + } + public static DocumentParseBridgeException resultFetchFailed(String message, Throwable cause) { return new DocumentParseBridgeException("result_fetch_failed", message, cause); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java index 2da2b084..909cfffc 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java @@ -5,6 +5,7 @@ import com.easyagents.document.core.entity.ParseResponse; import com.easyagents.document.core.entity.ParseResult; import com.easyagents.document.core.entity.ParseTaskInfo; import com.easyagents.document.core.entity.ParseTaskStatus; +import com.easyagents.document.core.exception.DocumentAsyncTaskNotFoundException; import com.easyagents.document.pdf.PdfDocumentParseService; import com.easyagents.document.pptx.PptxDocumentParseService; import com.easyagents.document.xlsx.XlsxDocumentParseService; @@ -140,6 +141,8 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic return parseResultMapper.map(taskStatus); } catch (DocumentParseBridgeException e) { throw e; + } catch (DocumentAsyncTaskNotFoundException e) { + throw DocumentParseBridgeException.taskNotFound("异步文档解析任务不存在", e); } catch (Exception e) { throw DocumentParseBridgeException.taskFailed("查询异步文档解析任务状态失败", e); } @@ -175,6 +178,9 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic } catch (DocumentParseBridgeException e) { LOG.error("桥接服务获取异步解析结果失败: providerTaskId={}", taskId, e); throw e; + } catch (DocumentAsyncTaskNotFoundException e) { + LOG.warn("桥接服务异步解析执行已丢失: providerTaskId={}", taskId); + throw DocumentParseBridgeException.taskNotFound("异步文档解析任务不存在", e); } catch (Exception e) { LOG.error("桥接服务获取异步解析结果异常: providerTaskId={}", taskId, e); throw DocumentParseBridgeException.resultFetchFailed("获取异步文档解析结果失败", e); @@ -212,6 +218,9 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic } catch (DocumentParseBridgeException e) { LOG.error("桥接服务查询异步解析任务状态失败: providerTaskId={}", taskId, e); throw e; + } catch (DocumentAsyncTaskNotFoundException e) { + LOG.warn("桥接服务异步解析执行已丢失: providerTaskId={}", taskId); + throw DocumentParseBridgeException.taskNotFound("异步文档解析任务不存在", e); } catch (Exception e) { LOG.error("桥接服务查询异步解析任务状态异常: providerTaskId={}", taskId, e); throw DocumentParseBridgeException.taskFailed("聚合查询异步文档解析任务信息失败", e); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppService.java index e9e3b3e9..2341bd7d 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppService.java @@ -80,6 +80,7 @@ public class DocumentImportBatchAppService { private final DocumentMapper documentMapper; private final RedisLockExecutor redisLockExecutor; private final DocumentImportBatchCircuitBreaker circuitBreaker; + private final DocumentImportRecoveryCoordinator recoveryCoordinator; @Resource(name = "default") private FileStorageService storageService; @@ -97,6 +98,7 @@ public class DocumentImportBatchAppService { * @param documentMapper 文档 Mapper * @param redisLockExecutor 分布式锁执行器 * @param circuitBreaker 自动导入批次熔断器 + * @param recoveryCoordinator 恢复令牌事务协调器 */ public DocumentImportBatchAppService(DocumentImportBatchService batchService, DocumentImportBatchItemService itemService, @@ -107,7 +109,8 @@ public class DocumentImportBatchAppService { DocumentImportBatchItemMapper itemMapper, DocumentMapper documentMapper, RedisLockExecutor redisLockExecutor, - DocumentImportBatchCircuitBreaker circuitBreaker) { + DocumentImportBatchCircuitBreaker circuitBreaker, + DocumentImportRecoveryCoordinator recoveryCoordinator) { this.batchService = batchService; this.itemService = itemService; this.batchTracker = batchTracker; @@ -118,6 +121,7 @@ public class DocumentImportBatchAppService { this.documentMapper = documentMapper; this.redisLockExecutor = redisLockExecutor; this.circuitBreaker = circuitBreaker; + this.recoveryCoordinator = recoveryCoordinator; } /** @@ -1246,13 +1250,10 @@ public class DocumentImportBatchAppService { Date claimedAt = new Date(); Date leaseUntil = new Date( claimedAt.getTime() + RECOVERY_DISPATCH_LEASE.toMillis()); - if (batchMapper.claimRecoveryPending( - batchId, recoveryToken, leaseUntil, claimedAt) <= 0) { - return; - } try { DocumentImportBatch claimedBatch = - batchMapper.selectClaimedRecovery(batchId, recoveryToken); + recoveryCoordinator.claim( + batchId, recoveryToken, leaseUntil, claimedAt); if (claimedBatch == null) { LOG.info( "批次恢复调度令牌已失效，旧持有者停止恢复: " @@ -1284,9 +1285,8 @@ public class DocumentImportBatchAppService { ); return; } - int finalized = batchMapper.finalizeRecoveryPending( - batchId, recoveryToken, new Date()); - if (finalized <= 0) { + if (!recoveryCoordinator.finalizeRecovery( + batchId, recoveryToken, new Date())) { LOG.info( "批次恢复待办已变更，当前实例跳过收尾: " + "batchId={}, recoveryToken={}", @@ -1299,7 +1299,8 @@ public class DocumentImportBatchAppService { if (!circuitBreaker.interruptRecoveryBatch( batchId, recoveryToken, error)) { LOG.info( - "批次恢复异常发生时令牌已失效，跳过旧持有者熔断: " + "批次恢复异常发生时令牌已失效或已有并发推进，" + + "跳过当前持有者熔断: " + "batchId={}, recoveryToken={}", batchId, recoveryToken @@ -1350,13 +1351,13 @@ public class DocumentImportBatchAppService { Date renewedLeaseUntil = new Date( nowMillis + RECOVERY_DISPATCH_LEASE.toMillis() ); - int renewed = batchMapper.renewRecoveryPendingLease( + boolean renewed = recoveryCoordinator.renew( batchId, recoveryToken, renewedLeaseUntil, now ); - if (renewed <= 0) { + if (!renewed) { return false; } renewAfter.set(nowMillis + renewIntervalMillis); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreaker.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreaker.java index e9d38668..6357295d 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreaker.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreaker.java @@ -9,7 +9,9 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.entity.DocumentImportBatchItem; import tech.easyflow.ai.entity.DocumentImportTask; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; import tech.easyflow.ai.enums.DocumentImportBatchStatus; import tech.easyflow.ai.enums.DocumentImportMode; import tech.easyflow.ai.enums.DocumentImportTaskStatus; @@ -23,6 +25,8 @@ import java.sql.SQLRecoverableException; import java.sql.SQLTimeoutException; import java.sql.SQLTransientConnectionException; import java.util.Date; +import java.util.List; +import java.util.Objects; import java.util.concurrent.RejectedExecutionException; /** @@ -135,13 +139,18 @@ public class DocumentImportBatchCircuitBreaker { if (recoveryToken == null || recoveryToken.isBlank()) { throw new IllegalArgumentException("恢复调度令牌不能为空"); } + DocumentImportRecoveryException recoveryError = + findCause(error, DocumentImportRecoveryException.class); return interruptBatch( batchId, resolveReason(error), null, null, error, - recoveryToken + recoveryToken, + recoveryError == null + ? List.of() + : recoveryError.getFailureFenceItemIds() ); } @@ -211,7 +220,7 @@ public class DocumentImportBatchCircuitBreaker { String phase, Throwable error) { return interruptBatch( - batchId, reason, taskId, phase, error, null); + batchId, reason, taskId, phase, error, null, List.of()); } /** @@ -223,6 +232,7 @@ public class DocumentImportBatchCircuitBreaker { * @param phase 触发任务阶段，可为空 * @param error 原始异常 * @param recoveryToken 恢复调度令牌，可为空 + * @param recoveryFailureItemIds 零重排时的初始失败项围栏 * @return 批次是否已经停止运行；围栏失效时返回 {@code false} */ private boolean interruptBatch(BigInteger batchId, @@ -230,11 +240,17 @@ public class DocumentImportBatchCircuitBreaker { BigInteger taskId, String phase, Throwable error, - String recoveryToken) { + String recoveryToken, + List recoveryFailureItemIds) { if (batchId == null) { return false; } - DocumentImportBatch batch = batchMapper.selectOneById(batchId); + boolean fencedRecoveryFailure = recoveryToken != null + && recoveryFailureItemIds != null + && !recoveryFailureItemIds.isEmpty(); + // 批次行是后续 task/item 状态收口的根锁；所有熔断路径先锁定 + // batch，避免多表围栏更新与任务完成事务形成 task -> batch 逆序。 + DocumentImportBatch batch = batchMapper.selectForUpdate(batchId); if (batch == null) { LOG.warn( "忽略无法关联批次的自动导入熔断请求: taskId={}, batchId={}", @@ -249,6 +265,39 @@ public class DocumentImportBatchCircuitBreaker { } Date now = new Date(); + if (fencedRecoveryFailure) { + if (!Boolean.TRUE.equals(batch.getRecoveryPending()) + || !Objects.equals(recoveryToken, batch.getRecoveryToken()) + || batch.getRecoveryLeaseUntil() == null + || !batch.getRecoveryLeaseUntil().after(now)) { + return false; + } + if (itemMapper.countActiveItems(batchId) > 0) { + LOG.info( + "恢复失败熔断发现批次已有活跃文件项，保留运行批次: " + + "batchId={}", + batchId + ); + return false; + } + for (BigInteger itemId : recoveryFailureItemIds) { + DocumentImportBatchItem item = + itemMapper.selectForUpdate(itemId); + if (item == null + || !batchId.equals(item.getBatchId()) + || !DocumentImportBatchItemStatus.FAILED + .name().equals(item.getStatus())) { + LOG.info( + "恢复失败熔断发现等价并发推进，保留运行批次: " + + "batchId={}, itemId={}, status={}", + batchId, + itemId, + item == null ? null : item.getStatus() + ); + return false; + } + } + } int interrupted; if (recoveryToken != null) { interrupted = batchMapper.interruptOwnedRecoveryBatch( @@ -305,6 +354,14 @@ public class DocumentImportBatchCircuitBreaker { * @return 稳定错误码与用户可见摘要 */ private InterruptionReason resolveReason(Throwable error) { + DocumentImportRecoveryException recoveryError = + findCause(error, DocumentImportRecoveryException.class); + if (recoveryError != null) { + return new InterruptionReason( + recoveryError.getCode(), + recoveryError.getUserMessage() + ); + } if (containsRedisFailure(error)) { return new InterruptionReason( REDIS_UNAVAILABLE, @@ -329,6 +386,25 @@ public class DocumentImportBatchCircuitBreaker { ); } + /** + * 从异常链中查找指定类型。 + * + * @param error 原始异常 + * @param type 目标异常类型 + * @param 异常类型 + * @return 匹配异常；不存在时返回 {@code null} + */ + private T findCause(Throwable error, Class type) { + Throwable current = error; + while (current != null) { + if (type.isInstance(current)) { + return type.cast(current); + } + current = current.getCause(); + } + return null; + } + /** * 判断异常链是否来自 Redis/Lettuce。 * diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java index e8e94849..b02affd9 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java @@ -27,6 +27,18 @@ import java.util.Date; @Service public class DocumentImportBatchTracker { + /** + * 单文件恢复失败原因的事务内处理结果。 + */ + public enum FailedRetryErrorOutcome { + /** 失败原因已写回，文件项仍等待后续人工恢复。 */ + ERROR_RECORDED, + /** 等价并发操作已把文件项推进出失败状态。 */ + ALREADY_ADVANCED, + /** 批次或文件项已不再允许当前恢复请求写回。 */ + REJECTED + } + private final DocumentImportBatchService batchService; private final DocumentImportBatchItemService itemService; private final DocumentImportBatchMapper batchMapper; @@ -64,6 +76,31 @@ public class DocumentImportBatchTracker { return batch; } + /** + * 在任务行变更前锁定所属批次，统一 batch -> task 锁序。 + * + *

自动导入的任务生命周期由运行批次驱动，批次结束后不允许 + * 旧任务继续推进。手动导入的上传批次可以先结束，用户之后再手动 + * 执行分块和索引，因此只参与锁序，不以批次终态阻断后续任务。

+ * + * @param batchId 批次 ID；无批次任务传入 {@code null} + * @return 批次存在且允许当前任务推进时为 {@code true} + */ + public boolean lockBatchForTaskMutation(BigInteger batchId) { + if (batchId == null) { + return true; + } + DocumentImportBatch batch = batchMapper.selectForUpdate(batchId); + if (batch == null) { + return false; + } + if (DocumentImportMode.AUTO.name().equals(batch.getImportMode())) { + return DocumentImportBatchStatus.RUNNING.name().equals( + batch.getStatus()); + } + return DocumentImportMode.MANUAL.name().equals(batch.getImportMode()); + } + /** * 查询批次文件项。 * @@ -107,10 +144,8 @@ public class DocumentImportBatchTracker { if (itemId == null) { return false; } - DocumentImportBatchItem current = requireItem(itemId); int attemptDelta = (status == DocumentImportBatchItemStatus.PENDING || status == DocumentImportBatchItemStatus.RUNNING) - && DocumentImportBatchItemStatus.FAILED.name().equals(current.getStatus()) ? 1 : 0; return transitionItem(itemId, stage, status, errorSummary, @@ -162,48 +197,52 @@ public class DocumentImportBatchTracker { if (itemId == null) { return false; } - for (int attempt = 0; attempt < 3; attempt++) { - DocumentImportBatchItem current = requireItem(itemId); - DocumentImportBatchItemStatus currentStatus = - DocumentImportBatchItemStatus.valueOf(current.getStatus()); - if (!isAllowedTransition(currentStatus, status)) { - return false; - } - String expectedStatus = current.getStatus(); - Date now = new Date(); - int updated = itemMapper.transitionStatus( - itemId, - expectedStatus, - stage.name(), - status.name(), - errorSummary, - failureCode, - retryable, - Math.max(0, attemptDelta), + LockedBatchItem locked = lockBatchThenItem(itemId); + if (DocumentImportBatchStatus.INTERRUPTED.name() + .equals(locked.batch.getStatus())) { + return false; + } + DocumentImportBatchItem current = locked.item; + DocumentImportBatchItemStatus currentStatus = + DocumentImportBatchItemStatus.valueOf(current.getStatus()); + if (!isAllowedTransition(currentStatus, status)) { + return false; + } + Date now = new Date(); + int effectiveAttemptDelta = currentStatus == DocumentImportBatchItemStatus.FAILED + ? Math.max(0, attemptDelta) + : 0; + int updated = itemMapper.transitionStatus( + itemId, + current.getStatus(), + stage.name(), + status.name(), + errorSummary, + failureCode, + retryable, + effectiveAttemptDelta, + now + ); + if (updated <= 0) { + return false; + } + CounterDelta delta = CounterDelta.between(current, status, retryable); + if (!delta.isZero()) { + batchMapper.adjustCounters( + current.getBatchId(), + delta.completed, + delta.processing, + delta.failed, + delta.pending, + delta.uploaded, + delta.skipped, + delta.cancelled, + delta.retryableFailed, now ); - if (updated <= 0) { - continue; - } - CounterDelta delta = CounterDelta.between(current, status, retryable); - if (!delta.isZero()) { - batchMapper.adjustCounters( - current.getBatchId(), - delta.completed, - delta.processing, - delta.failed, - delta.pending, - delta.uploaded, - delta.skipped, - delta.cancelled, - delta.retryableFailed, - now - ); - } - refreshBatch(current.getBatchId()); - return true; } - return false; + refreshBatch(current.getBatchId()); + return true; } /** @@ -243,7 +282,12 @@ public class DocumentImportBatchTracker { */ @org.springframework.transaction.annotation.Transactional public void bindDocument(BigInteger itemId, BigInteger documentId) { - DocumentImportBatchItem item = requireItem(itemId); + LockedBatchItem locked = lockBatchThenItem(itemId); + if (DocumentImportBatchStatus.INTERRUPTED.name() + .equals(locked.batch.getStatus())) { + throw new BusinessException("导入批次已中断，请继续批次后重试"); + } + DocumentImportBatchItem item = locked.item; if (documentId.equals(item.getDocumentId()) && DocumentImportBatchItemStatus.PENDING.name().equals(item.getStatus())) { return; @@ -252,6 +296,11 @@ public class DocumentImportBatchTracker { boolean recoveringFailedItem = DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus()) && item.getDocumentId() == null; + if (recoveringFailedItem + && !DocumentImportBatchStatus.RUNNING.name() + .equals(locked.batch.getStatus())) { + throw new BusinessException("导入批次状态已变化，请刷新后重试"); + } int updated = recoveringFailedItem ? itemMapper.bindFailedDocument(itemId, documentId, now) : itemMapper.bindDocument(itemId, documentId, now); @@ -268,6 +317,91 @@ public class DocumentImportBatchTracker { refreshBatch(item.getBatchId()); } + /** + * 将引用已丢失文档的失败项重新绑定到恢复创建的新文档。 + * + * @param itemId 文件项 ID + * @param missingDocumentId 已不存在的旧文档 ID + * @param documentId 恢复创建的新文档 ID + */ + @org.springframework.transaction.annotation.Transactional + public void replaceMissingDocument(BigInteger itemId, + BigInteger missingDocumentId, + BigInteger documentId) { + LockedBatchItem locked = lockBatchThenItem(itemId); + if (!DocumentImportBatchStatus.RUNNING.name() + .equals(locked.batch.getStatus())) { + throw new BusinessException("导入批次状态已变化，请刷新后重试"); + } + DocumentImportBatchItem item = locked.item; + if (!DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus()) + || !java.util.Objects.equals(missingDocumentId, item.getDocumentId())) { + throw new BusinessException("导入文件状态已变化，请刷新后重试"); + } + Date now = new Date(); + if (itemMapper.replaceMissingDocument( + itemId, missingDocumentId, documentId, now) <= 0) { + throw new BusinessException("原文档状态已变化，请刷新后重试"); + } + batchMapper.refreshCountersFromItems(item.getBatchId(), now); + refreshBatch(item.getBatchId()); + } + + /** + * 按批次到文件项的固定顺序取得失败项恢复权。 + * + * @param itemId 文件项 ID + * @return 可恢复文件项；状态已变化时返回 null + */ + @org.springframework.transaction.annotation.Transactional + public DocumentImportBatchItem lockFailedItemForRetry(BigInteger itemId) { + if (itemId == null) { + return null; + } + LockedBatchItem locked = lockBatchThenItem(itemId); + if (!DocumentImportBatchStatus.RUNNING.name() + .equals(locked.batch.getStatus()) + || !DocumentImportBatchItemStatus.FAILED.name() + .equals(locked.item.getStatus())) { + return null; + } + return locked.item; + } + + /** + * 在运行批次内保留单文件恢复失败原因。 + * + * @param itemId 文件项 ID + * @param batchId 批次 ID + * @param errorSummary 错误摘要 + * @return 失败原因写回、并发推进或拒绝结果 + */ + @org.springframework.transaction.annotation.Transactional( + propagation = org.springframework.transaction.annotation.Propagation.REQUIRES_NEW) + public FailedRetryErrorOutcome updateFailedRetryError( + BigInteger itemId, + BigInteger batchId, + String errorSummary + ) { + if (itemId == null || batchId == null) { + return FailedRetryErrorOutcome.REJECTED; + } + LockedBatchItem locked = lockBatchThenItem(itemId); + if (!batchId.equals(locked.batch.getId()) + || !DocumentImportBatchStatus.RUNNING.name() + .equals(locked.batch.getStatus())) { + return FailedRetryErrorOutcome.REJECTED; + } + if (!DocumentImportBatchItemStatus.FAILED.name() + .equals(locked.item.getStatus())) { + return FailedRetryErrorOutcome.ALREADY_ADVANCED; + } + return itemMapper.updateFailedRetryError( + itemId, batchId, errorSummary, new Date()) > 0 + ? FailedRetryErrorOutcome.ERROR_RECORDED + : FailedRetryErrorOutcome.REJECTED; + } + /** * 原子完成文件上传并增量更新批次上传数。 * @@ -280,7 +414,12 @@ public class DocumentImportBatchTracker { public boolean completeUpload(BigInteger itemId, String filePath, String storageLocator) { - DocumentImportBatchItem item = requireItem(itemId); + LockedBatchItem locked = lockBatchThenItem(itemId); + if (DocumentImportBatchStatus.INTERRUPTED.name() + .equals(locked.batch.getStatus())) { + return false; + } + DocumentImportBatchItem item = locked.item; if (DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus())) { return filePath.equals(item.getFilePath()) && storageLocator.equals(item.getStorageLocator()); @@ -304,6 +443,34 @@ public class DocumentImportBatchTracker { return true; } + /** + * 先锁批次再锁文件项，统一所有计数迁移路径的加锁顺序。 + * + * @param itemId 文件项 ID + * @return 已锁定的批次与文件项 + */ + private LockedBatchItem lockBatchThenItem(BigInteger itemId) { + DocumentImportBatchItem candidate = requireItem(itemId); + DocumentImportBatch batch = batchMapper.selectForUpdate(candidate.getBatchId()); + if (batch == null) { + throw new BusinessException("导入批次不存在"); + } + DocumentImportBatchItem item = itemMapper.selectForUpdate(itemId); + if (item == null) { + throw new BusinessException("导入文件不存在"); + } + if (!java.util.Objects.equals(batch.getId(), item.getBatchId())) { + throw new IllegalStateException("导入文件批次归属已变化"); + } + return new LockedBatchItem(batch, item); + } + + private record LockedBatchItem( + DocumentImportBatch batch, + DocumentImportBatchItem item + ) { + } + /** * 记录文件项成功后需要清理的历史文档。 * diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryCoordinator.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryCoordinator.java new file mode 100644 index 00000000..46c060c3 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryCoordinator.java @@ -0,0 +1,75 @@ +package tech.easyflow.ai.documentimport.task; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; + +import java.math.BigInteger; +import java.util.Date; + +/** + * 在独立短事务中维护批次恢复令牌。 + * + *

继续操作通常从原事务的 {@code afterCommit} 回调启动。将领取、续租和 + * 收尾放入独立事务，可保证恢复令牌在单文件重试事务开始前已经提交，并避免 + * 复用刚完成提交但尚未解绑的事务资源。

+ */ +@Service +public class DocumentImportRecoveryCoordinator { + + private final DocumentImportBatchMapper batchMapper; + + public DocumentImportRecoveryCoordinator(DocumentImportBatchMapper batchMapper) { + this.batchMapper = batchMapper; + } + + /** + * 领取恢复待办并读取本次令牌对应的恢复参数。 + * + * @param batchId 批次 ID + * @param recoveryToken 恢复令牌 + * @param leaseUntil 租约到期时间 + * @param claimedAt 领取时间 + * @return 已领取批次；领取失败时返回 null + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public DocumentImportBatch claim(BigInteger batchId, + String recoveryToken, + Date leaseUntil, + Date claimedAt) { + if (batchMapper.claimRecoveryPending( + batchId, recoveryToken, leaseUntil, claimedAt) <= 0) { + return null; + } + return batchMapper.selectClaimedRecovery(batchId, recoveryToken); + } + + /** + * 续期当前恢复令牌。 + * + * @return 当前令牌仍有效且续期成功时返回 true + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean renew(BigInteger batchId, + String recoveryToken, + Date leaseUntil, + Date renewedAt) { + return batchMapper.renewRecoveryPendingLease( + batchId, recoveryToken, leaseUntil, renewedAt) > 0; + } + + /** + * 按真实计数收尾并清除恢复待办。 + * + * @return 当前令牌仍有效且收尾成功时返回 true + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean finalizeRecovery(BigInteger batchId, + String recoveryToken, + Date finalizedAt) { + return batchMapper.finalizeRecoveryPending( + batchId, recoveryToken, finalizedAt) > 0; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryException.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryException.java new file mode 100644 index 00000000..fa49e6bb --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryException.java @@ -0,0 +1,71 @@ +package tech.easyflow.ai.documentimport.task; + +import java.math.BigInteger; +import java.util.Collection; +import java.util.List; + +/** + * 批次恢复未能重新排队任何失败项。 + * + *

该异常只携带稳定错误码和用户可见摘要，供恢复令牌持有者 + * 中断批次，避免“继续”请求在没有启动任何新任务时表现为成功。

+ */ +public class DocumentImportRecoveryException extends RuntimeException { + + private static final String ERROR_CODE = "document_import_recovery_failed"; + private static final String USER_MESSAGE = + "未能重新排队任何失败文件，请查看文件错误后继续"; + private final List failureFenceItemIds; + + /** + * 创建批次恢复失败异常。 + */ + public DocumentImportRecoveryException() { + this(List.of()); + } + + /** + * 创建带初始失败项快照的批次恢复失败异常。 + * + * @param failureFenceItemIds 领取恢复时仍为失败的文件项 ID + */ + public DocumentImportRecoveryException( + Collection failureFenceItemIds + ) { + super(USER_MESSAGE); + this.failureFenceItemIds = failureFenceItemIds == null + ? List.of() + : failureFenceItemIds.stream() + .filter(java.util.Objects::nonNull) + .distinct() + .sorted() + .toList(); + } + + /** + * 获取稳定错误码。 + * + * @return 稳定错误码 + */ + public String getCode() { + return ERROR_CODE; + } + + /** + * 获取用户可见摘要。 + * + * @return 用户可见摘要 + */ + public String getUserMessage() { + return USER_MESSAGE; + } + + /** + * 获取用于原子熔断围栏的初始失败项。 + * + * @return 按 ID 排序的不可变列表 + */ + public List getFailureFenceItemIds() { + return failureFenceItemIds; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java index 79b61987..16a8f512 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java @@ -79,6 +79,7 @@ import tech.easyflow.common.util.StringUtil; import tech.easyflow.common.web.exceptions.BusinessException; import javax.annotation.Resource; +import jakarta.annotation.PreDestroy; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; @@ -103,6 +104,11 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BooleanSupplier; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -126,6 +132,7 @@ public class KnowledgeDocumentImportTaskAppService { private static final String SEARCH_RENDER_MARKDOWN_METADATA_KEY = "renderMarkdown"; private static final String TASK_ERROR_PARSE_SERVICE_UNAVAILABLE = "parse_service_unavailable"; private static final String TASK_ERROR_PARSE_SERVICE_TIMEOUT = "parse_service_timeout"; + private static final String TASK_ERROR_PARSE_EXECUTION_LOST = "parse_execution_lost"; private static final String TASK_ERROR_DOCUMENT_SOURCE_UNAVAILABLE = "document_source_unavailable"; private static final String TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE = "unsupported_document_source"; private static final String TASK_ERROR_INVALID_PARSE_REQUEST = "invalid_parse_request"; @@ -134,13 +141,24 @@ public class KnowledgeDocumentImportTaskAppService { private static final String TASK_ERROR_EXECUTION_INTERRUPTED = "execution_interrupted"; private static final String TASK_ERROR_SPLIT_FAILED = "split_failed"; private static final String TASK_ERROR_INDEX_FAILED = "index_failed"; + private static final String PAYLOAD_REPLACE_EXISTING_INDEX = + "replaceExistingIndex"; private static final String DEFAULT_INDEX_FAILURE_MESSAGE = "分块或向量化失败，请重试"; private static final String ROLLBACK_FAILURE_SUFFIX = "；外部索引回滚未完成，请联系管理员处理"; + private static final String DOCUMENT_EXECUTION_LOCK_PREFIX = + "easyflow:lock:document-import:document:"; private static final Pattern HTTP_SERVER_ERROR_PATTERN = Pattern.compile("\\bstatus=5\\d{2}\\b"); private static final Pattern MARKDOWN_IMAGE_PATTERN = Pattern.compile("!\\[(?:[^\\]]*)\\]\\(([^)]+)\\)"); private final FlexIDKeyGenerator flexIdKeyGenerator = new FlexIDKeyGenerator(); private final TabularRowWindowChunkBuilder tabularChunkBuilder = new TabularRowWindowChunkBuilder(); + private final ScheduledExecutorService taskExecutionHeartbeatExecutor = + Executors.newScheduledThreadPool(2, runnable -> { + Thread thread = new Thread( + runnable, "easyflow-document-import-heartbeat"); + thread.setDaemon(true); + return thread; + }); @Resource private DocumentMapper documentMapper; @@ -306,7 +324,8 @@ public class KnowledgeDocumentImportTaskAppService { || !StringUtil.hasText(item.getFilePath())) { throw new BusinessException("批次包含未上传完成的文件"); } - DocumentImportTask task = createBatchImportTask(batch, item, knowledge); + DocumentImportTask task = createBatchImportTask( + batch, item, knowledge, false); taskIds.add(task.getId()); } // 启动接口返回后列表会统一刷新；批量建档阶段不逐文件推送，避免提交后形成 SSE 风暴。 @@ -330,13 +349,14 @@ public class KnowledgeDocumentImportTaskAppService { private DocumentImportTask createBatchImportTask( DocumentImportBatch batch, DocumentImportBatchItem item, - DocumentCollection knowledge) { + DocumentCollection knowledge, + boolean replaceMissingDocument) { boolean uploaded = DocumentImportBatchItemStatus.UPLOADED.name() .equals(item.getStatus()); - boolean failedWithoutDocument = + boolean failedForRecovery = DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus()) - && item.getDocumentId() == null; - if ((!uploaded && !failedWithoutDocument) + && (item.getDocumentId() == null || replaceMissingDocument); + if ((!uploaded && !failedForRecovery) || !StringUtil.hasText(item.getFilePath())) { throw new BusinessException("批次包含无法恢复的文件"); } @@ -378,7 +398,12 @@ public class KnowledgeDocumentImportTaskAppService { DocumentImportTaskPhase.PARSE, buildDocumentPayload(document) ); - documentImportBatchTracker.bindDocument(item.getId(), document.getId()); + if (replaceMissingDocument) { + documentImportBatchTracker.replaceMissingDocument( + item.getId(), item.getDocumentId(), document.getId()); + } else { + documentImportBatchTracker.bindDocument(item.getId(), document.getId()); + } return task; } @@ -421,12 +446,15 @@ public class KnowledgeDocumentImportTaskAppService { QueryWrapper query = QueryWrapper.create() .eq(DocumentImportBatchItem::getBatchId, batchId) .eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name()); + List allFailedItems = + documentImportBatchItemService.list(query); + List failedItems = allFailedItems; if (fileKeys != null && !fileKeys.isEmpty()) { - query.in(DocumentImportBatchItem::getClientFileKey, fileKeys); + failedItems = allFailedItems.stream() + .filter(item -> fileKeys.contains(item.getClientFileKey())) + .toList(); } - List failedItems = documentImportBatchItemService.list( - query - ); + int handledItems = 0; for (DocumentImportBatchItem item : failedItems) { if (recoveryLeaseGuard != null && !recoveryLeaseGuard.getAsBoolean()) { @@ -439,13 +467,20 @@ public class KnowledgeDocumentImportTaskAppService { return false; } try { - selfProxy.retryBatchItemInNewTransaction(item.getId()); + if (selfProxy.retryBatchItemInNewTransaction(item.getId())) { + handledItems++; + } } catch (BusinessException error) { LOG.error("批次失败项重试启动失败: batchId={}, itemId={}, documentId={}", batchId, item.getId(), item.getDocumentId(), error); - documentImportBatchItemMapper.updateFailedRetryError( + DocumentImportBatchTracker.FailedRetryErrorOutcome outcome = + documentImportBatchTracker.updateFailedRetryError( item.getId(), batchId, - truncateError(error.getMessage()), new Date()); + truncateError(error.getMessage())); + if (outcome + == DocumentImportBatchTracker.FailedRetryErrorOutcome.ALREADY_ADVANCED) { + handledItems++; + } } } // 最后一次校验覆盖“末项处理完成到批次收尾”之间的租约失效窗口。 @@ -458,6 +493,12 @@ public class KnowledgeDocumentImportTaskAppService { ); return false; } + if (!failedItems.isEmpty() && handledItems == 0) { + throw new DocumentImportRecoveryException( + allFailedItems.stream() + .map(DocumentImportBatchItem::getId) + .toList()); + } return true; } @@ -465,15 +506,22 @@ public class KnowledgeDocumentImportTaskAppService { * 在独立事务中重试一个批次失败项，避免单文件异常影响同批次其他文件。 * * @param itemId 批次文件项 ID + * @return 已重新排队，或已被等价并发操作推进出失败状态时返回 + * {@code true} */ @Transactional(propagation = Propagation.REQUIRES_NEW) - public void retryBatchItemInNewTransaction(BigInteger itemId) { + public boolean retryBatchItemInNewTransaction(BigInteger itemId) { DocumentImportBatchItem item = - documentImportBatchItemMapper.selectFailedForRetry(itemId); + documentImportBatchTracker.lockFailedItemForRetry(itemId); if (item == null) { - return; + DocumentImportBatchItem current = + documentImportBatchItemService.getById(itemId); + return current != null + && !DocumentImportBatchItemStatus.FAILED.name().equals( + current.getStatus()); } retryBatchItem(item); + return true; } /** @@ -486,8 +534,10 @@ public class KnowledgeDocumentImportTaskAppService { List tasks = documentImportTaskMapper.selectPendingFairly(redispatchBefore, limit); for (DocumentImportTask task : tasks) { - int touched = documentImportTaskMapper.touchPendingForDispatch( - task.getId(), redispatchBefore, new Date(), resolveOperatorId()); + KnowledgeDocumentImportTaskAppService executor = + selfProxy == null ? this : selfProxy; + int touched = executor.touchPendingTaskForDispatch( + task, redispatchBefore); if (touched != 1) { continue; } @@ -505,6 +555,24 @@ public class KnowledgeDocumentImportTaskAppService { } } + /** + * 取得批次行锁后刷新待处理任务的投递时间，统一 batch -> task 锁序。 + * + * @param task 待投递任务 + * @param redispatchBefore 允许重新投递的时间边界 + * @return 更新行数 + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public int touchPendingTaskForDispatch( + DocumentImportTask task, + Date redispatchBefore) { + if (!lockTaskBatchForMutation(task)) { + return 0; + } + return documentImportTaskMapper.touchPendingForDispatch( + task.getId(), redispatchBefore, new Date(), resolveOperatorId()); + } + /** * 处理待执行任务重新投递异常。 * @@ -596,6 +664,9 @@ public class KnowledgeDocumentImportTaskAppService { || current.getStartedAt().after(cutoff)) { return; } + if (!lockTaskBatchForMutation(current)) { + return; + } Date now = new Date(); String errorMessage = "文档解析服务响应超时，请重试"; DocumentImportTask update = new DocumentImportTask(); @@ -634,6 +705,9 @@ public class KnowledgeDocumentImportTaskAppService { || current.getCreated().after(cutoff)) { return; } + if (!lockTaskBatchForMutation(current)) { + return; + } Date now = new Date(); String errorMessage = "任务排队超时，请重试"; DocumentImportTask update = new DocumentImportTask(); @@ -766,6 +840,9 @@ public class KnowledgeDocumentImportTaskAppService { || !leaseExpired) { return; } + if (!lockTaskBatchForMutation(current)) { + return; + } String errorMessage = current.getBatchId() == null ? "任务执行中断，请重试" : "任务执行中断，请继续批次"; @@ -900,6 +977,20 @@ public class KnowledgeDocumentImportTaskAppService { */ @Transactional public Result startIndexTask(DocumentImportDtos.TaskStartIndexRequest request) { + return startIndexTask(request, false); + } + + /** + * 启动向量化任务。 + * + * @param request 启动请求 + * @param replaceExistingIndex 是否在写入前按稳定 chunk ID 清理上次失败尝试 + * @return 启动结果 + */ + private Result startIndexTask( + DocumentImportDtos.TaskStartIndexRequest request, + boolean replaceExistingIndex + ) { if (request.getDocumentId() == null) { throw new BusinessException("文档ID不能为空"); } @@ -946,6 +1037,9 @@ public class KnowledgeDocumentImportTaskAppService { } payload.put("chunkSnapshotPath", snapshotPath); payload.put("totalChunks", totalChunks); + if (replaceExistingIndex) { + payload.put(PAYLOAD_REPLACE_EXISTING_INDEX, true); + } DocumentImportTask task = createTask(document, DocumentImportTaskPhase.INDEX, payload); updateBatchItem(task.getBatchItemId(), DocumentImportBatchItemStage.INDEX, @@ -974,6 +1068,19 @@ public class KnowledgeDocumentImportTaskAppService { public Result retryParseTask(DocumentImportDtos.TaskRetryRequest request) { DocumentCollection knowledge = assertDocumentCollection(request.getKnowledgeId()); tech.easyflow.ai.entity.Document document = requireDocumentForKnowledge(request.getDocumentId(), knowledge.getId()); + lockAutomaticBatchItemForRetry(document); + return retryParseDocument(document); + } + + /** + * 在调用方已取得自动批次恢复权后创建新的解析尝试。 + * + * @param document 待重试文档 + * @return 任务结果 + */ + private Result retryParseDocument( + tech.easyflow.ai.entity.Document document + ) { if (!DocumentProcessStatus.PARSE_FAILED.name().equals(document.getProcessStatus())) { throw new BusinessException("当前文档不支持重试解析"); } @@ -982,10 +1089,14 @@ public class KnowledgeDocumentImportTaskAppService { } resetDocumentForParseRetry(document); DocumentImportTask task = createTask(document, DocumentImportTaskPhase.PARSE, buildDocumentPayload(document)); - updateBatchItem(task.getBatchItemId(), - DocumentImportBatchItemStage.PARSE, - DocumentImportBatchItemStatus.PENDING, - null); + if (task.getBatchItemId() != null + && !documentImportBatchTracker.updateItem( + task.getBatchItemId(), + DocumentImportBatchItemStage.PARSE, + DocumentImportBatchItemStatus.PENDING, + null)) { + throw new BusinessException("批次文件状态已变化，请刷新后重试"); + } dispatchParseTaskAfterCommit(task.getId()); if (task.getBatchId() == null) { scheduleParseTaskFallback(task.getId()); @@ -1004,16 +1115,14 @@ public class KnowledgeDocumentImportTaskAppService { DocumentCollection knowledge = assertDocumentCollection(request.getKnowledgeId()); tech.easyflow.ai.entity.Document document = requireDocumentForKnowledge(request.getDocumentId(), knowledge.getId()); - BigInteger batchId = optionAsBigInteger( - document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID); - if (DocumentProcessStatus.INDEX_FAILED.name().equals(document.getProcessStatus()) - && isAutomaticBatch(batchId)) { - return retryFailedTask(request); - } + lockAutomaticBatchItemForRetry(document); DocumentImportDtos.TaskStartIndexRequest startRequest = new DocumentImportDtos.TaskStartIndexRequest(); startRequest.setKnowledgeId(request.getKnowledgeId()); startRequest.setDocumentId(request.getDocumentId()); - return startIndexTask(startRequest); + return startIndexTask( + startRequest, + DocumentProcessStatus.INDEX_FAILED.name().equals( + document.getProcessStatus())); } /** @@ -1029,25 +1138,28 @@ public class KnowledgeDocumentImportTaskAppService { tech.easyflow.ai.entity.Document document = requireDocumentForKnowledge(request.getDocumentId(), knowledge.getId()); String status = document.getProcessStatus(); + lockAutomaticBatchItemForRetry(document); if (DocumentProcessStatus.PARSE_FAILED.name().equals(status)) { - return retryParseTask(request); + return retryParseDocument(document); } - BigInteger batchId = optionAsBigInteger( - document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID); - BigInteger batchItemId = optionAsBigInteger( - document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID); - if (DocumentProcessStatus.INDEX_FAILED.name().equals(status) - && !isAutomaticBatch(batchId)) { - return retryIndexTask(request); + if (DocumentProcessStatus.INDEX_FAILED.name().equals(status)) { + DocumentImportDtos.TaskStartIndexRequest startRequest = + new DocumentImportDtos.TaskStartIndexRequest(); + startRequest.setKnowledgeId(request.getKnowledgeId()); + startRequest.setDocumentId(request.getDocumentId()); + return startIndexTask(startRequest, true); } - if (!DocumentProcessStatus.SPLIT_FAILED.name().equals(status) - && !DocumentProcessStatus.INDEX_FAILED.name().equals(status)) { + if (!DocumentProcessStatus.SPLIT_FAILED.name().equals(status)) { throw new BusinessException("当前文档无需重试"); } if (!claimDocumentSplitRetry(document, status)) { throw new BusinessException("文档状态已变化，请刷新后重试"); } String staleSnapshotPath = invalidateChunkSnapshot(document); + BigInteger batchId = optionAsBigInteger( + document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID); + BigInteger batchItemId = optionAsBigInteger( + document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID); DocumentImportTask splitTask = enqueueAutomaticSplit(batchId, batchItemId, document); deleteChunkSnapshotAfterCommit(staleSnapshotPath); @@ -1073,11 +1185,25 @@ public class KnowledgeDocumentImportTaskAppService { DocumentCollection knowledge = assertDocumentCollection(item.getKnowledgeId()); DocumentImportTask task = - createBatchImportTask(batch, item, knowledge); + createBatchImportTask(batch, item, knowledge, false); + dispatchParseTaskAfterCommit(task.getId()); + return; + } + tech.easyflow.ai.entity.Document document = + documentMapper.selectOneById(item.getDocumentId()); + if (document == null) { + DocumentImportBatch batch = + documentImportBatchTracker.requireBatch(item.getBatchId()); + if (!Objects.equals(batch.getKnowledgeId(), item.getKnowledgeId())) { + throw new IllegalStateException("批次文件知识库归属不一致"); + } + DocumentCollection knowledge = + assertDocumentCollection(item.getKnowledgeId()); + DocumentImportTask task = + createBatchImportTask(batch, item, knowledge, true); dispatchParseTaskAfterCommit(task.getId()); return; } - tech.easyflow.ai.entity.Document document = requireDocument(item.getDocumentId()); DocumentImportDtos.TaskRetryRequest request = new DocumentImportDtos.TaskRetryRequest(); request.setKnowledgeId(item.getKnowledgeId()); request.setDocumentId(item.getDocumentId()); @@ -1113,16 +1239,46 @@ public class KnowledgeDocumentImportTaskAppService { document.setProgressPercent(0); clearDocumentTaskError(document); persistDocumentTaskState(document, now); - updateBatchItem(batchItemId, + if (!documentImportBatchTracker.updateItem( + batchItemId, DocumentImportBatchItemStage.SPLIT, DocumentImportBatchItemStatus.PENDING, - null); + null)) { + throw new BusinessException("批次文件状态已变化，请刷新后重试"); + } DocumentImportTask task = createTask(document, DocumentImportTaskPhase.SPLIT, payload); dispatchSplitTaskAfterCommit(task.getId()); return task; } + /** + * 自动批次的单文档重试必须在任何文档或任务写入前取得批次恢复权。 + * + * @param document 待重试文档 + */ + private void lockAutomaticBatchItemForRetry( + tech.easyflow.ai.entity.Document document + ) { + BigInteger batchId = optionAsBigInteger( + document.getOptions(), + DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID); + if (!isAutomaticBatch(batchId)) { + return; + } + BigInteger itemId = optionAsBigInteger( + document.getOptions(), + DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID); + DocumentImportBatchItem item = + documentImportBatchTracker.lockFailedItemForRetry(itemId); + if (item == null + || !Objects.equals(batchId, item.getBatchId()) + || !Objects.equals(document.getId(), item.getDocumentId()) + || !Objects.equals(document.getCollectionId(), item.getKnowledgeId())) { + throw new BusinessException("批次文件状态已变化，请刷新后重试"); + } + } + /** * 处理持久化分块任务消息。 * @@ -1140,11 +1296,8 @@ public class KnowledgeDocumentImportTaskAppService { taskId, task.getStatus()); return; } - LockHandle executionLock = redisLockExecutor.tryAcquire( - "easyflow:lock:document-import:document:" + task.getDocumentId(), - Duration.ZERO, - Duration.ofMinutes(30) - ); + LockHandle executionLock = + tryAcquireDocumentExecutionLock(task.getDocumentId()); if (executionLock == null) { LOG.info("文档已有导入任务执行中，延后分块任务: taskId={}, documentId={}", taskId, task.getDocumentId()); @@ -1152,12 +1305,17 @@ public class KnowledgeDocumentImportTaskAppService { } String snapshotPath = null; String previewSessionId = null; + TaskExecutionHeartbeat heartbeat = null; try { if (!selfProxy.tryMarkTaskRunning(taskId)) { LOG.info("分块任务未抢占成功，跳过本次执行: taskId={}", taskId); return; } task = requireTask(taskId); + if (!renewTaskExecutionLease(task, executionLock)) { + throw new TaskOwnershipLostException(task.getId()); + } + heartbeat = startTaskExecutionHeartbeat(task, executionLock); tech.easyflow.ai.entity.Document document = documentMapper.selectOneById(task.getDocumentId()); if (document == null) { @@ -1168,7 +1326,8 @@ public class KnowledgeDocumentImportTaskAppService { DocumentImportBatchItemStage.SPLIT, DocumentImportBatchItemStatus.RUNNING, null); - if (!touchRunningTask(task) || !executionLock.renew()) { + heartbeat.assertOwned(); + if (!renewTaskExecutionLease(task, executionLock)) { throw new TaskOwnershipLostException(task.getId()); } @@ -1187,7 +1346,8 @@ public class KnowledgeDocumentImportTaskAppService { snapshotPath = StringUtil.hasText(session.getChunkSnapshotPath()) ? session.getChunkSnapshotPath() : documentImportChunkSnapshotService.save(session); - if (!touchRunningTask(task) || !executionLock.renew()) { + heartbeat.assertOwned(); + if (!renewTaskExecutionLease(task, executionLock)) { throw new TaskOwnershipLostException(task.getId()); } if (!selfProxy.completeSplitTask( @@ -1212,6 +1372,9 @@ public class KnowledgeDocumentImportTaskAppService { LOG.warn("分块任务所有权已失效，忽略迟到失败: taskId={}", taskId); } } finally { + if (heartbeat != null) { + heartbeat.close(); + } executionLock.release(); } } @@ -1354,22 +1517,25 @@ public class KnowledgeDocumentImportTaskAppService { LOG.info("向量化任务已结束，跳过重复处理: taskId={}, status={}", taskId, task.getStatus()); return; } - LockHandle executionLock = redisLockExecutor.tryAcquire( - "easyflow:lock:document-import:document:" + task.getDocumentId(), - Duration.ZERO, - Duration.ofMinutes(30) - ); + boolean automaticBatch = isAutomaticBatch(task.getBatchId()); + LockHandle executionLock = + tryAcquireDocumentExecutionLock(task.getDocumentId()); if (executionLock == null) { LOG.info("文档已有导入任务执行中，延后向量化任务: taskId={}, documentId={}", taskId, task.getDocumentId()); return; } + TaskExecutionHeartbeat heartbeat = null; try { if (!selfProxy.tryMarkTaskRunning(taskId)) { LOG.info("向量化任务未抢占成功，跳过本次执行: taskId={}", taskId); return; } task = requireTask(taskId); + if (!renewTaskExecutionLease(task, executionLock)) { + throw new TaskOwnershipLostException(task.getId()); + } + heartbeat = startTaskExecutionHeartbeat(task, executionLock); tech.easyflow.ai.entity.Document document = documentMapper.selectOneById(task.getDocumentId()); if (document == null) { finishMissingDocumentTask(task); @@ -1386,13 +1552,17 @@ public class KnowledgeDocumentImportTaskAppService { List storedChunks = new ArrayList(); try { DocumentCollection knowledge = assertDocumentCollection(task.getKnowledgeId()); - if (!touchRunningTask(task) || !executionLock.renew()) { + heartbeat.assertOwned(); + if (!renewTaskExecutionLease(task, executionLock)) { throw new TaskOwnershipLostException(task.getId()); } clearPersistedChunks(document.getId()); storeContext = prepareStoreContext(document); String chunkSnapshotPath = asString(task.getPayloadJson().get("chunkSnapshotPath")); + boolean replaceExistingIndex = Boolean.TRUE.equals(asBoolean( + task.getPayloadJson().get(PAYLOAD_REPLACE_EXISTING_INDEX), + false)); DocumentImportDtos.PreviewSession session = resolveIndexPreviewSession( knowledge, @@ -1406,24 +1576,36 @@ public class KnowledgeDocumentImportTaskAppService { } StoreExecutionContext activeStoreContext = storeContext; DocumentImportTask activeTask = task; + TaskExecutionHeartbeat activeHeartbeat = heartbeat; Set uniqueChunkIds = new HashSet(Math.min(totalChunks, 65_536)); int[] completedChunks = new int[]{0}; java.util.function.Consumer> batchConsumer = batch -> { + activeHeartbeat.assertOwned(); assertUniqueChunkIds(batch, uniqueChunkIds); LOG.info("文档向量化任务开始处理批次: taskId={}, documentId={}, batchStart={}, batchEnd={}, batchSize={}, totalChunks={}", taskId, document.getId(), completedChunks[0], completedChunks[0] + batch.size(), batch.size(), totalChunks); // 提前登记当前批次，确保外部存储部分成功后仍能覆盖回滚范围。 storedChunks.addAll(toChunkIdMarkers(batch)); + if (replaceExistingIndex) { + deleteStoredChunksBeforeReplay( + activeTask.getId(), document.getId(), + activeStoreContext, batch); + } storeDocumentChunks(activeStoreContext, batch); - persistChunkBatch(document, batch); - completedChunks[0] += batch.size(); - updateDocumentIndexProgress( - document.getId(), totalChunks, completedChunks[0]); - if (!touchRunningTask(activeTask) || !executionLock.renew()) { + int nextCompletedChunks = completedChunks[0] + batch.size(); + if (!selfProxy.persistOwnedIndexBatch( + activeTask, + document, + batch, + totalChunks, + nextCompletedChunks, + executionLock + )) { throw new TaskOwnershipLostException(activeTask.getId()); } + completedChunks[0] = nextCompletedChunks; }; if (StringUtil.hasText(chunkSnapshotPath)) { documentImportChunkSnapshotService.forEachBatch( @@ -1441,29 +1623,25 @@ public class KnowledgeDocumentImportTaskAppService { } updateKnowledgeAfterStore(storeContext); selfProxy.completeIndexTask(task, document, totalChunks); - if (StringUtil.hasText(session.getSessionId())) { - documentImportPreviewService.remove(session.getSessionId()); - } - deleteChunkSnapshotAfterCompletion( - asString(task.getPayloadJson().get("chunkSnapshotPath"))); - cleanupCsvTableSnapshotAfterCompletion(document); + cleanupIndexArtifactsAfterCompletion(task, document, session); } catch (TaskOwnershipLostException ownershipLost) { - // 旧执行失去 token 后不能清理可能已由新执行写入的索引或分块。 - LOG.warn("向量化任务执行令牌已失效，停止迟到回滚: taskId={}, documentId={}", + LOG.warn("向量化任务执行令牌已失效，已停止旧执行；失败重试将复用稳定 chunk ID 收敛外部索引: taskId={}, documentId={}", taskId, document.getId()); } catch (Exception e) { LOG.error("文档向量化任务失败: taskId={}, documentId={}", taskId, document.getId(), e); if (!ownsRunningTask(task)) { - LOG.warn("向量化任务已失去执行权，跳过迟到失败和清理: taskId={}, documentId={}", + LOG.warn("向量化任务已失去执行权，跳过迟到失败；失败重试将复用稳定 chunk ID 收敛外部索引: taskId={}, documentId={}", taskId, document.getId()); return; } clearPersistedChunks(document.getId()); - boolean rollbackSucceeded = true; - if (storeContext != null && !storedChunks.isEmpty()) { - rollbackSucceeded = rollbackStoredChunks( - taskId, document.getId(), storeContext, storedChunks); - } + boolean rollbackSucceeded = rollbackFailedIndexAttempt( + automaticBatch, + taskId, + document.getId(), + storeContext, + storedChunks + ); String failureCode = e instanceof CsvImportException csvError ? csvError.getFailureCode() : TASK_ERROR_INDEX_FAILED; @@ -1476,6 +1654,9 @@ public class KnowledgeDocumentImportTaskAppService { closeStoreContext(storeContext); } } finally { + if (heartbeat != null) { + heartbeat.close(); + } executionLock.release(); } } @@ -1989,6 +2170,9 @@ public class KnowledgeDocumentImportTaskAppService { if ("service_not_enabled".equals(bridgeCode)) { return TASK_ERROR_PARSE_SERVICE_UNAVAILABLE; } + if ("task_not_found".equals(bridgeCode)) { + return TASK_ERROR_PARSE_EXECUTION_LOST; + } fallbackCode = TASK_ERROR_PARSE_FAILED; } if (current instanceof SocketTimeoutException) { @@ -2038,6 +2222,9 @@ public class KnowledgeDocumentImportTaskAppService { if (TASK_ERROR_PARSE_SERVICE_TIMEOUT.equals(errorCode)) { return "文档解析服务响应超时，请重试"; } + if (TASK_ERROR_PARSE_EXECUTION_LOST.equals(errorCode)) { + return "文档解析执行已中断，请点击继续重试"; + } if (TASK_ERROR_DOCUMENT_SOURCE_UNAVAILABLE.equals(errorCode)) { return "文档文件读取失败，请联系管理员"; } @@ -2732,6 +2919,9 @@ public class KnowledgeDocumentImportTaskAppService { return false; } try { + if (!lockTaskBatchForMutation(task)) { + return false; + } int globalLimit = resolvePhaseRunningLimit(phase); long globalRunning = documentImportTaskService.count( QueryWrapper.create() @@ -2946,7 +3136,7 @@ public class KnowledgeDocumentImportTaskAppService { clearDocumentTaskError(document); document.setProgressPercent(0); Map options = copyOptions(document.getOptions()); - clearDocumentParseProgress(options); + clearDocumentParseExecution(options); document.setOptions(options); persistDocumentTaskState(document, now); } @@ -3023,6 +3213,9 @@ public class KnowledgeDocumentImportTaskAppService { DocumentImportTaskStatus status, String errorSummary, String failureCode) { + if (!lockTaskBatchForMutation(task)) { + return false; + } int updated = documentImportTaskMapper.finishOwned( task.getId(), task.getExecutionToken(), @@ -4443,6 +4636,49 @@ public class KnowledgeDocumentImportTaskAppService { } } + /** + * 在同一短事务内校验两层执行权并持久化一批索引结果。 + * + *

先锁定 batch，再通过 execution token 续租 task，然后用 owner token + * 续租 Redis 文档锁。只有两层围栏均有效时才写入分块与进度； + * 事务持有 batch/task 行锁期间，watchdog 无法将本任务收口为失联并 + * 启动新执行，从而阻断迟到进度覆盖。

+ * + * @return 仍持有两层执行权且持久化完成时返回 {@code true} + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean persistOwnedIndexBatch( + DocumentImportTask task, + tech.easyflow.ai.entity.Document document, + List batch, + int totalChunks, + int completedChunks, + LockHandle executionLock) { + if (!lockTaskBatchForMutation(task)) { + return false; + } + Date now = new Date(); + Date leaseUntil = new Date(now.getTime() + + documentExecutionLeaseTimeout().toMillis()); + int renewed = documentImportTaskMapper.renewLease( + task.getId(), + task.getExecutionToken(), + leaseUntil, + now, + resolveOperatorId() + ); + if (renewed != 1) { + return false; + } + if (executionLock == null || !executionLock.renew()) { + throw new TaskOwnershipLostException(task.getId()); + } + persistChunkBatch(document, batch); + updateDocumentIndexProgress( + document.getId(), totalChunks, completedChunks); + return true; + } + /** * 回滚已写入的向量和关键词索引。 * @@ -4457,13 +4693,7 @@ public class KnowledgeDocumentImportTaskAppService { StoreExecutionContext storeContext, List documentChunks) { try { - Set uniqueIds = new LinkedHashSet(); - for (DocumentChunk chunk : documentChunks) { - if (chunk != null && chunk.getId() != null) { - uniqueIds.add(chunk.getId()); - } - } - List ids = new ArrayList(uniqueIds); + List ids = resolveChunkIds(documentChunks); if (ids.isEmpty()) { return true; } @@ -4492,6 +4722,110 @@ public class KnowledgeDocumentImportTaskAppService { } } + /** + * 按导入模式处置索引失败时已经写入外部存储的分块。 + * + *

自动批次必须保留稳定 ID，交由后继重放任务逐批定向覆盖；此处若主动 + * 删除，失权旧执行可能误删已经由后继任务写入的结果。手动任务没有批次级 + * 重放链路，继续沿用即时回滚。

+ */ + private boolean rollbackFailedIndexAttempt( + boolean automaticBatch, + BigInteger taskId, + BigInteger documentId, + StoreExecutionContext storeContext, + List documentChunks + ) { + if (storeContext == null || documentChunks == null + || documentChunks.isEmpty()) { + return true; + } + if (automaticBatch) { + LOG.warn("自动导入索引失败后保留稳定 chunk ID 的外部记录，等待可重放任务定向覆盖: taskId={}, documentId={}, chunkCount={}", + taskId, documentId, documentChunks.size()); + return true; + } + return rollbackStoredChunks( + taskId, documentId, storeContext, documentChunks); + } + + /** + * 索引失败重试写入前，按持久化快照中的稳定 chunk ID 清理旧外部记录。 + * + *

服务可能在外部存储成功、数据库记账前退出。重试保留原快照和 ID， + * 每批先删后写，使普通失败与 JVM 硬中断都能重复执行并最终收敛。

+ */ + private void deleteStoredChunksBeforeReplay( + BigInteger taskId, + BigInteger documentId, + StoreExecutionContext storeContext, + List documentChunks + ) { + List ids = resolveChunkIds(documentChunks); + if (ids.isEmpty()) { + return; + } + StoreResult deleteResult = + storeContext.documentStore.delete(ids, storeContext.options); + if (deleteResult == null || !deleteResult.isSuccess()) { + String reason = deleteResult == null + ? "未返回结果" + : deleteResult.getFailReason(); + throw new BusinessException("重试前清理旧向量索引失败: " + reason); + } + if (storeContext.searcher != null + && !storeContext.searcher.deleteDocuments(ids)) { + throw new BusinessException("重试前清理旧关键词索引失败"); + } + LOG.info("索引失败重试已清理旧外部记录: taskId={}, documentId={}, chunkCount={}", + taskId, documentId, ids.size()); + } + + private List resolveChunkIds(List documentChunks) { + Set uniqueIds = new LinkedHashSet(); + if (documentChunks != null) { + for (DocumentChunk chunk : documentChunks) { + if (chunk != null && chunk.getId() != null) { + uniqueIds.add(chunk.getId()); + } + } + } + return new ArrayList(uniqueIds); + } + + /** + * 最终业务状态已提交后尽力清理临时快照。 + * + *

预览缓存或文件清理失败不得倒退已完成的向量索引。 + * 缓存具有 TTL，文件快照失败保留诊断日志供后续清理。

+ */ + private void cleanupIndexArtifactsAfterCompletion( + DocumentImportTask task, + tech.easyflow.ai.entity.Document document, + DocumentImportDtos.PreviewSession session) { + if (session != null && StringUtil.hasText(session.getSessionId())) { + try { + documentImportPreviewService.remove(session.getSessionId()); + } catch (Exception cleanupError) { + LOG.warn("向量化已完成，但预览缓存清理失败: taskId={}, documentId={}, sessionId={}", + task.getId(), document.getId(), session.getSessionId(), cleanupError); + } + } + try { + deleteChunkSnapshotAfterCompletion( + asString(task.getPayloadJson().get("chunkSnapshotPath"))); + } catch (Exception cleanupError) { + LOG.warn("向量化已完成，但分块快照清理失败: taskId={}, documentId={}", + task.getId(), document.getId(), cleanupError); + } + try { + cleanupCsvTableSnapshotAfterCompletion(document); + } catch (Exception cleanupError) { + LOG.warn("向量化已完成，但 CSV 表快照清理失败: taskId={}, documentId={}", + task.getId(), document.getId(), cleanupError); + } + } + private void updateKnowledgeAfterStore(StoreExecutionContext storeContext) { DocumentCollection update = new DocumentCollection(); update.setId(storeContext.knowledge.getId()); @@ -4876,6 +5210,36 @@ public class KnowledgeDocumentImportTaskAppService { * @param task 任务实体 */ private boolean touchRunningTask(DocumentImportTask task) { + KnowledgeDocumentImportTaskAppService executor = + selfProxy == null ? this : selfProxy; + Date leaseUntil = executor.renewRunningTaskLease( + task.getId(), task.getBatchId(), task.getExecutionToken()); + if (leaseUntil != null) { + task.setModified(new Date()); + task.setLeaseUntil(leaseUntil); + return true; + } + return false; + } + + /** + * 取得批次行锁后使用 execution token 续租任务。 + * + * @param taskId 任务 ID + * @param batchId 批次 ID + * @param executionToken 当前执行令牌 + * @return 新租约到期时间；失去任务或批次执行权时返回 {@code null} + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public Date renewRunningTaskLease( + BigInteger taskId, + BigInteger batchId, + String executionToken) { + DocumentImportTask lockCandidate = new DocumentImportTask(); + lockCandidate.setBatchId(batchId); + if (!lockTaskBatchForMutation(lockCandidate)) { + return null; + } Date now = new Date(); Date leaseUntil = new Date( now.getTime() @@ -4883,18 +5247,13 @@ public class KnowledgeDocumentImportTaskAppService { 1L, bulkProperties.getInterruptionTimeout().toMillis())); int updated = documentImportTaskMapper.renewLease( - task.getId(), - task.getExecutionToken(), + taskId, + executionToken, leaseUntil, now, resolveOperatorId() ); - if (updated > 0) { - task.setModified(now); - task.setLeaseUntil(leaseUntil); - return true; - } - return false; + return updated > 0 ? leaseUntil : null; } /** @@ -5275,6 +5634,19 @@ public class KnowledgeDocumentImportTaskAppService { options.remove(DocumentImportKeys.KEY_DOCUMENT_PARSE_STATUS_MESSAGE); } + /** + * 清理已经终结的解析执行引用和进度，供重新解析前使用。 + * + * @param options 文档选项 + */ + private void clearDocumentParseExecution(Map options) { + if (options == null || options.isEmpty()) { + return; + } + options.remove(DocumentImportKeys.KEY_DOCUMENT_PROVIDER_TASK_ID); + clearDocumentParseProgress(options); + } + /** * 将 CSV 解析元信息提升到文档有界选项中，供后续分块阶段直接定位表格快照。 * @@ -5381,6 +5753,76 @@ public class KnowledgeDocumentImportTaskAppService { return markers; } + /** + * 在任务行变更前锁定所属批次，统一 batch -> task 锁序。 + * + * @param task 待变更任务 + * @return 无批次任务或运行中批次已锁定时为 {@code true} + */ + private boolean lockTaskBatchForMutation(DocumentImportTask task) { + if (task == null) { + return false; + } + return task.getBatchId() == null + || documentImportBatchTracker.lockBatchForTaskMutation( + task.getBatchId()); + } + + /** + * 获取单文档执行锁，租约与数据库任务的失联判定保持一致。 + * + *

进程异常退出时无法执行 finally 释放 Redis 锁。若锁租约长于任务租约， + * 看门狗已允许批次继续后，新任务仍会被旧锁额外阻塞。

+ * + * @param documentId 文档 ID + * @return 获取成功时返回锁句柄，否则返回 {@code null} + */ + private LockHandle tryAcquireDocumentExecutionLock(BigInteger documentId) { + return redisLockExecutor.tryAcquire( + DOCUMENT_EXECUTION_LOCK_PREFIX + documentId, + Duration.ZERO, + documentExecutionLeaseTimeout()); + } + + private Duration documentExecutionLeaseTimeout() { + return Duration.ofMillis(Math.max( + 1L, + bulkProperties.getInterruptionTimeout().toMillis())); + } + + /** + * 同步续租数据库任务与当前 owner-token 持有的 Redis 文档锁。 + */ + private boolean renewTaskExecutionLease( + DocumentImportTask task, + LockHandle executionLock) { + return touchRunningTask(task) && executionLock.renew(); + } + + private TaskExecutionHeartbeat startTaskExecutionHeartbeat( + DocumentImportTask task, + LockHandle executionLock) { + long renewIntervalMillis = Math.max( + 1L, + documentExecutionLeaseTimeout().toMillis() / 3L); + TaskExecutionHeartbeat heartbeat = + new TaskExecutionHeartbeat(task, executionLock); + heartbeat.schedule(taskExecutionHeartbeatExecutor.scheduleWithFixedDelay( + heartbeat::renew, + renewIntervalMillis, + renewIntervalMillis, + TimeUnit.MILLISECONDS)); + return heartbeat; + } + + /** + * 停止文档导入任务心跳线程。 + */ + @PreDestroy + public void shutdownTaskExecutionHeartbeatExecutor() { + taskExecutionHeartbeatExecutor.shutdownNow(); + } + private BigInteger resolveOperatorId() { try { return BigInteger.valueOf(StpUtil.getLoginIdAsLong()); @@ -5474,6 +5916,62 @@ public class KnowledgeDocumentImportTaskAppService { return value == null ? 0 : value; } + private final class TaskExecutionHeartbeat implements AutoCloseable { + + private final DocumentImportTask task; + private final LockHandle executionLock; + private final AtomicBoolean lost = new AtomicBoolean(); + private final AtomicBoolean closed = new AtomicBoolean(); + private volatile ScheduledFuture renewal; + + private TaskExecutionHeartbeat( + DocumentImportTask task, + LockHandle executionLock) { + this.task = task; + this.executionLock = executionLock; + } + + private void schedule(ScheduledFuture renewal) { + this.renewal = renewal; + } + + private void renew() { + if (closed.get() || lost.get()) { + return; + } + try { + if (!renewTaskExecutionLease(task, executionLock)) { + lost.set(true); + LOG.warn( + "文档导入任务心跳失去执行权: taskId={}, documentId={}", + task.getId(), task.getDocumentId()); + } + } catch (RuntimeException error) { + lost.set(true); + LOG.warn( + "文档导入任务心跳续租失败: taskId={}, documentId={}", + task.getId(), task.getDocumentId(), error); + } + } + + private void assertOwned() { + if (lost.get()) { + throw new TaskOwnershipLostException(task.getId()); + } + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + ScheduledFuture current = renewal; + if (current != null) { + current.cancel(false); + } + } + } + private static class ParsedKnowledgeContent { private final String documentLlmContent; private final String documentRenderMarkdown; diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java index 3b9cef6b..b8326ce2 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java @@ -17,22 +17,49 @@ import java.util.Date; */ public interface DocumentImportBatchItemMapper extends BaseMapper { + String SELECT_COLUMNS = "id, batch_id AS batchId, knowledge_id AS knowledgeId, " + + "document_id AS documentId, replaced_document_id AS replacedDocumentId, " + + "client_file_key AS clientFileKey, file_name AS fileName, " + + "relative_path AS relativePath, file_size AS fileSize, " + + "file_path AS filePath, storage_locator AS storageLocator, " + + "cleanup_pending AS cleanupPending, content_sha256 AS contentSha256, " + + "stage, status, error_summary AS errorSummary, " + + "failure_code AS failureCode, applied_strategy_code AS appliedStrategyCode, " + + "strategy_snapshot_json AS strategySnapshotJson, retryable, " + + "attempt_count AS attemptCount, created, created_by AS createdBy, " + + "modified, modified_by AS modifiedBy"; + /** - * 锁定运行批次中的失败项并取得本轮恢复权。 + * 锁定并读取批次文件项。 * - *

同时锁定批次和文件项，使恢复任务创建与批次熔断串行化。

+ *

调用方必须先锁定所属批次，再调用本方法，避免批次计数更新时 + * 出现共享锁升级死锁。

* * @param itemId 文件项 ID - * @return 可恢复文件项；状态已变化时返回 null + * @return 文件项；不存在时返回 null */ - @Select("SELECT item.* FROM tb_document_import_batch_item item " - + "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id " - + "WHERE item.id=#{itemId} AND item.status='FAILED' " - + "AND batch.status='RUNNING' FOR UPDATE") - DocumentImportBatchItem selectFailedForRetry( + @Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_batch_item " + + "WHERE id=#{itemId} FOR UPDATE") + DocumentImportBatchItem selectForUpdate( @Param("itemId") BigInteger itemId ); + /** + * 统计运行批次内已经由等价请求推进的活跃文件项。 + * + *

调用方必须先锁定批次；所有文件项状态迁移遵循批次到文件项 + * 的锁顺序，因此该当前状态检查到后续熔断之间不会新增活跃项。

+ * + * @param batchId 批次 ID + * @return 活跃文件项数量 + */ + @Select("SELECT COUNT(*) FROM tb_document_import_batch_item " + + "WHERE batch_id=#{batchId} " + + "AND status IN ('PENDING','UPLOADING','RUNNING','UPLOADED')") + int countActiveItems( + @Param("batchId") BigInteger batchId + ); + /** * 将中断批次中尚未结束的文件项统一收口为可恢复失败。 * @@ -100,14 +127,13 @@ public interface DocumentImportBatchItemMapper extends BaseMapper'INTERRUPTED'") + @Update("UPDATE tb_document_import_batch_item SET " + + "stage=#{stage}, status=#{status}, " + + "error_summary=#{errorSummary}, failure_code=#{failureCode}, " + + "retryable=#{retryable}, " + + "attempt_count=attempt_count + #{attemptDelta}, " + + "modified=#{modified} WHERE id=#{id} " + + "AND status=#{expectedStatus}") int transitionStatus( @Param("id") BigInteger id, @Param("expectedStatus") String expectedStatus, @@ -146,21 +172,46 @@ public interface DocumentImportBatchItemMapper extends BaseMapper { + String SELECT_COLUMNS = "id, knowledge_id AS knowledgeId, " + + "caller_type AS callerType, caller_id AS callerId, " + + "idempotency_key_hash AS idempotencyKeyHash, " + + "request_digest AS requestDigest, duplicate_policy AS duplicatePolicy, " + + "requested_strategy_json AS requestedStrategyJson, " + + "retry_generation AS retryGeneration, version, " + + "import_mode AS importMode, status, total_count AS totalCount, " + + "total_bytes AS totalBytes, completed_count AS completedCount, " + + "processing_count AS processingCount, failed_count AS failedCount, " + + "pending_count AS pendingCount, uploaded_count AS uploadedCount, " + + "skipped_count AS skippedCount, cancelled_count AS cancelledCount, " + + "retryable_failed_count AS retryableFailedCount, " + + "interrupt_code AS interruptCode, interrupt_message AS interruptMessage, " + + "interrupted_at AS interruptedAt, recovery_pending AS recoveryPending, " + + "recovery_file_keys_json AS recoveryFileKeysJson, " + + "recovery_token AS recoveryToken, " + + "recovery_lease_until AS recoveryLeaseUntil, " + + "started_at AS startedAt, finished_at AS finishedAt, created, " + + "created_by AS createdBy, modified, modified_by AS modifiedBy"; + + /** + * 锁定并读取批次，统一批次计数与文件项状态更新的加锁顺序。 + * + * @param batchId 批次 ID + * @return 批次；不存在时返回 null + */ + @Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_batch " + + "WHERE id=#{batchId} FOR UPDATE") + DocumentImportBatch selectForUpdate( + @Param("batchId") BigInteger batchId + ); + /** * 锁定并读取调用者拥有的批次，保证后续重试基于最新状态。 * @@ -26,7 +58,7 @@ public interface DocumentImportBatchMapper extends BaseMapper { + String SELECT_COLUMNS = "id, document_id AS documentId, " + + "knowledge_id AS knowledgeId, batch_id AS batchId, " + + "batch_item_id AS batchItemId, phase, status, " + + "provider_task_id AS providerTaskId, payload_json AS payloadJson, " + + "error_summary AS errorSummary, failure_code AS failureCode, " + + "attempt_no AS attemptNo, execution_token AS executionToken, " + + "lease_until AS leaseUntil, version, started_at AS startedAt, " + + "finished_at AS finishedAt, created, created_by AS createdBy, " + + "modified, modified_by AS modifiedBy"; + + String QUALIFIED_SELECT_COLUMNS = "task.id, task.document_id AS documentId, " + + "task.knowledge_id AS knowledgeId, task.batch_id AS batchId, " + + "task.batch_item_id AS batchItemId, task.phase, task.status, " + + "task.provider_task_id AS providerTaskId, task.payload_json AS payloadJson, " + + "task.error_summary AS errorSummary, task.failure_code AS failureCode, " + + "task.attempt_no AS attemptNo, task.execution_token AS executionToken, " + + "task.lease_until AS leaseUntil, task.version, task.started_at AS startedAt, " + + "task.finished_at AS finishedAt, task.created, task.created_by AS createdBy, " + + "task.modified, task.modified_by AS modifiedBy"; + /** * 按任务阶段和批次轮转查询待投递任务，避免大批次独占投递窗口。 * @@ -25,14 +45,16 @@ public interface DocumentImportTaskMapper extends BaseMapper * @param limit 最大任务数 * @return 公平排序后的待投递任务 */ - @Select("SELECT task.* FROM tb_document_import_task task JOIN (" + @Select("SELECT " + QUALIFIED_SELECT_COLUMNS + + " FROM tb_document_import_task task JOIN (" + "SELECT task.id, ROW_NUMBER() OVER (" + "PARTITION BY task.phase, COALESCE(task.batch_id, task.id) " + "ORDER BY task.created, task.id) AS lane_row " + "FROM tb_document_import_task task " + "LEFT JOIN tb_document_import_batch batch ON batch.id=task.batch_id " + "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore} " - + "AND (task.batch_id IS NULL OR batch.status='RUNNING')" + + "AND (task.batch_id IS NULL OR batch.import_mode='MANUAL' " + + "OR batch.status='RUNNING')" + ") ranked ON ranked.id=task.id " + "ORDER BY ranked.lane_row, task.created, task.id LIMIT #{limit}") List selectPendingFairly( @@ -52,11 +74,7 @@ public interface DocumentImportTaskMapper extends BaseMapper @Update("UPDATE tb_document_import_task SET modified=#{now}, " + "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 " + "WHERE id=#{id} AND status='PENDING' " - + "AND modified <= #{redispatchBefore} " - + "AND (batch_id IS NULL OR EXISTS (" - + "SELECT 1 FROM tb_document_import_batch batch " - + "WHERE batch.id=tb_document_import_task.batch_id " - + "AND batch.status='RUNNING'))") + + "AND modified <= #{redispatchBefore}") int touchPendingForDispatch( @Param("id") BigInteger id, @Param("redispatchBefore") Date redispatchBefore, @@ -72,7 +90,7 @@ public interface DocumentImportTaskMapper extends BaseMapper * @param limit 最大任务数 * @return 已失去执行租约的运行任务 */ - @Select("SELECT * FROM tb_document_import_task " + @Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_task " + "WHERE status='RUNNING' AND (" + "(lease_until IS NOT NULL AND lease_until <= #{now}) OR " + "(lease_until IS NULL AND modified <= #{legacyCutoff})) " @@ -99,11 +117,7 @@ public interface DocumentImportTaskMapper extends BaseMapper + "started_at=COALESCE(started_at, #{now}), finished_at=NULL, " + "error_summary=NULL, failure_code=NULL, modified=#{now}, " + "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 " - + "WHERE id=#{id} AND status='PENDING' " - + "AND (batch_id IS NULL OR EXISTS (" - + "SELECT 1 FROM tb_document_import_batch batch " - + "WHERE batch.id=tb_document_import_task.batch_id " - + "AND batch.status='RUNNING'))") + + "WHERE id=#{id} AND status='PENDING'") int claimPending( @Param("id") BigInteger id, @Param("executionToken") String executionToken, @@ -126,11 +140,7 @@ public interface DocumentImportTaskMapper extends BaseMapper + "modified=#{now}, modified_by=#{operatorId}, " + "version=COALESCE(version, 0) + 1 " + "WHERE id=#{id} AND status='RUNNING' " - + "AND execution_token=#{executionToken} " - + "AND (batch_id IS NULL OR EXISTS (" - + "SELECT 1 FROM tb_document_import_batch batch " - + "WHERE batch.id=tb_document_import_task.batch_id " - + "AND batch.status='RUNNING'))") + + "AND execution_token=#{executionToken}") int renewLease( @Param("id") BigInteger id, @Param("executionToken") String executionToken, @@ -156,11 +166,7 @@ public interface DocumentImportTaskMapper extends BaseMapper + "lease_until=NULL, finished_at=#{now}, modified=#{now}, " + "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 " + "WHERE id=#{id} AND status='RUNNING' " - + "AND execution_token=#{executionToken} " - + "AND (batch_id IS NULL OR EXISTS (" - + "SELECT 1 FROM tb_document_import_batch batch " - + "WHERE batch.id=tb_document_import_task.batch_id " - + "AND batch.status='RUNNING'))") + + "AND execution_token=#{executionToken}") int finishOwned( @Param("id") BigInteger id, @Param("executionToken") String executionToken, diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java index d3d5c948..f2b88e8b 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java @@ -6,6 +6,7 @@ import com.easyagents.document.core.entity.ParseResponse; import com.easyagents.document.core.entity.ParseResult; import com.easyagents.document.core.entity.ParseTaskInfo; import com.easyagents.document.core.entity.ParseTaskStatus; +import com.easyagents.document.core.exception.DocumentAsyncTaskNotFoundException; import com.easyagents.document.pdf.PdfDocumentParseService; import com.easyagents.document.pptx.PptxDocumentParseService; import com.easyagents.document.xlsx.XlsxDocumentParseService; @@ -128,6 +129,33 @@ public class DocumentParseBridgeServiceImplTest { Assert.assertEquals(0, pptxService.queryResultCallCount); } + /** + * 验证本地 Office 内存任务丢失会转换为稳定桥接错误码。 + */ + @Test + public void shouldExposeMissingOfficeTaskAsStableBridgeError() { + FakePptxDocumentParseService pptxService = + new FakePptxDocumentParseService(); + pptxService.queryTaskInfoError = + new DocumentAsyncTaskNotFoundException("lost-task"); + DocumentParseBridgeServiceImpl bridgeService = + buildBridgeService(null, pptxService, null, null); + + try { + bridgeService.queryTaskInfo( + "lost-task", + buildSource( + "slides.pptx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation" + ) + ); + Assert.fail("expected DocumentParseBridgeException"); + } catch (DocumentParseBridgeException error) { + Assert.assertEquals("task_not_found", error.getCode()); + Assert.assertSame(pptxService.queryTaskInfoError, error.getCause()); + } + } + /** * 验证缺少底层服务时抛出稳定错误码。 */ @@ -418,6 +446,7 @@ public class DocumentParseBridgeServiceImplTest { private int parseCallCount; private int queryTaskInfoCallCount; private int queryResultCallCount; + private RuntimeException queryTaskInfoError; @Override public ParseResponse parse(ParseRequest request) { @@ -450,6 +479,9 @@ public class DocumentParseBridgeServiceImplTest { @Override public ParseTaskInfo queryTaskInfo(String taskId) { queryTaskInfoCallCount++; + if (queryTaskInfoError != null) { + throw queryTaskInfoError; + } throw new UnsupportedOperationException(); } } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java index 6fd45211..d409112c 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java @@ -873,8 +873,8 @@ public class DocumentImportBatchAppServiceTest { Mockito.anyString(), Mockito.same(recoveryError) ); - Mockito.verify(context.batchMapper, Mockito.never()) - .finalizeRecoveryPending( + Mockito.verify(context.recoveryCoordinator, Mockito.never()) + .finalizeRecovery( Mockito.any(), Mockito.anyString(), Mockito.any(Date.class)); } @@ -892,8 +892,9 @@ public class DocumentImportBatchAppServiceTest { Mockito.when(context.batchMapper.selectRecoveryPendingBatches( Mockito.any(Date.class), Mockito.anyInt() )).thenReturn(List.of(batch)); - Mockito.when(context.batchMapper.selectClaimedRecovery( - Mockito.eq(batch.getId()), Mockito.anyString() + Mockito.when(context.recoveryCoordinator.claim( + Mockito.eq(batch.getId()), Mockito.anyString(), + Mockito.any(Date.class), Mockito.any(Date.class) )).thenReturn(null); int recovered = context.service.recoverPendingBatchRetries(); @@ -924,8 +925,9 @@ public class DocumentImportBatchAppServiceTest { Mockito.when(context.batchMapper.selectRecoveryPendingBatches( Mockito.any(Date.class), Mockito.anyInt() )).thenReturn(List.of(batch)); - Mockito.when(context.batchMapper.selectClaimedRecovery( - Mockito.eq(batch.getId()), Mockito.anyString() + Mockito.when(context.recoveryCoordinator.claim( + Mockito.eq(batch.getId()), Mockito.anyString(), + Mockito.any(Date.class), Mockito.any(Date.class) )).thenReturn(claimedBatch); int recovered = context.service.recoverPendingBatchRetries(); @@ -939,16 +941,14 @@ public class DocumentImportBatchAppServiceTest { ); ArgumentCaptor recoveryToken = ArgumentCaptor.forClass(String.class); - Mockito.verify(context.batchMapper).claimRecoveryPending( + Mockito.verify(context.recoveryCoordinator).claim( Mockito.eq(batch.getId()), recoveryToken.capture(), Mockito.any(Date.class), Mockito.any(Date.class) ); - Mockito.verify(context.batchMapper).selectClaimedRecovery( - batch.getId(), recoveryToken.getValue()); - Mockito.verify(context.batchMapper) - .finalizeRecoveryPending( + Mockito.verify(context.recoveryCoordinator) + .finalizeRecovery( Mockito.eq(batch.getId()), Mockito.eq(recoveryToken.getValue()), Mockito.any(Date.class)); @@ -975,8 +975,8 @@ public class DocumentImportBatchAppServiceTest { Assert.assertEquals(1, context.service.recoverPendingBatchRetries()); - Mockito.verify(context.batchMapper, Mockito.never()) - .finalizeRecoveryPending( + Mockito.verify(context.recoveryCoordinator, Mockito.never()) + .finalizeRecovery( Mockito.any(), Mockito.anyString(), Mockito.any(Date.class)); Mockito.verifyNoInteractions(context.circuitBreaker); } @@ -990,12 +990,12 @@ public class DocumentImportBatchAppServiceTest { BigInteger batchId = BigInteger.valueOf(84); String recoveryToken = "recovery-token"; AtomicLong now = new AtomicLong(1_000L); - Mockito.when(context.batchMapper.renewRecoveryPendingLease( + Mockito.when(context.recoveryCoordinator.renew( Mockito.eq(batchId), Mockito.eq(recoveryToken), Mockito.any(Date.class), Mockito.any(Date.class) - )).thenReturn(1, 0); + )).thenReturn(true, false); BooleanSupplier guard = context.service.createRecoveryLeaseGuard( batchId, recoveryToken, @@ -1004,8 +1004,8 @@ public class DocumentImportBatchAppServiceTest { ); Assert.assertTrue(guard.getAsBoolean()); - Mockito.verify(context.batchMapper, Mockito.never()) - .renewRecoveryPendingLease( + Mockito.verify(context.recoveryCoordinator, Mockito.never()) + .renew( Mockito.any(), Mockito.anyString(), Mockito.any(Date.class), @@ -1017,8 +1017,8 @@ public class DocumentImportBatchAppServiceTest { now.set(121_000L); Assert.assertFalse(guard.getAsBoolean()); - Mockito.verify(context.batchMapper, Mockito.times(2)) - .renewRecoveryPendingLease( + Mockito.verify(context.recoveryCoordinator, Mockito.times(2)) + .renew( Mockito.eq(batchId), Mockito.eq(recoveryToken), Mockito.any(Date.class), @@ -1198,6 +1198,8 @@ public class DocumentImportBatchAppServiceTest { RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class); DocumentImportBatchCircuitBreaker circuitBreaker = Mockito.mock(DocumentImportBatchCircuitBreaker.class); + DocumentImportRecoveryCoordinator recoveryCoordinator = + Mockito.mock(DocumentImportRecoveryCoordinator.class); RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class); Mockito.when(redisLockExecutor.tryAcquire( Mockito.anyString(), Mockito.any(), Mockito.any() @@ -1213,21 +1215,16 @@ public class DocumentImportBatchAppServiceTest { Mockito.when(batchMapper.selectOwnedForUpdate( Mockito.any(), Mockito.anyString(), Mockito.any() )).thenReturn(batch); - Mockito.when(batchMapper.claimRecoveryPending( - Mockito.any(), - Mockito.anyString(), - Mockito.any(Date.class), - Mockito.any(Date.class) - )).thenReturn(1); - Mockito.when(batchMapper.selectClaimedRecovery( - Mockito.any(), Mockito.anyString() + Mockito.when(recoveryCoordinator.claim( + Mockito.any(), Mockito.anyString(), + Mockito.any(Date.class), Mockito.any(Date.class) )).thenReturn(batch); Mockito.when(taskAppService.retryBatchFailures( Mockito.any(), Mockito.anySet(), Mockito.any(BooleanSupplier.class) )).thenReturn(true); - Mockito.when(batchMapper.finalizeRecoveryPending( + Mockito.when(recoveryCoordinator.finalizeRecovery( Mockito.any(), Mockito.anyString(), Mockito.any(Date.class) - )).thenReturn(1); + )).thenReturn(true); DocumentImportBatchAppService service = new DocumentImportBatchAppService( batchService, @@ -1239,12 +1236,13 @@ public class DocumentImportBatchAppServiceTest { itemMapper, documentMapper, redisLockExecutor, - circuitBreaker + circuitBreaker, + recoveryCoordinator ); return new TestContext( service, batchService, itemService, batchTracker, taskAppService, batchMapper, itemMapper, documentMapper, - circuitBreaker, lockHandle + circuitBreaker, recoveryCoordinator, lockHandle ); } @@ -1356,6 +1354,7 @@ public class DocumentImportBatchAppServiceTest { DocumentImportBatchItemMapper itemMapper, DocumentMapper documentMapper, DocumentImportBatchCircuitBreaker circuitBreaker, + DocumentImportRecoveryCoordinator recoveryCoordinator, RedisLockExecutor.LockHandle lockHandle ) { } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreakerTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreakerTest.java index 9f13c5cc..d7fa1bb5 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreakerTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreakerTest.java @@ -8,7 +8,9 @@ import org.springframework.dao.DataAccessResourceFailureException; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.data.redis.RedisSystemException; import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.entity.DocumentImportBatchItem; import tech.easyflow.ai.entity.DocumentImportTask; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; import tech.easyflow.ai.enums.DocumentImportBatchStatus; import tech.easyflow.ai.enums.DocumentImportMode; import tech.easyflow.ai.enums.DocumentImportTaskPhase; @@ -50,7 +52,7 @@ public class DocumentImportBatchCircuitBreakerTest { DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class); Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task); - Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch); + Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch); Mockito.when(batchMapper.interruptRunningBatchForActiveTask( Mockito.eq(batchId), Mockito.eq(taskId), @@ -161,6 +163,7 @@ public class DocumentImportBatchCircuitBreakerTest { DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class); Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch); + Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch); Mockito.when(batchMapper.interruptOwnedRecoveryBatch( Mockito.eq(batchId), Mockito.eq("stale-token"), @@ -183,6 +186,136 @@ public class DocumentImportBatchCircuitBreakerTest { Mockito.any(Date.class), Mockito.any()); } + /** + * 验证零重排熔断在批次与失败项锁内完成，且未发生并发推进时生效。 + */ + @Test + public void shouldInterruptRecoveryFailureWhenItemsRemainFailed() { + BigInteger batchId = BigInteger.valueOf(42); + BigInteger itemId = BigInteger.valueOf(43); + String recoveryToken = "current-token"; + DocumentImportBatch batch = runningAutoBatch(batchId); + batch.setRecoveryPending(true); + batch.setRecoveryToken(recoveryToken); + batch.setRecoveryLeaseUntil( + new Date(System.currentTimeMillis() + 60_000)); + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(itemId); + item.setBatchId(batchId); + item.setStatus(DocumentImportBatchItemStatus.FAILED.name()); + DocumentImportBatchMapper batchMapper = + Mockito.mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = + Mockito.mock(DocumentImportBatchItemMapper.class); + DocumentImportTaskMapper taskMapper = + Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch); + Mockito.when(itemMapper.selectForUpdate(itemId)).thenReturn(item); + Mockito.when(batchMapper.interruptOwnedRecoveryBatch( + Mockito.eq(batchId), Mockito.eq(recoveryToken), + Mockito.eq("document_import_recovery_failed"), + Mockito.anyString(), Mockito.any(Date.class) + )).thenReturn(1); + DocumentImportBatchCircuitBreaker circuitBreaker = + new DocumentImportBatchCircuitBreaker( + batchMapper, itemMapper, taskMapper); + + Assert.assertTrue(circuitBreaker.interruptRecoveryBatch( + batchId, + recoveryToken, + new DocumentImportRecoveryException(java.util.List.of(itemId)) + )); + + org.mockito.InOrder order = Mockito.inOrder(batchMapper, itemMapper); + order.verify(batchMapper).selectForUpdate(batchId); + order.verify(itemMapper).countActiveItems(batchId); + order.verify(itemMapper).selectForUpdate(itemId); + order.verify(batchMapper).interruptOwnedRecoveryBatch( + Mockito.eq(batchId), Mockito.eq(recoveryToken), + Mockito.eq("document_import_recovery_failed"), + Mockito.anyString(), Mockito.any(Date.class)); + } + + /** + * 验证错误写回提交后出现的等价并发推进会阻止旧请求熔断整批。 + */ + @Test + public void shouldKeepRunningBatchWhenRecoveryItemAdvanced() { + BigInteger batchId = BigInteger.valueOf(44); + BigInteger itemId = BigInteger.valueOf(45); + String recoveryToken = "current-token"; + DocumentImportBatch batch = runningAutoBatch(batchId); + batch.setRecoveryPending(true); + batch.setRecoveryToken(recoveryToken); + batch.setRecoveryLeaseUntil( + new Date(System.currentTimeMillis() + 60_000)); + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(itemId); + item.setBatchId(batchId); + item.setStatus(DocumentImportBatchItemStatus.PENDING.name()); + DocumentImportBatchMapper batchMapper = + Mockito.mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = + Mockito.mock(DocumentImportBatchItemMapper.class); + Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch); + Mockito.when(itemMapper.countActiveItems(batchId)).thenReturn(1); + DocumentImportBatchCircuitBreaker circuitBreaker = + new DocumentImportBatchCircuitBreaker( + batchMapper, + itemMapper, + Mockito.mock(DocumentImportTaskMapper.class)); + + Assert.assertFalse(circuitBreaker.interruptRecoveryBatch( + batchId, + recoveryToken, + new DocumentImportRecoveryException(java.util.List.of(itemId)) + )); + Mockito.verify(batchMapper, Mockito.never()) + .interruptOwnedRecoveryBatch( + Mockito.any(), Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.any(Date.class)); + Mockito.verify(itemMapper, Mockito.never()).selectForUpdate(itemId); + } + + /** + * 验证失败快照外已有活跃项时，选择性恢复也不能熔断并发任务。 + */ + @Test + public void shouldKeepRunningBatchWhenAnotherItemAdvancedBeforeSnapshot() { + BigInteger batchId = BigInteger.valueOf(46); + BigInteger failedItemId = BigInteger.valueOf(47); + String recoveryToken = "current-token"; + DocumentImportBatch batch = runningAutoBatch(batchId); + batch.setRecoveryPending(true); + batch.setRecoveryToken(recoveryToken); + batch.setRecoveryLeaseUntil( + new Date(System.currentTimeMillis() + 60_000)); + DocumentImportBatchMapper batchMapper = + Mockito.mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = + Mockito.mock(DocumentImportBatchItemMapper.class); + Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch); + Mockito.when(itemMapper.countActiveItems(batchId)).thenReturn(1); + DocumentImportBatchCircuitBreaker circuitBreaker = + new DocumentImportBatchCircuitBreaker( + batchMapper, + itemMapper, + Mockito.mock(DocumentImportTaskMapper.class)); + + Assert.assertFalse(circuitBreaker.interruptRecoveryBatch( + batchId, + recoveryToken, + new DocumentImportRecoveryException( + java.util.List.of(failedItemId)) + )); + Mockito.verify(itemMapper, Mockito.never()) + .selectForUpdate(failedItemId); + Mockito.verify(batchMapper, Mockito.never()) + .interruptOwnedRecoveryBatch( + Mockito.any(), Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.any(Date.class)); + } + /** * 验证 SQL 语法错误保留为通用系统异常，避免伪装成数据库不可用。 */ @@ -192,7 +325,7 @@ public class DocumentImportBatchCircuitBreakerTest { DocumentImportBatch batch = runningAutoBatch(batchId); DocumentImportBatchMapper batchMapper = Mockito.mock(DocumentImportBatchMapper.class); - Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch); + Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch); Mockito.when(batchMapper.interruptRunningBatch( Mockito.eq(batchId), Mockito.eq("document_import_infrastructure_failure"), @@ -222,7 +355,7 @@ public class DocumentImportBatchCircuitBreakerTest { DocumentImportBatch batch = runningAutoBatch(batchId); DocumentImportBatchMapper batchMapper = Mockito.mock(DocumentImportBatchMapper.class); - Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch); + Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch); Mockito.when(batchMapper.interruptRunningBatch( Mockito.eq(batchId), Mockito.eq("database_unavailable"), @@ -242,6 +375,36 @@ public class DocumentImportBatchCircuitBreakerTest { )); } + /** + * 验证没有文件重新排队时使用稳定恢复失败语义中断批次。 + */ + @Test + public void shouldExposeRecoveryFailureInsteadOfGenericInfrastructureError() { + BigInteger batchId = BigInteger.valueOf(71); + DocumentImportBatch batch = runningAutoBatch(batchId); + DocumentImportBatchMapper batchMapper = + Mockito.mock(DocumentImportBatchMapper.class); + Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch); + Mockito.when(batchMapper.interruptRunningBatch( + Mockito.eq(batchId), + Mockito.eq("document_import_recovery_failed"), + Mockito.eq("未能重新排队任何失败文件，请查看文件错误后继续"), + Mockito.any(Date.class) + )).thenReturn(1); + DocumentImportBatchCircuitBreaker circuitBreaker = + new DocumentImportBatchCircuitBreaker( + batchMapper, + Mockito.mock(DocumentImportBatchItemMapper.class), + Mockito.mock(DocumentImportTaskMapper.class) + ); + + Assert.assertTrue(circuitBreaker.interruptBatch( + batchId, + new IllegalStateException( + "recovery failed", new DocumentImportRecoveryException()) + )); + } + /** * 验证恢复与任务熔断 SQL 都包含对应所有权围栏。 * diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java index 73d8f279..b01940bc 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java @@ -1,6 +1,8 @@ package tech.easyflow.ai.documentimport.task; import org.junit.Test; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; import tech.easyflow.ai.documentimport.DocumentImportBatchDtos; import tech.easyflow.ai.entity.DocumentImportBatch; import tech.easyflow.ai.entity.DocumentImportBatchItem; @@ -13,6 +15,7 @@ import tech.easyflow.ai.mapper.DocumentImportBatchMapper; import tech.easyflow.ai.service.DocumentImportBatchItemService; import tech.easyflow.ai.service.DocumentImportBatchService; +import java.lang.reflect.Method; import java.math.BigInteger; import java.util.Date; @@ -25,6 +28,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -62,6 +66,39 @@ public class DocumentImportBatchTrackerTest { verify(batchService, never()).updateById(batch, false); } + /** + * 验证任务变更始终先锁批次，但只有自动批次受运行态门禁约束。 + */ + @Test + public void shouldAllowManualTaskAfterUploadBatchCompleted() { + BigInteger automaticBatchId = BigInteger.valueOf(101); + BigInteger manualBatchId = BigInteger.valueOf(102); + DocumentImportBatch automatic = new DocumentImportBatch(); + automatic.setId(automaticBatchId); + automatic.setImportMode(DocumentImportMode.AUTO.name()); + automatic.setStatus(DocumentImportBatchStatus.COMPLETED.name()); + DocumentImportBatch manual = new DocumentImportBatch(); + manual.setId(manualBatchId); + manual.setImportMode(DocumentImportMode.MANUAL.name()); + manual.setStatus(DocumentImportBatchStatus.COMPLETED.name()); + DocumentImportBatchMapper batchMapper = + mock(DocumentImportBatchMapper.class); + when(batchMapper.selectForUpdate(automaticBatchId)) + .thenReturn(automatic); + when(batchMapper.selectForUpdate(manualBatchId)).thenReturn(manual); + DocumentImportBatchTracker tracker = new DocumentImportBatchTracker( + mock(DocumentImportBatchService.class), + mock(DocumentImportBatchItemService.class), + batchMapper, + mock(DocumentImportBatchItemMapper.class) + ); + + assertFalse(tracker.lockBatchForTaskMutation(automaticBatchId)); + assertTrue(tracker.lockBatchForTaskMutation(manualBatchId)); + verify(batchMapper).selectForUpdate(automaticBatchId); + verify(batchMapper).selectForUpdate(manualBatchId); + } + /** * 验证兼容可重试计数与人工可继续的失败总数保持一致。 */ @@ -111,6 +148,8 @@ public class DocumentImportBatchTrackerTest { item.setStatus(DocumentImportBatchItemStatus.FAILED.name()); item.setRetryable(true); when(itemService.getById(item.getId())).thenReturn(item); + when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch); + when(itemMapper.selectForUpdate(item.getId())).thenReturn(item); when(itemMapper.transitionStatus( eq(item.getId()), eq(DocumentImportBatchItemStatus.FAILED.name()), @@ -143,6 +182,14 @@ public class DocumentImportBatchTrackerTest { eq(-1), any(Date.class) ); + org.mockito.InOrder lockOrder = inOrder(batchMapper, itemMapper); + lockOrder.verify(batchMapper).selectForUpdate(batch.getId()); + lockOrder.verify(itemMapper).selectForUpdate(item.getId()); + lockOrder.verify(itemMapper).transitionStatus( + eq(item.getId()), eq(DocumentImportBatchItemStatus.FAILED.name()), + eq(DocumentImportBatchItemStage.INDEX.name()), + eq(DocumentImportBatchItemStatus.PENDING.name()), eq(null), eq(null), + eq(false), eq(1), any(Date.class)); } /** @@ -167,6 +214,8 @@ public class DocumentImportBatchTrackerTest { item.setRetryable(true); BigInteger documentId = BigInteger.valueOf(99); when(itemService.getById(item.getId())).thenReturn(item); + when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch); + when(itemMapper.selectForUpdate(item.getId())).thenReturn(item); when(itemMapper.bindFailedDocument( eq(item.getId()), eq(documentId), any(Date.class) )).thenReturn(1); @@ -184,6 +233,35 @@ public class DocumentImportBatchTrackerTest { anyInt(), anyInt(), anyInt(), any(Date.class)); } + /** + * 验证失败项只能在运行批次内绑定恢复创建的文档。 + */ + @Test(expected = tech.easyflow.common.web.exceptions.BusinessException.class) + public void shouldRejectRecoveredDocumentOutsideRunningBatch() { + DocumentImportBatchService batchService = + mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = + mock(DocumentImportBatchItemService.class); + DocumentImportBatchMapper batchMapper = + mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = + mock(DocumentImportBatchItemMapper.class); + DocumentImportBatch batch = batch( + DocumentImportBatchStatus.PARTIAL_SUCCEEDED, 1); + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(BigInteger.TEN); + item.setBatchId(batch.getId()); + item.setStatus(DocumentImportBatchItemStatus.FAILED.name()); + when(itemService.getById(item.getId())).thenReturn(item); + when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch); + when(itemMapper.selectForUpdate(item.getId())).thenReturn(item); + DocumentImportBatchTracker tracker = + new DocumentImportBatchTracker( + batchService, itemService, batchMapper, itemMapper); + + tracker.bindDocument(item.getId(), BigInteger.valueOf(99)); + } + /** * 验证迟到任务不能把已完成文件重新改为处理中。 */ @@ -198,6 +276,9 @@ public class DocumentImportBatchTrackerTest { item.setBatchId(BigInteger.ONE); item.setStatus(DocumentImportBatchItemStatus.COMPLETED.name()); when(itemService.getById(item.getId())).thenReturn(item); + DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1); + when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch); + when(itemMapper.selectForUpdate(item.getId())).thenReturn(item); DocumentImportBatchTracker tracker = new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper); @@ -210,6 +291,87 @@ public class DocumentImportBatchTrackerTest { any(), any(), any(), any(), any(), any(), anyBoolean(), anyInt(), any(Date.class)); } + /** + * 验证批次中断后，已到达的工作线程不能再迁移文件项。 + */ + @Test + public void shouldFenceLateTransitionAfterBatchInterrupted() { + DocumentImportBatchService batchService = mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class); + DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class); + DocumentImportBatch batch = batch(DocumentImportBatchStatus.INTERRUPTED, 1); + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(BigInteger.TEN); + item.setBatchId(batch.getId()); + item.setStatus(DocumentImportBatchItemStatus.PENDING.name()); + when(itemService.getById(item.getId())).thenReturn(item); + when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch); + when(itemMapper.selectForUpdate(item.getId())).thenReturn(item); + + DocumentImportBatchTracker tracker = + new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper); + + assertFalse(tracker.updateItem(item.getId(), + DocumentImportBatchItemStage.PARSE, + DocumentImportBatchItemStatus.RUNNING, + null)); + verify(itemMapper, never()).transitionStatus( + any(), any(), any(), any(), any(), any(), anyBoolean(), anyInt(), any(Date.class)); + } + + /** + * 验证恢复失败原因在提交后回调内仍使用独立事务持久化。 + * + * @throws Exception 方法不存在时抛出 + */ + @Test + public void failedRetryErrorShouldUseIndependentTransaction() + throws Exception { + Method method = DocumentImportBatchTracker.class.getMethod( + "updateFailedRetryError", + BigInteger.class, + BigInteger.class, + String.class + ); + Transactional transactional = method.getAnnotation(Transactional.class); + + assertEquals(Propagation.REQUIRES_NEW, transactional.propagation()); + } + + /** + * 验证失败原因写回与等价并发推进在同一锁内完成判定。 + */ + @Test + public void failedRetryErrorShouldRecognizeConcurrentAdvance() { + DocumentImportBatchService batchService = + mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = + mock(DocumentImportBatchItemService.class); + DocumentImportBatchMapper batchMapper = + mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = + mock(DocumentImportBatchItemMapper.class); + DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1); + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(BigInteger.TEN); + item.setBatchId(batch.getId()); + item.setStatus(DocumentImportBatchItemStatus.PENDING.name()); + when(itemService.getById(item.getId())).thenReturn(item); + when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch); + when(itemMapper.selectForUpdate(item.getId())).thenReturn(item); + DocumentImportBatchTracker tracker = + new DocumentImportBatchTracker( + batchService, itemService, batchMapper, itemMapper); + + assertEquals( + DocumentImportBatchTracker.FailedRetryErrorOutcome.ALREADY_ADVANCED, + tracker.updateFailedRetryError( + item.getId(), batch.getId(), "状态已变化")); + verify(itemMapper, never()).updateFailedRetryError( + any(), any(), any(), any(Date.class)); + } + private DocumentImportBatch batch(DocumentImportBatchStatus status, int totalCount) { DocumentImportBatch batch = new DocumentImportBatch(); batch.setId(BigInteger.ONE); diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryCoordinatorTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryCoordinatorTest.java new file mode 100644 index 00000000..a4e59555 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportRecoveryCoordinatorTest.java @@ -0,0 +1,74 @@ +package tech.easyflow.ai.documentimport.task; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; + +import java.lang.reflect.Method; +import java.math.BigInteger; +import java.util.Date; + +/** + * {@link DocumentImportRecoveryCoordinator} 恢复令牌事务测试。 + */ +public class DocumentImportRecoveryCoordinatorTest { + + @Test + public void claimShouldReturnOnlyCurrentTokenBatch() { + DocumentImportBatchMapper mapper = + Mockito.mock(DocumentImportBatchMapper.class); + DocumentImportRecoveryCoordinator coordinator = + new DocumentImportRecoveryCoordinator(mapper); + BigInteger batchId = BigInteger.valueOf(81); + String token = "token"; + Date claimedAt = new Date(1_000L); + Date leaseUntil = new Date(121_000L); + DocumentImportBatch batch = new DocumentImportBatch(); + batch.setId(batchId); + Mockito.when(mapper.claimRecoveryPending( + batchId, token, leaseUntil, claimedAt)).thenReturn(1); + Mockito.when(mapper.selectClaimedRecovery(batchId, token)) + .thenReturn(batch); + + Assert.assertSame(batch, coordinator.claim( + batchId, token, leaseUntil, claimedAt)); + } + + @Test + public void claimShouldSkipReadWhenTokenWasNotAcquired() { + DocumentImportBatchMapper mapper = + Mockito.mock(DocumentImportBatchMapper.class); + DocumentImportRecoveryCoordinator coordinator = + new DocumentImportRecoveryCoordinator(mapper); + + Assert.assertNull(coordinator.claim( + BigInteger.ONE, "token", new Date(2_000L), new Date(1_000L))); + Mockito.verify(mapper, Mockito.never()) + .selectClaimedRecovery(Mockito.any(), Mockito.anyString()); + } + + @Test + public void tokenMutationsShouldAlwaysUseIndependentTransactions() + throws Exception { + assertRequiresNew("claim", BigInteger.class, String.class, + Date.class, Date.class); + assertRequiresNew("renew", BigInteger.class, String.class, + Date.class, Date.class); + assertRequiresNew("finalizeRecovery", BigInteger.class, + String.class, Date.class); + } + + private void assertRequiresNew(String methodName, Class... parameterTypes) + throws Exception { + Method method = DocumentImportRecoveryCoordinator.class.getMethod( + methodName, parameterTypes); + Transactional transactional = method.getAnnotation(Transactional.class); + Assert.assertNotNull(transactional); + Assert.assertEquals(Propagation.REQUIRES_NEW, + transactional.propagation()); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java index e71cbce9..56a42982 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java @@ -31,6 +31,7 @@ import tech.easyflow.ai.document.exception.DocumentParseBridgeException; import tech.easyflow.ai.document.model.DocumentParseArtifacts; import tech.easyflow.ai.document.model.DocumentParsedResult; import tech.easyflow.ai.document.model.DocumentSourceRef; +import tech.easyflow.ai.document.service.DocumentParseBridgeService; import tech.easyflow.ai.documentimport.DocumentImportDtos; import tech.easyflow.ai.documentimport.DocumentImportKeys; import tech.easyflow.ai.entity.DocumentChunk; @@ -45,14 +46,18 @@ import tech.easyflow.ai.enums.DocumentImportMode; import tech.easyflow.ai.enums.DocumentImportTaskStatus; import tech.easyflow.ai.enums.DocumentImportTaskPhase; import tech.easyflow.ai.enums.DocumentProcessStatus; +import tech.easyflow.ai.mapper.DocumentChunkMapper; import tech.easyflow.ai.mapper.DocumentImportTaskMapper; import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; import tech.easyflow.ai.mapper.DocumentMapper; import tech.easyflow.ai.service.DocumentImportBatchItemService; import tech.easyflow.ai.service.DocumentImportTaskService; import tech.easyflow.ai.service.DocumentCollectionService; +import tech.easyflow.ai.service.DocumentChunkService; import tech.easyflow.ai.service.DocumentService; import tech.easyflow.common.cache.RedisLockExecutor; +import tech.easyflow.common.domain.Result; import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.web.exceptions.BusinessException; @@ -66,6 +71,7 @@ import java.math.BigInteger; import java.net.SocketTimeoutException; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; import java.util.Base64; import java.util.Collection; @@ -84,6 +90,173 @@ import java.util.concurrent.atomic.AtomicReference; */ public class KnowledgeDocumentImportTaskAppServiceTest { + /** + * 验证单文档执行锁与数据库任务失联租约共用同一超时，避免重启后旧锁继续阻塞恢复任务。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void documentExecutionLockShouldUseTaskInterruptionTimeout() + throws Exception { + BigInteger documentId = BigInteger.valueOf(28); + Duration interruptionTimeout = Duration.ofSeconds(42); + DocumentImportBulkProperties properties = + new DocumentImportBulkProperties(); + properties.setInterruptionTimeout(interruptionTimeout); + RedisLockExecutor lockExecutor = Mockito.mock(RedisLockExecutor.class); + RedisLockExecutor.LockHandle lockHandle = + Mockito.mock(RedisLockExecutor.LockHandle.class); + Mockito.when(lockExecutor.tryAcquire( + Mockito.anyString(), Mockito.any(), Mockito.any() + )).thenReturn(lockHandle); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "bulkProperties", properties); + setField(service, "redisLockExecutor", lockExecutor); + + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod( + "tryAcquireDocumentExecutionLock", BigInteger.class); + method.setAccessible(true); + + Assert.assertSame(lockHandle, method.invoke(service, documentId)); + Mockito.verify(lockExecutor).tryAcquire( + "easyflow:lock:document-import:document:" + documentId, + Duration.ZERO, + interruptionTimeout); + } + + /** + * 验证业务调用阻塞期间后台心跳会同时续租数据库任务与 Redis 文档锁。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void taskExecutionHeartbeatShouldRenewBothLeases() + throws Exception { + BigInteger taskId = BigInteger.valueOf(281); + BigInteger documentId = BigInteger.valueOf(282); + DocumentImportBulkProperties properties = + new DocumentImportBulkProperties(); + properties.setInterruptionTimeout(Duration.ofMillis(90)); + DocumentImportTaskMapper taskMapper = + Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.renewLease( + Mockito.eq(taskId), Mockito.eq("execution-token"), + Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any() + )).thenReturn(1); + RedisLockExecutor.LockHandle lockHandle = + Mockito.mock(RedisLockExecutor.LockHandle.class); + Mockito.when(lockHandle.renew()).thenReturn(true); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "bulkProperties", properties); + setField(service, "documentImportTaskMapper", taskMapper); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setDocumentId(documentId); + task.setExecutionToken("execution-token"); + + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod( + "startTaskExecutionHeartbeat", + DocumentImportTask.class, + RedisLockExecutor.LockHandle.class); + method.setAccessible(true); + AutoCloseable heartbeat = null; + try { + heartbeat = (AutoCloseable) method.invoke(service, task, lockHandle); + Mockito.verify(taskMapper, Mockito.timeout(1_000).atLeastOnce()) + .renewLease( + Mockito.eq(taskId), Mockito.eq("execution-token"), + Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any()); + Mockito.verify(lockHandle, Mockito.timeout(1_000).atLeastOnce()) + .renew(); + } finally { + if (heartbeat != null) { + heartbeat.close(); + } + service.shutdownTaskExecutionHeartbeatExecutor(); + } + } + + /** + * 验证索引批次按 batch -> task -> Redis 顺序续租，且 Redis 所有权 + * 丢失时不会持久化 chunk 或文档进度。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void persistOwnedIndexBatchShouldFenceWritesWhenRedisLeaseIsLost() + throws Exception { + BigInteger batchId = BigInteger.valueOf(291); + BigInteger taskId = BigInteger.valueOf(292); + BigInteger documentId = BigInteger.valueOf(293); + DocumentImportBulkProperties properties = + new DocumentImportBulkProperties(); + properties.setInterruptionTimeout(Duration.ofSeconds(30)); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.lockBatchForTaskMutation(batchId)) + .thenReturn(true); + DocumentImportTaskMapper taskMapper = + Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.renewLease( + Mockito.eq(taskId), Mockito.eq("execution-token"), + Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any() + )).thenReturn(1); + RedisLockExecutor.LockHandle lockHandle = + Mockito.mock(RedisLockExecutor.LockHandle.class); + Mockito.when(lockHandle.renew()).thenReturn(false); + DocumentChunkService chunkService = + Mockito.mock(DocumentChunkService.class); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "bulkProperties", properties); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "documentImportTaskMapper", taskMapper); + setField(service, "documentChunkService", chunkService); + setField(service, "documentMapper", documentMapper); + + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setBatchId(batchId); + task.setExecutionToken("execution-token"); + tech.easyflow.ai.entity.Document document = + new tech.easyflow.ai.entity.Document(); + document.setId(documentId); + document.setCollectionId(BigInteger.valueOf(294)); + DocumentChunk chunk = new DocumentChunk(); + chunk.setId(BigInteger.valueOf(295)); + + try { + service.persistOwnedIndexBatch( + task, document, List.of(chunk), 1, 1, lockHandle); + Assert.fail("Redis 执行锁丢失时应拒绝持久化"); + } catch (RuntimeException error) { + Assert.assertEquals("TaskOwnershipLostException", + error.getClass().getSimpleName()); + } + + ArgumentCaptor leaseUntilCaptor = + ArgumentCaptor.forClass(Date.class); + ArgumentCaptor nowCaptor = ArgumentCaptor.forClass(Date.class); + Mockito.verify(taskMapper).renewLease( + Mockito.eq(taskId), Mockito.eq("execution-token"), + leaseUntilCaptor.capture(), nowCaptor.capture(), Mockito.any()); + Assert.assertTrue(leaseUntilCaptor.getValue().after( + nowCaptor.getValue())); + org.mockito.InOrder inOrder = Mockito.inOrder( + tracker, taskMapper, lockHandle); + inOrder.verify(tracker).lockBatchForTaskMutation(batchId); + inOrder.verify(taskMapper).renewLease( + Mockito.eq(taskId), Mockito.eq("execution-token"), + Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any()); + inOrder.verify(lockHandle).renew(); + Mockito.verifyNoInteractions(chunkService, documentMapper); + } + /** * 验证预览翻页直接读取分片快照，避免恢复完整会话。 * @@ -239,11 +412,16 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Mockito.mock(DocumentImportBatchCircuitBreaker.class); Mockito.when(circuitBreaker.interruptTaskBatch(taskId, failure)) .thenReturn(true); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.lockBatchForTaskMutation(task.getBatchId())) + .thenReturn(true); KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); setField(service, "documentImportTaskMapper", taskMapper); setField(service, "parseTaskProducer", producer); setField(service, "documentImportBatchCircuitBreaker", circuitBreaker); + setField(service, "documentImportBatchTracker", tracker); setField(service, "bulkProperties", new DocumentImportBulkProperties()); service.dispatchPendingTasks(); @@ -287,11 +465,16 @@ public class KnowledgeDocumentImportTaskAppServiceTest { new IllegalStateException("database unavailable"); Mockito.when(circuitBreaker.interruptTaskBatch(taskId, dispatchFailure)) .thenThrow(interruptionFailure); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.lockBatchForTaskMutation(task.getBatchId())) + .thenReturn(true); KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); setField(service, "documentImportTaskMapper", taskMapper); setField(service, "parseTaskProducer", producer); setField(service, "documentImportBatchCircuitBreaker", circuitBreaker); + setField(service, "documentImportBatchTracker", tracker); setField(service, "bulkProperties", new DocumentImportBulkProperties()); try { @@ -424,14 +607,13 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Mockito.doThrow(new BusinessException("格式不支持")) .when(selfProxy) .retryBatchItemInNewTransaction(first.getId()); - DocumentImportBatchItemMapper itemMapper = - Mockito.mock(DocumentImportBatchItemMapper.class); + Mockito.when(selfProxy.retryBatchItemInNewTransaction(second.getId())) + .thenReturn(true); DocumentImportBatchTracker batchTracker = Mockito.mock(DocumentImportBatchTracker.class); KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); setField(service, "documentImportBatchItemService", itemService); - setField(service, "documentImportBatchItemMapper", itemMapper); setField(service, "documentImportBatchTracker", batchTracker); setField(service, "selfProxy", selfProxy); @@ -439,13 +621,145 @@ public class KnowledgeDocumentImportTaskAppServiceTest { batchId, Set.of(), () -> true)); org.mockito.InOrder order = Mockito.inOrder( - selfProxy, itemMapper); + selfProxy, batchTracker); order.verify(selfProxy).retryBatchItemInNewTransaction(first.getId()); - order.verify(itemMapper).updateFailedRetryError( + order.verify(batchTracker).updateFailedRetryError( Mockito.eq(first.getId()), Mockito.eq(batchId), - Mockito.eq("格式不支持"), Mockito.any(Date.class)); + Mockito.eq("格式不支持")); order.verify(selfProxy).retryBatchItemInNewTransaction(second.getId()); - Mockito.verifyNoInteractions(batchTracker); + } + + /** + * 验证未取得失败项且状态仍为失败时不会被计为已重新排队。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void retryBatchItemShouldReportUnclaimedFailedItem() + throws Exception { + BigInteger itemId = BigInteger.valueOf(104); + DocumentImportBatchItem current = new DocumentImportBatchItem(); + current.setId(itemId); + current.setStatus(DocumentImportBatchItemStatus.FAILED.name()); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.lockFailedItemForRetry(itemId)).thenReturn(null); + DocumentImportBatchItemService itemService = + Mockito.mock(DocumentImportBatchItemService.class); + Mockito.when(itemService.getById(itemId)).thenReturn(current); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "documentImportBatchItemService", itemService); + + Assert.assertFalse(service.retryBatchItemInNewTransaction(itemId)); + } + + /** + * 验证并发操作已推进文件项时可视为等价恢复，避免误熔断新任务。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void retryBatchItemShouldAcceptConcurrentlyAdvancedItem() + throws Exception { + BigInteger itemId = BigInteger.valueOf(105); + DocumentImportBatchItem current = new DocumentImportBatchItem(); + current.setId(itemId); + current.setStatus(DocumentImportBatchItemStatus.PENDING.name()); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.lockFailedItemForRetry(itemId)).thenReturn(null); + DocumentImportBatchItemService itemService = + Mockito.mock(DocumentImportBatchItemService.class); + Mockito.when(itemService.getById(itemId)).thenReturn(current); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "documentImportBatchItemService", itemService); + + Assert.assertTrue(service.retryBatchItemInNewTransaction(itemId)); + } + + /** + * 验证全部失败项均无法重新排队时不会把“继续”表现为成功。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void retryBatchFailuresShouldExposeWhenNothingWasRestarted() + throws Exception { + BigInteger batchId = BigInteger.valueOf(101); + DocumentImportBatchItem failed = new DocumentImportBatchItem(); + failed.setId(BigInteger.valueOf(102)); + failed.setBatchId(batchId); + failed.setDocumentId(BigInteger.valueOf(103)); + DocumentImportBatchItemService itemService = + Mockito.mock(DocumentImportBatchItemService.class); + Mockito.when(itemService.list(Mockito.any(QueryWrapper.class))) + .thenReturn(List.of(failed)); + KnowledgeDocumentImportTaskAppService selfProxy = + Mockito.mock(KnowledgeDocumentImportTaskAppService.class); + Mockito.doThrow(new BusinessException("批次不存在")) + .when(selfProxy) + .retryBatchItemInNewTransaction(failed.getId()); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportBatchItemService", itemService); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "selfProxy", selfProxy); + + try { + service.retryBatchFailures(batchId, Set.of(), () -> true); + Assert.fail("没有失败项重新排队时必须暴露恢复失败"); + } catch (DocumentImportRecoveryException error) { + Assert.assertEquals("document_import_recovery_failed", error.getCode()); + Assert.assertEquals( + List.of(failed.getId()), error.getFailureFenceItemIds()); + } + + Mockito.verify(tracker).updateFailedRetryError( + Mockito.eq(failed.getId()), Mockito.eq(batchId), + Mockito.eq("批次不存在")); + } + + /** + * 验证错误写回前的等价并发推进不会误触发整批熔断。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void retryBatchFailuresShouldAcceptAdvanceDuringErrorWriteBack() + throws Exception { + BigInteger batchId = BigInteger.valueOf(106); + DocumentImportBatchItem failed = new DocumentImportBatchItem(); + failed.setId(BigInteger.valueOf(107)); + failed.setBatchId(batchId); + DocumentImportBatchItemService itemService = + Mockito.mock(DocumentImportBatchItemService.class); + Mockito.when(itemService.list(Mockito.any(QueryWrapper.class))) + .thenReturn(List.of(failed)); + KnowledgeDocumentImportTaskAppService selfProxy = + Mockito.mock(KnowledgeDocumentImportTaskAppService.class); + Mockito.doThrow(new BusinessException("状态已变化")) + .when(selfProxy) + .retryBatchItemInNewTransaction(failed.getId()); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.updateFailedRetryError( + failed.getId(), batchId, "状态已变化")) + .thenReturn( + DocumentImportBatchTracker.FailedRetryErrorOutcome.ALREADY_ADVANCED); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportBatchItemService", itemService); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "selfProxy", selfProxy); + + Assert.assertTrue(service.retryBatchFailures( + batchId, Set.of(), () -> true)); } /** @@ -499,14 +813,14 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Assert.assertTrue(sql.contains("status='PENDING'")); Assert.assertTrue(sql.contains("modified <= #{redispatchBefore}")); Assert.assertTrue(sql.contains("modified=#{now}")); - Assert.assertTrue(sql.contains("batch.status='RUNNING'")); + Assert.assertFalse(sql.contains("tb_document_import_batch")); } /** * 验证任务扫描、领取、续租和完成 SQL 均包含批次运行状态门禁。 */ @Test - public void activeTaskSqlShouldFenceInterruptedBatch() { + public void activeTaskSqlShouldAvoidImplicitTaskThenBatchLocks() { for (Method method : DocumentImportTaskMapper.class.getDeclaredMethods()) { String name = method.getName(); if (!"selectPendingFairly".equals(name) @@ -521,29 +835,104 @@ public class KnowledgeDocumentImportTaskAppServiceTest { ? String.join(" ", update.value()) : String.join(" ", select.value()); - Assert.assertTrue(name + " 缺少批次运行状态门禁", - sql.contains("batch.status='RUNNING'")); + if ("selectPendingFairly".equals(name)) { + Assert.assertTrue(sql.contains("batch.status='RUNNING'")); + Assert.assertTrue(sql.contains("batch.import_mode='MANUAL'")); + Assert.assertTrue(sql.contains("task.batch_id AS batchId")); + Assert.assertTrue(sql.contains( + "task.provider_task_id AS providerTaskId")); + Assert.assertFalse(sql.contains("SELECT task.*")); + } else { + Assert.assertFalse(name + " 不应隐式反向读取批次行", + sql.contains("tb_document_import_batch")); + } } } + /** + * 验证批次任务收口前先锁批次行，避免与 batch -> tasks 熔断路径形成反向等待。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void finishTaskShouldLockBatchBeforeTaskRow() throws Exception { + BigInteger batchId = BigInteger.valueOf(121); + BigInteger taskId = BigInteger.valueOf(122); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setBatchId(batchId); + task.setExecutionToken("execution-token"); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.lockBatchForTaskMutation(batchId)).thenReturn(true); + DocumentImportTaskMapper taskMapper = + Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.finishOwned( + Mockito.eq(taskId), Mockito.eq("execution-token"), + Mockito.eq(DocumentImportTaskStatus.COMPLETED.name()), + Mockito.isNull(), Mockito.isNull(), Mockito.any(Date.class), + Mockito.any() + )).thenReturn(1); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "documentImportTaskMapper", taskMapper); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod( + "finishTask", + DocumentImportTask.class, + Date.class, + DocumentImportTaskStatus.class, + String.class, + String.class); + method.setAccessible(true); + + Assert.assertEquals(Boolean.TRUE, method.invoke( + service, + task, + new Date(), + DocumentImportTaskStatus.COMPLETED, + null, + null)); + + org.mockito.InOrder lockOrder = Mockito.inOrder(tracker, taskMapper); + lockOrder.verify(tracker).lockBatchForTaskMutation(batchId); + lockOrder.verify(taskMapper).finishOwned( + Mockito.eq(taskId), Mockito.eq("execution-token"), + Mockito.eq(DocumentImportTaskStatus.COMPLETED.name()), + Mockito.isNull(), Mockito.isNull(), Mockito.any(Date.class), + Mockito.any()); + } + /** * 验证失败项恢复同时锁定文件项与运行批次，阻止熔断后继续建任务。 * * @throws Exception 映射方法不存在时抛出 */ @Test - public void retryItemSqlShouldRequireRunningBatchLock() + public void retryItemSqlShouldUseBatchThenItemLocks() throws Exception { - Method method = DocumentImportBatchItemMapper.class.getMethod( - "selectFailedForRetry", + Method batchLock = DocumentImportBatchMapper.class.getMethod( + "selectForUpdate", BigInteger.class ); - Select select = method.getAnnotation(Select.class); - String sql = String.join(" ", select.value()); + Method itemLock = DocumentImportBatchItemMapper.class.getMethod( + "selectForUpdate", + BigInteger.class + ); + String batchSql = String.join(" ", + batchLock.getAnnotation(Select.class).value()); + String itemSql = String.join(" ", + itemLock.getAnnotation(Select.class).value()); - Assert.assertTrue(sql.contains("item.status='FAILED'")); - Assert.assertTrue(sql.contains("batch.status='RUNNING'")); - Assert.assertTrue(sql.contains("FOR UPDATE")); + Assert.assertTrue(batchSql.contains("tb_document_import_batch")); + Assert.assertTrue(batchSql.contains("knowledge_id AS knowledgeId")); + Assert.assertTrue(batchSql.contains("recovery_token AS recoveryToken")); + Assert.assertTrue(batchSql.contains("FOR UPDATE")); + Assert.assertTrue(itemSql.contains("tb_document_import_batch_item")); + Assert.assertTrue(itemSql.contains("batch_id AS batchId")); + Assert.assertTrue(itemSql.contains("storage_locator AS storageLocator")); + Assert.assertTrue(itemSql.contains("FOR UPDATE")); } /** @@ -552,7 +941,7 @@ public class KnowledgeDocumentImportTaskAppServiceTest { * @throws Exception 映射方法不存在时抛出 */ @Test - public void itemTransitionSqlShouldFenceInterruptedBatch() + public void itemTransitionSqlShouldAvoidParentSharedLockUpgrade() throws Exception { Method method = DocumentImportBatchItemMapper.class.getMethod( "transitionStatus", @@ -569,9 +958,8 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Update update = method.getAnnotation(Update.class); String sql = String.join(" ", update.value()); - Assert.assertTrue(sql.contains( - "INNER JOIN tb_document_import_batch batch")); - Assert.assertTrue(sql.contains("batch.status<>'INTERRUPTED'")); + Assert.assertFalse(sql.contains("tb_document_import_batch batch")); + Assert.assertTrue(sql.contains("status=#{expectedStatus}")); } /** @@ -644,6 +1032,7 @@ public class KnowledgeDocumentImportTaskAppServiceTest { BigInteger taskId = BigInteger.valueOf(35); DocumentImportTask task = new DocumentImportTask(); task.setId(taskId); + task.setBatchId(BigInteger.valueOf(36)); task.setPhase(DocumentImportTaskPhase.SPLIT.name()); task.setStatus(DocumentImportTaskStatus.PENDING.name()); @@ -669,12 +1058,17 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Mockito.when(lockExecutor.tryAcquire( Mockito.anyString(), Mockito.any(), Mockito.any() )).thenReturn(lockHandle); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.lockBatchForTaskMutation(task.getBatchId())) + .thenReturn(true); KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); setField(service, "documentImportTaskService", taskService); setField(service, "documentImportTaskMapper", taskMapper); setField(service, "redisLockExecutor", lockExecutor); + setField(service, "documentImportBatchTracker", tracker); setField(service, "bulkProperties", new DocumentImportBulkProperties()); Assert.assertTrue(service.tryMarkTaskRunning(taskId)); @@ -691,6 +1085,11 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Assert.assertFalse(token.getValue().isBlank()); Assert.assertTrue(lease.getValue().after(new Date())); Mockito.verify(lockHandle).release(); + org.mockito.InOrder lockOrder = Mockito.inOrder(tracker, taskMapper); + lockOrder.verify(tracker).lockBatchForTaskMutation(task.getBatchId()); + lockOrder.verify(taskMapper).claimPending( + Mockito.eq(taskId), Mockito.anyString(), Mockito.any(Date.class), + Mockito.any(Date.class), Mockito.nullable(BigInteger.class)); } /** @@ -1106,6 +1505,322 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Assert.assertEquals("parse_failed", code); } + /** + * 验证进程重启导致的本地解析任务丢失会映射为可恢复错误。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void resolveParseFailureShouldRecognizeLostLocalExecution() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + Method codeMethod = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod("resolveParseFailureCode", Throwable.class); + Method messageMethod = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod( + "resolveParseFailureMessage", Throwable.class, String.class); + codeMethod.setAccessible(true); + messageMethod.setAccessible(true); + DocumentParseBridgeException error = + DocumentParseBridgeException.taskNotFound( + "异步文档解析任务不存在", + new IllegalStateException("process restarted") + ); + + String code = (String) codeMethod.invoke(service, error); + String message = (String) messageMethod.invoke(service, error, code); + + Assert.assertEquals("parse_execution_lost", code); + Assert.assertEquals("文档解析执行已中断，请点击继续重试", message); + } + + /** + * 验证监控发现 provider 执行丢失后会按稳定语义收口解析任务。 + * + * @throws Exception 依赖注入异常 + */ + @Test + public void handleRunningParseTaskShouldFailLostProviderExecution() + throws Exception { + BigInteger taskId = BigInteger.valueOf(711); + BigInteger documentId = BigInteger.valueOf(712); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setDocumentId(documentId); + task.setPhase(DocumentImportTaskPhase.PARSE.name()); + task.setStatus(DocumentImportTaskStatus.RUNNING.name()); + task.setProviderTaskId("lost-provider-task"); + tech.easyflow.ai.entity.Document document = + new tech.easyflow.ai.entity.Document(); + document.setId(documentId); + document.setTitle("recover.xlsx"); + document.setProcessStatus(DocumentProcessStatus.PARSING.name()); + + DocumentImportTaskService taskService = + Mockito.mock(DocumentImportTaskService.class); + Mockito.when(taskService.getById(taskId)).thenReturn(task); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneById(documentId)).thenReturn(document); + DocumentParseBridgeService bridgeService = + Mockito.mock(DocumentParseBridgeService.class); + Mockito.when(bridgeService.queryTaskInfo( + Mockito.eq("lost-provider-task"), Mockito.any(DocumentSourceRef.class) + )).thenThrow(DocumentParseBridgeException.taskNotFound( + "异步文档解析任务不存在", + new IllegalStateException("process restarted") + )); + KnowledgeDocumentImportTaskAppService selfProxy = + Mockito.mock(KnowledgeDocumentImportTaskAppService.class); + Mockito.when(selfProxy.failParseTask( + Mockito.eq(task), + Mockito.eq(document), + Mockito.anyString(), + Mockito.anyString(), + Mockito.any(Throwable.class) + )).thenReturn(true); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportTaskService", taskService); + setField(service, "documentMapper", documentMapper); + setField(service, "documentParseBridgeService", bridgeService); + setField(service, "selfProxy", selfProxy); + + service.handleRunningParseTask(taskId); + + Mockito.verify(selfProxy).failParseTask( + Mockito.eq(task), + Mockito.eq(document), + Mockito.eq("文档解析执行已中断，请点击继续重试"), + Mockito.eq("parse_execution_lost"), + Mockito.any(Throwable.class) + ); + } + + /** + * 验证重新解析前会移除已经失效的 provider 任务和进度信息。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void clearDocumentParseExecutionShouldRemoveLostProviderTask() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod("clearDocumentParseExecution", Map.class); + method.setAccessible(true); + Map options = new LinkedHashMap(); + options.put(DocumentImportKeys.KEY_DOCUMENT_PROVIDER_TASK_ID, "lost-task"); + options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_PROGRESS_PERCENT, 25); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID, "batch-1"); + + method.invoke(service, options); + + Assert.assertFalse(options.containsKey( + DocumentImportKeys.KEY_DOCUMENT_PROVIDER_TASK_ID)); + Assert.assertFalse(options.containsKey( + DocumentImportKeys.KEY_DOCUMENT_PARSE_PROGRESS_PERCENT)); + Assert.assertEquals("batch-1", options.get( + DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID)); + } + + /** + * 验证成功解析的 provider 任务审计信息不会随临时进度一起清除。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void clearDocumentParseProgressShouldPreserveProviderTask() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod("clearDocumentParseProgress", Map.class); + method.setAccessible(true); + Map options = new LinkedHashMap(); + options.put(DocumentImportKeys.KEY_DOCUMENT_PROVIDER_TASK_ID, "task-2"); + options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_PROGRESS_PERCENT, 100); + + method.invoke(service, options); + + Assert.assertEquals("task-2", options.get( + DocumentImportKeys.KEY_DOCUMENT_PROVIDER_TASK_ID)); + Assert.assertFalse(options.containsKey( + DocumentImportKeys.KEY_DOCUMENT_PARSE_PROGRESS_PERCENT)); + } + + /** + * 验证解析恢复会复用批次、文件项和文档，并创建新的持久化解析任务。 + * + * @throws Exception 依赖注入异常 + */ + @Test + public void retryParseTaskShouldCreateNewAttemptWithExistingBatchContext() + throws Exception { + BigInteger knowledgeId = BigInteger.valueOf(721); + BigInteger documentId = BigInteger.valueOf(722); + BigInteger batchId = BigInteger.valueOf(723); + BigInteger itemId = BigInteger.valueOf(724); + BigInteger newTaskId = BigInteger.valueOf(725); + DocumentCollection knowledge = new DocumentCollection(); + knowledge.setId(knowledgeId); + tech.easyflow.ai.entity.Document document = + new tech.easyflow.ai.entity.Document(); + document.setId(documentId); + document.setCollectionId(knowledgeId); + document.setTitle("recover.xlsx"); + document.setDocumentPath("/stored/recover.xlsx"); + document.setProcessStatus(DocumentProcessStatus.PARSE_FAILED.name()); + document.setLastTaskError("旧解析执行已丢失"); + Map options = new LinkedHashMap(); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID, + batchId.toString()); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID, + itemId.toString()); + options.put(DocumentImportKeys.KEY_DOCUMENT_PROVIDER_TASK_ID, + "lost-provider-task"); + options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_PROGRESS_PERCENT, 30); + options.put(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE, + "parse_execution_lost"); + document.setOptions(options); + + DocumentCollectionService knowledgeService = + Mockito.mock(DocumentCollectionService.class); + Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneById(documentId)).thenReturn(document); + Mockito.when(documentMapper.updateByQuery( + Mockito.any(tech.easyflow.ai.entity.Document.class), Mockito.any())) + .thenReturn(1); + DocumentImportTaskService taskService = + Mockito.mock(DocumentImportTaskService.class); + Mockito.when(taskService.save(Mockito.any(DocumentImportTask.class))) + .thenAnswer(invocation -> { + DocumentImportTask task = invocation.getArgument(0); + task.setId(newTaskId); + return true; + }); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + DocumentImportBatchItem failedItem = new DocumentImportBatchItem(); + failedItem.setId(itemId); + failedItem.setBatchId(batchId); + failedItem.setKnowledgeId(knowledgeId); + failedItem.setDocumentId(documentId); + failedItem.setStatus(DocumentImportBatchItemStatus.FAILED.name()); + Mockito.when(tracker.isAutoBatch(batchId)).thenReturn(true); + Mockito.when(tracker.lockFailedItemForRetry(itemId)) + .thenReturn(failedItem); + Mockito.when(tracker.updateItem( + Mockito.eq(itemId), + Mockito.eq(DocumentImportBatchItemStage.PARSE), + Mockito.eq(DocumentImportBatchItemStatus.PENDING), + Mockito.isNull() + )).thenReturn(true); + DocumentImportParseTaskProducer producer = + Mockito.mock(DocumentImportParseTaskProducer.class); + + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "knowledgeService", knowledgeService); + setField(service, "documentMapper", documentMapper); + setField(service, "documentImportTaskService", taskService); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "parseTaskProducer", producer); + setField(service, "documentImportTaskStatusStreamService", + new NoopTaskStatusStreamService()); + DocumentImportDtos.TaskRetryRequest request = + new DocumentImportDtos.TaskRetryRequest(); + request.setKnowledgeId(knowledgeId); + request.setDocumentId(documentId); + + Result result = + service.retryParseTask(request); + + ArgumentCaptor taskCaptor = + ArgumentCaptor.forClass(DocumentImportTask.class); + Mockito.verify(taskService).save(taskCaptor.capture()); + DocumentImportTask newTask = taskCaptor.getValue(); + Assert.assertEquals(newTaskId, result.getData().getTaskId()); + Assert.assertEquals(documentId, newTask.getDocumentId()); + Assert.assertEquals(batchId, newTask.getBatchId()); + Assert.assertEquals(itemId, newTask.getBatchItemId()); + Assert.assertEquals(DocumentImportTaskPhase.PARSE.name(), + newTask.getPhase()); + Assert.assertNull(newTask.getProviderTaskId()); + Assert.assertFalse(document.getOptions().containsKey( + DocumentImportKeys.KEY_DOCUMENT_PROVIDER_TASK_ID)); + Assert.assertFalse(document.getOptions().containsKey( + DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE)); + Mockito.verify(tracker).updateItem( + itemId, + DocumentImportBatchItemStage.PARSE, + DocumentImportBatchItemStatus.PENDING, + null + ); + Mockito.verify(tracker).lockFailedItemForRetry(itemId); + Mockito.verify(producer).send(newTaskId); + } + + /** + * 验证自动批次单文档重试未取得批次恢复权时不会先写文档或任务。 + * + * @throws Exception 依赖注入异常 + */ + @Test + public void retryParseTaskShouldLockBatchItemBeforeMutation() + throws Exception { + BigInteger knowledgeId = BigInteger.valueOf(726); + BigInteger documentId = BigInteger.valueOf(727); + BigInteger batchId = BigInteger.valueOf(728); + BigInteger itemId = BigInteger.valueOf(729); + DocumentCollection knowledge = new DocumentCollection(); + knowledge.setId(knowledgeId); + tech.easyflow.ai.entity.Document document = + new tech.easyflow.ai.entity.Document(); + document.setId(documentId); + document.setCollectionId(knowledgeId); + document.setProcessStatus(DocumentProcessStatus.PARSE_FAILED.name()); + Map options = new LinkedHashMap(); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID, + batchId.toString()); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID, + itemId.toString()); + document.setOptions(options); + DocumentCollectionService knowledgeService = + Mockito.mock(DocumentCollectionService.class); + Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneById(documentId)) + .thenReturn(document); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.isAutoBatch(batchId)).thenReturn(true); + Mockito.when(tracker.lockFailedItemForRetry(itemId)).thenReturn(null); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "knowledgeService", knowledgeService); + setField(service, "documentMapper", documentMapper); + setField(service, "documentImportBatchTracker", tracker); + DocumentImportDtos.TaskRetryRequest request = + new DocumentImportDtos.TaskRetryRequest(); + request.setKnowledgeId(knowledgeId); + request.setDocumentId(documentId); + + try { + service.retryParseTask(request); + Assert.fail("未取得批次恢复权时必须拒绝单文档重试"); + } catch (BusinessException error) { + Assert.assertEquals( + "批次文件状态已变化，请刷新后重试", error.getMessage()); + } + + Mockito.verify(documentMapper, Mockito.never()).updateByQuery( + Mockito.any(tech.easyflow.ai.entity.Document.class), + Mockito.any(QueryWrapper.class)); + } + /** * 验证人工批次继续不受历史尝试次数限制。 * @@ -1225,7 +1940,81 @@ public class KnowledgeDocumentImportTaskAppServiceTest { } /** - * 验证无文档失败项绑定 SQL 同时校验失败状态、空文档和运行批次。 + * 验证失败项引用的文档记录丢失后，会复用原批次项重新建档解析。 + * + * @throws Exception 反射调用或依赖注入异常 + */ + @Test + public void retryBatchItemWithMissingDocumentShouldRecreateImportTask() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + DocumentImportTaskService taskService = + Mockito.mock(DocumentImportTaskService.class); + DocumentCollectionService knowledgeService = + Mockito.mock(DocumentCollectionService.class); + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + DocumentImportParseTaskProducer producer = + Mockito.mock(DocumentImportParseTaskProducer.class); + BigInteger itemId = BigInteger.valueOf(721); + BigInteger batchId = BigInteger.valueOf(722); + BigInteger knowledgeId = BigInteger.valueOf(723); + BigInteger missingDocumentId = BigInteger.valueOf(724); + BigInteger taskId = BigInteger.valueOf(725); + DocumentImportBatchItem failed = new DocumentImportBatchItem(); + failed.setId(itemId); + failed.setBatchId(batchId); + failed.setKnowledgeId(knowledgeId); + failed.setDocumentId(missingDocumentId); + failed.setFileName("recover.pdf"); + failed.setRelativePath("folder/recover.pdf"); + failed.setFilePath("/stored/recover.pdf"); + failed.setStage(DocumentImportBatchItemStage.PARSE.name()); + failed.setStatus(DocumentImportBatchItemStatus.FAILED.name()); + DocumentImportBatch batch = new DocumentImportBatch(); + batch.setId(batchId); + batch.setKnowledgeId(knowledgeId); + batch.setImportMode(DocumentImportMode.AUTO.name()); + batch.setStatus(DocumentImportBatchStatus.RUNNING.name()); + DocumentCollection knowledge = new DocumentCollection(); + knowledge.setId(knowledgeId); + Mockito.when(documentMapper.selectOneById(missingDocumentId)) + .thenReturn(null); + Mockito.when(tracker.requireBatch(batchId)).thenReturn(batch); + Mockito.when(knowledgeService.getById(knowledgeId)) + .thenReturn(knowledge); + Mockito.when(taskService.save(Mockito.any(DocumentImportTask.class))) + .thenAnswer(invocation -> { + DocumentImportTask task = invocation.getArgument(0); + task.setId(taskId); + return true; + }); + setField(service, "documentMapper", documentMapper); + setField(service, "documentImportTaskService", taskService); + setField(service, "knowledgeService", knowledgeService); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "parseTaskProducer", producer); + + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod("retryBatchItem", DocumentImportBatchItem.class); + method.setAccessible(true); + method.invoke(service, failed); + + ArgumentCaptor documentCaptor = + ArgumentCaptor.forClass(tech.easyflow.ai.entity.Document.class); + Mockito.verify(documentMapper).insert(documentCaptor.capture()); + tech.easyflow.ai.entity.Document document = documentCaptor.getValue(); + Mockito.verify(tracker).replaceMissingDocument( + itemId, missingDocumentId, document.getId()); + Mockito.verify(tracker, Mockito.never()).bindDocument( + Mockito.any(), Mockito.any()); + Mockito.verify(producer).send(taskId); + } + + /** + * 验证无文档失败项绑定 SQL 保留文件项状态 CAS，批次运行态由 Tracker 锁定校验。 * * @throws Exception Mapper 方法不存在时抛出 */ @@ -1241,10 +2030,34 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Update update = method.getAnnotation(Update.class); String sql = String.join(" ", update.value()); - Assert.assertTrue(sql.contains("item.status='FAILED'")); - Assert.assertTrue(sql.contains("item.document_id IS NULL")); - Assert.assertTrue(sql.contains("batch.status='RUNNING'")); - Assert.assertTrue(sql.contains("item.status='PENDING'")); + Assert.assertTrue(sql.contains("status='FAILED'")); + Assert.assertTrue(sql.contains("document_id IS NULL")); + Assert.assertFalse(sql.contains("tb_document_import_batch batch")); + Assert.assertTrue(sql.contains("status='PENDING'")); + } + + /** + * 验证悬空文档引用只能在旧文档确实不存在时被替换。 + * + * @throws Exception Mapper 方法不存在时抛出 + */ + @Test + public void replaceMissingDocumentSqlShouldFenceOldReference() + throws Exception { + Method method = DocumentImportBatchItemMapper.class.getMethod( + "replaceMissingDocument", + BigInteger.class, + BigInteger.class, + BigInteger.class, + Date.class + ); + String sql = String.join(" ", + method.getAnnotation(Update.class).value()); + + Assert.assertTrue(sql.contains("status='FAILED'")); + Assert.assertTrue(sql.contains("document_id=#{missingDocumentId}")); + Assert.assertTrue(sql.contains("NOT EXISTS")); + Assert.assertTrue(sql.contains("tb_document")); } /** @@ -1392,18 +2205,19 @@ public class KnowledgeDocumentImportTaskAppServiceTest { } /** - * 验证自动导入向量化失败后从分块阶段重试，并废弃旧分块快照。 + * 验证自动导入向量化失败后复用持久化快照和稳定 chunk ID， + * 避免服务退出后遗留无法定位的外部索引。 * * @throws Exception 反射注入异常 */ @Test - public void retryIndexTaskShouldResplitAutoBatchAfterIndexFailure() + public void retryIndexTaskShouldReuseSnapshotAfterAutoBatchIndexFailure() throws Exception { BigInteger knowledgeId = BigInteger.valueOf(101); BigInteger documentId = BigInteger.valueOf(102); BigInteger batchId = BigInteger.valueOf(103); BigInteger batchItemId = BigInteger.valueOf(104); - BigInteger splitTaskId = BigInteger.valueOf(105); + BigInteger indexTaskId = BigInteger.valueOf(105); String staleSnapshotPath = "document-import/chunks/stale.snapshot"; DocumentCollection knowledge = new DocumentCollection(); @@ -1422,10 +2236,6 @@ public class KnowledgeDocumentImportTaskAppServiceTest { staleSnapshotPath); document.setOptions(options); - DocumentImportBatch batch = new DocumentImportBatch(); - batch.setId(batchId); - batch.setRequestedStrategyJson("{\"strategyCode\":\"AUTO\"}"); - DocumentCollectionService knowledgeService = Mockito.mock(DocumentCollectionService.class); Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge); @@ -1437,22 +2247,49 @@ public class KnowledgeDocumentImportTaskAppServiceTest { .thenReturn(1); DocumentImportBatchTracker tracker = Mockito.mock(DocumentImportBatchTracker.class); + DocumentImportBatchItem failedItem = new DocumentImportBatchItem(); + failedItem.setId(batchItemId); + failedItem.setBatchId(batchId); + failedItem.setKnowledgeId(knowledgeId); + failedItem.setDocumentId(documentId); + failedItem.setStatus(DocumentImportBatchItemStatus.FAILED.name()); Mockito.when(tracker.isAutoBatch(batchId)).thenReturn(true); - Mockito.when(tracker.requireBatch(batchId)).thenReturn(batch); + Mockito.when(tracker.lockFailedItemForRetry(batchItemId)) + .thenReturn(failedItem); + Mockito.when(tracker.updateItem( + batchItemId, + DocumentImportBatchItemStage.INDEX, + DocumentImportBatchItemStatus.PENDING, + null + )).thenReturn(true); + DocumentImportDtos.PreviewSession snapshot = + new DocumentImportDtos.PreviewSession(); + snapshot.setKnowledgeId(knowledgeId); + snapshot.setDocumentId(documentId); + snapshot.setChunkSnapshotPath(staleSnapshotPath); + snapshot.setSourceFormat("docx"); + snapshot.setStrategyConfig(StrategyConfig.defaults()); + snapshot.setTotalChunks(3); + DocumentImportChunkSnapshotService snapshotService = + Mockito.mock(DocumentImportChunkSnapshotService.class); + Mockito.when(snapshotService.loadHeader(staleSnapshotPath)) + .thenReturn(snapshot); DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class); AtomicReference createdTask = new AtomicReference(); Mockito.doAnswer(invocation -> { DocumentImportTask task = invocation.getArgument(0); - task.setId(splitTaskId); + task.setId(indexTaskId); createdTask.set(task); return true; }).when(taskService).save(Mockito.any(DocumentImportTask.class)); - DocumentImportSplitTaskProducer splitTaskProducer = - Mockito.mock(DocumentImportSplitTaskProducer.class); - DocumentImportSnapshotCleanupService cleanupService = - Mockito.mock(DocumentImportSnapshotCleanupService.class); + DocumentImportIndexTaskProducer indexTaskProducer = + Mockito.mock(DocumentImportIndexTaskProducer.class); + RagIngestionService ragIngestionService = + Mockito.mock(RagIngestionService.class); + Mockito.when(ragIngestionService.toStrategyLabel(Mockito.anyString())) + .thenReturn("自动分块"); ThreadPoolTaskExecutor taskExecutor = Mockito.mock(ThreadPoolTaskExecutor.class); @@ -1462,8 +2299,9 @@ public class KnowledgeDocumentImportTaskAppServiceTest { setField(service, "documentMapper", documentMapper); setField(service, "documentImportBatchTracker", tracker); setField(service, "documentImportTaskService", taskService); - setField(service, "splitTaskProducer", splitTaskProducer); - setField(service, "snapshotCleanupService", cleanupService); + setField(service, "documentImportChunkSnapshotService", snapshotService); + setField(service, "indexTaskProducer", indexTaskProducer); + setField(service, "ragIngestionService", ragIngestionService); setField(service, "documentImportTaskExecutor", taskExecutor); setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService()); @@ -1476,27 +2314,22 @@ public class KnowledgeDocumentImportTaskAppServiceTest { service.retryIndexTask(request).getData(); Assert.assertNotNull(response); - Assert.assertEquals(splitTaskId, response.getTaskId()); - Assert.assertEquals(DocumentProcessStatus.SPLITTING.name(), + Assert.assertEquals(indexTaskId, response.getTaskId()); + Assert.assertEquals(DocumentProcessStatus.INDEXING.name(), response.getProcessStatus()); - Assert.assertFalse(document.getOptions().containsKey( + Assert.assertEquals(staleSnapshotPath, document.getOptions().get( DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH)); Assert.assertNotNull(createdTask.get()); - Assert.assertEquals(DocumentImportTaskPhase.SPLIT.name(), + Assert.assertEquals(DocumentImportTaskPhase.INDEX.name(), createdTask.get().getPhase()); - Assert.assertEquals(batch.getRequestedStrategyJson(), - createdTask.get().getPayloadJson().get("strategyConfigJson")); - Mockito.verify(splitTaskProducer).send(splitTaskId); - Mockito.verify(cleanupService).scheduleChunkSnapshot( - staleSnapshotPath); - Mockito.verify(documentMapper).update( - Mockito.argThat(updated -> updated.getOptions() != null - && !updated.getOptions().containsKey( - DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH)), - Mockito.eq(false)); + Assert.assertEquals(staleSnapshotPath, + createdTask.get().getPayloadJson().get("chunkSnapshotPath")); + Assert.assertEquals(Boolean.TRUE, + createdTask.get().getPayloadJson().get("replaceExistingIndex")); + Mockito.verify(indexTaskProducer).send(indexTaskId); Mockito.verify(tracker).updateItem( batchItemId, - DocumentImportBatchItemStage.SPLIT, + DocumentImportBatchItemStage.INDEX, DocumentImportBatchItemStatus.PENDING, null); } @@ -2088,6 +2921,130 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Mockito.verify(searcher).deleteDocuments(List.of(chunk.getId())); } + /** + * 验证自动批次索引失败时不会主动删除稳定 ID 的外部记录，避免失权旧执行 + * 误删后继重放任务已经写入的结果。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void automaticIndexFailureShouldKeepExternalStableIds() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + DocumentStore documentStore = Mockito.mock(DocumentStore.class); + DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class); + StoreOptions storeOptions = + StoreOptions.ofCollectionName("knowledge-test"); + + DocumentCollection knowledge = new DocumentCollection(); + knowledge.setId(BigInteger.valueOf(781)); + Class contextClass = Class.forName( + KnowledgeDocumentImportTaskAppService.class.getName() + + "$StoreExecutionContext"); + Constructor constructor = contextClass.getDeclaredConstructor( + DocumentCollection.class, + EmbeddingModel.class, + DocumentStore.class, + StoreOptions.class, + DocumentSearcher.class + ); + constructor.setAccessible(true); + Object context = constructor.newInstance( + knowledge, null, documentStore, storeOptions, searcher); + + DocumentChunk chunk = new DocumentChunk(); + chunk.setId(BigInteger.valueOf(782)); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod( + "rollbackFailedIndexAttempt", + boolean.class, + BigInteger.class, + BigInteger.class, + contextClass, + List.class + ); + method.setAccessible(true); + + Object result = method.invoke( + service, + true, + BigInteger.valueOf(783), + BigInteger.valueOf(784), + context, + List.of(chunk) + ); + + Assert.assertEquals(Boolean.TRUE, result); + Mockito.verifyNoInteractions(documentStore, searcher); + } + + /** + * 验证索引失败重放会按持久化快照中的稳定 ID 清理两类外部索引。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void deleteStoredChunksBeforeReplayShouldCleanStableIds() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + DocumentStore documentStore = Mockito.mock(DocumentStore.class); + DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class); + StoreOptions storeOptions = StoreOptions.ofCollectionName("knowledge-test"); + Mockito.when(documentStore.delete( + Mockito.anyCollection(), Mockito.same(storeOptions))) + .thenReturn(StoreResult.success()); + Mockito.when(searcher.deleteDocuments(Mockito.anyCollection())) + .thenReturn(true); + + DocumentCollection knowledge = new DocumentCollection(); + knowledge.setId(BigInteger.valueOf(801)); + Class contextClass = Class.forName( + KnowledgeDocumentImportTaskAppService.class.getName() + + "$StoreExecutionContext"); + Constructor constructor = contextClass.getDeclaredConstructor( + DocumentCollection.class, + EmbeddingModel.class, + DocumentStore.class, + StoreOptions.class, + DocumentSearcher.class + ); + constructor.setAccessible(true); + Object context = constructor.newInstance( + knowledge, null, documentStore, storeOptions, searcher); + + BigInteger documentId = BigInteger.valueOf(802); + BigInteger taskId = BigInteger.valueOf(803); + DocumentChunk first = new DocumentChunk(); + first.setId(BigInteger.valueOf(805)); + DocumentChunk duplicate = new DocumentChunk(); + duplicate.setId(first.getId()); + DocumentChunk second = new DocumentChunk(); + second.setId(BigInteger.valueOf(806)); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod( + "deleteStoredChunksBeforeReplay", + BigInteger.class, + BigInteger.class, + contextClass, + List.class + ); + method.setAccessible(true); + method.invoke( + service, + taskId, + documentId, + context, + List.of(first, duplicate, second) + ); + + Mockito.verify(documentStore).delete( + List.of(first.getId(), second.getId()), storeOptions); + Mockito.verify(searcher).deleteDocuments( + List.of(first.getId(), second.getId())); + } + /** * 验证索引业务异常会保留准确原因，并在回滚失败时追加处置提示。 *