(fileKeys);
}
/**
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreaker.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreaker.java
new file mode 100644
index 00000000..e9d38668
--- /dev/null
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreaker.java
@@ -0,0 +1,440 @@
+package tech.easyflow.ai.documentimport.task;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessResourceFailureException;
+import org.springframework.dao.QueryTimeoutException;
+import org.springframework.dao.TransientDataAccessResourceException;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Propagation;
+import org.springframework.transaction.annotation.Transactional;
+import tech.easyflow.ai.entity.DocumentImportBatch;
+import tech.easyflow.ai.entity.DocumentImportTask;
+import tech.easyflow.ai.enums.DocumentImportBatchStatus;
+import tech.easyflow.ai.enums.DocumentImportMode;
+import tech.easyflow.ai.enums.DocumentImportTaskStatus;
+import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
+import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
+import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
+
+import java.math.BigInteger;
+import java.sql.SQLException;
+import java.sql.SQLRecoverableException;
+import java.sql.SQLTimeoutException;
+import java.sql.SQLTransientConnectionException;
+import java.util.Date;
+import java.util.concurrent.RejectedExecutionException;
+
+/**
+ * 知识库自动导入批次熔断器。
+ *
+ * 消费者或调度器发生未被业务流程收口的基础设施异常时，通过
+ * 数据库 CAS 立即停止整个自动导入批次，撤销活跃任务执行令牌，
+ * 并把未完成文件保留为可恢复失败。
+ */
+@Service
+public class DocumentImportBatchCircuitBreaker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DocumentImportBatchCircuitBreaker.class);
+ private static final String REDIS_UNAVAILABLE = "redis_unavailable";
+ private static final String DATABASE_UNAVAILABLE = "database_unavailable";
+ private static final String WORKER_OVERLOADED = "worker_overloaded";
+ private static final String INFRASTRUCTURE_FAILURE =
+ "document_import_infrastructure_failure";
+
+ private final DocumentImportBatchMapper batchMapper;
+ private final DocumentImportBatchItemMapper itemMapper;
+ private final DocumentImportTaskMapper taskMapper;
+
+ /**
+ * 创建自动导入批次熔断器。
+ *
+ * @param batchMapper 批次 Mapper
+ * @param itemMapper 批次项 Mapper
+ * @param taskMapper 导入任务 Mapper
+ */
+ public DocumentImportBatchCircuitBreaker(
+ DocumentImportBatchMapper batchMapper,
+ DocumentImportBatchItemMapper itemMapper,
+ DocumentImportTaskMapper taskMapper) {
+ this.batchMapper = batchMapper;
+ this.itemMapper = itemMapper;
+ this.taskMapper = taskMapper;
+ }
+
+ /**
+ * 根据异常任务强制中断所属自动导入批次。
+ *
+ * 该方法使用独立事务，保证调用方业务事务已经回滚时仍能写入
+ * 可恢复终态。技术异常通过完整堆栈写入后端日志；数据库仅保存
+ * 稳定错误码和面向用户的安全摘要。
+ *
+ * @param taskId 触发异常的任务 ID
+ * @param error 原始异常
+ * @return 批次已经中断或已经结束时返回 {@code true}；无自动批次
+ * 归属时返回 {@code false}
+ */
+ @Transactional(propagation = Propagation.REQUIRES_NEW)
+ public boolean interruptTaskBatch(BigInteger taskId, Throwable error) {
+ DocumentImportTask task =
+ taskId == null ? null : taskMapper.selectOneById(taskId);
+ if (task == null) {
+ LOG.warn("忽略无法关联任务的自动导入熔断请求: taskId={}", taskId);
+ return true;
+ }
+ if (task.getBatchId() == null) {
+ return false;
+ }
+ if (!isActiveTask(task)) {
+ LOG.info(
+ "忽略已结束任务的迟到熔断请求: taskId={}, batchId={}, status={}",
+ task.getId(), task.getBatchId(), task.getStatus());
+ return true;
+ }
+ return interruptBatch(
+ task.getBatchId(),
+ resolveReason(error),
+ task.getId(),
+ task.getPhase(),
+ error
+ );
+ }
+
+ /**
+ * 根据系统异常强制中断指定自动导入批次。
+ *
+ * @param batchId 批次 ID
+ * @param error 原始异常
+ * @return 批次已经中断、结束或不存在时返回 {@code true}
+ */
+ @Transactional(propagation = Propagation.REQUIRES_NEW)
+ public boolean interruptBatch(BigInteger batchId, Throwable error) {
+ return interruptBatch(
+ batchId,
+ resolveReason(error),
+ null,
+ null,
+ error
+ );
+ }
+
+ /**
+ * 仅在恢复令牌仍由当前实例持有时中断指定批次。
+ *
+ * @param batchId 批次 ID
+ * @param recoveryToken 当前恢复调度令牌
+ * @param error 原始异常
+ * @return 批次已停止时返回 {@code true}；令牌失效时返回 {@code false}
+ * @throws IllegalArgumentException 恢复令牌为空时抛出
+ */
+ @Transactional(propagation = Propagation.REQUIRES_NEW)
+ public boolean interruptRecoveryBatch(BigInteger batchId,
+ String recoveryToken,
+ Throwable error) {
+ if (recoveryToken == null || recoveryToken.isBlank()) {
+ throw new IllegalArgumentException("恢复调度令牌不能为空");
+ }
+ return interruptBatch(
+ batchId,
+ resolveReason(error),
+ null,
+ null,
+ error,
+ recoveryToken
+ );
+ }
+
+ /**
+ * 使用指定稳定原因强制中断自动导入批次。
+ *
+ * @param batchId 批次 ID
+ * @param interruptCode 稳定中断码
+ * @param interruptMessage 用户可见中断原因
+ * @param error 原始异常
+ * @return 批次已经中断、结束或不存在时返回 {@code true}
+ */
+ @Transactional(propagation = Propagation.REQUIRES_NEW)
+ public boolean interruptBatch(BigInteger batchId,
+ String interruptCode,
+ String interruptMessage,
+ Throwable error) {
+ return interruptBatch(
+ batchId,
+ new InterruptionReason(interruptCode, interruptMessage),
+ null,
+ null,
+ error
+ );
+ }
+
+ /**
+ * 加入调用方事务并使用指定原因中断批次。
+ *
+ * 供超时回收等已经持有任务行锁的事务使用，避免开启新事务
+ * 等待调用方自身尚未提交的行锁。
+ *
+ * @param batchId 批次 ID
+ * @param interruptCode 稳定中断码
+ * @param interruptMessage 用户可见中断原因
+ * @param error 原始异常
+ * @return 批次是否已经停止运行
+ */
+ @Transactional(propagation = Propagation.MANDATORY)
+ public boolean interruptBatchInCurrentTransaction(
+ BigInteger batchId,
+ String interruptCode,
+ String interruptMessage,
+ Throwable error) {
+ return interruptBatch(
+ batchId,
+ new InterruptionReason(interruptCode, interruptMessage),
+ null,
+ null,
+ error
+ );
+ }
+
+ /**
+ * 原子中断批次并统一收口关联任务、文档和文件项。
+ *
+ * @param batchId 批次 ID
+ * @param reason 稳定中断原因
+ * @param taskId 触发任务 ID，可为空
+ * @param phase 触发任务阶段，可为空
+ * @param error 原始异常
+ * @return 批次是否已经停止运行
+ */
+ private boolean interruptBatch(BigInteger batchId,
+ InterruptionReason reason,
+ BigInteger taskId,
+ String phase,
+ Throwable error) {
+ return interruptBatch(
+ batchId, reason, taskId, phase, error, null);
+ }
+
+ /**
+ * 按调用来源围栏原子中断批次并收口关联状态。
+ *
+ * @param batchId 批次 ID
+ * @param reason 稳定中断原因
+ * @param taskId 触发任务 ID，可为空
+ * @param phase 触发任务阶段，可为空
+ * @param error 原始异常
+ * @param recoveryToken 恢复调度令牌，可为空
+ * @return 批次是否已经停止运行；围栏失效时返回 {@code false}
+ */
+ private boolean interruptBatch(BigInteger batchId,
+ InterruptionReason reason,
+ BigInteger taskId,
+ String phase,
+ Throwable error,
+ String recoveryToken) {
+ if (batchId == null) {
+ return false;
+ }
+ DocumentImportBatch batch = batchMapper.selectOneById(batchId);
+ if (batch == null) {
+ LOG.warn(
+ "忽略无法关联批次的自动导入熔断请求: taskId={}, batchId={}",
+ taskId, batchId);
+ return true;
+ }
+ if (!DocumentImportMode.AUTO.name().equals(batch.getImportMode())) {
+ return false;
+ }
+ if (!DocumentImportBatchStatus.RUNNING.name().equals(batch.getStatus())) {
+ return true;
+ }
+
+ Date now = new Date();
+ int interrupted;
+ if (recoveryToken != null) {
+ interrupted = batchMapper.interruptOwnedRecoveryBatch(
+ batch.getId(), recoveryToken, reason.code, reason.message, now);
+ } else if (taskId != null) {
+ interrupted = batchMapper.interruptRunningBatchForActiveTask(
+ batch.getId(), taskId, reason.code, reason.message, now);
+ } else {
+ interrupted = batchMapper.interruptRunningBatch(
+ batch.getId(), reason.code, reason.message, now);
+ }
+ if (interrupted <= 0) {
+ DocumentImportBatch current = batchMapper.selectOneById(batch.getId());
+ if (current == null
+ || !DocumentImportBatchStatus.RUNNING.name()
+ .equals(current.getStatus())) {
+ return true;
+ }
+ if (taskId != null) {
+ DocumentImportTask currentTask = taskMapper.selectOneById(taskId);
+ return currentTask == null || !isActiveTask(currentTask);
+ }
+ return false;
+ }
+
+ // 先同步文档列表，再撤销任务令牌；两个更新处于同一事务中。
+ int interruptedDocuments = taskMapper.interruptActiveDocuments(
+ batch.getId(), reason.message, reason.code, now, BigInteger.ZERO);
+ int interruptedTasks = taskMapper.interruptActiveTasks(
+ batch.getId(), reason.message, reason.code, now, BigInteger.ZERO);
+ int interruptedItems = itemMapper.interruptActiveItems(
+ batch.getId(), reason.message, reason.code, now);
+ batchMapper.refreshCountersFromItems(batch.getId(), now);
+
+ LOG.error(
+ "知识库自动导入批次已强制中断: batchId={}, taskId={}, phase={}, "
+ + "interruptCode={}, interruptedTasks={}, interruptedItems={}, "
+ + "interruptedDocuments={}",
+ batch.getId(),
+ taskId,
+ phase,
+ reason.code,
+ interruptedTasks,
+ interruptedItems,
+ interruptedDocuments,
+ error);
+ return true;
+ }
+
+ /**
+ * 将异常链归类为稳定中断原因。
+ *
+ * @param error 原始异常
+ * @return 稳定错误码与用户可见摘要
+ */
+ private InterruptionReason resolveReason(Throwable error) {
+ if (containsRedisFailure(error)) {
+ return new InterruptionReason(
+ REDIS_UNAVAILABLE,
+ "缓存与消息服务异常，自动导入已中断，请确认服务恢复后继续"
+ );
+ }
+ if (containsWorkerOverload(error)) {
+ return new InterruptionReason(
+ WORKER_OVERLOADED,
+ "导入任务处理资源已满，批次已中断，请稍后继续"
+ );
+ }
+ if (containsDataAccessFailure(error)) {
+ return new InterruptionReason(
+ DATABASE_UNAVAILABLE,
+ "数据库服务异常，自动导入已中断，请确认服务恢复后继续"
+ );
+ }
+ return new InterruptionReason(
+ INFRASTRUCTURE_FAILURE,
+ "自动导入发生系统异常，批次已中断，请联系管理员排查后继续"
+ );
+ }
+
+ /**
+ * 判断异常链是否来自 Redis/Lettuce。
+ *
+ * @param error 原始异常
+ * @return 是否是 Redis 故障
+ */
+ private boolean containsRedisFailure(Throwable error) {
+ Throwable current = error;
+ while (current != null) {
+ String className = current.getClass().getName();
+ if (className.startsWith("io.lettuce.")
+ || className.contains(".redis.")
+ || className.contains("RedisCommand")) {
+ return true;
+ }
+ current = current.getCause();
+ }
+ return false;
+ }
+
+ /**
+ * 判断异常链是否包含线程池拒绝。
+ *
+ * @param error 原始异常
+ * @return 是否是任务资源过载
+ */
+ private boolean containsWorkerOverload(Throwable error) {
+ Throwable current = error;
+ while (current != null) {
+ if (current instanceof RejectedExecutionException
+ || current.getClass().getSimpleName()
+ .contains("TaskRejectedException")) {
+ return true;
+ }
+ current = current.getCause();
+ }
+ return false;
+ }
+
+ /**
+ * 判断异常链是否包含数据库连接、资源或超时异常。
+ *
+ * @param error 原始异常
+ * @return 是否是数据库故障
+ */
+ private boolean containsDataAccessFailure(Throwable error) {
+ Throwable current = error;
+ while (current != null) {
+ if (current instanceof DataAccessResourceFailureException
+ || current instanceof TransientDataAccessResourceException
+ || current instanceof QueryTimeoutException
+ || current instanceof SQLTransientConnectionException
+ || current instanceof SQLRecoverableException
+ || current instanceof SQLTimeoutException
+ || isSqlConnectionFailure(current)
+ || "org.hibernate.exception.JDBCConnectionException"
+ .equals(current.getClass().getName())) {
+ return true;
+ }
+ current = current.getCause();
+ }
+ return false;
+ }
+
+ /**
+ * 根据 SQLState 判断是否属于数据库连接类故障。
+ *
+ * @param error 当前异常
+ * @return SQLState 使用连接异常类别 08 时返回 {@code true}
+ */
+ private boolean isSqlConnectionFailure(Throwable error) {
+ if (!(error instanceof SQLException sqlError)) {
+ return false;
+ }
+ String sqlState = sqlError.getSQLState();
+ return sqlState != null && sqlState.startsWith("08");
+ }
+
+ /**
+ * 判断任务是否仍可能代表当前批次的活跃执行。
+ *
+ * @param task 导入任务
+ * @return 待处理或运行中返回 {@code true}
+ */
+ private boolean isActiveTask(DocumentImportTask task) {
+ return task != null
+ && (DocumentImportTaskStatus.PENDING.name().equals(task.getStatus())
+ || DocumentImportTaskStatus.RUNNING.name().equals(task.getStatus()));
+ }
+
+ /**
+ * 稳定中断码与安全展示文案。
+ */
+ private static final class InterruptionReason {
+ private final String code;
+ private final String message;
+
+ /**
+ * 创建中断原因。
+ *
+ * @param code 稳定错误码
+ * @param message 用户可见摘要
+ */
+ private InterruptionReason(String code, String message) {
+ this.code = code;
+ this.message = message;
+ }
+ }
+}
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java
index e2bed47b..e8e94849 100644
--- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java
@@ -235,10 +235,11 @@ public class DocumentImportBatchTracker {
}
/**
- * 将批次文件项绑定到创建后的文档。
+ * 将已上传或缺少文档的失败项绑定到创建后的文档。
*
* @param itemId 文件项 ID
* @param documentId 文档 ID
+ * @throws BusinessException 文件项状态变化或绑定失败时抛出
*/
@org.springframework.transaction.annotation.Transactional
public void bindDocument(BigInteger itemId, BigInteger documentId) {
@@ -248,12 +249,22 @@ public class DocumentImportBatchTracker {
return;
}
Date now = new Date();
- int updated = itemMapper.bindDocument(itemId, documentId, now);
+ boolean recoveringFailedItem =
+ DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
+ && item.getDocumentId() == null;
+ int updated = recoveringFailedItem
+ ? itemMapper.bindFailedDocument(itemId, documentId, now)
+ : itemMapper.bindDocument(itemId, documentId, now);
if (updated <= 0) {
throw new BusinessException("导入文件状态已变化，请刷新后重试");
}
- batchMapper.adjustCounters(item.getBatchId(),
- 0, 0, 0, 0, -1, 0, 0, 0, now);
+ if (recoveringFailedItem) {
+ // 防御性恢复属于极低频路径，按文件项真实状态汇总可避免手工计数漂移。
+ batchMapper.refreshCountersFromItems(item.getBatchId(), now);
+ } else {
+ batchMapper.adjustCounters(item.getBatchId(),
+ 0, 0, 0, 0, -1, 0, 0, 0, now);
+ }
refreshBatch(item.getBatchId());
}
@@ -369,12 +380,33 @@ public class DocumentImportBatchTracker {
* @param batchId 批次 ID
*/
public void markInterrupted(BigInteger batchId) {
+ markInterrupted(
+ batchId,
+ "execution_interrupted",
+ "任务执行中断，请确认服务恢复后继续批次"
+ );
+ }
+
+ /**
+ * 将批次标记为已中断并记录可展示的稳定原因。
+ *
+ * @param batchId 批次 ID
+ * @param interruptCode 稳定中断码
+ * @param interruptMessage 用户可见中断原因
+ */
+ public void markInterrupted(BigInteger batchId,
+ String interruptCode,
+ String interruptMessage) {
if (batchId == null) {
return;
}
Date now = new Date();
DocumentImportBatch update = new DocumentImportBatch();
update.setStatus(DocumentImportBatchStatus.INTERRUPTED.name());
+ update.setInterruptCode(interruptCode);
+ update.setInterruptMessage(interruptMessage);
+ update.setInterruptedAt(now);
+ update.setRecoveryPending(false);
update.setModified(now);
batchMapper.updateByQuery(update,
QueryWrapper.create()
@@ -409,6 +441,9 @@ public class DocumentImportBatchTracker {
+ valueOrZero(batch.getSkippedCount())
+ valueOrZero(batch.getCancelledCount());
response.setProgressPercent(Math.min(100, terminalCount * 100 / total));
+ response.setInterruptCode(batch.getInterruptCode());
+ response.setInterruptMessage(batch.getInterruptMessage());
+ response.setInterruptedAt(batch.getInterruptedAt());
response.setStartedAt(batch.getStartedAt());
response.setFinishedAt(batch.getFinishedAt());
return response;
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportIndexTaskConsumer.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportIndexTaskConsumer.java
index c276905f..5a533c8c 100644
--- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportIndexTaskConsumer.java
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportIndexTaskConsumer.java
@@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.common.mq.config.MQProperties;
import tech.easyflow.common.mq.core.MQConsumerHandler;
+import tech.easyflow.common.mq.core.MQDeferException;
import tech.easyflow.common.mq.core.MQMessage;
import tech.easyflow.common.mq.core.MQSubscription;
@@ -23,11 +24,21 @@ public class DocumentImportIndexTaskConsumer implements MQConsumerHandler {
private static final Logger LOG = LoggerFactory.getLogger(DocumentImportIndexTaskConsumer.class);
private final KnowledgeDocumentImportTaskAppService appService;
+ private final DocumentImportBatchCircuitBreaker circuitBreaker;
private final MQProperties mqProperties;
+ /**
+ * 创建文档向量化任务消费者。
+ *
+ * @param appService 文档导入任务应用服务
+ * @param circuitBreaker 自动导入批次熔断器
+ * @param mqProperties MQ 配置
+ */
public DocumentImportIndexTaskConsumer(KnowledgeDocumentImportTaskAppService appService,
+ DocumentImportBatchCircuitBreaker circuitBreaker,
MQProperties mqProperties) {
this.appService = appService;
+ this.circuitBreaker = circuitBreaker;
this.mqProperties = mqProperties;
}
@@ -37,6 +48,7 @@ public class DocumentImportIndexTaskConsumer implements MQConsumerHandler {
subscription.setTopic(DocumentImportTaskMqConstants.INDEX_TOPIC);
subscription.setConsumerGroup(DocumentImportTaskMqConstants.INDEX_GROUP);
subscription.setShardCount(resolveShardCount());
+ subscription.setBatchEnabled(false);
return subscription;
}
@@ -58,13 +70,43 @@ public class DocumentImportIndexTaskConsumer implements MQConsumerHandler {
LOG.info("文档向量化消费者处理完成: taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(), message.getStreamMessageId());
} catch (Exception exception) {
- LOG.error("文档向量化消费者处理失败: taskId={}, messageId={}, streamMessageId={}",
- event.getTaskId(), message.getMessageId(), message.getStreamMessageId(), exception);
- throw exception;
+ handleInfrastructureFailure(event, message, exception);
}
}
}
+ /**
+ * 记录完整异常并中断自动导入批次；中断状态暂时无法持久化时保留消息 pending。
+ *
+ * @param event 任务事件
+ * @param message MQ 消息
+ * @param error 原始异常
+ */
+ private void handleInfrastructureFailure(DocumentImportTaskMessage event,
+ MQMessage message,
+ Exception error) {
+ try {
+ if (circuitBreaker.interruptTaskBatch(event.getTaskId(), error)) {
+ return;
+ }
+ } catch (Exception interruptError) {
+ interruptError.addSuppressed(error);
+ LOG.error(
+ "文档向量化异常且批次中断状态写入失败，消息将暂缓确认: "
+ + "taskId={}, messageId={}, streamMessageId={}",
+ event.getTaskId(), message.getMessageId(),
+ message.getStreamMessageId(), interruptError);
+ throw new MQDeferException(
+ "文档向量化批次中断状态暂时无法持久化", interruptError);
+ }
+ LOG.error(
+ "文档向量化任务发生基础设施异常，消息将暂缓确认: "
+ + "taskId={}, messageId={}, streamMessageId={}",
+ event.getTaskId(), message.getMessageId(),
+ message.getStreamMessageId(), error);
+ throw new MQDeferException("文档向量化任务暂时无法处理", error);
+ }
+
/**
* 向量化消费者需覆盖生产端的所有分片，避免消息落入未订阅分片。
*
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportParseTaskConsumer.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportParseTaskConsumer.java
index a4981c4c..5fe84bb9 100644
--- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportParseTaskConsumer.java
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportParseTaskConsumer.java
@@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.common.mq.config.MQProperties;
import tech.easyflow.common.mq.core.MQConsumerHandler;
+import tech.easyflow.common.mq.core.MQDeferException;
import tech.easyflow.common.mq.core.MQMessage;
import tech.easyflow.common.mq.core.MQSubscription;
@@ -23,11 +24,21 @@ public class DocumentImportParseTaskConsumer implements MQConsumerHandler {
private static final Logger LOG = LoggerFactory.getLogger(DocumentImportParseTaskConsumer.class);
private final KnowledgeDocumentImportTaskAppService appService;
+ private final DocumentImportBatchCircuitBreaker circuitBreaker;
private final MQProperties mqProperties;
+ /**
+ * 创建文档解析任务消费者。
+ *
+ * @param appService 文档导入任务应用服务
+ * @param circuitBreaker 自动导入批次熔断器
+ * @param mqProperties MQ 配置
+ */
public DocumentImportParseTaskConsumer(KnowledgeDocumentImportTaskAppService appService,
+ DocumentImportBatchCircuitBreaker circuitBreaker,
MQProperties mqProperties) {
this.appService = appService;
+ this.circuitBreaker = circuitBreaker;
this.mqProperties = mqProperties;
}
@@ -37,6 +48,7 @@ public class DocumentImportParseTaskConsumer implements MQConsumerHandler {
subscription.setTopic(DocumentImportTaskMqConstants.PARSE_TOPIC);
subscription.setConsumerGroup(DocumentImportTaskMqConstants.PARSE_GROUP);
subscription.setShardCount(resolveShardCount());
+ subscription.setBatchEnabled(false);
return subscription;
}
@@ -58,13 +70,43 @@ public class DocumentImportParseTaskConsumer implements MQConsumerHandler {
LOG.info("文档解析消费者处理完成: taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(), message.getStreamMessageId());
} catch (Exception exception) {
- LOG.error("文档解析消费者处理失败: taskId={}, messageId={}, streamMessageId={}",
- event.getTaskId(), message.getMessageId(), message.getStreamMessageId(), exception);
- throw exception;
+ handleInfrastructureFailure(event, message, exception);
}
}
}
+ /**
+ * 记录完整异常并中断自动导入批次；中断状态暂时无法持久化时保留消息 pending。
+ *
+ * @param event 任务事件
+ * @param message MQ 消息
+ * @param error 原始异常
+ */
+ private void handleInfrastructureFailure(DocumentImportTaskMessage event,
+ MQMessage message,
+ Exception error) {
+ try {
+ if (circuitBreaker.interruptTaskBatch(event.getTaskId(), error)) {
+ return;
+ }
+ } catch (Exception interruptError) {
+ interruptError.addSuppressed(error);
+ LOG.error(
+ "文档解析异常且批次中断状态写入失败，消息将暂缓确认: "
+ + "taskId={}, messageId={}, streamMessageId={}",
+ event.getTaskId(), message.getMessageId(),
+ message.getStreamMessageId(), interruptError);
+ throw new MQDeferException(
+ "文档解析批次中断状态暂时无法持久化", interruptError);
+ }
+ LOG.error(
+ "文档解析任务发生基础设施异常，消息将暂缓确认: "
+ + "taskId={}, messageId={}, streamMessageId={}",
+ event.getTaskId(), message.getMessageId(),
+ message.getStreamMessageId(), error);
+ throw new MQDeferException("文档解析任务暂时无法处理", error);
+ }
+
/**
* 解析消费者需覆盖生产端的所有分片，避免消息落入未订阅分片。
*
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitor.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitor.java
index 26306ef0..cf67cb35 100644
--- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitor.java
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitor.java
@@ -1,5 +1,7 @@
package tech.easyflow.ai.documentimport.task;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tech.easyflow.common.cache.DistributedScheduledLock;
@@ -13,15 +15,28 @@ import tech.easyflow.common.cache.DistributedScheduledLock;
@Component
public class DocumentImportPendingTaskMonitor {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DocumentImportPendingTaskMonitor.class);
+ private static final long FAILURE_COOLDOWN_MILLIS = 30_000L;
+ private static final long FULL_ERROR_LOG_INTERVAL_MILLIS = 60_000L;
+
private final KnowledgeDocumentImportTaskAppService appService;
+ private final DocumentImportBatchAppService batchAppService;
+ private long retryAfter;
+ private long nextFullErrorLogAt;
+ private int suppressedFailures;
/**
* 创建文档导入恢复调度器。
*
* @param appService 文档导入任务服务
+ * @param batchAppService 文档导入批次服务
*/
- public DocumentImportPendingTaskMonitor(KnowledgeDocumentImportTaskAppService appService) {
+ public DocumentImportPendingTaskMonitor(
+ KnowledgeDocumentImportTaskAppService appService,
+ DocumentImportBatchAppService batchAppService) {
this.appService = appService;
+ this.batchAppService = batchAppService;
}
/**
@@ -31,12 +46,39 @@ public class DocumentImportPendingTaskMonitor {
fixedDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}",
initialDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}"
)
- @DistributedScheduledLock(
- key = "easyflow:schedule:document-import:pending-dispatch",
- leaseSeconds = 30L
- )
public void dispatchPendingTasks() {
- appService.dispatchPendingTasks();
+ long now = System.currentTimeMillis();
+ if (now < retryAfter) {
+ return;
+ }
+ try {
+ batchAppService.recoverPendingBatchRetries();
+ appService.dispatchPendingTasks();
+ if (suppressedFailures > 0) {
+ LOG.info(
+ "文档导入待处理调度已恢复: suppressedFailures={}",
+ suppressedFailures
+ );
+ }
+ retryAfter = 0L;
+ nextFullErrorLogAt = 0L;
+ suppressedFailures = 0;
+ } catch (RuntimeException error) {
+ retryAfter = now + FAILURE_COOLDOWN_MILLIS;
+ if (now >= nextFullErrorLogAt) {
+ LOG.error(
+ "文档导入待处理调度异常，30 秒后重试: "
+ + "suppressedFailures={}",
+ suppressedFailures,
+ error
+ );
+ nextFullErrorLogAt =
+ now + FULL_ERROR_LOG_INTERVAL_MILLIS;
+ suppressedFailures = 0;
+ } else {
+ suppressedFailures++;
+ }
+ }
}
/**
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskConsumer.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskConsumer.java
index 9cb88345..c0e8fe5e 100644
--- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskConsumer.java
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskConsumer.java
@@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.common.mq.config.MQProperties;
import tech.easyflow.common.mq.core.MQConsumerHandler;
+import tech.easyflow.common.mq.core.MQDeferException;
import tech.easyflow.common.mq.core.MQMessage;
import tech.easyflow.common.mq.core.MQSubscription;
@@ -24,18 +25,22 @@ public class DocumentImportSplitTaskConsumer implements MQConsumerHandler {
LoggerFactory.getLogger(DocumentImportSplitTaskConsumer.class);
private final KnowledgeDocumentImportTaskAppService appService;
+ private final DocumentImportBatchCircuitBreaker circuitBreaker;
private final MQProperties mqProperties;
/**
* 创建分块任务消费者。
*
* @param appService 文档导入应用服务
+ * @param circuitBreaker 自动导入批次熔断器
* @param mqProperties MQ 配置
*/
public DocumentImportSplitTaskConsumer(
KnowledgeDocumentImportTaskAppService appService,
+ DocumentImportBatchCircuitBreaker circuitBreaker,
MQProperties mqProperties) {
this.appService = appService;
+ this.circuitBreaker = circuitBreaker;
this.mqProperties = mqProperties;
}
@@ -50,6 +55,7 @@ public class DocumentImportSplitTaskConsumer implements MQConsumerHandler {
subscription.setTopic(DocumentImportTaskMqConstants.SPLIT_TOPIC);
subscription.setConsumerGroup(DocumentImportTaskMqConstants.SPLIT_GROUP);
subscription.setShardCount(resolveShardCount());
+ subscription.setBatchEnabled(false);
return subscription;
}
@@ -74,14 +80,43 @@ public class DocumentImportSplitTaskConsumer implements MQConsumerHandler {
try {
appService.handleSplitTask(event.getTaskId());
} catch (Exception error) {
- LOG.error("文档分块消费者处理失败: taskId={}, messageId={}, streamMessageId={}",
- event.getTaskId(), message.getMessageId(),
- message.getStreamMessageId(), error);
- throw error;
+ handleInfrastructureFailure(event, message, error);
}
}
}
+ /**
+ * 记录完整异常并中断自动导入批次；中断状态暂时无法持久化时保留消息 pending。
+ *
+ * @param event 任务事件
+ * @param message MQ 消息
+ * @param error 原始异常
+ */
+ private void handleInfrastructureFailure(DocumentImportTaskMessage event,
+ MQMessage message,
+ Exception error) {
+ try {
+ if (circuitBreaker.interruptTaskBatch(event.getTaskId(), error)) {
+ return;
+ }
+ } catch (Exception interruptError) {
+ interruptError.addSuppressed(error);
+ LOG.error(
+ "文档分块异常且批次中断状态写入失败，消息将暂缓确认: "
+ + "taskId={}, messageId={}, streamMessageId={}",
+ event.getTaskId(), message.getMessageId(),
+ message.getStreamMessageId(), interruptError);
+ throw new MQDeferException(
+ "文档分块批次中断状态暂时无法持久化", interruptError);
+ }
+ LOG.error(
+ "文档分块任务发生基础设施异常，消息将暂缓确认: "
+ + "taskId={}, messageId={}, streamMessageId={}",
+ event.getTaskId(), message.getMessageId(),
+ message.getStreamMessageId(), error);
+ throw new MQDeferException("文档分块任务暂时无法处理", error);
+ }
+
/**
* 获取当前 Redis Stream 分片数。
*
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java
index c3a58941..cf4a26a3 100644
--- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java
@@ -59,6 +59,7 @@ import tech.easyflow.ai.enums.DocumentImportTaskStatus;
import tech.easyflow.ai.enums.DocumentProcessStatus;
import tech.easyflow.ai.entity.Model;
import tech.easyflow.ai.mapper.DocumentChunkMapper;
+import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.ai.service.DocumentChunkService;
@@ -101,6 +102,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
+import java.util.function.BooleanSupplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -163,6 +165,9 @@ public class KnowledgeDocumentImportTaskAppService {
@Resource
private DocumentImportBatchItemService documentImportBatchItemService;
+ @Resource
+ private DocumentImportBatchItemMapper documentImportBatchItemMapper;
+
@Autowired
@Lazy
private DocumentService documentService;
@@ -185,6 +190,9 @@ public class KnowledgeDocumentImportTaskAppService {
@Resource
private DocumentImportChunkSnapshotService documentImportChunkSnapshotService;
+ @Resource
+ private DocumentImportBatchCircuitBreaker documentImportBatchCircuitBreaker;
+
@Resource
private CsvTableSnapshotService csvTableSnapshotService;
@@ -297,36 +305,7 @@ public class KnowledgeDocumentImportTaskAppService {
|| !StringUtil.hasText(item.getFilePath())) {
throw new BusinessException("批次包含未上传完成的文件");
}
- String fileExt = normalizeFileExtension(item.getFileName(), item.getFilePath());
- assertSupportedImportFile(fileExt);
- Date now = new Date();
- tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
- document.setId(generateId(document));
- document.setCollectionId(knowledge.getId());
- document.setDocumentPath(item.getFilePath());
- document.setTitle(item.getFileName());
- document.setDocumentType(fileExt);
- document.setCreated(now);
- document.setModified(now);
- document.setCreatedBy(resolveOperatorId());
- document.setModifiedBy(resolveOperatorId());
- document.setProcessStatus(DocumentProcessStatus.PARSING.name());
- document.setTotalChunks(0);
- document.setCompletedChunks(0);
- document.setFailedChunks(0);
- document.setProgressPercent(0);
- document.setTaskModifiedAt(now);
- Map options = buildInitialOptions(fileExt);
- options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_MODE, batch.getImportMode());
- options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID, batch.getId().toString());
- options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID, item.getId().toString());
- options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_RELATIVE_PATH, item.getRelativePath());
- document.setOptions(options);
- documentMapper.insert(document);
-
- DocumentImportTask task = createTask(document, DocumentImportTaskPhase.PARSE,
- buildDocumentPayload(document));
- documentImportBatchTracker.bindDocument(item.getId(), document.getId());
+ DocumentImportTask task = createBatchImportTask(batch, item, knowledge);
taskIds.add(task.getId());
}
// 启动接口返回后列表会统一刷新；批量建档阶段不逐文件推送，避免提交后形成 SSE 风暴。
@@ -339,21 +318,105 @@ public class KnowledgeDocumentImportTaskAppService {
}
/**
- * 重试批次中所有失败文件，单个文件重试失败不会中止其他文件。
+ * 为一个已上传文件或缺少文档的失败项创建文档与解析任务。
+ *
+ * @param batch 导入批次
+ * @param item 批次文件项
+ * @param knowledge 目标知识库
+ * @return 新创建的解析任务
+ * @throws BusinessException 文件状态、路径或格式不允许建档时抛出
+ */
+ private DocumentImportTask createBatchImportTask(
+ DocumentImportBatch batch,
+ DocumentImportBatchItem item,
+ DocumentCollection knowledge) {
+ boolean uploaded = DocumentImportBatchItemStatus.UPLOADED.name()
+ .equals(item.getStatus());
+ boolean failedWithoutDocument =
+ DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
+ && item.getDocumentId() == null;
+ if ((!uploaded && !failedWithoutDocument)
+ || !StringUtil.hasText(item.getFilePath())) {
+ throw new BusinessException("批次包含无法恢复的文件");
+ }
+ String fileExt = normalizeFileExtension(
+ item.getFileName(), item.getFilePath());
+ assertSupportedImportFile(fileExt);
+ Date now = new Date();
+ tech.easyflow.ai.entity.Document document =
+ new tech.easyflow.ai.entity.Document();
+ document.setId(generateId(document));
+ document.setCollectionId(knowledge.getId());
+ document.setDocumentPath(item.getFilePath());
+ document.setTitle(item.getFileName());
+ document.setDocumentType(fileExt);
+ document.setCreated(now);
+ document.setModified(now);
+ document.setCreatedBy(resolveOperatorId());
+ document.setModifiedBy(resolveOperatorId());
+ document.setProcessStatus(DocumentProcessStatus.PARSING.name());
+ document.setTotalChunks(0);
+ document.setCompletedChunks(0);
+ document.setFailedChunks(0);
+ document.setProgressPercent(0);
+ document.setTaskModifiedAt(now);
+ Map options = buildInitialOptions(fileExt);
+ options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_MODE,
+ batch.getImportMode());
+ options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID,
+ batch.getId().toString());
+ options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID,
+ item.getId().toString());
+ options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_RELATIVE_PATH,
+ item.getRelativePath());
+ document.setOptions(options);
+ documentMapper.insert(document);
+
+ DocumentImportTask task = createTask(
+ document,
+ DocumentImportTaskPhase.PARSE,
+ buildDocumentPayload(document)
+ );
+ documentImportBatchTracker.bindDocument(item.getId(), document.getId());
+ return task;
+ }
+
+ /**
+ * 重试批次中所有失败文件，单文件业务失败不会中止其他文件。
*
* @param batchId 批次 ID
+ * @throws RuntimeException 基础设施异常需要交由恢复令牌持有者熔断时抛出
*/
public void retryBatchFailures(BigInteger batchId) {
retryBatchFailures(batchId, Set.of());
}
/**
- * 重试批次中选定的失败文件，单个文件重试失败不会中止其他文件。
+ * 重试批次中选定的失败文件，单文件业务失败不会中止其他文件。
*
* @param batchId 批次 ID
* @param fileKeys 指定文件键；为空时重试全部失败项
+ * @throws RuntimeException 基础设施异常需要交由恢复令牌持有者熔断时抛出
*/
public void retryBatchFailures(BigInteger batchId, Set fileKeys) {
+ retryBatchFailures(batchId, fileKeys, () -> true);
+ }
+
+ /**
+ * 在恢复租约仍有效时重试批次中的选定失败文件。
+ *
+ * 租约检查发生在每个文件重试前，旧持有者失去令牌后立即停止，
+ * 避免多实例继续重复扫描和竞争文件项行锁。
+ *
+ * @param batchId 批次 ID
+ * @param fileKeys 指定文件键；为空时重试全部失败项
+ * @param recoveryLeaseGuard 恢复租约检查器
+ * @return 全部文件处理完成且最终仍持有恢复租约时返回 {@code true}
+ * @throws RuntimeException 基础设施异常需要交由恢复令牌持有者熔断时抛出
+ */
+ boolean retryBatchFailures(BigInteger batchId,
+ Set fileKeys,
+ BooleanSupplier recoveryLeaseGuard) {
QueryWrapper query = QueryWrapper.create()
.eq(DocumentImportBatchItem::getBatchId, batchId)
.eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name());
@@ -364,17 +427,37 @@ public class KnowledgeDocumentImportTaskAppService {
query
);
for (DocumentImportBatchItem item : failedItems) {
+ if (recoveryLeaseGuard != null
+ && !recoveryLeaseGuard.getAsBoolean()) {
+ LOG.info(
+ "批次失败项恢复租约已失效，停止当前实例重试: "
+ + "batchId={}, itemId={}",
+ batchId,
+ item.getId()
+ );
+ return false;
+ }
try {
selfProxy.retryBatchItemInNewTransaction(item.getId());
- } catch (Exception error) {
+ } catch (BusinessException error) {
LOG.error("批次失败项重试启动失败: batchId={}, itemId={}, documentId={}",
batchId, item.getId(), item.getDocumentId(), error);
- updateBatchItem(item.getId(),
- DocumentImportBatchItemStage.valueOf(item.getStage()),
- DocumentImportBatchItemStatus.FAILED,
- truncateError(error.getMessage()));
+ documentImportBatchItemMapper.updateFailedRetryError(
+ item.getId(), batchId,
+ truncateError(error.getMessage()), new Date());
}
}
+ // 最后一次校验覆盖“末项处理完成到批次收尾”之间的租约失效窗口。
+ if (recoveryLeaseGuard != null
+ && !recoveryLeaseGuard.getAsBoolean()) {
+ LOG.info(
+ "批次失败项处理完成时恢复租约已失效，跳过旧持有者收尾: "
+ + "batchId={}",
+ batchId
+ );
+ return false;
+ }
+ return true;
}
/**
@@ -384,7 +467,12 @@ public class KnowledgeDocumentImportTaskAppService {
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void retryBatchItemInNewTransaction(BigInteger itemId) {
- retryBatchItem(documentImportBatchTracker.requireItem(itemId));
+ DocumentImportBatchItem item =
+ documentImportBatchItemMapper.selectFailedForRetry(itemId);
+ if (item == null) {
+ return;
+ }
+ retryBatchItem(item);
}
/**
@@ -402,16 +490,57 @@ public class KnowledgeDocumentImportTaskAppService {
if (touched != 1) {
continue;
}
- if (DocumentImportTaskPhase.PARSE.name().equals(task.getPhase())) {
- parseTaskProducer.send(task.getId());
- } else if (DocumentImportTaskPhase.SPLIT.name().equals(task.getPhase())) {
- splitTaskProducer.send(task.getId());
- } else if (DocumentImportTaskPhase.INDEX.name().equals(task.getPhase())) {
- indexTaskProducer.send(task.getId());
+ try {
+ if (DocumentImportTaskPhase.PARSE.name().equals(task.getPhase())) {
+ parseTaskProducer.send(task.getId());
+ } else if (DocumentImportTaskPhase.SPLIT.name().equals(task.getPhase())) {
+ splitTaskProducer.send(task.getId());
+ } else if (DocumentImportTaskPhase.INDEX.name().equals(task.getPhase())) {
+ indexTaskProducer.send(task.getId());
+ }
+ } catch (Exception error) {
+ handlePendingDispatchFailure(task, error);
}
}
}
+ /**
+ * 处理待执行任务重新投递异常。
+ *
+ * 自动导入任务会触发批次熔断；未能完成熔断或无批次任务
+ * 保留 PENDING，并向调度器传播异常以触发有界退避。
+ *
+ * @param task 投递失败任务
+ * @param error 原始异常
+ */
+ private void handlePendingDispatchFailure(DocumentImportTask task,
+ Exception error) {
+ try {
+ if (documentImportBatchCircuitBreaker != null
+ && documentImportBatchCircuitBreaker.interruptTaskBatch(
+ task.getId(), error)) {
+ return;
+ }
+ } catch (Exception interruptError) {
+ IllegalStateException dispatchError = new IllegalStateException(
+ "文档导入任务投递失败且批次中断状态写入失败: "
+ + "taskId=" + task.getId()
+ + ", batchId=" + task.getBatchId()
+ + ", phase=" + task.getPhase(),
+ interruptError
+ );
+ dispatchError.addSuppressed(error);
+ throw dispatchError;
+ }
+ throw new IllegalStateException(
+ "文档导入任务投递失败，等待调度退避后重试: "
+ + "taskId=" + task.getId()
+ + ", batchId=" + task.getBatchId()
+ + ", phase=" + task.getPhase(),
+ error
+ );
+ }
+
/**
* 检测超过最长排队时间的待处理任务。
*/
@@ -824,7 +953,9 @@ public class KnowledgeDocumentImportTaskAppService {
LOG.info("文档向量化任务已创建: knowledgeId={}, documentId={}, taskId={}, previewSessionId={}, totalChunks={}",
knowledge.getId(), document.getId(), task.getId(), session.getSessionId(), totalChunks);
dispatchIndexTaskAfterCommit(task.getId());
- scheduleIndexTaskFallback(task.getId());
+ if (task.getBatchId() == null) {
+ scheduleIndexTaskFallback(task.getId());
+ }
DocumentImportDtos.TaskStartIndexResponse response = new DocumentImportDtos.TaskStartIndexResponse();
response.setTaskId(task.getId());
@@ -855,7 +986,9 @@ public class KnowledgeDocumentImportTaskAppService {
DocumentImportBatchItemStatus.PENDING,
null);
dispatchParseTaskAfterCommit(task.getId());
- scheduleParseTaskFallback(task.getId());
+ if (task.getBatchId() == null) {
+ scheduleParseTaskFallback(task.getId());
+ }
return Result.ok(buildTaskStartResponse(task, DocumentProcessStatus.PARSING));
}
@@ -923,9 +1056,25 @@ public class KnowledgeDocumentImportTaskAppService {
return Result.ok(response);
}
+ /**
+ * 恢复一个失败批次项；缺少文档时先原子重新建档。
+ *
+ * @param item 已锁定的失败批次项
+ * @throws BusinessException 文件或文档状态不允许恢复时抛出
+ */
private void retryBatchItem(DocumentImportBatchItem item) {
if (item.getDocumentId() == null) {
- throw new BusinessException("失败文件尚未生成文档");
+ DocumentImportBatch batch =
+ documentImportBatchTracker.requireBatch(item.getBatchId());
+ if (!Objects.equals(batch.getKnowledgeId(), item.getKnowledgeId())) {
+ throw new IllegalStateException("批次文件知识库归属不一致");
+ }
+ DocumentCollection knowledge =
+ assertDocumentCollection(item.getKnowledgeId());
+ DocumentImportTask task =
+ createBatchImportTask(batch, item, knowledge);
+ dispatchParseTaskAfterCommit(task.getId());
+ return;
}
tech.easyflow.ai.entity.Document document = requireDocument(item.getDocumentId());
DocumentImportDtos.TaskRetryRequest request = new DocumentImportDtos.TaskRetryRequest();
@@ -970,7 +1119,6 @@ public class KnowledgeDocumentImportTaskAppService {
DocumentImportTask task =
createTask(document, DocumentImportTaskPhase.SPLIT, payload);
dispatchSplitTaskAfterCommit(task.getId());
- scheduleSplitTaskFallback(task.getId());
return task;
}
@@ -1052,14 +1200,14 @@ public class KnowledgeDocumentImportTaskAppService {
LOG.warn("分块任务执行令牌已失效: taskId={}", taskId);
} catch (Exception error) {
deleteOwnedSplitArtifacts(previewSessionId, snapshotPath);
- String errorMessage = truncateError(error.getMessage());
+ String errorMessage = resolveSplitFailureMessage(error);
LOG.error("文档分块任务失败: taskId={}, documentId={}",
taskId, task.getDocumentId(), error);
String failureCode = error instanceof CsvImportException csvError
? csvError.getFailureCode()
: TASK_ERROR_SPLIT_FAILED;
if (!selfProxy.failSplitTask(
- task, task.getDocumentId(), errorMessage, failureCode)) {
+ task, task.getDocumentId(), errorMessage, failureCode, error)) {
LOG.warn("分块任务所有权已失效，忽略迟到失败: taskId={}", taskId);
}
} finally {
@@ -1111,7 +1259,13 @@ public class KnowledgeDocumentImportTaskAppService {
} catch (Exception e) {
LOG.error("文档解析任务失败: taskId={}, documentId={}", taskId, document.getId(), e);
String errorCode = resolveParseFailureCode(e);
- markParseFailed(task, document, resolveParseFailureMessage(e, errorCode), errorCode);
+ markParseFailed(
+ task,
+ document,
+ resolveParseFailureMessage(e, errorCode),
+ errorCode,
+ e
+ );
}
}
@@ -1173,7 +1327,13 @@ public class KnowledgeDocumentImportTaskAppService {
} catch (Exception e) {
LOG.error("文档解析任务收敛失败: taskId={}, documentId={}", taskId, document.getId(), e);
String errorCode = resolveParseFailureCode(e);
- markParseFailed(task, document, resolveParseFailureMessage(e, errorCode), errorCode);
+ markParseFailed(
+ task,
+ document,
+ resolveParseFailureMessage(e, errorCode),
+ errorCode,
+ e
+ );
}
}
@@ -1309,7 +1469,8 @@ public class KnowledgeDocumentImportTaskAppService {
markIndexFailed(
task, document,
resolveIndexFailureMessage(e, rollbackSucceeded),
- failureCode);
+ failureCode,
+ e);
} finally {
closeStoreContext(storeContext);
}
@@ -1523,7 +1684,6 @@ public class KnowledgeDocumentImportTaskAppService {
DocumentImportBatchItemStatus.PENDING,
null);
dispatchIndexTaskAfterCommit(indexTask.getId());
- scheduleIndexTaskFallback(indexTask.getId());
return true;
}
@@ -1534,6 +1694,7 @@ public class KnowledgeDocumentImportTaskAppService {
* @param documentId 文档 ID
* @param errorMessage 用户可见错误信息
* @param failureCode 稳定失败码
+ * @param error 原始异常
* @return 是否仍持有本轮分块任务执行权
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
@@ -1541,7 +1702,8 @@ public class KnowledgeDocumentImportTaskAppService {
DocumentImportTask task,
BigInteger documentId,
String errorMessage,
- String failureCode) {
+ String failureCode,
+ Throwable error) {
Date now = new Date();
if (!finishTask(
task, now, DocumentImportTaskStatus.FAILED,
@@ -1553,12 +1715,15 @@ public class KnowledgeDocumentImportTaskAppService {
current.setProgressPercent(0);
setDocumentTaskError(current, errorMessage, failureCode);
persistDocumentTaskState(current, now);
- updateBatchItemFailure(
- task.getBatchItemId(),
- DocumentImportBatchItemStage.SPLIT,
- errorMessage,
- failureCode
- );
+ if (!interruptFailedTaskBatchInCurrentTransaction(
+ task, errorMessage, failureCode, error)) {
+ updateBatchItemFailure(
+ task.getBatchItemId(),
+ DocumentImportBatchItemStage.SPLIT,
+ errorMessage,
+ failureCode
+ );
+ }
return true;
}
@@ -1664,6 +1829,31 @@ public class KnowledgeDocumentImportTaskAppService {
* @param task 孤儿任务
*/
private void finishMissingDocumentTask(DocumentImportTask task) {
+ if (TransactionSynchronizationManager.isActualTransactionActive()) {
+ finishMissingDocumentTaskInCurrentTransaction(task);
+ return;
+ }
+ selfProxy.finishMissingDocumentTaskInNewTransaction(task);
+ }
+
+ /**
+ * 在独立事务中原子收口缺少关联文档的任务与自动导入批次。
+ *
+ * @param task 孤儿任务
+ */
+ @Transactional(propagation = Propagation.REQUIRES_NEW)
+ public void finishMissingDocumentTaskInNewTransaction(
+ DocumentImportTask task) {
+ finishMissingDocumentTaskInCurrentTransaction(task);
+ }
+
+ /**
+ * 在当前事务中原子收口缺少关联文档的任务与自动导入批次。
+ *
+ * @param task 孤儿任务
+ */
+ private void finishMissingDocumentTaskInCurrentTransaction(
+ DocumentImportTask task) {
Date now = new Date();
String errorMessage = "关联文档已删除，任务已自动清理";
if (!finishTask(task, now, DocumentImportTaskStatus.FAILED, errorMessage)) {
@@ -1687,22 +1877,90 @@ public class KnowledgeDocumentImportTaskAppService {
private void finishRecoveredBatchState(DocumentImportTask task,
DocumentImportBatchItemStage stage,
String errorMessage,
- String failureCode) {
+ String failureCode) {
try {
- updateBatchItemFailure(
- task.getBatchItemId(),
- stage,
- errorMessage,
- failureCode);
- if (task.getBatchId() != null) {
- documentImportBatchTracker.markInterrupted(task.getBatchId());
+ boolean batchStopped = task.getBatchId() != null
+ && interruptRecoveredBatch(
+ task, errorMessage, failureCode);
+ if (!batchStopped) {
+ updateBatchItemFailure(
+ task.getBatchItemId(),
+ stage,
+ errorMessage,
+ failureCode);
}
} catch (Exception error) {
LOG.error("同步已回收任务的批次状态失败: taskId={}, batchId={}, batchItemId={}",
task.getId(), task.getBatchId(), task.getBatchItemId(), error);
+ if (task.getBatchId() != null) {
+ throw error instanceof RuntimeException
+ ? (RuntimeException) error
+ : new IllegalStateException(
+ "同步已回收任务的批次状态失败", error);
+ }
}
}
+ /**
+ * 在任务回收事务中强制中断所属批次。
+ *
+ * @param task 已回收任务
+ * @param errorMessage 用户可见错误原因
+ * @param failureCode 稳定失败码
+ * @return 所属批次是否已经停止运行
+ */
+ private boolean interruptRecoveredBatch(DocumentImportTask task,
+ String errorMessage,
+ String failureCode) {
+ Throwable error = new IllegalStateException(
+ "文档导入任务恢复时触发批次中断: taskId=" + task.getId());
+ if (TransactionSynchronizationManager.isActualTransactionActive()) {
+ return documentImportBatchCircuitBreaker
+ .interruptBatchInCurrentTransaction(
+ task.getBatchId(),
+ failureCode,
+ errorMessage,
+ error
+ );
+ } else {
+ return documentImportBatchCircuitBreaker.interruptBatch(
+ task.getBatchId(),
+ failureCode,
+ errorMessage,
+ error
+ );
+ }
+ }
+
+ /**
+ * 在任务失败状态事务内同步中断所属自动导入批次。
+ *
+ * 任务、文档、批次项与批次必须原子提交。批次中断写入失败时，
+ * 当前失败事务整体回滚，消息保留待重试，避免留下永久 RUNNING 批次。
+ *
+ * @param task 失败任务
+ * @param errorMessage 用户可见错误原因
+ * @param failureCode 稳定失败码
+ * @param error 原始异常
+ * @return 所属批次是否已经停止运行
+ */
+ private boolean interruptFailedTaskBatchInCurrentTransaction(
+ DocumentImportTask task,
+ String errorMessage,
+ String failureCode,
+ Throwable error) {
+ if (task.getBatchId() == null) {
+ return false;
+ }
+ return documentImportBatchCircuitBreaker
+ .interruptBatchInCurrentTransaction(
+ task.getBatchId(),
+ failureCode,
+ errorMessage,
+ error
+ );
+ }
+
/**
* 将底层解析异常转换为稳定错误码。
*
@@ -1743,6 +2001,9 @@ public class KnowledgeDocumentImportTaskAppService {
String message = current.getMessage();
if (StringUtil.hasText(message)) {
String normalized = message.toLowerCase(Locale.ROOT);
+ if (normalized.contains("unsupported file type")) {
+ return TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE;
+ }
if (normalized.contains("timed out")
|| normalized.contains("timeout")
|| normalized.contains("超时")) {
@@ -1779,7 +2040,29 @@ public class KnowledgeDocumentImportTaskAppService {
if (TASK_ERROR_DOCUMENT_SOURCE_UNAVAILABLE.equals(errorCode)) {
return "文档文件读取失败，请联系管理员";
}
- return truncateError(error == null ? null : error.getMessage());
+ if (TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE.equals(errorCode)) {
+ return "文档格式或来源不受解析服务支持，请检查文件后继续";
+ }
+ if (TASK_ERROR_INVALID_PARSE_REQUEST.equals(errorCode)) {
+ return "文档解析请求无效，请检查文件后继续";
+ }
+ if (error instanceof BusinessException) {
+ return truncateError(error.getMessage());
+ }
+ return "文档解析失败，请检查文件或联系管理员后继续";
+ }
+
+ /**
+ * 将分块异常转换为可安全展示的错误信息。
+ *
+ * @param error 原始异常
+ * @return 用户可见错误信息
+ */
+ private String resolveSplitFailureMessage(Throwable error) {
+ if (error instanceof BusinessException) {
+ return truncateError(error.getMessage());
+ }
+ return "文档分块失败，请联系管理员排查后继续";
}
/**
@@ -1789,13 +2072,16 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档实体
* @param errorMessage 用户可见错误信息
* @param errorCode 稳定错误码
+ * @param error 原始异常
*/
private void markParseFailed(DocumentImportTask task,
tech.easyflow.ai.entity.Document document,
String errorMessage,
- String errorCode) {
+ String errorCode,
+ Throwable error) {
KnowledgeDocumentImportTaskAppService executor = selfProxy == null ? this : selfProxy;
- if (!executor.failParseTask(task, document, errorMessage, errorCode)) {
+ if (!executor.failParseTask(
+ task, document, errorMessage, errorCode, error)) {
LOG.warn("解析任务所有权已失效，忽略迟到失败: taskId={}, documentId={}",
task.getId(), document.getId());
}
@@ -1808,13 +2094,15 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档实体
* @param errorMessage 用户可见错误信息
* @param errorCode 稳定错误码
+ * @param error 原始异常
* @return 是否仍持有任务终态写入权
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean failParseTask(DocumentImportTask task,
tech.easyflow.ai.entity.Document document,
String errorMessage,
- String errorCode) {
+ String errorCode,
+ Throwable error) {
Date now = new Date();
if (!finishTask(
task, now, DocumentImportTaskStatus.FAILED,
@@ -1827,11 +2115,14 @@ public class KnowledgeDocumentImportTaskAppService {
LOG.warn("文档解析任务失败: taskId={}, documentId={}, processStatus={}, error={}",
task.getId(), document.getId(), DocumentProcessStatus.PARSE_FAILED.name(), errorMessage);
- updateBatchItemFailure(
- task.getBatchItemId(),
- DocumentImportBatchItemStage.PARSE,
- errorMessage,
- errorCode);
+ if (!interruptFailedTaskBatchInCurrentTransaction(
+ task, errorMessage, errorCode, error)) {
+ updateBatchItemFailure(
+ task.getBatchItemId(),
+ DocumentImportBatchItemStage.PARSE,
+ errorMessage,
+ errorCode);
+ }
return true;
}
@@ -2304,7 +2595,12 @@ public class KnowledgeDocumentImportTaskAppService {
tech.easyflow.ai.entity.Document document,
String errorMessage) {
markIndexFailed(
- task, document, errorMessage, TASK_ERROR_INDEX_FAILED);
+ task,
+ document,
+ errorMessage,
+ TASK_ERROR_INDEX_FAILED,
+ new IllegalStateException(errorMessage)
+ );
}
/**
@@ -2314,14 +2610,17 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档
* @param errorMessage 用户可见错误信息
* @param failureCode 稳定失败码
+ * @param error 原始异常
*/
private void markIndexFailed(
DocumentImportTask task,
tech.easyflow.ai.entity.Document document,
String errorMessage,
- String failureCode) {
+ String failureCode,
+ Throwable error) {
KnowledgeDocumentImportTaskAppService executor = selfProxy == null ? this : selfProxy;
- if (!executor.failIndexTask(task, document, errorMessage, failureCode)) {
+ if (!executor.failIndexTask(
+ task, document, errorMessage, failureCode, error)) {
LOG.warn("向量化任务所有权已失效，忽略迟到失败: taskId={}, documentId={}",
task.getId(), document.getId());
}
@@ -2355,7 +2654,12 @@ public class KnowledgeDocumentImportTaskAppService {
tech.easyflow.ai.entity.Document document,
String errorMessage) {
return failIndexTask(
- task, document, errorMessage, TASK_ERROR_INDEX_FAILED);
+ task,
+ document,
+ errorMessage,
+ TASK_ERROR_INDEX_FAILED,
+ new IllegalStateException(errorMessage)
+ );
}
/**
@@ -2365,13 +2669,15 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档实体
* @param errorMessage 用户可见错误信息
* @param failureCode 稳定失败码
+ * @param error 原始异常
* @return 是否仍持有任务终态写入权
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean failIndexTask(DocumentImportTask task,
tech.easyflow.ai.entity.Document document,
String errorMessage,
- String failureCode) {
+ String failureCode,
+ Throwable error) {
String resolvedFailureCode = StringUtil.hasText(failureCode)
? failureCode
: TASK_ERROR_INDEX_FAILED;
@@ -2397,11 +2703,14 @@ public class KnowledgeDocumentImportTaskAppService {
defaultInt(current.getTotalChunks()),
errorMessage);
- updateBatchItemFailure(
- task.getBatchItemId(),
- DocumentImportBatchItemStage.INDEX,
- errorMessage,
- resolvedFailureCode);
+ if (!interruptFailedTaskBatchInCurrentTransaction(
+ task, errorMessage, resolvedFailureCode, error)) {
+ updateBatchItemFailure(
+ task.getBatchItemId(),
+ DocumentImportBatchItemStage.INDEX,
+ errorMessage,
+ resolvedFailureCode);
+ }
return true;
}
@@ -4487,15 +4796,6 @@ public class KnowledgeDocumentImportTaskAppService {
scheduleTaskFallback(taskId, "index", () -> selfProxy.handleIndexTask(taskId));
}
- /**
- * 在事务提交后调度本地分块兜底执行。
- *
- * @param taskId 任务 ID
- */
- private void scheduleSplitTaskFallback(BigInteger taskId) {
- scheduleTaskFallback(taskId, "split", () -> selfProxy.handleSplitTask(taskId));
- }
-
/**
* 在事务提交后调度本地解析兜底执行。
*
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatch.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatch.java
index 3735d85b..4c815e01 100644
--- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatch.java
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatch.java
@@ -85,6 +85,27 @@ public class DocumentImportBatch extends DateEntity implements Serializable {
@Column(comment = "可重试失败数")
private Integer retryableFailedCount;
+ @Column(comment = "中断错误码")
+ private String interruptCode;
+
+ @Column(comment = "中断原因")
+ private String interruptMessage;
+
+ @Column(comment = "中断时间")
+ private Date interruptedAt;
+
+ @Column(comment = "是否存在待恢复调度")
+ private Boolean recoveryPending;
+
+ @Column(comment = "待恢复文件键 JSON")
+ private String recoveryFileKeysJson;
+
+ @Column(comment = "恢复调度令牌")
+ private String recoveryToken;
+
+ @Column(comment = "恢复调度租约到期时间")
+ private Date recoveryLeaseUntil;
+
@Column(comment = "开始时间")
private Date startedAt;
@@ -279,6 +300,62 @@ public class DocumentImportBatch extends DateEntity implements Serializable {
this.retryableFailedCount = retryableFailedCount;
}
+ public String getInterruptCode() {
+ return interruptCode;
+ }
+
+ public void setInterruptCode(String interruptCode) {
+ this.interruptCode = interruptCode;
+ }
+
+ public String getInterruptMessage() {
+ return interruptMessage;
+ }
+
+ public void setInterruptMessage(String interruptMessage) {
+ this.interruptMessage = interruptMessage;
+ }
+
+ public Date getInterruptedAt() {
+ return interruptedAt;
+ }
+
+ public void setInterruptedAt(Date interruptedAt) {
+ this.interruptedAt = interruptedAt;
+ }
+
+ public Boolean getRecoveryPending() {
+ return recoveryPending;
+ }
+
+ public void setRecoveryPending(Boolean recoveryPending) {
+ this.recoveryPending = recoveryPending;
+ }
+
+ public String getRecoveryFileKeysJson() {
+ return recoveryFileKeysJson;
+ }
+
+ public void setRecoveryFileKeysJson(String recoveryFileKeysJson) {
+ this.recoveryFileKeysJson = recoveryFileKeysJson;
+ }
+
+ public String getRecoveryToken() {
+ return recoveryToken;
+ }
+
+ public void setRecoveryToken(String recoveryToken) {
+ this.recoveryToken = recoveryToken;
+ }
+
+ public Date getRecoveryLeaseUntil() {
+ return recoveryLeaseUntil;
+ }
+
+ public void setRecoveryLeaseUntil(Date recoveryLeaseUntil) {
+ this.recoveryLeaseUntil = recoveryLeaseUntil;
+ }
+
public Date getStartedAt() {
return startedAt;
}
diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java
index 41433c4a..3b9cef6b 100644
--- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java
+++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java
@@ -2,6 +2,7 @@ package tech.easyflow.ai.mapper;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
@@ -16,6 +17,45 @@ import java.util.Date;
*/
public interface DocumentImportBatchItemMapper extends BaseMapper {
+ /**
+ * 锁定运行批次中的失败项并取得本轮恢复权。
+ *
+ * 同时锁定批次和文件项，使恢复任务创建与批次熔断串行化。
+ *
+ * @param itemId 文件项 ID
+ * @return 可恢复文件项；状态已变化时返回 null
+ */
+ @Select("SELECT item.* FROM tb_document_import_batch_item item "
+ + "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id "
+ + "WHERE item.id=#{itemId} AND item.status='FAILED' "
+ + "AND batch.status='RUNNING' FOR UPDATE")
+ DocumentImportBatchItem selectFailedForRetry(
+ @Param("itemId") BigInteger itemId
+ );
+
+ /**
+ * 将中断批次中尚未结束的文件项统一收口为可恢复失败。
+ *
+ * @param batchId 批次 ID
+ * @param errorSummary 用户可见错误摘要
+ * @param failureCode 稳定失败码
+ * @param modified 修改时间
+ * @return 更新文件项数量
+ */
+ @Update("UPDATE tb_document_import_batch_item item "
+ + "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id "
+ + "SET item.status='FAILED', item.error_summary=#{errorSummary}, "
+ + "item.failure_code=#{failureCode}, item.retryable=1, "
+ + "item.modified=#{modified} "
+ + "WHERE item.batch_id=#{batchId} AND batch.status='INTERRUPTED' "
+ + "AND item.status IN ('PENDING','RUNNING','UPLOADED')")
+ int interruptActiveItems(
+ @Param("batchId") BigInteger batchId,
+ @Param("errorSummary") String errorSummary,
+ @Param("failureCode") String failureCode,
+ @Param("modified") Date modified
+ );
+
/**
* 原子领取文件上传权并刷新批次进度时间。
*
@@ -60,11 +100,14 @@ public interface DocumentImportBatchItemMapper extends BaseMapper'INTERRUPTED'")
int transitionStatus(
@Param("id") BigInteger id,
@Param("expectedStatus") String expectedStatus,
@@ -95,6 +138,51 @@ public interface DocumentImportBatchItemMapper extends BaseMapper #{modified}")
+ int interruptOwnedRecoveryBatch(
+ @Param("batchId") BigInteger batchId,
+ @Param("recoveryToken") String recoveryToken,
+ @Param("interruptCode") String interruptCode,
+ @Param("interruptMessage") String interruptMessage,
+ @Param("modified") Date modified
+ );
+
+ /**
+ * 仅在触发任务仍是当前活跃任务时中断运行批次。
+ *
+ * @param batchId 批次 ID
+ * @param taskId 触发异常的任务 ID
+ * @param interruptCode 稳定中断码
+ * @param interruptMessage 用户可见中断原因
+ * @param modified 中断时间
+ * @return 任务仍活跃且中断成功时返回 1
+ */
+ @Update("UPDATE tb_document_import_batch batch "
+ + "INNER JOIN tb_document_import_task task ON task.batch_id=batch.id SET "
+ + "batch.status='INTERRUPTED', batch.interrupt_code=#{interruptCode}, "
+ + "batch.interrupt_message=#{interruptMessage}, "
+ + "batch.interrupted_at=#{modified}, batch.recovery_pending=0, "
+ + "batch.recovery_file_keys_json=NULL, batch.recovery_token=NULL, "
+ + "batch.recovery_lease_until=NULL, batch.finished_at=NULL, "
+ + "batch.modified=#{modified}, batch.version=batch.version + 1 "
+ + "WHERE batch.id=#{batchId} AND batch.import_mode='AUTO' "
+ + "AND batch.status='RUNNING' AND task.id=#{taskId} "
+ + "AND task.status IN ('PENDING','RUNNING')")
+ int interruptRunningBatchForActiveTask(
+ @Param("batchId") BigInteger batchId,
+ @Param("taskId") BigInteger taskId,
+ @Param("interruptCode") String interruptCode,
+ @Param("interruptMessage") String interruptMessage,
+ @Param("modified") Date modified
+ );
+
+ /**
+ * 根据批次项真实状态重新汇总计数。
+ *
+ * @param batchId 批次 ID
+ * @param modified 修改时间
+ * @return 更新行数
+ */
+ @Update("UPDATE tb_document_import_batch batch INNER JOIN ("
+ + "SELECT batch_id, "
+ + "SUM(status='COMPLETED') completed_count, "
+ + "SUM(status='RUNNING') processing_count, "
+ + "SUM(status='FAILED') failed_count, "
+ + "SUM(status='PENDING') pending_count, "
+ + "SUM(status='UPLOADED') uploaded_count, "
+ + "SUM(status='SKIPPED') skipped_count, "
+ + "SUM(status='CANCELLED') cancelled_count, "
+ + "SUM(status='FAILED' AND retryable=1) retryable_failed_count "
+ + "FROM tb_document_import_batch_item WHERE batch_id=#{batchId} "
+ + "GROUP BY batch_id"
+ + ") counters ON counters.batch_id=batch.id SET "
+ + "batch.completed_count=counters.completed_count, "
+ + "batch.processing_count=counters.processing_count, "
+ + "batch.failed_count=counters.failed_count, "
+ + "batch.pending_count=counters.pending_count, "
+ + "batch.uploaded_count=counters.uploaded_count, "
+ + "batch.skipped_count=counters.skipped_count, "
+ + "batch.cancelled_count=counters.cancelled_count, "
+ + "batch.retryable_failed_count=counters.retryable_failed_count, "
+ + "batch.modified=#{modified} WHERE batch.id=#{batchId}")
+ int refreshCountersFromItems(
+ @Param("batchId") BigInteger batchId,
+ @Param("modified") Date modified
+ );
+
+ /**
+ * 查询已经提交但尚未完成恢复调度的运行批次。
+ *
+ * @param now 当前时间
+ * @param limit 最大批次数
+ * @return 待恢复批次
+ */
+ @Select("SELECT * FROM tb_document_import_batch "
+ + "WHERE status='RUNNING' AND recovery_pending=1 "
+ + "AND (recovery_token IS NULL OR recovery_lease_until <= #{now}) "
+ + "ORDER BY modified, id LIMIT #{limit}")
+ List selectRecoveryPendingBatches(
+ @Param("now") Date now,
+ @Param("limit") int limit
+ );
+
+ /**
+ * 使用租约令牌领取一个批次恢复待办。
+ *
+ * @param batchId 批次 ID
+ * @param recoveryToken 恢复调度令牌
+ * @param recoveryLeaseUntil 租约到期时间
+ * @param modified 领取时间
+ * @return 更新行数
+ */
+ @Update("UPDATE tb_document_import_batch SET "
+ + "recovery_token=#{recoveryToken}, "
+ + "recovery_lease_until=#{recoveryLeaseUntil}, "
+ + "modified=#{modified}, version=version + 1 "
+ + "WHERE id=#{batchId} AND status='RUNNING' "
+ + "AND recovery_pending=1 AND (recovery_token IS NULL "
+ + "OR recovery_lease_until <= #{modified})")
+ int claimRecoveryPending(
+ @Param("batchId") BigInteger batchId,
+ @Param("recoveryToken") String recoveryToken,
+ @Param("recoveryLeaseUntil") Date recoveryLeaseUntil,
+ @Param("modified") Date modified
+ );
+
+ /**
+ * 续期当前持有者的批次恢复租约。
+ *
+ * @param batchId 批次 ID
+ * @param recoveryToken 恢复调度令牌
+ * @param recoveryLeaseUntil 新租约到期时间
+ * @param modified 续租时间
+ * @return 当前令牌仍有效且续租成功时返回 1
+ */
+ @Update("UPDATE tb_document_import_batch SET "
+ + "recovery_lease_until=#{recoveryLeaseUntil}, "
+ + "modified=#{modified}, version=version + 1 "
+ + "WHERE id=#{batchId} AND status='RUNNING' "
+ + "AND recovery_pending=1 AND recovery_token=#{recoveryToken} "
+ + "AND recovery_lease_until > #{modified}")
+ int renewRecoveryPendingLease(
+ @Param("batchId") BigInteger batchId,
+ @Param("recoveryToken") String recoveryToken,
+ @Param("recoveryLeaseUntil") Date recoveryLeaseUntil,
+ @Param("modified") Date modified
+ );
+
+ /**
+ * 按恢复令牌读取当前持有者领取后的精确恢复参数。
+ *
+ * @param batchId 批次 ID
+ * @param recoveryToken 恢复调度令牌
+ * @return 当前令牌持有的批次；令牌失效时返回空
+ */
+ @Select("SELECT * FROM tb_document_import_batch "
+ + "WHERE id=#{batchId} AND status='RUNNING' "
+ + "AND recovery_pending=1 AND recovery_token=#{recoveryToken}")
+ DocumentImportBatch selectClaimedRecovery(
+ @Param("batchId") BigInteger batchId,
+ @Param("recoveryToken") String recoveryToken
+ );
+
+ /**
+ * 原子完成当前恢复调度并按真实计数收口批次状态。
+ *
+ * 终态汇总与令牌清理在同一条 SQL 中完成，避免进程在两次写入
+ * 之间退出后留下无恢复待办的运行批次。
+ *
+ * @param batchId 批次 ID
+ * @param recoveryToken 恢复调度令牌
+ * @param modified 修改时间
+ * @return 当前令牌和租约仍有效且收尾成功时返回 1
+ */
+ @Update("UPDATE tb_document_import_batch SET "
+ + "status=CASE WHEN "
+ + "completed_count + failed_count + skipped_count + cancelled_count "
+ + ">= total_count AND processing_count=0 AND pending_count=0 "
+ + "THEN CASE WHEN failed_count > 0 THEN 'PARTIAL_SUCCEEDED' "
+ + "ELSE 'COMPLETED' END ELSE 'RUNNING' END, "
+ + "finished_at=CASE WHEN "
+ + "completed_count + failed_count + skipped_count + cancelled_count "
+ + ">= total_count AND processing_count=0 AND pending_count=0 "
+ + "THEN #{modified} ELSE NULL END, "
+ + "recovery_pending=0, recovery_file_keys_json=NULL, "
+ + "recovery_token=NULL, recovery_lease_until=NULL, "
+ + "modified=#{modified}, version=version + 1 "
+ + "WHERE id=#{batchId} AND import_mode='AUTO' AND status='RUNNING' "
+ + "AND recovery_pending=1 AND recovery_token=#{recoveryToken} "
+ + "AND recovery_lease_until > #{modified}")
+ int finalizeRecoveryPending(
+ @Param("batchId") BigInteger batchId,
+ @Param("recoveryToken") String recoveryToken,
+ @Param("modified") Date modified
+ );
+
+ /**
+ * 原子领取管理端人工继续权并清除上一轮中断信息。
+ *
+ * @param batchId 批次 ID
+ * @param modified 修改时间
+ * @return 更新行数
+ */
+ @Update("UPDATE tb_document_import_batch SET status='RUNNING', "
+ + "finished_at=NULL, interrupt_code=NULL, interrupt_message=NULL, "
+ + "interrupted_at=NULL, recovery_pending=1, "
+ + "recovery_file_keys_json=NULL, "
+ + "recovery_token=NULL, recovery_lease_until=NULL, "
+ + "modified=#{modified}, version=version + 1 "
+ + "WHERE id=#{batchId} "
+ + "AND status IN ('PARTIAL_SUCCEEDED','INTERRUPTED')")
+ int claimContinue(
+ @Param("batchId") BigInteger batchId,
+ @Param("modified") Date modified
+ );
+
/**
* 原子领取 Public API 重试代次。
*
@@ -80,11 +324,16 @@ public interface DocumentImportBatchMapper extends BaseMapper
+ "PARTITION BY task.phase, COALESCE(task.batch_id, task.id) "
+ "ORDER BY task.created, task.id) AS lane_row "
+ "FROM tb_document_import_task task "
- + "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore}"
+ + "LEFT JOIN tb_document_import_batch batch ON batch.id=task.batch_id "
+ + "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore} "
+ + "AND (task.batch_id IS NULL OR batch.status='RUNNING')"
+ ") ranked ON ranked.id=task.id "
+ "ORDER BY ranked.lane_row, task.created, task.id LIMIT #{limit}")
List selectPendingFairly(
@@ -50,7 +52,11 @@ public interface DocumentImportTaskMapper extends BaseMapper
@Update("UPDATE tb_document_import_task SET modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='PENDING' "
- + "AND modified <= #{redispatchBefore}")
+ + "AND modified <= #{redispatchBefore} "
+ + "AND (batch_id IS NULL OR EXISTS ("
+ + "SELECT 1 FROM tb_document_import_batch batch "
+ + "WHERE batch.id=tb_document_import_task.batch_id "
+ + "AND batch.status='RUNNING'))")
int touchPendingForDispatch(
@Param("id") BigInteger id,
@Param("redispatchBefore") Date redispatchBefore,
@@ -93,7 +99,11 @@ public interface DocumentImportTaskMapper extends BaseMapper
+ "started_at=COALESCE(started_at, #{now}), finished_at=NULL, "
+ "error_summary=NULL, failure_code=NULL, modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
- + "WHERE id=#{id} AND status='PENDING'")
+ + "WHERE id=#{id} AND status='PENDING' "
+ + "AND (batch_id IS NULL OR EXISTS ("
+ + "SELECT 1 FROM tb_document_import_batch batch "
+ + "WHERE batch.id=tb_document_import_task.batch_id "
+ + "AND batch.status='RUNNING'))")
int claimPending(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@@ -116,7 +126,11 @@ public interface DocumentImportTaskMapper extends BaseMapper
+ "modified=#{now}, modified_by=#{operatorId}, "
+ "version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='RUNNING' "
- + "AND execution_token=#{executionToken}")
+ + "AND execution_token=#{executionToken} "
+ + "AND (batch_id IS NULL OR EXISTS ("
+ + "SELECT 1 FROM tb_document_import_batch batch "
+ + "WHERE batch.id=tb_document_import_task.batch_id "
+ + "AND batch.status='RUNNING'))")
int renewLease(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@@ -142,7 +156,11 @@ public interface DocumentImportTaskMapper extends BaseMapper
+ "lease_until=NULL, finished_at=#{now}, modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='RUNNING' "
- + "AND execution_token=#{executionToken}")
+ + "AND execution_token=#{executionToken} "
+ + "AND (batch_id IS NULL OR EXISTS ("
+ + "SELECT 1 FROM tb_document_import_batch batch "
+ + "WHERE batch.id=tb_document_import_task.batch_id "
+ + "AND batch.status='RUNNING'))")
int finishOwned(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@@ -182,4 +200,68 @@ public interface DocumentImportTaskMapper extends BaseMapper
@Param("legacyCutoff") Date legacyCutoff,
@Param("operatorId") BigInteger operatorId
);
+
+ /**
+ * 将中断批次中所有活跃任务强制收口并撤销执行令牌。
+ *
+ * @param batchId 批次 ID
+ * @param errorSummary 用户可见错误摘要
+ * @param failureCode 稳定失败码
+ * @param now 中断时间
+ * @param operatorId 操作人 ID
+ * @return 更新任务数量
+ */
+ @Update("UPDATE tb_document_import_task task "
+ + "INNER JOIN tb_document_import_batch batch ON batch.id=task.batch_id "
+ + "SET task.status='FAILED', task.error_summary=#{errorSummary}, "
+ + "task.failure_code=#{failureCode}, task.execution_token=NULL, "
+ + "task.lease_until=NULL, task.finished_at=#{now}, "
+ + "task.modified=#{now}, task.modified_by=#{operatorId}, "
+ + "task.version=COALESCE(task.version, 0) + 1 "
+ + "WHERE task.batch_id=#{batchId} AND batch.status='INTERRUPTED' "
+ + "AND task.status IN ('PENDING','RUNNING')")
+ int interruptActiveTasks(
+ @Param("batchId") BigInteger batchId,
+ @Param("errorSummary") String errorSummary,
+ @Param("failureCode") String failureCode,
+ @Param("now") Date now,
+ @Param("operatorId") BigInteger operatorId
+ );
+
+ /**
+ * 将中断任务关联文档同步为对应阶段失败，供文档列表展示。
+ *
+ * @param batchId 批次 ID
+ * @param errorSummary 用户可见错误摘要
+ * @param failureCode 稳定失败码
+ * @param now 中断时间
+ * @param operatorId 操作人 ID
+ * @return 更新文档数量
+ */
+ @Update("UPDATE tb_document document "
+ + "INNER JOIN tb_document_import_task task "
+ + "ON task.document_id=document.id "
+ + "INNER JOIN tb_document_import_batch batch "
+ + "ON batch.id=task.batch_id SET "
+ + "document.process_status=CASE task.phase "
+ + "WHEN 'PARSE' THEN 'PARSE_FAILED' "
+ + "WHEN 'SPLIT' THEN 'SPLIT_FAILED' "
+ + "ELSE 'INDEX_FAILED' END, "
+ + "document.progress_percent=0, "
+ + "document.last_task_error=#{errorSummary}, "
+ + "document.options=JSON_SET("
+ + "CASE WHEN JSON_VALID(document.options) "
+ + "THEN document.options ELSE JSON_OBJECT() END, "
+ + "'$.\"task.errorCode\"', #{failureCode}), "
+ + "document.task_modified_at=#{now}, document.modified=#{now}, "
+ + "document.modified_by=#{operatorId} "
+ + "WHERE task.batch_id=#{batchId} AND batch.status='INTERRUPTED' "
+ + "AND task.status IN ('PENDING','RUNNING')")
+ int interruptActiveDocuments(
+ @Param("batchId") BigInteger batchId,
+ @Param("errorSummary") String errorSummary,
+ @Param("failureCode") String failureCode,
+ @Param("now") Date now,
+ @Param("operatorId") BigInteger operatorId
+ );
}
diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java
index c2156476..d1715b5b 100644
--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java
@@ -1,8 +1,10 @@
package tech.easyflow.ai.documentimport.task;
import com.mybatisflex.core.query.QueryWrapper;
+import org.apache.ibatis.annotations.Update;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
@@ -29,10 +31,13 @@ import tech.easyflow.common.filestorage.FileStorageWriteResult;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.lang.reflect.Field;
+import java.lang.reflect.Method;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BooleanSupplier;
/**
* {@link DocumentImportBatchAppService} 批次启动与重复策略回归测试。
@@ -577,6 +582,7 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
+ Mockito.nullable(String.class),
Mockito.any(Date.class)
)).thenReturn(1);
@@ -601,6 +607,7 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.eq(0),
+ Mockito.eq("[\"file-key\"]"),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
@@ -633,6 +640,7 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
+ Mockito.nullable(String.class),
Mockito.any(Date.class)
)).thenReturn(1);
@@ -666,6 +674,7 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.eq(0),
+ Mockito.isNull(),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
@@ -693,9 +702,8 @@ public class DocumentImportBatchAppServiceTest {
failed.setRetryable(false);
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(failed));
- Mockito.when(context.batchMapper.updateByQuery(
- Mockito.any(DocumentImportBatch.class),
- Mockito.any(QueryWrapper.class)
+ Mockito.when(context.batchMapper.claimContinue(
+ Mockito.eq(batch.getId()), Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.batchTracker.toStatusResponse(batch))
.thenAnswer(invocation -> {
@@ -719,15 +727,261 @@ public class DocumentImportBatchAppServiceTest {
Assert.assertEquals(DocumentImportBatchStatus.RUNNING.name(),
response.getStatus());
+ Mockito.verify(context.batchMapper).claimContinue(
+ Mockito.eq(batch.getId()), Mockito.any(Date.class));
Mockito.verify(context.batchTracker, Mockito.never())
.refreshBatch(batch.getId());
Mockito.verify(context.taskAppService).retryBatchFailures(
- batch.getId(),
- Set.of(failed.getClientFileKey())
+ Mockito.eq(batch.getId()),
+ Mockito.eq(Set.of()),
+ Mockito.any(BooleanSupplier.class)
);
Mockito.verify(context.lockHandle).release();
}
+ /**
+ * 验证继续后的恢复调度再次异常时通过统一熔断器收口整个批次。
+ */
+ @Test
+ public void continueRecoveryFailureShouldInterruptWholeBatch() {
+ TestContext context = createContext();
+ DocumentImportBatch batch = context.batchService.getOne(
+ QueryWrapper.create()
+ );
+ batch.setImportMode(DocumentImportMode.AUTO.name());
+ batch.setStatus(DocumentImportBatchStatus.INTERRUPTED.name());
+ DocumentImportBatchItem failed = uploadedItem(
+ BigInteger.valueOf(34),
+ batch.getId()
+ );
+ failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
+ Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
+ .thenReturn(List.of(failed));
+ Mockito.when(context.batchMapper.claimContinue(
+ Mockito.eq(batch.getId()), Mockito.any(Date.class)
+ )).thenReturn(1);
+ Mockito.when(context.batchTracker.toStatusResponse(batch))
+ .thenReturn(new DocumentImportBatchDtos.StatusResponse());
+ IllegalStateException recoveryError =
+ new IllegalStateException("Redis unavailable");
+ Mockito.doThrow(recoveryError)
+ .when(context.taskAppService)
+ .retryBatchFailures(
+ Mockito.eq(batch.getId()),
+ Mockito.eq(Set.of()),
+ Mockito.any(BooleanSupplier.class)
+ );
+
+ beginTransactionSynchronization();
+ try {
+ context.service.continueBatch(
+ batch.getKnowledgeId(), batch.getId()
+ );
+ } finally {
+ completeTransactionSynchronization(
+ TransactionSynchronization.STATUS_COMMITTED
+ );
+ }
+
+ Mockito.verify(context.circuitBreaker)
+ .interruptRecoveryBatch(
+ Mockito.eq(batch.getId()),
+ Mockito.anyString(),
+ Mockito.same(recoveryError)
+ );
+ Mockito.verify(context.batchMapper, Mockito.never())
+ .finalizeRecoveryPending(
+ Mockito.any(), Mockito.anyString(), Mockito.any(Date.class));
+ }
+
+ /**
+ * 验证领取后恢复令牌已经失效时旧持有者直接退出且不触发批次熔断。
+ */
+ @Test
+ public void staleRecoveryOwnerShouldStopWithoutInterruptingBatch() {
+ TestContext context = createContext();
+ DocumentImportBatch batch = context.batchService.getOne(
+ QueryWrapper.create()
+ );
+ batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
+ batch.setRecoveryPending(true);
+ Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
+ Mockito.any(Date.class), Mockito.anyInt()
+ )).thenReturn(List.of(batch));
+ Mockito.when(context.batchMapper.selectClaimedRecovery(
+ Mockito.eq(batch.getId()), Mockito.anyString()
+ )).thenReturn(null);
+
+ int recovered = context.service.recoverPendingBatchRetries();
+
+ Assert.assertEquals(1, recovered);
+ Mockito.verifyNoInteractions(context.circuitBreaker);
+ Mockito.verify(context.taskAppService, Mockito.never())
+ .retryBatchFailures(
+ Mockito.any(), Mockito.anySet(),
+ Mockito.any(BooleanSupplier.class));
+ }
+
+ /**
+ * 验证进程错过提交后回调时，调度器可从持久恢复待办重放任务创建。
+ */
+ @Test
+ public void pendingRecoveryShouldReplayAndClearDurableMarker() {
+ TestContext context = createContext();
+ DocumentImportBatch batch = context.batchService.getOne(
+ QueryWrapper.create()
+ );
+ batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
+ batch.setRecoveryPending(true);
+ batch.setRecoveryFileKeysJson("[\"stale-file\"]");
+ DocumentImportBatch claimedBatch = new DocumentImportBatch();
+ claimedBatch.setId(batch.getId());
+ claimedBatch.setRecoveryFileKeysJson("[\"selected-file\"]");
+ Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
+ Mockito.any(Date.class), Mockito.anyInt()
+ )).thenReturn(List.of(batch));
+ Mockito.when(context.batchMapper.selectClaimedRecovery(
+ Mockito.eq(batch.getId()), Mockito.anyString()
+ )).thenReturn(claimedBatch);
+
+ int recovered = context.service.recoverPendingBatchRetries();
+
+ Assert.assertEquals(1, recovered);
+ Mockito.verify(context.taskAppService)
+ .retryBatchFailures(
+ Mockito.eq(batch.getId()),
+ Mockito.eq(Set.of("selected-file")),
+ Mockito.any(BooleanSupplier.class)
+ );
+ ArgumentCaptor recoveryToken =
+ ArgumentCaptor.forClass(String.class);
+ Mockito.verify(context.batchMapper).claimRecoveryPending(
+ Mockito.eq(batch.getId()),
+ recoveryToken.capture(),
+ Mockito.any(Date.class),
+ Mockito.any(Date.class)
+ );
+ Mockito.verify(context.batchMapper).selectClaimedRecovery(
+ batch.getId(), recoveryToken.getValue());
+ Mockito.verify(context.batchMapper)
+ .finalizeRecoveryPending(
+ Mockito.eq(batch.getId()),
+ Mockito.eq(recoveryToken.getValue()),
+ Mockito.any(Date.class));
+ }
+
+ /**
+ * 验证文件恢复期间租约失效后旧持有者不会清理持久恢复待办。
+ */
+ @Test
+ public void lostRecoveryLeaseShouldKeepDurableMarker() {
+ TestContext context = createContext();
+ DocumentImportBatch batch = context.batchService.getOne(
+ QueryWrapper.create()
+ );
+ batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
+ batch.setRecoveryPending(true);
+ Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
+ Mockito.any(Date.class), Mockito.anyInt()
+ )).thenReturn(List.of(batch));
+ Mockito.when(context.taskAppService.retryBatchFailures(
+ Mockito.eq(batch.getId()), Mockito.anySet(),
+ Mockito.any(BooleanSupplier.class)
+ )).thenReturn(false);
+
+ Assert.assertEquals(1, context.service.recoverPendingBatchRetries());
+
+ Mockito.verify(context.batchMapper, Mockito.never())
+ .finalizeRecoveryPending(
+ Mockito.any(), Mockito.anyString(), Mockito.any(Date.class));
+ Mockito.verifyNoInteractions(context.circuitBreaker);
+ }
+
+ /**
+ * 验证长时间恢复会在半租期续租，并在令牌失效后停止旧实例。
+ */
+ @Test
+ public void recoveryLeaseGuardShouldRenewAndStopWhenTokenIsLost() {
+ TestContext context = createContext();
+ BigInteger batchId = BigInteger.valueOf(84);
+ String recoveryToken = "recovery-token";
+ AtomicLong now = new AtomicLong(1_000L);
+ Mockito.when(context.batchMapper.renewRecoveryPendingLease(
+ Mockito.eq(batchId),
+ Mockito.eq(recoveryToken),
+ Mockito.any(Date.class),
+ Mockito.any(Date.class)
+ )).thenReturn(1, 0);
+ BooleanSupplier guard = context.service.createRecoveryLeaseGuard(
+ batchId,
+ recoveryToken,
+ 121_000L,
+ now::get
+ );
+
+ Assert.assertTrue(guard.getAsBoolean());
+ Mockito.verify(context.batchMapper, Mockito.never())
+ .renewRecoveryPendingLease(
+ Mockito.any(),
+ Mockito.anyString(),
+ Mockito.any(Date.class),
+ Mockito.any(Date.class)
+ );
+
+ now.set(61_000L);
+ Assert.assertTrue(guard.getAsBoolean());
+ now.set(121_000L);
+ Assert.assertFalse(guard.getAsBoolean());
+
+ Mockito.verify(context.batchMapper, Mockito.times(2))
+ .renewRecoveryPendingLease(
+ Mockito.eq(batchId),
+ Mockito.eq(recoveryToken),
+ Mockito.any(Date.class),
+ Mockito.any(Date.class)
+ );
+ }
+
+ /**
+ * 验证恢复租约续期 SQL 同时校验批次状态、令牌和未过期租约。
+ *
+ * @throws Exception Mapper 方法不存在时抛出
+ */
+ @Test
+ public void recoveryLeaseRenewalSqlShouldFenceCurrentOwner()
+ throws Exception {
+ Method method = DocumentImportBatchMapper.class.getMethod(
+ "renewRecoveryPendingLease",
+ BigInteger.class,
+ String.class,
+ Date.class,
+ Date.class
+ );
+ Update update = method.getAnnotation(Update.class);
+ String sql = String.join(" ", update.value());
+
+ Assert.assertTrue(sql.contains("status='RUNNING'"));
+ Assert.assertTrue(sql.contains("recovery_pending=1"));
+ Assert.assertTrue(sql.contains("recovery_token=#{recoveryToken}"));
+ Assert.assertTrue(sql.contains("recovery_lease_until > #{modified}"));
+
+ Method finalizeMethod = DocumentImportBatchMapper.class.getMethod(
+ "finalizeRecoveryPending",
+ BigInteger.class,
+ String.class,
+ Date.class
+ );
+ String finalizeSql = String.join(" ",
+ finalizeMethod.getAnnotation(Update.class).value());
+ Assert.assertTrue(finalizeSql.contains(
+ "status=CASE WHEN"));
+ Assert.assertTrue(finalizeSql.contains("recovery_pending=0"));
+ Assert.assertTrue(finalizeSql.contains(
+ "recovery_token=#{recoveryToken}"));
+ Assert.assertTrue(finalizeSql.contains(
+ "recovery_lease_until > #{modified}"));
+ }
+
/**
* 验证运行中的任务拒绝再次领取，调用方可继续查询原 taskId。
*/
@@ -761,6 +1015,50 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
+ Mockito.nullable(String.class),
+ Mockito.any(Date.class)
+ );
+ Mockito.verify(context.lockHandle).release();
+ }
+
+ /**
+ * 验证 Public API 不会在同一知识库已有运行批次时恢复旧批次。
+ */
+ @Test
+ public void retryShouldRejectWhenAnotherAutoBatchIsRunning() {
+ TestContext context = createContext();
+ DocumentImportBatch batch = context.batchService.getOne(
+ QueryWrapper.create()
+ );
+ batch.setImportMode(DocumentImportMode.AUTO.name());
+ batch.setStatus(DocumentImportBatchStatus.INTERRUPTED.name());
+ Mockito.when(context.batchService.count(Mockito.any(QueryWrapper.class)))
+ .thenReturn(1L);
+
+ beginTransactionSynchronization();
+ try {
+ context.service.retryOwnedBatch(
+ batch.getId(),
+ publicCaller(),
+ Set.of()
+ );
+ Assert.fail("Expected active automatic batch rejection");
+ } catch (BusinessException expected) {
+ Assert.assertTrue(
+ expected.getMessage().contains("当前知识库已有自动导入批次")
+ );
+ } finally {
+ completeTransactionSynchronization(
+ TransactionSynchronization.STATUS_ROLLED_BACK
+ );
+ }
+
+ Mockito.verify(context.batchMapper, Mockito.never()).claimRetry(
+ Mockito.any(),
+ Mockito.anyString(),
+ Mockito.any(),
+ Mockito.anyInt(),
+ Mockito.nullable(String.class),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
@@ -814,6 +1112,8 @@ public class DocumentImportBatchAppServiceTest {
Mockito.mock(DocumentImportBatchItemMapper.class);
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ Mockito.mock(DocumentImportBatchCircuitBreaker.class);
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
Mockito.when(redisLockExecutor.tryAcquire(
Mockito.anyString(), Mockito.any(), Mockito.any()
@@ -829,6 +1129,21 @@ public class DocumentImportBatchAppServiceTest {
Mockito.when(batchMapper.selectOwnedForUpdate(
Mockito.any(), Mockito.anyString(), Mockito.any()
)).thenReturn(batch);
+ Mockito.when(batchMapper.claimRecoveryPending(
+ Mockito.any(),
+ Mockito.anyString(),
+ Mockito.any(Date.class),
+ Mockito.any(Date.class)
+ )).thenReturn(1);
+ Mockito.when(batchMapper.selectClaimedRecovery(
+ Mockito.any(), Mockito.anyString()
+ )).thenReturn(batch);
+ Mockito.when(taskAppService.retryBatchFailures(
+ Mockito.any(), Mockito.anySet(), Mockito.any(BooleanSupplier.class)
+ )).thenReturn(true);
+ Mockito.when(batchMapper.finalizeRecoveryPending(
+ Mockito.any(), Mockito.anyString(), Mockito.any(Date.class)
+ )).thenReturn(1);
DocumentImportBatchAppService service = new DocumentImportBatchAppService(
batchService,
@@ -839,11 +1154,13 @@ public class DocumentImportBatchAppServiceTest {
batchMapper,
itemMapper,
documentMapper,
- redisLockExecutor
+ redisLockExecutor,
+ circuitBreaker
);
return new TestContext(
service, batchService, itemService, batchTracker,
- taskAppService, batchMapper, itemMapper, documentMapper, lockHandle
+ taskAppService, batchMapper, itemMapper, documentMapper,
+ circuitBreaker, lockHandle
);
}
@@ -954,6 +1271,7 @@ public class DocumentImportBatchAppServiceTest {
DocumentImportBatchMapper batchMapper,
DocumentImportBatchItemMapper itemMapper,
DocumentMapper documentMapper,
+ DocumentImportBatchCircuitBreaker circuitBreaker,
RedisLockExecutor.LockHandle lockHandle
) {
}
diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreakerTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreakerTest.java
new file mode 100644
index 00000000..9f13c5cc
--- /dev/null
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreakerTest.java
@@ -0,0 +1,297 @@
+package tech.easyflow.ai.documentimport.task;
+
+import org.apache.ibatis.annotations.Update;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.springframework.dao.DataAccessResourceFailureException;
+import org.springframework.jdbc.BadSqlGrammarException;
+import org.springframework.data.redis.RedisSystemException;
+import tech.easyflow.ai.entity.DocumentImportBatch;
+import tech.easyflow.ai.entity.DocumentImportTask;
+import tech.easyflow.ai.enums.DocumentImportBatchStatus;
+import tech.easyflow.ai.enums.DocumentImportMode;
+import tech.easyflow.ai.enums.DocumentImportTaskPhase;
+import tech.easyflow.ai.enums.DocumentImportTaskStatus;
+import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
+import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
+import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
+
+import java.math.BigInteger;
+import java.lang.reflect.Method;
+import java.sql.SQLException;
+import java.util.Date;
+
+/**
+ * {@link DocumentImportBatchCircuitBreaker} 批次熔断回归测试。
+ */
+public class DocumentImportBatchCircuitBreakerTest {
+
+ /**
+ * 验证 Redis 超时会强制中断批次、撤销任务并刷新真实计数。
+ */
+ @Test
+ public void shouldInterruptAutomaticBatchOnRedisFailure() {
+ BigInteger taskId = BigInteger.valueOf(11);
+ BigInteger batchId = BigInteger.valueOf(12);
+ DocumentImportTask task = new DocumentImportTask();
+ task.setId(taskId);
+ task.setBatchId(batchId);
+ task.setPhase(DocumentImportTaskPhase.PARSE.name());
+ task.setStatus(DocumentImportTaskStatus.PENDING.name());
+ DocumentImportBatch batch = new DocumentImportBatch();
+ batch.setId(batchId);
+ batch.setImportMode(DocumentImportMode.AUTO.name());
+ batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
+ DocumentImportBatchMapper batchMapper =
+ Mockito.mock(DocumentImportBatchMapper.class);
+ DocumentImportBatchItemMapper itemMapper =
+ Mockito.mock(DocumentImportBatchItemMapper.class);
+ DocumentImportTaskMapper taskMapper =
+ Mockito.mock(DocumentImportTaskMapper.class);
+ Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
+ Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
+ Mockito.when(batchMapper.interruptRunningBatchForActiveTask(
+ Mockito.eq(batchId),
+ Mockito.eq(taskId),
+ Mockito.eq("redis_unavailable"),
+ Mockito.anyString(),
+ Mockito.any(Date.class)
+ )).thenReturn(1);
+ Mockito.when(taskMapper.interruptActiveDocuments(
+ Mockito.eq(batchId), Mockito.anyString(),
+ Mockito.eq("redis_unavailable"), Mockito.any(Date.class),
+ Mockito.eq(BigInteger.ZERO)
+ )).thenReturn(1);
+ Mockito.when(taskMapper.interruptActiveTasks(
+ Mockito.eq(batchId), Mockito.anyString(),
+ Mockito.eq("redis_unavailable"), Mockito.any(Date.class),
+ Mockito.eq(BigInteger.ZERO)
+ )).thenReturn(2);
+ Mockito.when(itemMapper.interruptActiveItems(
+ Mockito.eq(batchId), Mockito.anyString(),
+ Mockito.eq("redis_unavailable"), Mockito.any(Date.class)
+ )).thenReturn(2);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ new DocumentImportBatchCircuitBreaker(
+ batchMapper, itemMapper, taskMapper);
+
+ boolean interrupted = circuitBreaker.interruptTaskBatch(
+ taskId,
+ new RedisSystemException(
+ "Redis command timed out", new IllegalStateException("timeout"))
+ );
+
+ Assert.assertTrue(interrupted);
+ Mockito.verify(batchMapper).refreshCountersFromItems(
+ Mockito.eq(batchId), Mockito.any(Date.class));
+ }
+
+ /**
+ * 验证无批次任务不会误触发批次熔断。
+ */
+ @Test
+ public void shouldLeaveStandaloneTaskForDeferredRetry() {
+ BigInteger taskId = BigInteger.valueOf(21);
+ DocumentImportTask task = new DocumentImportTask();
+ task.setId(taskId);
+ DocumentImportBatchMapper batchMapper =
+ Mockito.mock(DocumentImportBatchMapper.class);
+ DocumentImportBatchItemMapper itemMapper =
+ Mockito.mock(DocumentImportBatchItemMapper.class);
+ DocumentImportTaskMapper taskMapper =
+ Mockito.mock(DocumentImportTaskMapper.class);
+ Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ new DocumentImportBatchCircuitBreaker(
+ batchMapper, itemMapper, taskMapper);
+
+ Assert.assertFalse(circuitBreaker.interruptTaskBatch(
+ taskId, new IllegalStateException("temporary failure")));
+ Mockito.verify(batchMapper, Mockito.never()).interruptRunningBatch(
+ Mockito.any(), Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(Date.class));
+ }
+
+ /**
+ * 验证旧任务已结束后，迟到消费者异常不会中断新一轮运行批次。
+ */
+ @Test
+ public void shouldIgnoreLateFailureFromFinishedTask() {
+ BigInteger taskId = BigInteger.valueOf(31);
+ DocumentImportTask task = new DocumentImportTask();
+ task.setId(taskId);
+ task.setBatchId(BigInteger.valueOf(32));
+ task.setStatus(DocumentImportTaskStatus.FAILED.name());
+ DocumentImportBatchMapper batchMapper =
+ Mockito.mock(DocumentImportBatchMapper.class);
+ DocumentImportBatchItemMapper itemMapper =
+ Mockito.mock(DocumentImportBatchItemMapper.class);
+ DocumentImportTaskMapper taskMapper =
+ Mockito.mock(DocumentImportTaskMapper.class);
+ Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ new DocumentImportBatchCircuitBreaker(
+ batchMapper, itemMapper, taskMapper);
+
+ Assert.assertTrue(circuitBreaker.interruptTaskBatch(
+ taskId, new IllegalStateException("迟到异常")));
+
+ Mockito.verify(batchMapper, Mockito.never())
+ .interruptRunningBatchForActiveTask(
+ Mockito.any(), Mockito.any(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.any(Date.class));
+ Mockito.verify(batchMapper, Mockito.never()).selectOneById(Mockito.any());
+ }
+
+ /**
+ * 验证恢复令牌失效后旧持有者无法中断仍由新持有者运行的批次。
+ */
+ @Test
+ public void shouldRejectInterruptionFromStaleRecoveryOwner() {
+ BigInteger batchId = BigInteger.valueOf(41);
+ DocumentImportBatch batch = new DocumentImportBatch();
+ batch.setId(batchId);
+ batch.setImportMode(DocumentImportMode.AUTO.name());
+ batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
+ DocumentImportBatchMapper batchMapper =
+ Mockito.mock(DocumentImportBatchMapper.class);
+ DocumentImportBatchItemMapper itemMapper =
+ Mockito.mock(DocumentImportBatchItemMapper.class);
+ DocumentImportTaskMapper taskMapper =
+ Mockito.mock(DocumentImportTaskMapper.class);
+ Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
+ Mockito.when(batchMapper.interruptOwnedRecoveryBatch(
+ Mockito.eq(batchId),
+ Mockito.eq("stale-token"),
+ Mockito.anyString(),
+ Mockito.anyString(),
+ Mockito.any(Date.class)
+ )).thenReturn(0);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ new DocumentImportBatchCircuitBreaker(
+ batchMapper, itemMapper, taskMapper);
+
+ Assert.assertFalse(circuitBreaker.interruptRecoveryBatch(
+ batchId,
+ "stale-token",
+ new IllegalStateException("旧持有者异常")
+ ));
+ Mockito.verifyNoInteractions(itemMapper);
+ Mockito.verify(taskMapper, Mockito.never()).interruptActiveTasks(
+ Mockito.any(), Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(Date.class), Mockito.any());
+ }
+
+ /**
+ * 验证 SQL 语法错误保留为通用系统异常，避免伪装成数据库不可用。
+ */
+ @Test
+ public void shouldKeepBadSqlGrammarAsInfrastructureFailure() {
+ BigInteger batchId = BigInteger.valueOf(51);
+ DocumentImportBatch batch = runningAutoBatch(batchId);
+ DocumentImportBatchMapper batchMapper =
+ Mockito.mock(DocumentImportBatchMapper.class);
+ Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
+ Mockito.when(batchMapper.interruptRunningBatch(
+ Mockito.eq(batchId),
+ Mockito.eq("document_import_infrastructure_failure"),
+ Mockito.anyString(),
+ Mockito.any(Date.class)
+ )).thenReturn(1);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ new DocumentImportBatchCircuitBreaker(
+ batchMapper,
+ Mockito.mock(DocumentImportBatchItemMapper.class),
+ Mockito.mock(DocumentImportTaskMapper.class)
+ );
+
+ Assert.assertTrue(circuitBreaker.interruptBatch(
+ batchId,
+ new BadSqlGrammarException(
+ "query", "SELECT broken", new SQLException("syntax", "42000"))
+ ));
+ }
+
+ /**
+ * 验证明确的数据库资源故障仍归类为数据库不可用。
+ */
+ @Test
+ public void shouldClassifyDatabaseResourceFailureAsUnavailable() {
+ BigInteger batchId = BigInteger.valueOf(61);
+ DocumentImportBatch batch = runningAutoBatch(batchId);
+ DocumentImportBatchMapper batchMapper =
+ Mockito.mock(DocumentImportBatchMapper.class);
+ Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
+ Mockito.when(batchMapper.interruptRunningBatch(
+ Mockito.eq(batchId),
+ Mockito.eq("database_unavailable"),
+ Mockito.anyString(),
+ Mockito.any(Date.class)
+ )).thenReturn(1);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ new DocumentImportBatchCircuitBreaker(
+ batchMapper,
+ Mockito.mock(DocumentImportBatchItemMapper.class),
+ Mockito.mock(DocumentImportTaskMapper.class)
+ );
+
+ Assert.assertTrue(circuitBreaker.interruptBatch(
+ batchId,
+ new DataAccessResourceFailureException("connection unavailable")
+ ));
+ }
+
+ /**
+ * 验证恢复与任务熔断 SQL 都包含对应所有权围栏。
+ *
+ * @throws Exception Mapper 方法不存在时抛出
+ */
+ @Test
+ public void interruptionSqlShouldFenceRecoveryAndActiveTask()
+ throws Exception {
+ Method recoveryMethod = DocumentImportBatchMapper.class.getMethod(
+ "interruptOwnedRecoveryBatch",
+ BigInteger.class,
+ String.class,
+ String.class,
+ String.class,
+ Date.class
+ );
+ String recoverySql = String.join(" ",
+ recoveryMethod.getAnnotation(Update.class).value());
+ Assert.assertTrue(recoverySql.contains("recovery_pending=1"));
+ Assert.assertTrue(recoverySql.contains(
+ "recovery_token=#{recoveryToken}"));
+ Assert.assertTrue(recoverySql.contains(
+ "recovery_lease_until > #{modified}"));
+
+ Method taskMethod = DocumentImportBatchMapper.class.getMethod(
+ "interruptRunningBatchForActiveTask",
+ BigInteger.class,
+ BigInteger.class,
+ String.class,
+ String.class,
+ Date.class
+ );
+ String taskSql = String.join(" ",
+ taskMethod.getAnnotation(Update.class).value());
+ Assert.assertTrue(taskSql.contains("task.id=#{taskId}"));
+ Assert.assertTrue(taskSql.contains(
+ "task.status IN ('PENDING','RUNNING')"));
+ }
+
+ /**
+ * 创建运行中的自动导入批次。
+ *
+ * @param batchId 批次 ID
+ * @return 测试批次
+ */
+ private DocumentImportBatch runningAutoBatch(BigInteger batchId) {
+ DocumentImportBatch batch = new DocumentImportBatch();
+ batch.setId(batchId);
+ batch.setImportMode(DocumentImportMode.AUTO.name());
+ batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
+ return batch;
+ }
+}
diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java
index e5e26aa8..73d8f279 100644
--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java
@@ -75,6 +75,9 @@ public class DocumentImportBatchTrackerTest {
batch.setCompletedCount(1);
batch.setFailedCount(1);
batch.setRetryableFailedCount(0);
+ batch.setInterruptCode("redis_unavailable");
+ batch.setInterruptMessage("缓存与消息服务异常");
+ batch.setInterruptedAt(new Date());
when(batchService.getById(batch.getId())).thenReturn(batch);
DocumentImportBatchTracker tracker =
@@ -84,6 +87,9 @@ public class DocumentImportBatchTrackerTest {
assertEquals(DocumentImportBatchStatus.INTERRUPTED.name(), response.getStatus());
assertEquals(100, response.getProgressPercent().intValue());
assertEquals(1, response.getRetryableFailedCount().intValue());
+ assertEquals("redis_unavailable", response.getInterruptCode());
+ assertEquals("缓存与消息服务异常", response.getInterruptMessage());
+ assertEquals(batch.getInterruptedAt(), response.getInterruptedAt());
}
/**
@@ -139,6 +145,45 @@ public class DocumentImportBatchTrackerTest {
);
}
+ /**
+ * 验证无文档失败项绑定后按真实文件状态刷新批次计数。
+ */
+ @Test
+ public void shouldBindRecoveredDocumentAndRefreshCounters() {
+ DocumentImportBatchService batchService =
+ mock(DocumentImportBatchService.class);
+ DocumentImportBatchItemService itemService =
+ mock(DocumentImportBatchItemService.class);
+ DocumentImportBatchMapper batchMapper =
+ mock(DocumentImportBatchMapper.class);
+ DocumentImportBatchItemMapper itemMapper =
+ mock(DocumentImportBatchItemMapper.class);
+ DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1);
+ batch.setPendingCount(1);
+ DocumentImportBatchItem item = new DocumentImportBatchItem();
+ item.setId(BigInteger.TEN);
+ item.setBatchId(batch.getId());
+ item.setStatus(DocumentImportBatchItemStatus.FAILED.name());
+ item.setRetryable(true);
+ BigInteger documentId = BigInteger.valueOf(99);
+ when(itemService.getById(item.getId())).thenReturn(item);
+ when(itemMapper.bindFailedDocument(
+ eq(item.getId()), eq(documentId), any(Date.class)
+ )).thenReturn(1);
+ when(batchService.getById(batch.getId())).thenReturn(batch);
+ DocumentImportBatchTracker tracker =
+ new DocumentImportBatchTracker(
+ batchService, itemService, batchMapper, itemMapper);
+
+ tracker.bindDocument(item.getId(), documentId);
+
+ verify(batchMapper).refreshCountersFromItems(
+ eq(batch.getId()), any(Date.class));
+ verify(batchMapper, never()).adjustCounters(
+ any(), anyInt(), anyInt(), anyInt(), anyInt(), anyInt(),
+ anyInt(), anyInt(), anyInt(), any(Date.class));
+ }
+
/**
* 验证迟到任务不能把已完成文件重新改为处理中。
*/
diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitorTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitorTest.java
new file mode 100644
index 00000000..95fd2b14
--- /dev/null
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitorTest.java
@@ -0,0 +1,36 @@
+package tech.easyflow.ai.documentimport.task;
+
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * {@link DocumentImportPendingTaskMonitor} 调度退避回归测试。
+ *
+ * @author Codex
+ * @since 2026-08-07
+ */
+public class DocumentImportPendingTaskMonitorTest {
+
+ /**
+ * 验证任务投递异常后立即进入冷却期，避免短周期重复扫描。
+ */
+ @Test
+ public void dispatchFailureShouldEnterCooldown() {
+ KnowledgeDocumentImportTaskAppService appService =
+ Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
+ DocumentImportBatchAppService batchAppService =
+ Mockito.mock(DocumentImportBatchAppService.class);
+ Mockito.doThrow(new IllegalStateException("Redis unavailable"))
+ .when(appService)
+ .dispatchPendingTasks();
+ DocumentImportPendingTaskMonitor monitor =
+ new DocumentImportPendingTaskMonitor(appService, batchAppService);
+
+ monitor.dispatchPendingTasks();
+ monitor.dispatchPendingTasks();
+
+ Mockito.verify(batchAppService, Mockito.times(1))
+ .recoverPendingBatchRetries();
+ Mockito.verify(appService, Mockito.times(1)).dispatchPendingTasks();
+ }
+}
diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java
index 260e3429..ce6761b0 100644
--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java
@@ -14,11 +14,15 @@ import com.easyagents.rag.ingestion.RagIngestionService;
import com.easyagents.rag.ingestion.model.AnalysisResult;
import com.easyagents.rag.ingestion.model.StrategyConfig;
import com.easyagents.search.engine.service.DocumentSearcher;
+import com.mybatisflex.core.query.QueryWrapper;
+import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
+import org.springframework.data.redis.RedisSystemException;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
import tech.easyflow.ai.document.model.DocumentParseArtifacts;
@@ -28,20 +32,26 @@ import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.DocumentImportKeys;
import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.ai.entity.DocumentCollection;
+import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.entity.DocumentImportTask;
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
+import tech.easyflow.ai.enums.DocumentImportBatchStatus;
+import tech.easyflow.ai.enums.DocumentImportMode;
import tech.easyflow.ai.enums.DocumentImportTaskStatus;
import tech.easyflow.ai.enums.DocumentImportTaskPhase;
import tech.easyflow.ai.enums.DocumentProcessStatus;
import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
+import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
import tech.easyflow.ai.service.DocumentImportTaskService;
+import tech.easyflow.ai.service.DocumentCollectionService;
import tech.easyflow.ai.service.DocumentService;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.common.filestorage.FileStorageService;
+import tech.easyflow.common.web.exceptions.BusinessException;
import java.io.ByteArrayInputStream;
import java.lang.reflect.Constructor;
@@ -59,6 +69,7 @@ import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -192,6 +203,277 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Mockito.verify(producer, Mockito.never()).send(Mockito.any());
}
+ /**
+ * 验证自动导入任务投递异常会触发批次熔断，且不会阻断本轮扫描。
+ *
+ * @throws Exception 反射注入异常
+ */
+ @Test
+ public void dispatchPendingTasksShouldInterruptBatchWhenProducerFails()
+ throws Exception {
+ BigInteger taskId = BigInteger.valueOf(31);
+ DocumentImportTask task = new DocumentImportTask();
+ task.setId(taskId);
+ task.setBatchId(BigInteger.valueOf(32));
+ task.setPhase(DocumentImportTaskPhase.PARSE.name());
+ task.setStatus(DocumentImportTaskStatus.PENDING.name());
+ DocumentImportTaskMapper taskMapper =
+ Mockito.mock(DocumentImportTaskMapper.class);
+ Mockito.when(taskMapper.selectPendingFairly(
+ Mockito.any(Date.class), Mockito.anyInt()))
+ .thenReturn(List.of(task));
+ Mockito.when(taskMapper.touchPendingForDispatch(
+ Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
+ Mockito.any(BigInteger.class)))
+ .thenReturn(1);
+ DocumentImportParseTaskProducer producer =
+ Mockito.mock(DocumentImportParseTaskProducer.class);
+ IllegalStateException failure =
+ new IllegalStateException("Redis command timed out");
+ Mockito.doThrow(failure).when(producer).send(taskId);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ Mockito.mock(DocumentImportBatchCircuitBreaker.class);
+ Mockito.when(circuitBreaker.interruptTaskBatch(taskId, failure))
+ .thenReturn(true);
+ KnowledgeDocumentImportTaskAppService service =
+ new KnowledgeDocumentImportTaskAppService();
+ setField(service, "documentImportTaskMapper", taskMapper);
+ setField(service, "parseTaskProducer", producer);
+ setField(service, "documentImportBatchCircuitBreaker", circuitBreaker);
+ setField(service, "bulkProperties", new DocumentImportBulkProperties());
+
+ service.dispatchPendingTasks();
+
+ Mockito.verify(circuitBreaker).interruptTaskBatch(taskId, failure);
+ }
+
+ /**
+ * 验证投递失败后若批次中断写入也失败，异常会传播给调度器触发退避。
+ *
+ * @throws Exception 反射注入异常
+ */
+ @Test
+ public void dispatchPendingTasksShouldPropagateInterruptionWriteFailure()
+ throws Exception {
+ BigInteger taskId = BigInteger.valueOf(33);
+ DocumentImportTask task = new DocumentImportTask();
+ task.setId(taskId);
+ task.setBatchId(BigInteger.valueOf(34));
+ task.setPhase(DocumentImportTaskPhase.PARSE.name());
+ task.setStatus(DocumentImportTaskStatus.PENDING.name());
+ DocumentImportTaskMapper taskMapper =
+ Mockito.mock(DocumentImportTaskMapper.class);
+ Mockito.when(taskMapper.selectPendingFairly(
+ Mockito.any(Date.class), Mockito.anyInt()))
+ .thenReturn(List.of(task));
+ Mockito.when(taskMapper.touchPendingForDispatch(
+ Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
+ Mockito.any(BigInteger.class)))
+ .thenReturn(1);
+ DocumentImportParseTaskProducer producer =
+ Mockito.mock(DocumentImportParseTaskProducer.class);
+ RedisSystemException dispatchFailure = new RedisSystemException(
+ "Redis command timed out",
+ new IllegalStateException("timeout")
+ );
+ Mockito.doThrow(dispatchFailure).when(producer).send(taskId);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ Mockito.mock(DocumentImportBatchCircuitBreaker.class);
+ IllegalStateException interruptionFailure =
+ new IllegalStateException("database unavailable");
+ Mockito.when(circuitBreaker.interruptTaskBatch(taskId, dispatchFailure))
+ .thenThrow(interruptionFailure);
+ KnowledgeDocumentImportTaskAppService service =
+ new KnowledgeDocumentImportTaskAppService();
+ setField(service, "documentImportTaskMapper", taskMapper);
+ setField(service, "parseTaskProducer", producer);
+ setField(service, "documentImportBatchCircuitBreaker", circuitBreaker);
+ setField(service, "bulkProperties", new DocumentImportBulkProperties());
+
+ try {
+ service.dispatchPendingTasks();
+ Assert.fail("批次中断写入失败必须传播给外层调度器");
+ } catch (IllegalStateException error) {
+ Assert.assertSame(interruptionFailure, error.getCause());
+ Assert.assertEquals(1, error.getSuppressed().length);
+ Assert.assertSame(dispatchFailure, error.getSuppressed()[0]);
+ Assert.assertTrue(error.getMessage().contains("taskId=33"));
+ Assert.assertTrue(error.getMessage().contains("batchId=34"));
+ }
+ }
+
+ /**
+ * 验证无批次任务投递失败时保留任务并传播异常，避免两秒热循环。
+ *
+ * @throws Exception 反射注入异常
+ */
+ @Test
+ public void dispatchPendingTasksShouldPropagateUnbatchedFailure()
+ throws Exception {
+ BigInteger taskId = BigInteger.valueOf(35);
+ DocumentImportTask task = new DocumentImportTask();
+ task.setId(taskId);
+ task.setPhase(DocumentImportTaskPhase.PARSE.name());
+ task.setStatus(DocumentImportTaskStatus.PENDING.name());
+ DocumentImportTaskMapper taskMapper =
+ Mockito.mock(DocumentImportTaskMapper.class);
+ Mockito.when(taskMapper.selectPendingFairly(
+ Mockito.any(Date.class), Mockito.anyInt()))
+ .thenReturn(List.of(task));
+ Mockito.when(taskMapper.touchPendingForDispatch(
+ Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
+ Mockito.any(BigInteger.class)))
+ .thenReturn(1);
+ DocumentImportParseTaskProducer producer =
+ Mockito.mock(DocumentImportParseTaskProducer.class);
+ IllegalStateException dispatchFailure =
+ new IllegalStateException("Redis unavailable");
+ Mockito.doThrow(dispatchFailure).when(producer).send(taskId);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ Mockito.mock(DocumentImportBatchCircuitBreaker.class);
+ Mockito.when(circuitBreaker.interruptTaskBatch(taskId, dispatchFailure))
+ .thenReturn(false);
+ KnowledgeDocumentImportTaskAppService service =
+ new KnowledgeDocumentImportTaskAppService();
+ setField(service, "documentImportTaskMapper", taskMapper);
+ setField(service, "parseTaskProducer", producer);
+ setField(service, "documentImportBatchCircuitBreaker", circuitBreaker);
+ setField(service, "bulkProperties", new DocumentImportBulkProperties());
+
+ try {
+ service.dispatchPendingTasks();
+ Assert.fail("无批次投递失败必须传播给外层调度器");
+ } catch (IllegalStateException error) {
+ Assert.assertSame(dispatchFailure, error.getCause());
+ Assert.assertTrue(error.getMessage().contains("taskId=35"));
+ Assert.assertTrue(error.getMessage().contains("batchId=null"));
+ }
+ }
+
+ /**
+ * 验证批次恢复遇到基础设施异常后向令牌持有者传播并停止后续重试。
+ *
+ * @throws Exception 反射注入异常
+ */
+ @Test
+ public void retryBatchInfrastructureFailureShouldPropagateToOwner()
+ throws Exception {
+ BigInteger batchId = BigInteger.valueOf(91);
+ DocumentImportBatchItem first = new DocumentImportBatchItem();
+ first.setId(BigInteger.valueOf(92));
+ first.setBatchId(batchId);
+ DocumentImportBatchItem second = new DocumentImportBatchItem();
+ second.setId(BigInteger.valueOf(93));
+ second.setBatchId(batchId);
+ DocumentImportBatchItemService itemService =
+ Mockito.mock(DocumentImportBatchItemService.class);
+ Mockito.when(itemService.list(Mockito.any(QueryWrapper.class)))
+ .thenReturn(List.of(first, second));
+ KnowledgeDocumentImportTaskAppService selfProxy =
+ Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
+ RedisSystemException failure = new RedisSystemException(
+ "Redis command timed out",
+ new IllegalStateException("timeout")
+ );
+ Mockito.doThrow(failure)
+ .when(selfProxy)
+ .retryBatchItemInNewTransaction(first.getId());
+ KnowledgeDocumentImportTaskAppService service =
+ new KnowledgeDocumentImportTaskAppService();
+ setField(service, "documentImportBatchItemService", itemService);
+ setField(service, "selfProxy", selfProxy);
+
+ try {
+ service.retryBatchFailures(batchId, Set.of());
+ Assert.fail("基础设施异常必须传播给恢复令牌持有者");
+ } catch (RedisSystemException error) {
+ Assert.assertSame(failure, error);
+ }
+
+ Mockito.verify(selfProxy, Mockito.never())
+ .retryBatchItemInNewTransaction(second.getId());
+ }
+
+ /**
+ * 验证单文件业务失败不会提前关闭批次并跳过后续失败项。
+ *
+ * @throws Exception 反射注入异常
+ */
+ @Test
+ public void retryBatchBusinessFailureShouldContinueRemainingItems()
+ throws Exception {
+ BigInteger batchId = BigInteger.valueOf(96);
+ DocumentImportBatchItem first = new DocumentImportBatchItem();
+ first.setId(BigInteger.valueOf(97));
+ first.setBatchId(batchId);
+ first.setDocumentId(BigInteger.valueOf(98));
+ DocumentImportBatchItem second = new DocumentImportBatchItem();
+ second.setId(BigInteger.valueOf(99));
+ second.setBatchId(batchId);
+ second.setDocumentId(BigInteger.valueOf(100));
+ DocumentImportBatchItemService itemService =
+ Mockito.mock(DocumentImportBatchItemService.class);
+ Mockito.when(itemService.list(Mockito.any(QueryWrapper.class)))
+ .thenReturn(List.of(first, second));
+ KnowledgeDocumentImportTaskAppService selfProxy =
+ Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
+ Mockito.doThrow(new BusinessException("格式不支持"))
+ .when(selfProxy)
+ .retryBatchItemInNewTransaction(first.getId());
+ DocumentImportBatchItemMapper itemMapper =
+ Mockito.mock(DocumentImportBatchItemMapper.class);
+ DocumentImportBatchTracker batchTracker =
+ Mockito.mock(DocumentImportBatchTracker.class);
+ KnowledgeDocumentImportTaskAppService service =
+ new KnowledgeDocumentImportTaskAppService();
+ setField(service, "documentImportBatchItemService", itemService);
+ setField(service, "documentImportBatchItemMapper", itemMapper);
+ setField(service, "documentImportBatchTracker", batchTracker);
+ setField(service, "selfProxy", selfProxy);
+
+ Assert.assertTrue(service.retryBatchFailures(
+ batchId, Set.of(), () -> true));
+
+ org.mockito.InOrder order = Mockito.inOrder(
+ selfProxy, itemMapper);
+ order.verify(selfProxy).retryBatchItemInNewTransaction(first.getId());
+ order.verify(itemMapper).updateFailedRetryError(
+ Mockito.eq(first.getId()), Mockito.eq(batchId),
+ Mockito.eq("格式不支持"), Mockito.any(Date.class));
+ order.verify(selfProxy).retryBatchItemInNewTransaction(second.getId());
+ Mockito.verifyNoInteractions(batchTracker);
+ }
+
+ /**
+ * 验证恢复租约失效后旧实例停止重试剩余文件。
+ *
+ * @throws Exception 反射注入异常
+ */
+ @Test
+ public void retryBatchFailuresShouldStopWhenRecoveryLeaseIsLost()
+ throws Exception {
+ BigInteger batchId = BigInteger.valueOf(94);
+ DocumentImportBatchItem failed = new DocumentImportBatchItem();
+ failed.setId(BigInteger.valueOf(95));
+ failed.setBatchId(batchId);
+ DocumentImportBatchItemService itemService =
+ Mockito.mock(DocumentImportBatchItemService.class);
+ Mockito.when(itemService.list(Mockito.any(QueryWrapper.class)))
+ .thenReturn(List.of(failed));
+ KnowledgeDocumentImportTaskAppService selfProxy =
+ Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
+ KnowledgeDocumentImportTaskAppService service =
+ new KnowledgeDocumentImportTaskAppService();
+ setField(service, "documentImportBatchItemService", itemService);
+ setField(service, "selfProxy", selfProxy);
+
+ Assert.assertFalse(service.retryBatchFailures(
+ batchId, Set.of(), () -> false));
+
+ Mockito.verify(selfProxy, Mockito.never())
+ .retryBatchItemInNewTransaction(Mockito.any());
+ }
+
/**
* 验证重新投递资格更新包含待处理状态和过期时间边界，确保竞争更新仅一个成功。
*
@@ -213,6 +495,102 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Assert.assertTrue(sql.contains("status='PENDING'"));
Assert.assertTrue(sql.contains("modified <= #{redispatchBefore}"));
Assert.assertTrue(sql.contains("modified=#{now}"));
+ Assert.assertTrue(sql.contains("batch.status='RUNNING'"));
+ }
+
+ /**
+ * 验证任务扫描、领取、续租和完成 SQL 均包含批次运行状态门禁。
+ */
+ @Test
+ public void activeTaskSqlShouldFenceInterruptedBatch() {
+ for (Method method : DocumentImportTaskMapper.class.getDeclaredMethods()) {
+ String name = method.getName();
+ if (!"selectPendingFairly".equals(name)
+ && !"claimPending".equals(name)
+ && !"renewLease".equals(name)
+ && !"finishOwned".equals(name)) {
+ continue;
+ }
+ Select select = method.getAnnotation(Select.class);
+ Update update = method.getAnnotation(Update.class);
+ String sql = select == null
+ ? String.join(" ", update.value())
+ : String.join(" ", select.value());
+
+ Assert.assertTrue(name + " 缺少批次运行状态门禁",
+ sql.contains("batch.status='RUNNING'"));
+ }
+ }
+
+ /**
+ * 验证失败项恢复同时锁定文件项与运行批次，阻止熔断后继续建任务。
+ *
+ * @throws Exception 映射方法不存在时抛出
+ */
+ @Test
+ public void retryItemSqlShouldRequireRunningBatchLock()
+ throws Exception {
+ Method method = DocumentImportBatchItemMapper.class.getMethod(
+ "selectFailedForRetry",
+ BigInteger.class
+ );
+ Select select = method.getAnnotation(Select.class);
+ String sql = String.join(" ", select.value());
+
+ Assert.assertTrue(sql.contains("item.status='FAILED'"));
+ Assert.assertTrue(sql.contains("batch.status='RUNNING'"));
+ Assert.assertTrue(sql.contains("FOR UPDATE"));
+ }
+
+ /**
+ * 验证批次中断后，迟到 worker 不能重新打开文件项或覆盖中断原因。
+ *
+ * @throws Exception 映射方法不存在时抛出
+ */
+ @Test
+ public void itemTransitionSqlShouldFenceInterruptedBatch()
+ throws Exception {
+ Method method = DocumentImportBatchItemMapper.class.getMethod(
+ "transitionStatus",
+ BigInteger.class,
+ String.class,
+ String.class,
+ String.class,
+ String.class,
+ String.class,
+ boolean.class,
+ int.class,
+ Date.class
+ );
+ Update update = method.getAnnotation(Update.class);
+ String sql = String.join(" ", update.value());
+
+ Assert.assertTrue(sql.contains(
+ "INNER JOIN tb_document_import_batch batch"));
+ Assert.assertTrue(sql.contains("batch.status<>'INTERRUPTED'"));
+ }
+
+ /**
+ * 验证批次中断同步文档状态时兼容历史无效 JSON 扩展字段。
+ *
+ * @throws Exception 映射方法不存在时抛出
+ */
+ @Test
+ public void interruptDocumentSqlShouldHandleInvalidOptions()
+ throws Exception {
+ Method method = DocumentImportTaskMapper.class.getMethod(
+ "interruptActiveDocuments",
+ BigInteger.class,
+ String.class,
+ String.class,
+ Date.class,
+ BigInteger.class
+ );
+ Update update = method.getAnnotation(Update.class);
+ String sql = String.join(" ", update.value());
+
+ Assert.assertTrue(sql.contains("JSON_VALID(document.options)"));
+ Assert.assertTrue(sql.contains("ELSE JSON_OBJECT()"));
}
/**
@@ -423,6 +801,7 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
BigInteger batchId = BigInteger.valueOf(61);
BigInteger itemId = BigInteger.valueOf(62);
DocumentImportTask task = new DocumentImportTask();
+ task.setId(BigInteger.valueOf(60));
task.setBatchId(batchId);
task.setBatchItemId(itemId);
DocumentImportBatchItem item = new DocumentImportBatchItem();
@@ -431,10 +810,13 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
DocumentImportBatchTracker tracker =
Mockito.mock(DocumentImportBatchTracker.class);
+ DocumentImportBatchCircuitBreaker circuitBreaker =
+ Mockito.mock(DocumentImportBatchCircuitBreaker.class);
Mockito.when(tracker.requireItem(itemId)).thenReturn(item);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportBatchTracker", tracker);
+ setField(service, "documentImportBatchCircuitBreaker", circuitBreaker);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod(
@@ -446,13 +828,20 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
);
method.setAccessible(true);
- method.invoke(
- service,
- task,
- DocumentImportBatchItemStage.PARSE,
- "任务执行中断，请继续批次",
- "execution_interrupted"
- );
+ TransactionSynchronizationManager.setActualTransactionActive(true);
+ TransactionSynchronizationManager.initSynchronization();
+ try {
+ method.invoke(
+ service,
+ task,
+ DocumentImportBatchItemStage.PARSE,
+ "任务执行中断，请继续批次",
+ "execution_interrupted"
+ );
+ } finally {
+ TransactionSynchronizationManager.clearSynchronization();
+ TransactionSynchronizationManager.setActualTransactionActive(false);
+ }
Mockito.verify(tracker).transitionItem(
itemId,
@@ -463,7 +852,12 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
0,
"execution_interrupted"
);
- Mockito.verify(tracker).markInterrupted(batchId);
+ Mockito.verify(circuitBreaker).interruptBatchInCurrentTransaction(
+ Mockito.eq(batchId),
+ Mockito.eq("execution_interrupted"),
+ Mockito.eq("任务执行中断，请继续批次"),
+ Mockito.any(IllegalStateException.class)
+ );
}
/**
@@ -657,6 +1051,38 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Assert.assertEquals("invalid_parse_request", invalidRequest);
}
+ /**
+ * 验证 MinerU 不支持文件类型的响应经桥接异常包装后仍能安全归类。
+ *
+ * @throws Exception 反射调用异常
+ */
+ @Test
+ public void resolveParseFailureShouldRecognizeWrappedUnsupportedFileType()
+ throws Exception {
+ KnowledgeDocumentImportTaskAppService service =
+ new KnowledgeDocumentImportTaskAppService();
+ Method codeMethod = KnowledgeDocumentImportTaskAppService.class
+ .getDeclaredMethod("resolveParseFailureCode", Throwable.class);
+ Method messageMethod = KnowledgeDocumentImportTaskAppService.class
+ .getDeclaredMethod("resolveParseFailureMessage", Throwable.class, String.class);
+ codeMethod.setAccessible(true);
+ messageMethod.setAccessible(true);
+ DocumentParseBridgeException error = DocumentParseBridgeException.taskFailed(
+ "异步解析任务失败",
+ new RuntimeException(
+ "MinerU request failed: path=/tasks, status=400, "
+ + "body={\"detail\":\"Unsupported file type: doc\"}"
+ )
+ );
+
+ String code = (String) codeMethod.invoke(service, error);
+ String message = (String) messageMethod.invoke(service, error, code);
+
+ Assert.assertEquals("unsupported_document_source", code);
+ Assert.assertEquals("文档格式或来源不受解析服务支持，请检查文件后继续", message);
+ Assert.assertFalse(message.contains("doc"));
+ }
+
/**
* 验证未识别的系统异常仍会获得稳定错误码并允许后续人工重试。
*
@@ -677,27 +1103,144 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
}
/**
- * 验证未知系统异常和临时源读取失败允许人工重试，确定性输入错误除外。
+ * 验证人工批次继续不受历史尝试次数限制。
*
- * @throws Exception 反射调用异常
+ * @throws Exception 反射调用或依赖注入异常
*/
@Test
- public void parseRetryabilityShouldDefaultToRecoverable()
+ public void retryBatchItemShouldIgnoreHistoricalAttemptCount()
+ throws Exception {
+ KnowledgeDocumentImportTaskAppService service = Mockito.spy(
+ new KnowledgeDocumentImportTaskAppService());
+ DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
+ setField(service, "documentMapper", documentMapper);
+ DocumentImportBatchItem failed = new DocumentImportBatchItem();
+ failed.setId(BigInteger.valueOf(701));
+ failed.setKnowledgeId(BigInteger.valueOf(702));
+ failed.setDocumentId(BigInteger.valueOf(703));
+ failed.setStage(DocumentImportBatchItemStage.INDEX.name());
+ failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
+ failed.setAttemptCount(99);
+ tech.easyflow.ai.entity.Document document =
+ new tech.easyflow.ai.entity.Document();
+ document.setId(failed.getDocumentId());
+ document.setProcessStatus(DocumentProcessStatus.INDEX_FAILED.name());
+ Mockito.when(documentMapper.selectOneById(failed.getDocumentId()))
+ .thenReturn(document);
+ Mockito.doReturn(null).when(service).retryFailedTask(
+ Mockito.any(DocumentImportDtos.TaskRetryRequest.class));
+
+ Method method = KnowledgeDocumentImportTaskAppService.class
+ .getDeclaredMethod("retryBatchItem", DocumentImportBatchItem.class);
+ method.setAccessible(true);
+ method.invoke(service, failed);
+
+ Mockito.verify(service).retryFailedTask(Mockito.argThat(request ->
+ failed.getKnowledgeId().equals(request.getKnowledgeId())
+ && failed.getDocumentId().equals(request.getDocumentId())
+ ));
+ }
+
+ /**
+ * 验证中断时形成的无文档失败项可以重新建档并创建解析任务。
+ *
+ * @throws Exception 反射调用或依赖注入异常
+ */
+ @Test
+ public void retryBatchItemWithoutDocumentShouldRecreateImportTask()
throws Exception {
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
- Method method = KnowledgeDocumentImportTaskAppService.class
- .getDeclaredMethod("isRetryableParseFailure", String.class);
- method.setAccessible(true);
+ DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
+ DocumentImportTaskService taskService =
+ Mockito.mock(DocumentImportTaskService.class);
+ DocumentCollectionService knowledgeService =
+ Mockito.mock(DocumentCollectionService.class);
+ DocumentImportBatchTracker tracker =
+ Mockito.mock(DocumentImportBatchTracker.class);
+ DocumentImportParseTaskProducer producer =
+ Mockito.mock(DocumentImportParseTaskProducer.class);
+ BigInteger itemId = BigInteger.valueOf(711);
+ BigInteger batchId = BigInteger.valueOf(712);
+ BigInteger knowledgeId = BigInteger.valueOf(713);
+ BigInteger taskId = BigInteger.valueOf(714);
+ DocumentImportBatchItem failed = new DocumentImportBatchItem();
+ failed.setId(itemId);
+ failed.setBatchId(batchId);
+ failed.setKnowledgeId(knowledgeId);
+ failed.setFileName("recover.docx");
+ failed.setRelativePath("folder/recover.docx");
+ failed.setFilePath("/stored/recover.docx");
+ failed.setStage(DocumentImportBatchItemStage.UPLOAD.name());
+ failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
+ DocumentImportBatch batch = new DocumentImportBatch();
+ batch.setId(batchId);
+ batch.setKnowledgeId(knowledgeId);
+ batch.setImportMode(DocumentImportMode.AUTO.name());
+ batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
+ DocumentCollection knowledge = new DocumentCollection();
+ knowledge.setId(knowledgeId);
+ Mockito.when(tracker.requireBatch(batchId)).thenReturn(batch);
+ Mockito.when(knowledgeService.getById(knowledgeId))
+ .thenReturn(knowledge);
+ Mockito.when(taskService.save(Mockito.any(DocumentImportTask.class)))
+ .thenAnswer(invocation -> {
+ DocumentImportTask task = invocation.getArgument(0);
+ task.setId(taskId);
+ return true;
+ });
+ setField(service, "documentMapper", documentMapper);
+ setField(service, "documentImportTaskService", taskService);
+ setField(service, "knowledgeService", knowledgeService);
+ setField(service, "documentImportBatchTracker", tracker);
+ setField(service, "parseTaskProducer", producer);
- Assert.assertTrue((Boolean) method.invoke(service, new Object[] {null}));
- Assert.assertTrue((Boolean) method.invoke(
- service, "document_source_unavailable"));
- Assert.assertTrue((Boolean) method.invoke(service, "parse_failed"));
- Assert.assertFalse((Boolean) method.invoke(
- service, "unsupported_document_source"));
- Assert.assertFalse((Boolean) method.invoke(
- service, "invalid_parse_request"));
+ Method method = KnowledgeDocumentImportTaskAppService.class
+ .getDeclaredMethod("retryBatchItem", DocumentImportBatchItem.class);
+ method.setAccessible(true);
+ method.invoke(service, failed);
+
+ ArgumentCaptor documentCaptor =
+ ArgumentCaptor.forClass(tech.easyflow.ai.entity.Document.class);
+ Mockito.verify(documentMapper).insert(documentCaptor.capture());
+ tech.easyflow.ai.entity.Document document = documentCaptor.getValue();
+ Assert.assertNotNull(document.getId());
+ Assert.assertEquals(knowledgeId, document.getCollectionId());
+ Assert.assertEquals(DocumentProcessStatus.PARSING.name(),
+ document.getProcessStatus());
+ ArgumentCaptor taskCaptor =
+ ArgumentCaptor.forClass(DocumentImportTask.class);
+ Mockito.verify(taskService).save(taskCaptor.capture());
+ Assert.assertEquals(taskId, taskCaptor.getValue().getId());
+ Assert.assertEquals(batchId, taskCaptor.getValue().getBatchId());
+ Assert.assertEquals(itemId, taskCaptor.getValue().getBatchItemId());
+ Assert.assertEquals(DocumentImportTaskPhase.PARSE.name(),
+ taskCaptor.getValue().getPhase());
+ Mockito.verify(tracker).bindDocument(itemId, document.getId());
+ Mockito.verify(producer).send(taskId);
+ }
+
+ /**
+ * 验证无文档失败项绑定 SQL 同时校验失败状态、空文档和运行批次。
+ *
+ * @throws Exception Mapper 方法不存在时抛出
+ */
+ @Test
+ public void bindFailedDocumentSqlShouldFenceRecoveryState()
+ throws Exception {
+ Method method = DocumentImportBatchItemMapper.class.getMethod(
+ "bindFailedDocument",
+ BigInteger.class,
+ BigInteger.class,
+ Date.class
+ );
+ Update update = method.getAnnotation(Update.class);
+ String sql = String.join(" ", update.value());
+
+ Assert.assertTrue(sql.contains("item.status='FAILED'"));
+ Assert.assertTrue(sql.contains("item.document_id IS NULL"));
+ Assert.assertTrue(sql.contains("batch.status='RUNNING'"));
+ Assert.assertTrue(sql.contains("item.status='PENDING'"));
}
/**
diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/application-prod.yml b/easyflow-starter/easyflow-starter-all/src/main/resources/application-prod.yml
index 2abbeca0..fd9606c4 100644
--- a/easyflow-starter/easyflow-starter-all/src/main/resources/application-prod.yml
+++ b/easyflow-starter/easyflow-starter-all/src/main/resources/application-prod.yml
@@ -44,6 +44,9 @@ easyflow:
consumer-batch-size: 200
consumer-block-timeout: 2000ms
pending-claim-idle: 60000ms
+ consumer-failure-initial-backoff: 1s
+ consumer-failure-max-backoff: 30s
+ consumer-failure-log-interval: 60s
max-retry: 16
consumer-executor:
core-size: 4
diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml b/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml
index a0939ded..00d3bf26 100644
--- a/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml
+++ b/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml
@@ -128,6 +128,9 @@ easyflow:
consumer-batch-size: 200
consumer-block-timeout: 2000ms
pending-claim-idle: 60000ms
+ consumer-failure-initial-backoff: 1s
+ consumer-failure-max-backoff: 30s
+ consumer-failure-log-interval: 60s
max-retry: 16
consumer-executor:
core-size: 16
diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V55__mysql_sys_api_key_name.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V52__mysql_sys_api_key_name.sql
similarity index 100%
rename from easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V55__mysql_sys_api_key_name.sql
rename to easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V52__mysql_sys_api_key_name.sql
diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V54__mysql_document_import_batch_interruption.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V54__mysql_document_import_batch_interruption.sql
new file mode 100644
index 00000000..ff691f76
--- /dev/null
+++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V54__mysql_document_import_batch_interruption.sql
@@ -0,0 +1,21 @@
+ALTER TABLE `tb_document_import_batch`
+ ADD COLUMN `interrupt_code` varchar(64) NULL DEFAULT NULL COMMENT '中断错误码'
+ AFTER `retryable_failed_count`,
+ ADD COLUMN `interrupt_message` varchar(500) NULL DEFAULT NULL COMMENT '中断原因'
+ AFTER `interrupt_code`,
+ ADD COLUMN `interrupted_at` datetime NULL DEFAULT NULL COMMENT '中断时间'
+ AFTER `interrupt_message`,
+ ADD COLUMN `recovery_pending` tinyint(1) NOT NULL DEFAULT 0
+ COMMENT '是否存在待恢复调度'
+ AFTER `interrupted_at`,
+ ADD COLUMN `recovery_file_keys_json` longtext NULL
+ COMMENT '待恢复文件键 JSON'
+ AFTER `recovery_pending`,
+ ADD COLUMN `recovery_token` varchar(64) NULL
+ COMMENT '恢复调度令牌'
+ AFTER `recovery_file_keys_json`,
+ ADD COLUMN `recovery_lease_until` datetime NULL
+ COMMENT '恢复调度租约到期时间'
+ AFTER `recovery_token`,
+ ADD INDEX `idx_document_import_batch_recovery`
+ (`status`, `recovery_pending`, `modified`, `id`) USING BTREE;
diff --git a/easyflow-ui-admin/app/src/components/page/PageData.test.ts b/easyflow-ui-admin/app/src/components/page/PageData.test.ts
index 1714a5c4..6b10c86c 100644
--- a/easyflow-ui-admin/app/src/components/page/PageData.test.ts
+++ b/easyflow-ui-admin/app/src/components/page/PageData.test.ts
@@ -59,4 +59,35 @@ describe('page data recovery', () => {
expect(get).toHaveBeenCalledTimes(2);
expect(wrapper.text()).not.toContain('数据加载失败，请重试');
});
+
+ it('coalesces repeated reloads while a page request is still running', async () => {
+ let resolveInitialRequest: (value: {
+ data: { records: never[]; totalRow: number };
+ }) => void = () => {};
+ const initialRequest = new Promise<{
+ data: { records: never[]; totalRow: number };
+ }>((resolve) => {
+ resolveInitialRequest = resolve;
+ });
+ const get = vi
+ .fn()
+ .mockReturnValueOnce(initialRequest)
+ .mockResolvedValue({ data: { records: [], totalRow: 0 } });
+ const wrapper = mount(PageData, {
+ global: {
+ directives: { loading: {} },
+ },
+ props: { pageUrl: '/page', requestClient: { get } },
+ });
+
+ const firstReload = (wrapper.vm as any).reload({ silent: true });
+ const secondReload = (wrapper.vm as any).reload({ silent: true });
+
+ expect(get).toHaveBeenCalledTimes(1);
+ resolveInitialRequest({ data: { records: [], totalRow: 0 } });
+ await Promise.all([firstReload, secondReload]);
+ await flushPromises();
+
+ expect(get).toHaveBeenCalledTimes(2);
+ });
});
diff --git a/easyflow-ui-admin/app/src/components/page/PageData.vue b/easyflow-ui-admin/app/src/components/page/PageData.vue
index 0a878471..9953b2fa 100644
--- a/easyflow-ui-admin/app/src/components/page/PageData.vue
+++ b/easyflow-ui-admin/app/src/components/page/PageData.vue
@@ -24,6 +24,15 @@ interface PageDataState {
pageSize: number;
}
+interface PageDataReloadOptions {
+ silent?: boolean;
+}
+
+interface PageDataRequest {
+ silent: boolean;
+ version: number;
+}
+
const props = withDefaults(defineProps(), {
pageSize: 10,
pageSizes: () => [10, 20, 50, 100],
@@ -41,7 +50,9 @@ const pageList = ref([]);
const loading = ref(false);
const loadError = ref();
const queryParams = ref>({ ...props.initialQueryParams });
-let pageRequest = 0;
+let activePageRequest: null | Promise = null;
+let pendingPageRequest: null | PageDataRequest = null;
+let pageRequestVersion = 0;
const pageInfo = reactive({
pageNumber: Math.max(1, Math.trunc(props.initialPageNumber)),
@@ -60,11 +71,7 @@ const doGet = async (params: Record) => {
return { data };
};
-// 获取页面数据
-const getPageList = async () => {
- const request = ++pageRequest;
- loading.value = true;
- loadError.value = undefined;
+const loadPageListOnce = async (request: PageDataRequest) => {
try {
const res = await doGet({
pageNumber: pageInfo.pageNumber,
@@ -72,21 +79,61 @@ const getPageList = async () => {
...props.extraQueryParams,
...queryParams.value,
});
- if (request === pageRequest) {
+ if (request.version === pageRequestVersion) {
pageList.value = res.data?.records || [];
pageInfo.total = res.data?.totalRow || 0;
}
} catch (error) {
- if (request === pageRequest) {
+ if (request.version === pageRequestVersion) {
loadError.value = error;
- pageList.value = [];
- pageInfo.total = 0;
+ if (!request.silent || pageList.value.length === 0) {
+ pageList.value = [];
+ pageInfo.total = 0;
+ }
}
- } finally {
- if (request === pageRequest) loading.value = false;
}
};
+const requestPageList = (silent: boolean) => {
+ const request: PageDataRequest = {
+ silent: silent && pageList.value.length > 0,
+ version: ++pageRequestVersion,
+ };
+ loadError.value = undefined;
+ if (!request.silent) {
+ loading.value = true;
+ }
+ if (activePageRequest) {
+ pendingPageRequest = pendingPageRequest
+ ? {
+ silent: pendingPageRequest.silent && request.silent,
+ version: request.version,
+ }
+ : request;
+ return activePageRequest;
+ }
+
+ const drainPageRequests = async () => {
+ let currentRequest: null | PageDataRequest = request;
+ while (currentRequest) {
+ pendingPageRequest = null;
+ await loadPageListOnce(currentRequest);
+ currentRequest = pendingPageRequest;
+ }
+ };
+ activePageRequest = drainPageRequests().finally(() => {
+ activePageRequest = null;
+ loading.value = false;
+ });
+ return activePageRequest;
+};
+
+// 获取页面数据
+const getPageList = () => requestPageList(false);
+
+const reload = (options: PageDataReloadOptions = {}) =>
+ requestPageList(Boolean(options.silent));
+
// 分页事件处理
const handleSizeChange = (newSize: number) => {
pageInfo.pageSize = newSize;
@@ -138,7 +185,7 @@ const setQuery = (newQueryParams: Record) => {
// 暴露方法给父组件
defineExpose({
getPageState,
- reload: getPageList,
+ reload,
patchRowById,
setQuery,
});
diff --git a/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json b/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json
index 411e12e8..46b68831 100644
--- a/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json
+++ b/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json
@@ -116,6 +116,10 @@
"pendingCount": "Pending",
"skippedCount": "Skipped",
"continueBatch": "Continue",
+ "batchStatusLoadFailed": "Unable to load the automatic import status. Retrying automatically.",
+ "interruptFallback": "Automatic import was interrupted. Resume after the service recovers.",
+ "interruptCode": "Error code",
+ "interruptedAt": "Interrupted at",
"splitOrIndexFailed": "Chunking or indexing failed. Please retry.",
"documentSourceUnavailable": "Unable to read the document file. Please contact the administrator.",
"parseServiceUnavailable": "The document parsing service is temporarily unavailable. Please retry later.",
diff --git a/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json b/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json
index f0fd760f..8771a0d3 100644
--- a/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json
+++ b/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json
@@ -116,6 +116,10 @@
"pendingCount": "等待",
"skippedCount": "跳过",
"continueBatch": "继续",
+ "batchStatusLoadFailed": "自动导入状态暂时无法获取，将自动重试",
+ "interruptFallback": "自动导入发生异常，批次已中断，请确认服务恢复后继续",
+ "interruptCode": "错误标识",
+ "interruptedAt": "中断时间",
"splitOrIndexFailed": "分块或向量化失败，请重试",
"documentSourceUnavailable": "文档文件读取失败，请联系管理员",
"parseServiceUnavailable": "文档解析服务暂不可用，请稍后重试",
diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.test.ts b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.test.ts
index 78533358..052e4b6b 100644
--- a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.test.ts
+++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.test.ts
@@ -153,4 +153,95 @@ describe('documentImportBatchStatus', () => {
expect(wrapper.find('.batch-status__continue').exists()).toBe(false);
wrapper.unmount();
});
+
+ it('中断批次展示异常原因、错误标识和继续入口', async () => {
+ apiMocks.get.mockResolvedValue({
+ data: {
+ ...createBatch('RUNNING', 0),
+ failedCount: 2,
+ interruptCode: 'redis_unavailable',
+ interruptedAt: '2026-08-07T09:25:28+08:00',
+ interruptMessage: '缓存与消息服务异常，自动导入已中断',
+ pendingCount: 0,
+ status: 'INTERRUPTED',
+ },
+ errorCode: 0,
+ });
+
+ const wrapper = mount(DocumentImportBatchStatus, {
+ props: {
+ knowledgeId: 'knowledge-1',
+ manageable: true,
+ },
+ });
+ await flushPromises();
+
+ const alert = wrapper.find('.batch-status__alert');
+ expect(alert.exists()).toBe(true);
+ expect(alert.text()).toContain('缓存与消息服务异常，自动导入已中断');
+ expect(alert.text()).toContain('redis_unavailable');
+ expect(wrapper.find('.batch-status__count').text()).toContain('2 / 2');
+ expect(wrapper.find('.batch-status__continue').exists()).toBe(true);
+ wrapper.unmount();
+ });
+
+ it('状态接口暂时失败时展示错误并自动退避恢复轮询', async () => {
+ vi.useFakeTimers();
+ apiMocks.get
+ .mockRejectedValueOnce(new Error('network unavailable'))
+ .mockResolvedValueOnce({
+ data: createBatch('RUNNING', 1),
+ errorCode: 0,
+ });
+
+ const wrapper = mount(DocumentImportBatchStatus, {
+ props: { knowledgeId: 'knowledge-1' },
+ });
+ await flushPromises();
+
+ expect(wrapper.find('.batch-status--error').exists()).toBe(true);
+
+ await vi.advanceTimersByTimeAsync(3000);
+ await flushPromises();
+
+ expect(apiMocks.get).toHaveBeenCalledTimes(2);
+ expect(wrapper.find('.batch-status--error').exists()).toBe(false);
+ expect(wrapper.find('.batch-status').exists()).toBe(true);
+ wrapper.unmount();
+ });
+
+ it('运行批次轮询失败时保留旧状态、展示错误并自动恢复', async () => {
+ vi.useFakeTimers();
+ apiMocks.get
+ .mockResolvedValueOnce({
+ data: createBatch('RUNNING', 1),
+ errorCode: 0,
+ })
+ .mockRejectedValueOnce(new Error('network unavailable'))
+ .mockResolvedValueOnce({
+ data: createBatch('COMPLETED', 2),
+ errorCode: 0,
+ });
+
+ const wrapper = mount(DocumentImportBatchStatus, {
+ props: { knowledgeId: 'knowledge-1' },
+ });
+ await flushPromises();
+
+ await vi.advanceTimersByTimeAsync(3000);
+ await flushPromises();
+
+ expect(wrapper.find('.batch-status__summary').exists()).toBe(true);
+ expect(wrapper.find('.batch-status__load-error').exists()).toBe(true);
+
+ await vi.advanceTimersByTimeAsync(3000);
+ await flushPromises();
+
+ expect(apiMocks.get).toHaveBeenCalledTimes(3);
+ expect(wrapper.find('.batch-status__load-error').exists()).toBe(false);
+ expect(wrapper.text()).toContain(
+ 'documentCollection.importDoc.batchCompleted',
+ );
+ wrapper.unmount();
+ });
});
diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.vue
index 4dfdbac9..d387329e 100644
--- a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.vue
+++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.vue
@@ -3,15 +3,19 @@ import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue';
import { $t } from '@easyflow/locales';
-import { ElButton, ElProgress } from 'element-plus';
+import { ElAlert, ElButton, ElProgress } from 'element-plus';
import { api } from '#/api/request';
interface BatchStatus {
batchId: string;
+ cancelledCount?: number;
completedCount: number;
failedCount: number;
importMode: 'AUTO' | 'MANUAL';
+ interruptCode?: string;
+ interruptedAt?: string;
+ interruptMessage?: string;
pendingCount: number;
processingCount: number;
progressPercent: number;
@@ -43,9 +47,12 @@ const props = defineProps({
const emit = defineEmits(['continued']);
const batch = ref();
const continuing = ref(false);
+const loadError = ref('');
+const refreshing = ref(false);
let pollTimer: null | ReturnType = null;
let disposed = false;
let refreshGeneration = 0;
+let pollDelayMs = 3000;
const canContinue = computed(
() =>
@@ -55,11 +62,15 @@ const canContinue = computed(
Number(batch.value?.failedCount || 0) > 0,
);
-const processedCount = computed(
- () =>
+const processedCount = computed(() => {
+ const processed =
Number(batch.value?.completedCount || 0) +
- Number(batch.value?.processingCount || 0),
-);
+ Number(batch.value?.processingCount || 0) +
+ Number(batch.value?.failedCount || 0) +
+ Number(batch.value?.skippedCount || 0) +
+ Number(batch.value?.cancelledCount || 0);
+ return Math.min(Number(batch.value?.totalCount || 0), processed);
+});
const allFailed = computed(
() =>
@@ -86,9 +97,35 @@ const statusLabel = computed(() => {
return $t('documentCollection.importDoc.batchRunning');
});
+const interruptMetadata = computed(() => {
+ if (batch.value?.status !== 'INTERRUPTED') {
+ return '';
+ }
+ const details: string[] = [];
+ if (batch.value.interruptCode) {
+ details.push(
+ `${$t('documentCollection.importDoc.interruptCode')} ${
+ batch.value.interruptCode
+ }`,
+ );
+ }
+ if (batch.value.interruptedAt) {
+ const interruptedAt = new Date(batch.value.interruptedAt);
+ details.push(
+ `${$t('documentCollection.importDoc.interruptedAt')} ${
+ Number.isNaN(interruptedAt.getTime())
+ ? batch.value.interruptedAt
+ : interruptedAt.toLocaleString()
+ }`,
+ );
+ }
+ return details.join(' · ');
+});
+
async function refresh(hideCompletedOnRestore = false) {
if (!props.knowledgeId) return;
const currentGeneration = ++refreshGeneration;
+ refreshing.value = true;
try {
const response = await api.get('/api/v1/document/import/batch/current', {
params: { knowledgeId: props.knowledgeId },
@@ -96,14 +133,28 @@ async function refresh(hideCompletedOnRestore = false) {
if (disposed || currentGeneration !== refreshGeneration) {
return;
}
- const restoredBatch =
- response.errorCode === 0 ? response.data || undefined : undefined;
+ if (response.errorCode !== 0) {
+ loadError.value =
+ response.message ||
+ $t('documentCollection.importDoc.batchStatusLoadFailed');
+ return;
+ }
+ loadError.value = '';
+ pollDelayMs = 3000;
+ const restoredBatch = response.data || undefined;
batch.value =
hideCompletedOnRestore && restoredBatch?.status === 'COMPLETED'
? undefined
: restoredBatch;
+ } catch {
+ if (!disposed && currentGeneration === refreshGeneration) {
+ loadError.value = $t(
+ 'documentCollection.importDoc.batchStatusLoadFailed',
+ );
+ }
} finally {
if (!disposed && currentGeneration === refreshGeneration) {
+ refreshing.value = false;
schedulePoll();
}
}
@@ -112,8 +163,16 @@ async function refresh(hideCompletedOnRestore = false) {
function schedulePoll() {
if (pollTimer) clearTimeout(pollTimer);
pollTimer = null;
+ if (disposed) {
+ return;
+ }
+ if (loadError.value) {
+ const retryDelay = pollDelayMs;
+ pollDelayMs = Math.min(pollDelayMs * 2, 30_000);
+ pollTimer = setTimeout(refresh, retryDelay);
+ return;
+ }
if (
- disposed ||
!batch.value ||
batch.value.status === 'COMPLETED' ||
batch.value.status === 'PARTIAL_SUCCEEDED' ||
@@ -160,6 +219,8 @@ watch(
const knowledgeChanged = knowledgeId !== previousKnowledgeId;
if (knowledgeChanged) {
batch.value = undefined;
+ loadError.value = '';
+ pollDelayMs = 3000;
}
refresh(knowledgeChanged);
},
@@ -230,6 +291,46 @@ watch(
>
{{ $t('documentCollection.importDoc.continueBatch') }}
+
+
+
+
+ {{ $t('documentCollection.importDoc.retry') }}
+
+
+
+
+
+
+ {{ $t('documentCollection.importDoc.retry') }}
+
@@ -237,6 +338,7 @@ watch(
.batch-status {
display: flex;
flex: 1 1 360px;
+ flex-wrap: wrap;
gap: 12px;
align-items: center;
min-width: min(360px, 100%);
@@ -301,6 +403,25 @@ watch(
min-width: 48px;
}
+.batch-status__alert {
+ flex: 1 0 100%;
+}
+
+.batch-status__load-error {
+ display: flex;
+ flex: 1 0 100%;
+ gap: 8px;
+ align-items: center;
+}
+
+.batch-status__load-error .batch-status__alert {
+ flex-basis: auto;
+}
+
+.batch-status--error .batch-status__alert {
+ flex-basis: 320px;
+}
+
@media (max-width: 900px) {
.batch-status {
flex-basis: 100%;
diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentTable.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentTable.vue
index e76c34c4..8283b388 100644
--- a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentTable.vue
+++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentTable.vue
@@ -85,7 +85,7 @@ const props = defineProps({
const emits = defineEmits(['viewDoc', 'continueProcess']);
const STREAM_RECONNECT_DELAY = 1500;
-const STREAM_RELOAD_DELAY = 250;
+const STREAM_RELOAD_DELAY = 3000;
const pageDataRef = ref();
const retryingDocumentIds = ref>(new Set());
@@ -254,7 +254,7 @@ const scheduleReload = () => {
}
reloadTimer = setTimeout(() => {
reloadTimer = null;
- pageDataRef.value?.reload?.();
+ pageDataRef.value?.reload?.({ silent: true });
}, STREAM_RELOAD_DELAY);
};
@@ -535,6 +535,7 @@ watch(
if (disposed) {
return;
}
+ clearReloadTimer();
openTaskStatusStream();
},
);