EasyFlow-Tech-Dev/EasyFlow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 完善自动导入异常中断与恢复

Browse Source 
- 自动导入基础设施异常触发批次熔断，保留完整日志并输出安全错误信息

- 增加恢复令牌与租约围栏、无文档失败项重建及消息退避机制

- 前端展示中断状态并在状态请求失败后自动恢复轮询

- 补充批次中断迁移、配置与并发异常路径测试
This commit is contained in:
陈子默
2026-08-10 11:26:00 +08:00
parent 54d85ae460
commit bae7b18977
33 changed files with 3725 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchDtos.java
View File

@@ -262,6 +262,9 @@ public final class DocumentImportBatchDtos {
private Integer cancelledCount;
private Integer retryableFailedCount;
private Integer progressPercent;
private String interruptCode;
private String interruptMessage;
private Date interruptedAt;
private Date startedAt;
private Date finishedAt;
@@ -369,6 +372,30 @@ public final class DocumentImportBatchDtos {
this.progressPercent = progressPercent;
}
public String getInterruptCode() {
return interruptCode;
}
public void setInterruptCode(String interruptCode) {
this.interruptCode = interruptCode;
}
public String getInterruptMessage() {
return interruptMessage;
}
public void setInterruptMessage(String interruptMessage) {
this.interruptMessage = interruptMessage;
}
public Date getInterruptedAt() {
return interruptedAt;
}
public void setInterruptedAt(Date interruptedAt) {
this.interruptedAt = interruptedAt;
}
public Date getStartedAt() {
return startedAt;
}

218
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppService.java
View File

@@ -1,6 +1,7 @@
package tech.easyflow.ai.documentimport.task;
import cn.dev33.satoken.stp.StpUtil;
import com.alibaba.fastjson2.JSON;
import com.mybatisflex.core.query.QueryWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,6 +46,10 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BooleanSupplier;
import java.util.function.LongSupplier;
/**
* 知识库文档批量导入应用服务。
@@ -59,6 +64,9 @@ public class DocumentImportBatchAppService {
private static final Set<String> SUPPORTED_EXTENSIONS =
DocumentImportFormatPolicy.supportedExtensions();
private static final Duration BATCH_MUTATION_LOCK_LEASE = Duration.ofMinutes(30);
private static final Duration RECOVERY_DISPATCH_LEASE = Duration.ofMinutes(2);
private static final Duration RECOVERY_LEASE_RENEW_INTERVAL =
RECOVERY_DISPATCH_LEASE.dividedBy(2);
private final DocumentImportBatchService batchService;
private final DocumentImportBatchItemService itemService;
@@ -69,6 +77,7 @@ public class DocumentImportBatchAppService {
private final DocumentImportBatchItemMapper itemMapper;
private final DocumentMapper documentMapper;
private final RedisLockExecutor redisLockExecutor;
private final DocumentImportBatchCircuitBreaker circuitBreaker;
@Resource(name = "default")
private FileStorageService storageService;
@@ -85,6 +94,7 @@ public class DocumentImportBatchAppService {
* @param itemMapper 批次项 Mapper
* @param documentMapper 文档 Mapper
* @param redisLockExecutor 分布式锁执行器
* @param circuitBreaker 自动导入批次熔断器
*/
public DocumentImportBatchAppService(DocumentImportBatchService batchService,
DocumentImportBatchItemService itemService,
@@ -94,7 +104,8 @@ public class DocumentImportBatchAppService {
DocumentImportBatchMapper batchMapper,
DocumentImportBatchItemMapper itemMapper,
DocumentMapper documentMapper,
RedisLockExecutor redisLockExecutor) {
RedisLockExecutor redisLockExecutor,
DocumentImportBatchCircuitBreaker circuitBreaker) {
this.batchService = batchService;
this.itemService = itemService;
this.batchTracker = batchTracker;
@@ -104,6 +115,7 @@ public class DocumentImportBatchAppService {
this.itemMapper = itemMapper;
this.documentMapper = documentMapper;
this.redisLockExecutor = redisLockExecutor;
this.circuitBreaker = circuitBreaker;
}
/**
@@ -177,6 +189,10 @@ public class DocumentImportBatchAppService {
batch.setSkippedCount(0);
batch.setCancelledCount(0);
batch.setRetryableFailedCount(0);
batch.setRecoveryPending(false);
batch.setRecoveryFileKeysJson(null);
batch.setRecoveryToken(null);
batch.setRecoveryLeaseUntil(null);
batch.setCreated(now);
batch.setModified(now);
batch.setCreatedBy(operatorId);
@@ -446,28 +462,22 @@ public class DocumentImportBatchAppService {
if (failedItems.isEmpty()) {
throw new BusinessException("当前批次没有失败项");
}
Set<String> selectedKeys = failedItems.stream()
.map(DocumentImportBatchItem::getClientFileKey)
.collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new));
Date now = new Date();
DocumentImportBatch update = new DocumentImportBatch();
update.setStatus(DocumentImportBatchStatus.RUNNING.name());
update.setFinishedAt(null);
update.setModified(now);
int claimed = batchMapper.updateByQuery(update,
QueryWrapper.create()
.eq(DocumentImportBatch::getId, batchId)
.in(DocumentImportBatch::getStatus, List.of(
DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name(),
DocumentImportBatchStatus.INTERRUPTED.name()
)));
int claimed = batchMapper.claimContinue(batchId, now);
if (claimed <= 0) {
throw new BusinessException("批次状态已变化，请刷新后重试");
}
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
batch.setFinishedAt(null);
batch.setInterruptCode(null);
batch.setInterruptMessage(null);
batch.setInterruptedAt(null);
batch.setRecoveryPending(true);
batch.setRecoveryFileKeysJson(null);
batch.setRecoveryToken(null);
batch.setRecoveryLeaseUntil(null);
batch.setModified(now);
runAfterCommit(() -> resumeBatchFailures(batchId, selectedKeys));
runAfterCommit(() -> resumeBatchFailures(batchId));
return batchTracker.toStatusResponse(batch);
}
@@ -765,6 +775,7 @@ public class DocumentImportBatchAppService {
&& !DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())) {
throw new BusinessException(409, 40904, "当前任务状态不允许重试");
}
assertNoOtherActiveAutoBatch(batch);
Set<String> requestedFileKeys = fileKeys == null ? Set.of() : fileKeys;
List<DocumentImportBatchItem> retryItems =
listFailedItems(batch.getId(), requestedFileKeys);
@@ -774,20 +785,24 @@ public class DocumentImportBatchAppService {
Date now = new Date();
int expectedGeneration = valueOrZero(batch.getRetryGeneration());
int claimedGeneration = expectedGeneration + 1;
Set<String> selectedKeys = retryItems.stream()
.map(DocumentImportBatchItem::getClientFileKey)
.collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new));
String recoveryFileKeysJson = requestedFileKeys.isEmpty()
? null
: JSON.toJSONString(selectedKeys);
int claimed = batchMapper.claimRetry(
batch.getId(),
caller.getCallerType().name(),
caller.getCallerId(),
expectedGeneration,
recoveryFileKeysJson,
now
);
if (claimed <= 0) {
throw new BusinessException(409, 40903, "任务已被其他请求重试，请刷新任务状态");
}
Set<String> selectedKeys = retryItems.stream()
.map(DocumentImportBatchItem::getClientFileKey)
.collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new));
runAfterCommit(() -> resumeBatchFailures(batch.getId(), selectedKeys));
runAfterCommit(() -> resumeBatchFailures(batch.getId()));
return new DocumentImportBatchRetryResult(
batch.getId(),
DocumentImportBatchStatus.RUNNING.name(),
@@ -1151,15 +1166,164 @@ public class DocumentImportBatchAppService {
* 在批次状态提交后启动选定失败项重试。
*
* @param batchId 批次 ID
* @param fileKeys 指定文件键；为空时重试全部
*/
private void resumeBatchFailures(BigInteger batchId, Set<String> fileKeys) {
try {
taskAppService.retryBatchFailures(batchId, fileKeys);
} catch (RuntimeException error) {
LOG.error("批次失败项恢复调度异常: batchId={}", batchId, error);
batchTracker.markInterrupted(batchId);
private void resumeBatchFailures(BigInteger batchId) {
String recoveryToken = UUID.randomUUID().toString();
Date claimedAt = new Date();
Date leaseUntil = new Date(
claimedAt.getTime() + RECOVERY_DISPATCH_LEASE.toMillis());
if (batchMapper.claimRecoveryPending(
batchId, recoveryToken, leaseUntil, claimedAt) <= 0) {
return;
}
try {
DocumentImportBatch claimedBatch =
batchMapper.selectClaimedRecovery(batchId, recoveryToken);
if (claimedBatch == null) {
LOG.info(
"批次恢复调度令牌已失效，旧持有者停止恢复: "
+ "batchId={}, recoveryToken={}",
batchId,
recoveryToken
);
return;
}
Set<String> fileKeys =
parseRecoveryFileKeys(claimedBatch.getRecoveryFileKeysJson());
BooleanSupplier recoveryLeaseGuard = createRecoveryLeaseGuard(
batchId,
recoveryToken,
leaseUntil.getTime(),
System::currentTimeMillis
);
boolean recoveryCompleted = taskAppService.retryBatchFailures(
batchId,
fileKeys,
recoveryLeaseGuard
);
if (!recoveryCompleted) {
LOG.info(
"批次恢复租约已失效，旧持有者跳过批次收尾: "
+ "batchId={}, recoveryToken={}",
batchId,
recoveryToken
);
return;
}
int finalized = batchMapper.finalizeRecoveryPending(
batchId, recoveryToken, new Date());
if (finalized <= 0) {
LOG.info(
"批次恢复待办已变更，当前实例跳过收尾: "
+ "batchId={}, recoveryToken={}",
batchId,
recoveryToken
);
}
} catch (RuntimeException error) {
try {
if (!circuitBreaker.interruptRecoveryBatch(
batchId, recoveryToken, error)) {
LOG.info(
"批次恢复异常发生时令牌已失效，跳过旧持有者熔断: "
+ "batchId={}, recoveryToken={}",
batchId,
recoveryToken
);
}
} catch (RuntimeException interruptError) {
interruptError.addSuppressed(error);
LOG.error(
"批次失败项恢复调度异常且强制中断失败: batchId={}",
batchId,
interruptError
);
throw interruptError;
}
}
}
/**
* 创建按时间续期的恢复租约检查器。
*
* <p>检查器在租约剩余一半时执行带令牌 CAS 的续租。续租未命中
* 表示当前实例已失去恢复权，调用方应停止后续文件重试。</p>
*
* @param batchId 批次 ID
* @param recoveryToken 恢复调度令牌
* @param leaseUntilMillis 当前租约到期毫秒时间
* @param clock 毫秒时钟
* @return 恢复租约检查器
*/
BooleanSupplier createRecoveryLeaseGuard(BigInteger batchId,
String recoveryToken,
long leaseUntilMillis,
LongSupplier clock) {
long renewIntervalMillis =
RECOVERY_LEASE_RENEW_INTERVAL.toMillis();
AtomicLong renewAfter = new AtomicLong(
Math.max(
clock.getAsLong(),
leaseUntilMillis - renewIntervalMillis
)
);
return () -> {
long nowMillis = clock.getAsLong();
if (nowMillis < renewAfter.get()) {
return true;
}
Date now = new Date(nowMillis);
Date renewedLeaseUntil = new Date(
nowMillis + RECOVERY_DISPATCH_LEASE.toMillis()
);
int renewed = batchMapper.renewRecoveryPendingLease(
batchId,
recoveryToken,
renewedLeaseUntil,
now
);
if (renewed <= 0) {
return false;
}
renewAfter.set(nowMillis + renewIntervalMillis);
return true;
};
}
/**
* 重放事务已经提交但进程尚未完成的批次恢复调度。
*
* <p>恢复待办持久化在批次表中；多实例重复扫描由文件项与批次
* 的行锁门禁保证幂等。</p>
*
* @return 本轮扫描批次数
*/
public int recoverPendingBatchRetries() {
int limit = Math.max(1, properties.getPendingDispatchBatchSize());
List<DocumentImportBatch> batches =
batchMapper.selectRecoveryPendingBatches(new Date(), limit);
for (DocumentImportBatch batch : batches) {
// 中断状态写入失败属于系统性故障，交由外层调度冷却后重试。
resumeBatchFailures(batch.getId());
}
return batches.size();
}
/**
* 解析持久化的恢复文件选择。
*
* @param recoveryFileKeysJson 文件键 JSON；为空表示恢复全部失败项
* @return 本轮恢复文件键
*/
private Set<String> parseRecoveryFileKeys(String recoveryFileKeysJson) {
if (!StringUtil.hasText(recoveryFileKeysJson)) {
return Set.of();
}
List<String> fileKeys =
JSON.parseArray(recoveryFileKeysJson, String.class);
return fileKeys == null
? Set.of()
: new LinkedHashSet<String>(fileKeys);
}
/**

440
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreaker.java Normal file
View File

@@ -0,0 +1,440 @@
package tech.easyflow.ai.documentimport.task;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.dao.QueryTimeoutException;
import org.springframework.dao.TransientDataAccessResourceException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.entity.DocumentImportTask;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.enums.DocumentImportMode;
import tech.easyflow.ai.enums.DocumentImportTaskStatus;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
import java.math.BigInteger;
import java.sql.SQLException;
import java.sql.SQLRecoverableException;
import java.sql.SQLTimeoutException;
import java.sql.SQLTransientConnectionException;
import java.util.Date;
import java.util.concurrent.RejectedExecutionException;
/**
* 知识库自动导入批次熔断器。
*
* <p>消费者或调度器发生未被业务流程收口的基础设施异常时，通过
* 数据库 CAS 立即停止整个自动导入批次，撤销活跃任务执行令牌，
* 并把未完成文件保留为可恢复失败。</p>
*/
@Service
public class DocumentImportBatchCircuitBreaker {
private static final Logger LOG =
LoggerFactory.getLogger(DocumentImportBatchCircuitBreaker.class);
private static final String REDIS_UNAVAILABLE = "redis_unavailable";
private static final String DATABASE_UNAVAILABLE = "database_unavailable";
private static final String WORKER_OVERLOADED = "worker_overloaded";
private static final String INFRASTRUCTURE_FAILURE =
"document_import_infrastructure_failure";
private final DocumentImportBatchMapper batchMapper;
private final DocumentImportBatchItemMapper itemMapper;
private final DocumentImportTaskMapper taskMapper;
/**
* 创建自动导入批次熔断器。
*
* @param batchMapper 批次 Mapper
* @param itemMapper 批次项 Mapper
* @param taskMapper 导入任务 Mapper
*/
public DocumentImportBatchCircuitBreaker(
DocumentImportBatchMapper batchMapper,
DocumentImportBatchItemMapper itemMapper,
DocumentImportTaskMapper taskMapper) {
this.batchMapper = batchMapper;
this.itemMapper = itemMapper;
this.taskMapper = taskMapper;
}
/**
* 根据异常任务强制中断所属自动导入批次。
*
* <p>该方法使用独立事务，保证调用方业务事务已经回滚时仍能写入
* 可恢复终态。技术异常通过完整堆栈写入后端日志；数据库仅保存
* 稳定错误码和面向用户的安全摘要。</p>
*
* @param taskId 触发异常的任务 ID
* @param error 原始异常
* @return 批次已经中断或已经结束时返回 {@code true}；无自动批次
* 归属时返回 {@code false}
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean interruptTaskBatch(BigInteger taskId, Throwable error) {
DocumentImportTask task =
taskId == null ? null : taskMapper.selectOneById(taskId);
if (task == null) {
LOG.warn("忽略无法关联任务的自动导入熔断请求: taskId={}", taskId);
return true;
}
if (task.getBatchId() == null) {
return false;
}
if (!isActiveTask(task)) {
LOG.info(
"忽略已结束任务的迟到熔断请求: taskId={}, batchId={}, status={}",
task.getId(), task.getBatchId(), task.getStatus());
return true;
}
return interruptBatch(
task.getBatchId(),
resolveReason(error),
task.getId(),
task.getPhase(),
error
);
}
/**
* 根据系统异常强制中断指定自动导入批次。
*
* @param batchId 批次 ID
* @param error 原始异常
* @return 批次已经中断、结束或不存在时返回 {@code true}
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean interruptBatch(BigInteger batchId, Throwable error) {
return interruptBatch(
batchId,
resolveReason(error),
null,
null,
error
);
}
/**
* 仅在恢复令牌仍由当前实例持有时中断指定批次。
*
* @param batchId 批次 ID
* @param recoveryToken 当前恢复调度令牌
* @param error 原始异常
* @return 批次已停止时返回 {@code true}；令牌失效时返回 {@code false}
* @throws IllegalArgumentException 恢复令牌为空时抛出
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean interruptRecoveryBatch(BigInteger batchId,
String recoveryToken,
Throwable error) {
if (recoveryToken == null || recoveryToken.isBlank()) {
throw new IllegalArgumentException("恢复调度令牌不能为空");
}
return interruptBatch(
batchId,
resolveReason(error),
null,
null,
error,
recoveryToken
);
}
/**
* 使用指定稳定原因强制中断自动导入批次。
*
* @param batchId 批次 ID
* @param interruptCode 稳定中断码
* @param interruptMessage 用户可见中断原因
* @param error 原始异常
* @return 批次已经中断、结束或不存在时返回 {@code true}
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean interruptBatch(BigInteger batchId,
String interruptCode,
String interruptMessage,
Throwable error) {
return interruptBatch(
batchId,
new InterruptionReason(interruptCode, interruptMessage),
null,
null,
error
);
}
/**
* 加入调用方事务并使用指定原因中断批次。
*
* <p>供超时回收等已经持有任务行锁的事务使用，避免开启新事务
* 等待调用方自身尚未提交的行锁。</p>
*
* @param batchId 批次 ID
* @param interruptCode 稳定中断码
* @param interruptMessage 用户可见中断原因
* @param error 原始异常
* @return 批次是否已经停止运行
*/
@Transactional(propagation = Propagation.MANDATORY)
public boolean interruptBatchInCurrentTransaction(
BigInteger batchId,
String interruptCode,
String interruptMessage,
Throwable error) {
return interruptBatch(
batchId,
new InterruptionReason(interruptCode, interruptMessage),
null,
null,
error
);
}
/**
* 原子中断批次并统一收口关联任务、文档和文件项。
*
* @param batchId 批次 ID
* @param reason 稳定中断原因
* @param taskId 触发任务 ID，可为空
* @param phase 触发任务阶段，可为空
* @param error 原始异常
* @return 批次是否已经停止运行
*/
private boolean interruptBatch(BigInteger batchId,
InterruptionReason reason,
BigInteger taskId,
String phase,
Throwable error) {
return interruptBatch(
batchId, reason, taskId, phase, error, null);
}
/**
* 按调用来源围栏原子中断批次并收口关联状态。
*
* @param batchId 批次 ID
* @param reason 稳定中断原因
* @param taskId 触发任务 ID，可为空
* @param phase 触发任务阶段，可为空
* @param error 原始异常
* @param recoveryToken 恢复调度令牌，可为空
* @return 批次是否已经停止运行；围栏失效时返回 {@code false}
*/
private boolean interruptBatch(BigInteger batchId,
InterruptionReason reason,
BigInteger taskId,
String phase,
Throwable error,
String recoveryToken) {
if (batchId == null) {
return false;
}
DocumentImportBatch batch = batchMapper.selectOneById(batchId);
if (batch == null) {
LOG.warn(
"忽略无法关联批次的自动导入熔断请求: taskId={}, batchId={}",
taskId, batchId);
return true;
}
if (!DocumentImportMode.AUTO.name().equals(batch.getImportMode())) {
return false;
}
if (!DocumentImportBatchStatus.RUNNING.name().equals(batch.getStatus())) {
return true;
}
Date now = new Date();
int interrupted;
if (recoveryToken != null) {
interrupted = batchMapper.interruptOwnedRecoveryBatch(
batch.getId(), recoveryToken, reason.code, reason.message, now);
} else if (taskId != null) {
interrupted = batchMapper.interruptRunningBatchForActiveTask(
batch.getId(), taskId, reason.code, reason.message, now);
} else {
interrupted = batchMapper.interruptRunningBatch(
batch.getId(), reason.code, reason.message, now);
}
if (interrupted <= 0) {
DocumentImportBatch current = batchMapper.selectOneById(batch.getId());
if (current == null
|| !DocumentImportBatchStatus.RUNNING.name()
.equals(current.getStatus())) {
return true;
}
if (taskId != null) {
DocumentImportTask currentTask = taskMapper.selectOneById(taskId);
return currentTask == null || !isActiveTask(currentTask);
}
return false;
}
// 先同步文档列表，再撤销任务令牌；两个更新处于同一事务中。
int interruptedDocuments = taskMapper.interruptActiveDocuments(
batch.getId(), reason.message, reason.code, now, BigInteger.ZERO);
int interruptedTasks = taskMapper.interruptActiveTasks(
batch.getId(), reason.message, reason.code, now, BigInteger.ZERO);
int interruptedItems = itemMapper.interruptActiveItems(
batch.getId(), reason.message, reason.code, now);
batchMapper.refreshCountersFromItems(batch.getId(), now);
LOG.error(
"知识库自动导入批次已强制中断: batchId={}, taskId={}, phase={}, "
+ "interruptCode={}, interruptedTasks={}, interruptedItems={}, "
+ "interruptedDocuments={}",
batch.getId(),
taskId,
phase,
reason.code,
interruptedTasks,
interruptedItems,
interruptedDocuments,
error);
return true;
}
/**
* 将异常链归类为稳定中断原因。
*
* @param error 原始异常
* @return 稳定错误码与用户可见摘要
*/
private InterruptionReason resolveReason(Throwable error) {
if (containsRedisFailure(error)) {
return new InterruptionReason(
REDIS_UNAVAILABLE,
"缓存与消息服务异常，自动导入已中断，请确认服务恢复后继续"
);
}
if (containsWorkerOverload(error)) {
return new InterruptionReason(
WORKER_OVERLOADED,
"导入任务处理资源已满，批次已中断，请稍后继续"
);
}
if (containsDataAccessFailure(error)) {
return new InterruptionReason(
DATABASE_UNAVAILABLE,
"数据库服务异常，自动导入已中断，请确认服务恢复后继续"
);
}
return new InterruptionReason(
INFRASTRUCTURE_FAILURE,
"自动导入发生系统异常，批次已中断，请联系管理员排查后继续"
);
}
/**
* 判断异常链是否来自 Redis/Lettuce。
*
* @param error 原始异常
* @return 是否是 Redis 故障
*/
private boolean containsRedisFailure(Throwable error) {
Throwable current = error;
while (current != null) {
String className = current.getClass().getName();
if (className.startsWith("io.lettuce.")
|| className.contains(".redis.")
|| className.contains("RedisCommand")) {
return true;
}
current = current.getCause();
}
return false;
}
/**
* 判断异常链是否包含线程池拒绝。
*
* @param error 原始异常
* @return 是否是任务资源过载
*/
private boolean containsWorkerOverload(Throwable error) {
Throwable current = error;
while (current != null) {
if (current instanceof RejectedExecutionException
|| current.getClass().getSimpleName()
.contains("TaskRejectedException")) {
return true;
}
current = current.getCause();
}
return false;
}
/**
* 判断异常链是否包含数据库连接、资源或超时异常。
*
* @param error 原始异常
* @return 是否是数据库故障
*/
private boolean containsDataAccessFailure(Throwable error) {
Throwable current = error;
while (current != null) {
if (current instanceof DataAccessResourceFailureException
|| current instanceof TransientDataAccessResourceException
|| current instanceof QueryTimeoutException
|| current instanceof SQLTransientConnectionException
|| current instanceof SQLRecoverableException
|| current instanceof SQLTimeoutException
|| isSqlConnectionFailure(current)
|| "org.hibernate.exception.JDBCConnectionException"
.equals(current.getClass().getName())) {
return true;
}
current = current.getCause();
}
return false;
}
/**
* 根据 SQLState 判断是否属于数据库连接类故障。
*
* @param error 当前异常
* @return SQLState 使用连接异常类别 08 时返回 {@code true}
*/
private boolean isSqlConnectionFailure(Throwable error) {
if (!(error instanceof SQLException sqlError)) {
return false;
}
String sqlState = sqlError.getSQLState();
return sqlState != null && sqlState.startsWith("08");
}
/**
* 判断任务是否仍可能代表当前批次的活跃执行。
*
* @param task 导入任务
* @return 待处理或运行中返回 {@code true}
*/
private boolean isActiveTask(DocumentImportTask task) {
return task != null
&& (DocumentImportTaskStatus.PENDING.name().equals(task.getStatus())
|| DocumentImportTaskStatus.RUNNING.name().equals(task.getStatus()));
}
/**
* 稳定中断码与安全展示文案。
*/
private static final class InterruptionReason {
private final String code;
private final String message;
/**
* 创建中断原因。
*
* @param code 稳定错误码
* @param message 用户可见摘要
*/
private InterruptionReason(String code, String message) {
this.code = code;
this.message = message;
}
}
}

43
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java
View File

@@ -235,10 +235,11 @@ public class DocumentImportBatchTracker {
}
/**
* 将批次文件项绑定到创建后的文档。
* 将已上传或缺少文档的失败项绑定到创建后的文档。
*
* @param itemId 文件项 ID
* @param documentId 文档 ID
* @throws BusinessException 文件项状态变化或绑定失败时抛出
*/
@org.springframework.transaction.annotation.Transactional
public void bindDocument(BigInteger itemId, BigInteger documentId) {
@@ -248,12 +249,22 @@ public class DocumentImportBatchTracker {
return;
}
Date now = new Date();
int updated = itemMapper.bindDocument(itemId, documentId, now);
boolean recoveringFailedItem =
DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
&& item.getDocumentId() == null;
int updated = recoveringFailedItem
? itemMapper.bindFailedDocument(itemId, documentId, now)
: itemMapper.bindDocument(itemId, documentId, now);
if (updated <= 0) {
throw new BusinessException("导入文件状态已变化，请刷新后重试");
}
batchMapper.adjustCounters(item.getBatchId(),
0, 0, 0, 0, -1, 0, 0, 0, now);
if (recoveringFailedItem) {
// 防御性恢复属于极低频路径，按文件项真实状态汇总可避免手工计数漂移。
batchMapper.refreshCountersFromItems(item.getBatchId(), now);
} else {
batchMapper.adjustCounters(item.getBatchId(),
0, 0, 0, 0, -1, 0, 0, 0, now);
}
refreshBatch(item.getBatchId());
}
@@ -369,12 +380,33 @@ public class DocumentImportBatchTracker {
* @param batchId 批次 ID
*/
public void markInterrupted(BigInteger batchId) {
markInterrupted(
batchId,
"execution_interrupted",
"任务执行中断，请确认服务恢复后继续批次"
);
}
/**
* 将批次标记为已中断并记录可展示的稳定原因。
*
* @param batchId 批次 ID
* @param interruptCode 稳定中断码
* @param interruptMessage 用户可见中断原因
*/
public void markInterrupted(BigInteger batchId,
String interruptCode,
String interruptMessage) {
if (batchId == null) {
return;
}
Date now = new Date();
DocumentImportBatch update = new DocumentImportBatch();
update.setStatus(DocumentImportBatchStatus.INTERRUPTED.name());
update.setInterruptCode(interruptCode);
update.setInterruptMessage(interruptMessage);
update.setInterruptedAt(now);
update.setRecoveryPending(false);
update.setModified(now);
batchMapper.updateByQuery(update,
QueryWrapper.create()
@@ -409,6 +441,9 @@ public class DocumentImportBatchTracker {
+ valueOrZero(batch.getSkippedCount())
+ valueOrZero(batch.getCancelledCount());
response.setProgressPercent(Math.min(100, terminalCount * 100 / total));
response.setInterruptCode(batch.getInterruptCode());
response.setInterruptMessage(batch.getInterruptMessage());
response.setInterruptedAt(batch.getInterruptedAt());
response.setStartedAt(batch.getStartedAt());
response.setFinishedAt(batch.getFinishedAt());
return response;

48
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportIndexTaskConsumer.java
View File

@@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.common.mq.config.MQProperties;
import tech.easyflow.common.mq.core.MQConsumerHandler;
import tech.easyflow.common.mq.core.MQDeferException;
import tech.easyflow.common.mq.core.MQMessage;
import tech.easyflow.common.mq.core.MQSubscription;
@@ -23,11 +24,21 @@ public class DocumentImportIndexTaskConsumer implements MQConsumerHandler {
private static final Logger LOG = LoggerFactory.getLogger(DocumentImportIndexTaskConsumer.class);
private final KnowledgeDocumentImportTaskAppService appService;
private final DocumentImportBatchCircuitBreaker circuitBreaker;
private final MQProperties mqProperties;
/**
* 创建文档向量化任务消费者。
*
* @param appService 文档导入任务应用服务
* @param circuitBreaker 自动导入批次熔断器
* @param mqProperties MQ 配置
*/
public DocumentImportIndexTaskConsumer(KnowledgeDocumentImportTaskAppService appService,
DocumentImportBatchCircuitBreaker circuitBreaker,
MQProperties mqProperties) {
this.appService = appService;
this.circuitBreaker = circuitBreaker;
this.mqProperties = mqProperties;
}
@@ -37,6 +48,7 @@ public class DocumentImportIndexTaskConsumer implements MQConsumerHandler {
subscription.setTopic(DocumentImportTaskMqConstants.INDEX_TOPIC);
subscription.setConsumerGroup(DocumentImportTaskMqConstants.INDEX_GROUP);
subscription.setShardCount(resolveShardCount());
subscription.setBatchEnabled(false);
return subscription;
}
@@ -58,13 +70,43 @@ public class DocumentImportIndexTaskConsumer implements MQConsumerHandler {
LOG.info("文档向量化消费者处理完成: taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(), message.getStreamMessageId());
} catch (Exception exception) {
LOG.error("文档向量化消费者处理失败: taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(), message.getStreamMessageId(), exception);
throw exception;
handleInfrastructureFailure(event, message, exception);
}
}
}
/**
* 记录完整异常并中断自动导入批次；中断状态暂时无法持久化时保留消息 pending。
*
* @param event 任务事件
* @param message MQ 消息
* @param error 原始异常
*/
private void handleInfrastructureFailure(DocumentImportTaskMessage event,
MQMessage message,
Exception error) {
try {
if (circuitBreaker.interruptTaskBatch(event.getTaskId(), error)) {
return;
}
} catch (Exception interruptError) {
interruptError.addSuppressed(error);
LOG.error(
"文档向量化异常且批次中断状态写入失败，消息将暂缓确认: "
+ "taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(),
message.getStreamMessageId(), interruptError);
throw new MQDeferException(
"文档向量化批次中断状态暂时无法持久化", interruptError);
}
LOG.error(
"文档向量化任务发生基础设施异常，消息将暂缓确认: "
+ "taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(),
message.getStreamMessageId(), error);
throw new MQDeferException("文档向量化任务暂时无法处理", error);
}
/**
* 向量化消费者需覆盖生产端的所有分片，避免消息落入未订阅分片。
*

48
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportParseTaskConsumer.java
View File

@@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.common.mq.config.MQProperties;
import tech.easyflow.common.mq.core.MQConsumerHandler;
import tech.easyflow.common.mq.core.MQDeferException;
import tech.easyflow.common.mq.core.MQMessage;
import tech.easyflow.common.mq.core.MQSubscription;
@@ -23,11 +24,21 @@ public class DocumentImportParseTaskConsumer implements MQConsumerHandler {
private static final Logger LOG = LoggerFactory.getLogger(DocumentImportParseTaskConsumer.class);
private final KnowledgeDocumentImportTaskAppService appService;
private final DocumentImportBatchCircuitBreaker circuitBreaker;
private final MQProperties mqProperties;
/**
* 创建文档解析任务消费者。
*
* @param appService 文档导入任务应用服务
* @param circuitBreaker 自动导入批次熔断器
* @param mqProperties MQ 配置
*/
public DocumentImportParseTaskConsumer(KnowledgeDocumentImportTaskAppService appService,
DocumentImportBatchCircuitBreaker circuitBreaker,
MQProperties mqProperties) {
this.appService = appService;
this.circuitBreaker = circuitBreaker;
this.mqProperties = mqProperties;
}
@@ -37,6 +48,7 @@ public class DocumentImportParseTaskConsumer implements MQConsumerHandler {
subscription.setTopic(DocumentImportTaskMqConstants.PARSE_TOPIC);
subscription.setConsumerGroup(DocumentImportTaskMqConstants.PARSE_GROUP);
subscription.setShardCount(resolveShardCount());
subscription.setBatchEnabled(false);
return subscription;
}
@@ -58,13 +70,43 @@ public class DocumentImportParseTaskConsumer implements MQConsumerHandler {
LOG.info("文档解析消费者处理完成: taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(), message.getStreamMessageId());
} catch (Exception exception) {
LOG.error("文档解析消费者处理失败: taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(), message.getStreamMessageId(), exception);
throw exception;
handleInfrastructureFailure(event, message, exception);
}
}
}
/**
* 记录完整异常并中断自动导入批次；中断状态暂时无法持久化时保留消息 pending。
*
* @param event 任务事件
* @param message MQ 消息
* @param error 原始异常
*/
private void handleInfrastructureFailure(DocumentImportTaskMessage event,
MQMessage message,
Exception error) {
try {
if (circuitBreaker.interruptTaskBatch(event.getTaskId(), error)) {
return;
}
} catch (Exception interruptError) {
interruptError.addSuppressed(error);
LOG.error(
"文档解析异常且批次中断状态写入失败，消息将暂缓确认: "
+ "taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(),
message.getStreamMessageId(), interruptError);
throw new MQDeferException(
"文档解析批次中断状态暂时无法持久化", interruptError);
}
LOG.error(
"文档解析任务发生基础设施异常，消息将暂缓确认: "
+ "taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(),
message.getStreamMessageId(), error);
throw new MQDeferException("文档解析任务暂时无法处理", error);
}
/**
* 解析消费者需覆盖生产端的所有分片，避免消息落入未订阅分片。
*

54
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitor.java
View File

@@ -1,5 +1,7 @@
package tech.easyflow.ai.documentimport.task;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tech.easyflow.common.cache.DistributedScheduledLock;
@@ -13,15 +15,28 @@ import tech.easyflow.common.cache.DistributedScheduledLock;
@Component
public class DocumentImportPendingTaskMonitor {
private static final Logger LOG =
LoggerFactory.getLogger(DocumentImportPendingTaskMonitor.class);
private static final long FAILURE_COOLDOWN_MILLIS = 30_000L;
private static final long FULL_ERROR_LOG_INTERVAL_MILLIS = 60_000L;
private final KnowledgeDocumentImportTaskAppService appService;
private final DocumentImportBatchAppService batchAppService;
private long retryAfter;
private long nextFullErrorLogAt;
private int suppressedFailures;
/**
* 创建文档导入恢复调度器。
*
* @param appService 文档导入任务服务
* @param batchAppService 文档导入批次服务
*/
public DocumentImportPendingTaskMonitor(KnowledgeDocumentImportTaskAppService appService) {
public DocumentImportPendingTaskMonitor(
KnowledgeDocumentImportTaskAppService appService,
DocumentImportBatchAppService batchAppService) {
this.appService = appService;
this.batchAppService = batchAppService;
}
/**
@@ -31,12 +46,39 @@ public class DocumentImportPendingTaskMonitor {
fixedDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}",
initialDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}"
)
@DistributedScheduledLock(
key = "easyflow:schedule:document-import:pending-dispatch",
leaseSeconds = 30L
)
public void dispatchPendingTasks() {
appService.dispatchPendingTasks();
long now = System.currentTimeMillis();
if (now < retryAfter) {
return;
}
try {
batchAppService.recoverPendingBatchRetries();
appService.dispatchPendingTasks();
if (suppressedFailures > 0) {
LOG.info(
"文档导入待处理调度已恢复: suppressedFailures={}",
suppressedFailures
);
}
retryAfter = 0L;
nextFullErrorLogAt = 0L;
suppressedFailures = 0;
} catch (RuntimeException error) {
retryAfter = now + FAILURE_COOLDOWN_MILLIS;
if (now >= nextFullErrorLogAt) {
LOG.error(
"文档导入待处理调度异常，30 秒后重试: "
+ "suppressedFailures={}",
suppressedFailures,
error
);
nextFullErrorLogAt =
now + FULL_ERROR_LOG_INTERVAL_MILLIS;
suppressedFailures = 0;
} else {
suppressedFailures++;
}
}
}
/**

43
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskConsumer.java
View File

@@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.common.mq.config.MQProperties;
import tech.easyflow.common.mq.core.MQConsumerHandler;
import tech.easyflow.common.mq.core.MQDeferException;
import tech.easyflow.common.mq.core.MQMessage;
import tech.easyflow.common.mq.core.MQSubscription;
@@ -24,18 +25,22 @@ public class DocumentImportSplitTaskConsumer implements MQConsumerHandler {
LoggerFactory.getLogger(DocumentImportSplitTaskConsumer.class);
private final KnowledgeDocumentImportTaskAppService appService;
private final DocumentImportBatchCircuitBreaker circuitBreaker;
private final MQProperties mqProperties;
/**
* 创建分块任务消费者。
*
* @param appService 文档导入应用服务
* @param circuitBreaker 自动导入批次熔断器
* @param mqProperties MQ 配置
*/
public DocumentImportSplitTaskConsumer(
KnowledgeDocumentImportTaskAppService appService,
DocumentImportBatchCircuitBreaker circuitBreaker,
MQProperties mqProperties) {
this.appService = appService;
this.circuitBreaker = circuitBreaker;
this.mqProperties = mqProperties;
}
@@ -50,6 +55,7 @@ public class DocumentImportSplitTaskConsumer implements MQConsumerHandler {
subscription.setTopic(DocumentImportTaskMqConstants.SPLIT_TOPIC);
subscription.setConsumerGroup(DocumentImportTaskMqConstants.SPLIT_GROUP);
subscription.setShardCount(resolveShardCount());
subscription.setBatchEnabled(false);
return subscription;
}
@@ -74,14 +80,43 @@ public class DocumentImportSplitTaskConsumer implements MQConsumerHandler {
try {
appService.handleSplitTask(event.getTaskId());
} catch (Exception error) {
LOG.error("文档分块消费者处理失败: taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(),
message.getStreamMessageId(), error);
throw error;
handleInfrastructureFailure(event, message, error);
}
}
}
/**
* 记录完整异常并中断自动导入批次；中断状态暂时无法持久化时保留消息 pending。
*
* @param event 任务事件
* @param message MQ 消息
* @param error 原始异常
*/
private void handleInfrastructureFailure(DocumentImportTaskMessage event,
MQMessage message,
Exception error) {
try {
if (circuitBreaker.interruptTaskBatch(event.getTaskId(), error)) {
return;
}
} catch (Exception interruptError) {
interruptError.addSuppressed(error);
LOG.error(
"文档分块异常且批次中断状态写入失败，消息将暂缓确认: "
+ "taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(),
message.getStreamMessageId(), interruptError);
throw new MQDeferException(
"文档分块批次中断状态暂时无法持久化", interruptError);
}
LOG.error(
"文档分块任务发生基础设施异常，消息将暂缓确认: "
+ "taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(),
message.getStreamMessageId(), error);
throw new MQDeferException("文档分块任务暂时无法处理", error);
}
/**
* 获取当前 Redis Stream 分片数。
*

494
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java
View File

@@ -59,6 +59,7 @@ import tech.easyflow.ai.enums.DocumentImportTaskStatus;
import tech.easyflow.ai.enums.DocumentProcessStatus;
import tech.easyflow.ai.entity.Model;
import tech.easyflow.ai.mapper.DocumentChunkMapper;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.ai.service.DocumentChunkService;
@@ -101,6 +102,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.function.BooleanSupplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -163,6 +165,9 @@ public class KnowledgeDocumentImportTaskAppService {
@Resource
private DocumentImportBatchItemService documentImportBatchItemService;
@Resource
private DocumentImportBatchItemMapper documentImportBatchItemMapper;
@Autowired
@Lazy
private DocumentService documentService;
@@ -185,6 +190,9 @@ public class KnowledgeDocumentImportTaskAppService {
@Resource
private DocumentImportChunkSnapshotService documentImportChunkSnapshotService;
@Resource
private DocumentImportBatchCircuitBreaker documentImportBatchCircuitBreaker;
@Resource
private CsvTableSnapshotService csvTableSnapshotService;
@@ -297,36 +305,7 @@ public class KnowledgeDocumentImportTaskAppService {
|| !StringUtil.hasText(item.getFilePath())) {
throw new BusinessException("批次包含未上传完成的文件");
}
String fileExt = normalizeFileExtension(item.getFileName(), item.getFilePath());
assertSupportedImportFile(fileExt);
Date now = new Date();
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
document.setId(generateId(document));
document.setCollectionId(knowledge.getId());
document.setDocumentPath(item.getFilePath());
document.setTitle(item.getFileName());
document.setDocumentType(fileExt);
document.setCreated(now);
document.setModified(now);
document.setCreatedBy(resolveOperatorId());
document.setModifiedBy(resolveOperatorId());
document.setProcessStatus(DocumentProcessStatus.PARSING.name());
document.setTotalChunks(0);
document.setCompletedChunks(0);
document.setFailedChunks(0);
document.setProgressPercent(0);
document.setTaskModifiedAt(now);
Map<String, Object> options = buildInitialOptions(fileExt);
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_MODE, batch.getImportMode());
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID, batch.getId().toString());
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID, item.getId().toString());
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_RELATIVE_PATH, item.getRelativePath());
document.setOptions(options);
documentMapper.insert(document);
DocumentImportTask task = createTask(document, DocumentImportTaskPhase.PARSE,
buildDocumentPayload(document));
documentImportBatchTracker.bindDocument(item.getId(), document.getId());
DocumentImportTask task = createBatchImportTask(batch, item, knowledge);
taskIds.add(task.getId());
}
// 启动接口返回后列表会统一刷新；批量建档阶段不逐文件推送，避免提交后形成 SSE 风暴。
@@ -339,21 +318,105 @@ public class KnowledgeDocumentImportTaskAppService {
}
/**
* 重试批次中所有失败文件，单个文件重试失败不会中止其他文件。
* 为一个已上传文件或缺少文档的失败项创建文档与解析任务。
*
* @param batch 导入批次
* @param item 批次文件项
* @param knowledge 目标知识库
* @return 新创建的解析任务
* @throws BusinessException 文件状态、路径或格式不允许建档时抛出
*/
private DocumentImportTask createBatchImportTask(
DocumentImportBatch batch,
DocumentImportBatchItem item,
DocumentCollection knowledge) {
boolean uploaded = DocumentImportBatchItemStatus.UPLOADED.name()
.equals(item.getStatus());
boolean failedWithoutDocument =
DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
&& item.getDocumentId() == null;
if ((!uploaded && !failedWithoutDocument)
|| !StringUtil.hasText(item.getFilePath())) {
throw new BusinessException("批次包含无法恢复的文件");
}
String fileExt = normalizeFileExtension(
item.getFileName(), item.getFilePath());
assertSupportedImportFile(fileExt);
Date now = new Date();
tech.easyflow.ai.entity.Document document =
new tech.easyflow.ai.entity.Document();
document.setId(generateId(document));
document.setCollectionId(knowledge.getId());
document.setDocumentPath(item.getFilePath());
document.setTitle(item.getFileName());
document.setDocumentType(fileExt);
document.setCreated(now);
document.setModified(now);
document.setCreatedBy(resolveOperatorId());
document.setModifiedBy(resolveOperatorId());
document.setProcessStatus(DocumentProcessStatus.PARSING.name());
document.setTotalChunks(0);
document.setCompletedChunks(0);
document.setFailedChunks(0);
document.setProgressPercent(0);
document.setTaskModifiedAt(now);
Map<String, Object> options = buildInitialOptions(fileExt);
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_MODE,
batch.getImportMode());
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID,
batch.getId().toString());
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID,
item.getId().toString());
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_RELATIVE_PATH,
item.getRelativePath());
document.setOptions(options);
documentMapper.insert(document);
DocumentImportTask task = createTask(
document,
DocumentImportTaskPhase.PARSE,
buildDocumentPayload(document)
);
documentImportBatchTracker.bindDocument(item.getId(), document.getId());
return task;
}
/**
* 重试批次中所有失败文件，单文件业务失败不会中止其他文件。
*
* @param batchId 批次 ID
* @throws RuntimeException 基础设施异常需要交由恢复令牌持有者熔断时抛出
*/
public void retryBatchFailures(BigInteger batchId) {
retryBatchFailures(batchId, Set.of());
}
/**
* 重试批次中选定的失败文件，单个文件重试失败不会中止其他文件。
* 重试批次中选定的失败文件，单文件业务失败不会中止其他文件。
*
* @param batchId 批次 ID
* @param fileKeys 指定文件键；为空时重试全部失败项
* @throws RuntimeException 基础设施异常需要交由恢复令牌持有者熔断时抛出
*/
public void retryBatchFailures(BigInteger batchId, Set<String> fileKeys) {
retryBatchFailures(batchId, fileKeys, () -> true);
}
/**
* 在恢复租约仍有效时重试批次中的选定失败文件。
*
* <p>租约检查发生在每个文件重试前，旧持有者失去令牌后立即停止，
* 避免多实例继续重复扫描和竞争文件项行锁。</p>
*
* @param batchId 批次 ID
* @param fileKeys 指定文件键；为空时重试全部失败项
* @param recoveryLeaseGuard 恢复租约检查器
* @return 全部文件处理完成且最终仍持有恢复租约时返回 {@code true}
* @throws RuntimeException 基础设施异常需要交由恢复令牌持有者熔断时抛出
*/
boolean retryBatchFailures(BigInteger batchId,
Set<String> fileKeys,
BooleanSupplier recoveryLeaseGuard) {
QueryWrapper query = QueryWrapper.create()
.eq(DocumentImportBatchItem::getBatchId, batchId)
.eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name());
@@ -364,17 +427,37 @@ public class KnowledgeDocumentImportTaskAppService {
query
);
for (DocumentImportBatchItem item : failedItems) {
if (recoveryLeaseGuard != null
&& !recoveryLeaseGuard.getAsBoolean()) {
LOG.info(
"批次失败项恢复租约已失效，停止当前实例重试: "
+ "batchId={}, itemId={}",
batchId,
item.getId()
);
return false;
}
try {
selfProxy.retryBatchItemInNewTransaction(item.getId());
} catch (Exception error) {
} catch (BusinessException error) {
LOG.error("批次失败项重试启动失败: batchId={}, itemId={}, documentId={}",
batchId, item.getId(), item.getDocumentId(), error);
updateBatchItem(item.getId(),
DocumentImportBatchItemStage.valueOf(item.getStage()),
DocumentImportBatchItemStatus.FAILED,
truncateError(error.getMessage()));
documentImportBatchItemMapper.updateFailedRetryError(
item.getId(), batchId,
truncateError(error.getMessage()), new Date());
}
}
// 最后一次校验覆盖“末项处理完成到批次收尾”之间的租约失效窗口。
if (recoveryLeaseGuard != null
&& !recoveryLeaseGuard.getAsBoolean()) {
LOG.info(
"批次失败项处理完成时恢复租约已失效，跳过旧持有者收尾: "
+ "batchId={}",
batchId
);
return false;
}
return true;
}
/**
@@ -384,7 +467,12 @@ public class KnowledgeDocumentImportTaskAppService {
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void retryBatchItemInNewTransaction(BigInteger itemId) {
retryBatchItem(documentImportBatchTracker.requireItem(itemId));
DocumentImportBatchItem item =
documentImportBatchItemMapper.selectFailedForRetry(itemId);
if (item == null) {
return;
}
retryBatchItem(item);
}
/**
@@ -402,16 +490,57 @@ public class KnowledgeDocumentImportTaskAppService {
if (touched != 1) {
continue;
}
if (DocumentImportTaskPhase.PARSE.name().equals(task.getPhase())) {
parseTaskProducer.send(task.getId());
} else if (DocumentImportTaskPhase.SPLIT.name().equals(task.getPhase())) {
splitTaskProducer.send(task.getId());
} else if (DocumentImportTaskPhase.INDEX.name().equals(task.getPhase())) {
indexTaskProducer.send(task.getId());
try {
if (DocumentImportTaskPhase.PARSE.name().equals(task.getPhase())) {
parseTaskProducer.send(task.getId());
} else if (DocumentImportTaskPhase.SPLIT.name().equals(task.getPhase())) {
splitTaskProducer.send(task.getId());
} else if (DocumentImportTaskPhase.INDEX.name().equals(task.getPhase())) {
indexTaskProducer.send(task.getId());
}
} catch (Exception error) {
handlePendingDispatchFailure(task, error);
}
}
}
/**
* 处理待执行任务重新投递异常。
*
* <p>自动导入任务会触发批次熔断；未能完成熔断或无批次任务
* 保留 PENDING，并向调度器传播异常以触发有界退避。</p>
*
* @param task 投递失败任务
* @param error 原始异常
*/
private void handlePendingDispatchFailure(DocumentImportTask task,
Exception error) {
try {
if (documentImportBatchCircuitBreaker != null
&& documentImportBatchCircuitBreaker.interruptTaskBatch(
task.getId(), error)) {
return;
}
} catch (Exception interruptError) {
IllegalStateException dispatchError = new IllegalStateException(
"文档导入任务投递失败且批次中断状态写入失败: "
+ "taskId=" + task.getId()
+ ", batchId=" + task.getBatchId()
+ ", phase=" + task.getPhase(),
interruptError
);
dispatchError.addSuppressed(error);
throw dispatchError;
}
throw new IllegalStateException(
"文档导入任务投递失败，等待调度退避后重试: "
+ "taskId=" + task.getId()
+ ", batchId=" + task.getBatchId()
+ ", phase=" + task.getPhase(),
error
);
}
/**
* 检测超过最长排队时间的待处理任务。
*/
@@ -824,7 +953,9 @@ public class KnowledgeDocumentImportTaskAppService {
LOG.info("文档向量化任务已创建: knowledgeId={}, documentId={}, taskId={}, previewSessionId={}, totalChunks={}",
knowledge.getId(), document.getId(), task.getId(), session.getSessionId(), totalChunks);
dispatchIndexTaskAfterCommit(task.getId());
scheduleIndexTaskFallback(task.getId());
if (task.getBatchId() == null) {
scheduleIndexTaskFallback(task.getId());
}
DocumentImportDtos.TaskStartIndexResponse response = new DocumentImportDtos.TaskStartIndexResponse();
response.setTaskId(task.getId());
@@ -855,7 +986,9 @@ public class KnowledgeDocumentImportTaskAppService {
DocumentImportBatchItemStatus.PENDING,
null);
dispatchParseTaskAfterCommit(task.getId());
scheduleParseTaskFallback(task.getId());
if (task.getBatchId() == null) {
scheduleParseTaskFallback(task.getId());
}
return Result.ok(buildTaskStartResponse(task, DocumentProcessStatus.PARSING));
}
@@ -923,9 +1056,25 @@ public class KnowledgeDocumentImportTaskAppService {
return Result.ok(response);
}
/**
* 恢复一个失败批次项；缺少文档时先原子重新建档。
*
* @param item 已锁定的失败批次项
* @throws BusinessException 文件或文档状态不允许恢复时抛出
*/
private void retryBatchItem(DocumentImportBatchItem item) {
if (item.getDocumentId() == null) {
throw new BusinessException("失败文件尚未生成文档");
DocumentImportBatch batch =
documentImportBatchTracker.requireBatch(item.getBatchId());
if (!Objects.equals(batch.getKnowledgeId(), item.getKnowledgeId())) {
throw new IllegalStateException("批次文件知识库归属不一致");
}
DocumentCollection knowledge =
assertDocumentCollection(item.getKnowledgeId());
DocumentImportTask task =
createBatchImportTask(batch, item, knowledge);
dispatchParseTaskAfterCommit(task.getId());
return;
}
tech.easyflow.ai.entity.Document document = requireDocument(item.getDocumentId());
DocumentImportDtos.TaskRetryRequest request = new DocumentImportDtos.TaskRetryRequest();
@@ -970,7 +1119,6 @@ public class KnowledgeDocumentImportTaskAppService {
DocumentImportTask task =
createTask(document, DocumentImportTaskPhase.SPLIT, payload);
dispatchSplitTaskAfterCommit(task.getId());
scheduleSplitTaskFallback(task.getId());
return task;
}
@@ -1052,14 +1200,14 @@ public class KnowledgeDocumentImportTaskAppService {
LOG.warn("分块任务执行令牌已失效: taskId={}", taskId);
} catch (Exception error) {
deleteOwnedSplitArtifacts(previewSessionId, snapshotPath);
String errorMessage = truncateError(error.getMessage());
String errorMessage = resolveSplitFailureMessage(error);
LOG.error("文档分块任务失败: taskId={}, documentId={}",
taskId, task.getDocumentId(), error);
String failureCode = error instanceof CsvImportException csvError
? csvError.getFailureCode()
: TASK_ERROR_SPLIT_FAILED;
if (!selfProxy.failSplitTask(
task, task.getDocumentId(), errorMessage, failureCode)) {
task, task.getDocumentId(), errorMessage, failureCode, error)) {
LOG.warn("分块任务所有权已失效，忽略迟到失败: taskId={}", taskId);
}
} finally {
@@ -1111,7 +1259,13 @@ public class KnowledgeDocumentImportTaskAppService {
} catch (Exception e) {
LOG.error("文档解析任务失败: taskId={}, documentId={}", taskId, document.getId(), e);
String errorCode = resolveParseFailureCode(e);
markParseFailed(task, document, resolveParseFailureMessage(e, errorCode), errorCode);
markParseFailed(
task,
document,
resolveParseFailureMessage(e, errorCode),
errorCode,
e
);
}
}
@@ -1173,7 +1327,13 @@ public class KnowledgeDocumentImportTaskAppService {
} catch (Exception e) {
LOG.error("文档解析任务收敛失败: taskId={}, documentId={}", taskId, document.getId(), e);
String errorCode = resolveParseFailureCode(e);
markParseFailed(task, document, resolveParseFailureMessage(e, errorCode), errorCode);
markParseFailed(
task,
document,
resolveParseFailureMessage(e, errorCode),
errorCode,
e
);
}
}
@@ -1309,7 +1469,8 @@ public class KnowledgeDocumentImportTaskAppService {
markIndexFailed(
task, document,
resolveIndexFailureMessage(e, rollbackSucceeded),
failureCode);
failureCode,
e);
} finally {
closeStoreContext(storeContext);
}
@@ -1523,7 +1684,6 @@ public class KnowledgeDocumentImportTaskAppService {
DocumentImportBatchItemStatus.PENDING,
null);
dispatchIndexTaskAfterCommit(indexTask.getId());
scheduleIndexTaskFallback(indexTask.getId());
return true;
}
@@ -1534,6 +1694,7 @@ public class KnowledgeDocumentImportTaskAppService {
* @param documentId 文档 ID
* @param errorMessage 用户可见错误信息
* @param failureCode 稳定失败码
* @param error 原始异常
* @return 是否仍持有本轮分块任务执行权
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
@@ -1541,7 +1702,8 @@ public class KnowledgeDocumentImportTaskAppService {
DocumentImportTask task,
BigInteger documentId,
String errorMessage,
String failureCode) {
String failureCode,
Throwable error) {
Date now = new Date();
if (!finishTask(
task, now, DocumentImportTaskStatus.FAILED,
@@ -1553,12 +1715,15 @@ public class KnowledgeDocumentImportTaskAppService {
current.setProgressPercent(0);
setDocumentTaskError(current, errorMessage, failureCode);
persistDocumentTaskState(current, now);
updateBatchItemFailure(
task.getBatchItemId(),
DocumentImportBatchItemStage.SPLIT,
errorMessage,
failureCode
);
if (!interruptFailedTaskBatchInCurrentTransaction(
task, errorMessage, failureCode, error)) {
updateBatchItemFailure(
task.getBatchItemId(),
DocumentImportBatchItemStage.SPLIT,
errorMessage,
failureCode
);
}
return true;
}
@@ -1664,6 +1829,31 @@ public class KnowledgeDocumentImportTaskAppService {
* @param task 孤儿任务
*/
private void finishMissingDocumentTask(DocumentImportTask task) {
if (TransactionSynchronizationManager.isActualTransactionActive()) {
finishMissingDocumentTaskInCurrentTransaction(task);
return;
}
selfProxy.finishMissingDocumentTaskInNewTransaction(task);
}
/**
* 在独立事务中原子收口缺少关联文档的任务与自动导入批次。
*
* @param task 孤儿任务
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void finishMissingDocumentTaskInNewTransaction(
DocumentImportTask task) {
finishMissingDocumentTaskInCurrentTransaction(task);
}
/**
* 在当前事务中原子收口缺少关联文档的任务与自动导入批次。
*
* @param task 孤儿任务
*/
private void finishMissingDocumentTaskInCurrentTransaction(
DocumentImportTask task) {
Date now = new Date();
String errorMessage = "关联文档已删除，任务已自动清理";
if (!finishTask(task, now, DocumentImportTaskStatus.FAILED, errorMessage)) {
@@ -1687,22 +1877,90 @@ public class KnowledgeDocumentImportTaskAppService {
private void finishRecoveredBatchState(DocumentImportTask task,
DocumentImportBatchItemStage stage,
String errorMessage,
String failureCode) {
String failureCode) {
try {
updateBatchItemFailure(
task.getBatchItemId(),
stage,
errorMessage,
failureCode);
if (task.getBatchId() != null) {
documentImportBatchTracker.markInterrupted(task.getBatchId());
boolean batchStopped = task.getBatchId() != null
&& interruptRecoveredBatch(
task, errorMessage, failureCode);
if (!batchStopped) {
updateBatchItemFailure(
task.getBatchItemId(),
stage,
errorMessage,
failureCode);
}
} catch (Exception error) {
LOG.error("同步已回收任务的批次状态失败: taskId={}, batchId={}, batchItemId={}",
task.getId(), task.getBatchId(), task.getBatchItemId(), error);
if (task.getBatchId() != null) {
throw error instanceof RuntimeException
? (RuntimeException) error
: new IllegalStateException(
"同步已回收任务的批次状态失败", error);
}
}
}
/**
* 在任务回收事务中强制中断所属批次。
*
* @param task 已回收任务
* @param errorMessage 用户可见错误原因
* @param failureCode 稳定失败码
* @return 所属批次是否已经停止运行
*/
private boolean interruptRecoveredBatch(DocumentImportTask task,
String errorMessage,
String failureCode) {
Throwable error = new IllegalStateException(
"文档导入任务恢复时触发批次中断: taskId=" + task.getId());
if (TransactionSynchronizationManager.isActualTransactionActive()) {
return documentImportBatchCircuitBreaker
.interruptBatchInCurrentTransaction(
task.getBatchId(),
failureCode,
errorMessage,
error
);
} else {
return documentImportBatchCircuitBreaker.interruptBatch(
task.getBatchId(),
failureCode,
errorMessage,
error
);
}
}
/**
* 在任务失败状态事务内同步中断所属自动导入批次。
*
* <p>任务、文档、批次项与批次必须原子提交。批次中断写入失败时，
* 当前失败事务整体回滚，消息保留待重试，避免留下永久 RUNNING 批次。</p>
*
* @param task 失败任务
* @param errorMessage 用户可见错误原因
* @param failureCode 稳定失败码
* @param error 原始异常
* @return 所属批次是否已经停止运行
*/
private boolean interruptFailedTaskBatchInCurrentTransaction(
DocumentImportTask task,
String errorMessage,
String failureCode,
Throwable error) {
if (task.getBatchId() == null) {
return false;
}
return documentImportBatchCircuitBreaker
.interruptBatchInCurrentTransaction(
task.getBatchId(),
failureCode,
errorMessage,
error
);
}
/**
* 将底层解析异常转换为稳定错误码。
*
@@ -1743,6 +2001,9 @@ public class KnowledgeDocumentImportTaskAppService {
String message = current.getMessage();
if (StringUtil.hasText(message)) {
String normalized = message.toLowerCase(Locale.ROOT);
if (normalized.contains("unsupported file type")) {
return TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE;
}
if (normalized.contains("timed out")
|| normalized.contains("timeout")
|| normalized.contains("超时")) {
@@ -1779,7 +2040,29 @@ public class KnowledgeDocumentImportTaskAppService {
if (TASK_ERROR_DOCUMENT_SOURCE_UNAVAILABLE.equals(errorCode)) {
return "文档文件读取失败，请联系管理员";
}
return truncateError(error == null ? null : error.getMessage());
if (TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE.equals(errorCode)) {
return "文档格式或来源不受解析服务支持，请检查文件后继续";
}
if (TASK_ERROR_INVALID_PARSE_REQUEST.equals(errorCode)) {
return "文档解析请求无效，请检查文件后继续";
}
if (error instanceof BusinessException) {
return truncateError(error.getMessage());
}
return "文档解析失败，请检查文件或联系管理员后继续";
}
/**
* 将分块异常转换为可安全展示的错误信息。
*
* @param error 原始异常
* @return 用户可见错误信息
*/
private String resolveSplitFailureMessage(Throwable error) {
if (error instanceof BusinessException) {
return truncateError(error.getMessage());
}
return "文档分块失败，请联系管理员排查后继续";
}
/**
@@ -1789,13 +2072,16 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档实体
* @param errorMessage 用户可见错误信息
* @param errorCode 稳定错误码
* @param error 原始异常
*/
private void markParseFailed(DocumentImportTask task,
tech.easyflow.ai.entity.Document document,
String errorMessage,
String errorCode) {
String errorCode,
Throwable error) {
KnowledgeDocumentImportTaskAppService executor = selfProxy == null ? this : selfProxy;
if (!executor.failParseTask(task, document, errorMessage, errorCode)) {
if (!executor.failParseTask(
task, document, errorMessage, errorCode, error)) {
LOG.warn("解析任务所有权已失效，忽略迟到失败: taskId={}, documentId={}",
task.getId(), document.getId());
}
@@ -1808,13 +2094,15 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档实体
* @param errorMessage 用户可见错误信息
* @param errorCode 稳定错误码
* @param error 原始异常
* @return 是否仍持有任务终态写入权
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean failParseTask(DocumentImportTask task,
tech.easyflow.ai.entity.Document document,
String errorMessage,
String errorCode) {
String errorCode,
Throwable error) {
Date now = new Date();
if (!finishTask(
task, now, DocumentImportTaskStatus.FAILED,
@@ -1827,11 +2115,14 @@ public class KnowledgeDocumentImportTaskAppService {
LOG.warn("文档解析任务失败: taskId={}, documentId={}, processStatus={}, error={}",
task.getId(), document.getId(), DocumentProcessStatus.PARSE_FAILED.name(), errorMessage);
updateBatchItemFailure(
task.getBatchItemId(),
DocumentImportBatchItemStage.PARSE,
errorMessage,
errorCode);
if (!interruptFailedTaskBatchInCurrentTransaction(
task, errorMessage, errorCode, error)) {
updateBatchItemFailure(
task.getBatchItemId(),
DocumentImportBatchItemStage.PARSE,
errorMessage,
errorCode);
}
return true;
}
@@ -2304,7 +2595,12 @@ public class KnowledgeDocumentImportTaskAppService {
tech.easyflow.ai.entity.Document document,
String errorMessage) {
markIndexFailed(
task, document, errorMessage, TASK_ERROR_INDEX_FAILED);
task,
document,
errorMessage,
TASK_ERROR_INDEX_FAILED,
new IllegalStateException(errorMessage)
);
}
/**
@@ -2314,14 +2610,17 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档
* @param errorMessage 用户可见错误信息
* @param failureCode 稳定失败码
* @param error 原始异常
*/
private void markIndexFailed(
DocumentImportTask task,
tech.easyflow.ai.entity.Document document,
String errorMessage,
String failureCode) {
String failureCode,
Throwable error) {
KnowledgeDocumentImportTaskAppService executor = selfProxy == null ? this : selfProxy;
if (!executor.failIndexTask(task, document, errorMessage, failureCode)) {
if (!executor.failIndexTask(
task, document, errorMessage, failureCode, error)) {
LOG.warn("向量化任务所有权已失效，忽略迟到失败: taskId={}, documentId={}",
task.getId(), document.getId());
}
@@ -2355,7 +2654,12 @@ public class KnowledgeDocumentImportTaskAppService {
tech.easyflow.ai.entity.Document document,
String errorMessage) {
return failIndexTask(
task, document, errorMessage, TASK_ERROR_INDEX_FAILED);
task,
document,
errorMessage,
TASK_ERROR_INDEX_FAILED,
new IllegalStateException(errorMessage)
);
}
/**
@@ -2365,13 +2669,15 @@ public class KnowledgeDocumentImportTaskAppService {
* @param document 文档实体
* @param errorMessage 用户可见错误信息
* @param failureCode 稳定失败码
* @param error 原始异常
* @return 是否仍持有任务终态写入权
*/
@Transactional(propagation = Propagation.REQUIRES_NEW)
public boolean failIndexTask(DocumentImportTask task,
tech.easyflow.ai.entity.Document document,
String errorMessage,
String failureCode) {
String failureCode,
Throwable error) {
String resolvedFailureCode = StringUtil.hasText(failureCode)
? failureCode
: TASK_ERROR_INDEX_FAILED;
@@ -2397,11 +2703,14 @@ public class KnowledgeDocumentImportTaskAppService {
defaultInt(current.getTotalChunks()),
errorMessage);
updateBatchItemFailure(
task.getBatchItemId(),
DocumentImportBatchItemStage.INDEX,
errorMessage,
resolvedFailureCode);
if (!interruptFailedTaskBatchInCurrentTransaction(
task, errorMessage, resolvedFailureCode, error)) {
updateBatchItemFailure(
task.getBatchItemId(),
DocumentImportBatchItemStage.INDEX,
errorMessage,
resolvedFailureCode);
}
return true;
}
@@ -4487,15 +4796,6 @@ public class KnowledgeDocumentImportTaskAppService {
scheduleTaskFallback(taskId, "index", () -> selfProxy.handleIndexTask(taskId));
}
/**
* 在事务提交后调度本地分块兜底执行。
*
* @param taskId 任务 ID
*/
private void scheduleSplitTaskFallback(BigInteger taskId) {
scheduleTaskFallback(taskId, "split", () -> selfProxy.handleSplitTask(taskId));
}
/**
* 在事务提交后调度本地解析兜底执行。
*

77
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatch.java
View File

@@ -85,6 +85,27 @@ public class DocumentImportBatch extends DateEntity implements Serializable {
@Column(comment = "可重试失败数")
private Integer retryableFailedCount;
@Column(comment = "中断错误码")
private String interruptCode;
@Column(comment = "中断原因")
private String interruptMessage;
@Column(comment = "中断时间")
private Date interruptedAt;
@Column(comment = "是否存在待恢复调度")
private Boolean recoveryPending;
@Column(comment = "待恢复文件键 JSON")
private String recoveryFileKeysJson;
@Column(comment = "恢复调度令牌")
private String recoveryToken;
@Column(comment = "恢复调度租约到期时间")
private Date recoveryLeaseUntil;
@Column(comment = "开始时间")
private Date startedAt;
@@ -279,6 +300,62 @@ public class DocumentImportBatch extends DateEntity implements Serializable {
this.retryableFailedCount = retryableFailedCount;
}
public String getInterruptCode() {
return interruptCode;
}
public void setInterruptCode(String interruptCode) {
this.interruptCode = interruptCode;
}
public String getInterruptMessage() {
return interruptMessage;
}
public void setInterruptMessage(String interruptMessage) {
this.interruptMessage = interruptMessage;
}
public Date getInterruptedAt() {
return interruptedAt;
}
public void setInterruptedAt(Date interruptedAt) {
this.interruptedAt = interruptedAt;
}
public Boolean getRecoveryPending() {
return recoveryPending;
}
public void setRecoveryPending(Boolean recoveryPending) {
this.recoveryPending = recoveryPending;
}
public String getRecoveryFileKeysJson() {
return recoveryFileKeysJson;
}
public void setRecoveryFileKeysJson(String recoveryFileKeysJson) {
this.recoveryFileKeysJson = recoveryFileKeysJson;
}
public String getRecoveryToken() {
return recoveryToken;
}
public void setRecoveryToken(String recoveryToken) {
this.recoveryToken = recoveryToken;
}
public Date getRecoveryLeaseUntil() {
return recoveryLeaseUntil;
}
public void setRecoveryLeaseUntil(Date recoveryLeaseUntil) {
this.recoveryLeaseUntil = recoveryLeaseUntil;
}
public Date getStartedAt() {
return startedAt;
}

98
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java
View File

@@ -2,6 +2,7 @@ package tech.easyflow.ai.mapper;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
@@ -16,6 +17,45 @@ import java.util.Date;
*/
public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImportBatchItem> {
/**
* 锁定运行批次中的失败项并取得本轮恢复权。
*
* <p>同时锁定批次和文件项，使恢复任务创建与批次熔断串行化。</p>
*
* @param itemId 文件项 ID
* @return 可恢复文件项；状态已变化时返回 null
*/
@Select("SELECT item.* FROM tb_document_import_batch_item item "
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id "
+ "WHERE item.id=#{itemId} AND item.status='FAILED' "
+ "AND batch.status='RUNNING' FOR UPDATE")
DocumentImportBatchItem selectFailedForRetry(
@Param("itemId") BigInteger itemId
);
/**
* 将中断批次中尚未结束的文件项统一收口为可恢复失败。
*
* @param batchId 批次 ID
* @param errorSummary 用户可见错误摘要
* @param failureCode 稳定失败码
* @param modified 修改时间
* @return 更新文件项数量
*/
@Update("UPDATE tb_document_import_batch_item item "
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id "
+ "SET item.status='FAILED', item.error_summary=#{errorSummary}, "
+ "item.failure_code=#{failureCode}, item.retryable=1, "
+ "item.modified=#{modified} "
+ "WHERE item.batch_id=#{batchId} AND batch.status='INTERRUPTED' "
+ "AND item.status IN ('PENDING','RUNNING','UPLOADED')")
int interruptActiveItems(
@Param("batchId") BigInteger batchId,
@Param("errorSummary") String errorSummary,
@Param("failureCode") String failureCode,
@Param("modified") Date modified
);
/**
* 原子领取文件上传权并刷新批次进度时间。
*
@@ -60,11 +100,14 @@ public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImport
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "stage=#{stage}, status=#{status}, error_summary=#{errorSummary}, "
+ "failure_code=#{failureCode}, "
+ "retryable=#{retryable}, attempt_count=attempt_count + #{attemptDelta}, "
+ "modified=#{modified} WHERE id=#{id} AND status=#{expectedStatus}")
@Update("UPDATE tb_document_import_batch_item item "
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id SET "
+ "item.stage=#{stage}, item.status=#{status}, "
+ "item.error_summary=#{errorSummary}, item.failure_code=#{failureCode}, "
+ "item.retryable=#{retryable}, "
+ "item.attempt_count=item.attempt_count + #{attemptDelta}, "
+ "item.modified=#{modified} WHERE item.id=#{id} "
+ "AND item.status=#{expectedStatus} AND batch.status<>'INTERRUPTED'")
int transitionStatus(
@Param("id") BigInteger id,
@Param("expectedStatus") String expectedStatus,
@@ -95,6 +138,51 @@ public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImport
@Param("modified") Date modified
);
/**
* 将缺少文档的失败项原子绑定到恢复创建的文档。
*
* @param id 文件项 ID
* @param documentId 恢复创建的文档 ID
* @param modified 修改时间
* @return 文件项仍属于运行批次且绑定成功时返回 1
*/
@Update("UPDATE tb_document_import_batch_item item "
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id SET "
+ "item.document_id=#{documentId}, item.stage='PARSE', "
+ "item.status='PENDING', item.error_summary=NULL, "
+ "item.failure_code=NULL, item.retryable=0, "
+ "item.attempt_count=COALESCE(item.attempt_count, 0) + 1, "
+ "item.modified=#{modified} "
+ "WHERE item.id=#{id} AND item.status='FAILED' "
+ "AND item.document_id IS NULL AND batch.status='RUNNING'")
int bindFailedDocument(
@Param("id") BigInteger id,
@Param("documentId") BigInteger documentId,
@Param("modified") Date modified
);
/**
* 更新恢复失败项的业务错误，但不提前汇总批次终态。
*
* @param id 文件项 ID
* @param batchId 批次 ID
* @param errorSummary 用户可见错误摘要
* @param modified 修改时间
* @return 文件项仍属于运行批次且更新成功时返回 1
*/
@Update("UPDATE tb_document_import_batch_item item "
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id SET "
+ "item.error_summary=#{errorSummary}, item.failure_code=NULL, "
+ "item.retryable=1, item.modified=#{modified} "
+ "WHERE item.id=#{id} AND item.batch_id=#{batchId} "
+ "AND item.status='FAILED' AND batch.status='RUNNING'")
int updateFailedRetryError(
@Param("id") BigInteger id,
@Param("batchId") BigInteger batchId,
@Param("errorSummary") String errorSummary,
@Param("modified") Date modified
);
/**
* 在物理写入前持久化可恢复存储定位符。
*

250
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchMapper.java
View File

@@ -73,6 +73,250 @@ public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatc
@Param("modified") Date modified
);
/**
* 原子熔断仍在运行的自动导入批次。
*
* @param batchId 批次 ID
* @param interruptCode 稳定中断码
* @param interruptMessage 用户可见中断原因
* @param modified 中断时间
* @return 成功取得熔断权时返回 1
*/
@Update("UPDATE tb_document_import_batch SET "
+ "status='INTERRUPTED', interrupt_code=#{interruptCode}, "
+ "interrupt_message=#{interruptMessage}, interrupted_at=#{modified}, "
+ "recovery_pending=0, recovery_file_keys_json=NULL, "
+ "recovery_token=NULL, recovery_lease_until=NULL, "
+ "finished_at=NULL, modified=#{modified}, "
+ "version=version + 1 "
+ "WHERE id=#{batchId} AND import_mode='AUTO' AND status='RUNNING'")
int interruptRunningBatch(
@Param("batchId") BigInteger batchId,
@Param("interruptCode") String interruptCode,
@Param("interruptMessage") String interruptMessage,
@Param("modified") Date modified
);
/**
* 仅允许当前恢复令牌持有者中断运行批次。
*
* @param batchId 批次 ID
* @param recoveryToken 当前恢复调度令牌
* @param interruptCode 稳定中断码
* @param interruptMessage 用户可见中断原因
* @param modified 中断时间
* @return 当前令牌仍持有恢复权且中断成功时返回 1
*/
@Update("UPDATE tb_document_import_batch SET "
+ "status='INTERRUPTED', interrupt_code=#{interruptCode}, "
+ "interrupt_message=#{interruptMessage}, interrupted_at=#{modified}, "
+ "recovery_pending=0, recovery_file_keys_json=NULL, "
+ "recovery_token=NULL, recovery_lease_until=NULL, "
+ "finished_at=NULL, modified=#{modified}, "
+ "version=version + 1 "
+ "WHERE id=#{batchId} AND import_mode='AUTO' AND status='RUNNING' "
+ "AND recovery_pending=1 AND recovery_token=#{recoveryToken} "
+ "AND recovery_lease_until > #{modified}")
int interruptOwnedRecoveryBatch(
@Param("batchId") BigInteger batchId,
@Param("recoveryToken") String recoveryToken,
@Param("interruptCode") String interruptCode,
@Param("interruptMessage") String interruptMessage,
@Param("modified") Date modified
);
/**
* 仅在触发任务仍是当前活跃任务时中断运行批次。
*
* @param batchId 批次 ID
* @param taskId 触发异常的任务 ID
* @param interruptCode 稳定中断码
* @param interruptMessage 用户可见中断原因
* @param modified 中断时间
* @return 任务仍活跃且中断成功时返回 1
*/
@Update("UPDATE tb_document_import_batch batch "
+ "INNER JOIN tb_document_import_task task ON task.batch_id=batch.id SET "
+ "batch.status='INTERRUPTED', batch.interrupt_code=#{interruptCode}, "
+ "batch.interrupt_message=#{interruptMessage}, "
+ "batch.interrupted_at=#{modified}, batch.recovery_pending=0, "
+ "batch.recovery_file_keys_json=NULL, batch.recovery_token=NULL, "
+ "batch.recovery_lease_until=NULL, batch.finished_at=NULL, "
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
+ "WHERE batch.id=#{batchId} AND batch.import_mode='AUTO' "
+ "AND batch.status='RUNNING' AND task.id=#{taskId} "
+ "AND task.status IN ('PENDING','RUNNING')")
int interruptRunningBatchForActiveTask(
@Param("batchId") BigInteger batchId,
@Param("taskId") BigInteger taskId,
@Param("interruptCode") String interruptCode,
@Param("interruptMessage") String interruptMessage,
@Param("modified") Date modified
);
/**
* 根据批次项真实状态重新汇总计数。
*
* @param batchId 批次 ID
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch batch INNER JOIN ("
+ "SELECT batch_id, "
+ "SUM(status='COMPLETED') completed_count, "
+ "SUM(status='RUNNING') processing_count, "
+ "SUM(status='FAILED') failed_count, "
+ "SUM(status='PENDING') pending_count, "
+ "SUM(status='UPLOADED') uploaded_count, "
+ "SUM(status='SKIPPED') skipped_count, "
+ "SUM(status='CANCELLED') cancelled_count, "
+ "SUM(status='FAILED' AND retryable=1) retryable_failed_count "
+ "FROM tb_document_import_batch_item WHERE batch_id=#{batchId} "
+ "GROUP BY batch_id"
+ ") counters ON counters.batch_id=batch.id SET "
+ "batch.completed_count=counters.completed_count, "
+ "batch.processing_count=counters.processing_count, "
+ "batch.failed_count=counters.failed_count, "
+ "batch.pending_count=counters.pending_count, "
+ "batch.uploaded_count=counters.uploaded_count, "
+ "batch.skipped_count=counters.skipped_count, "
+ "batch.cancelled_count=counters.cancelled_count, "
+ "batch.retryable_failed_count=counters.retryable_failed_count, "
+ "batch.modified=#{modified} WHERE batch.id=#{batchId}")
int refreshCountersFromItems(
@Param("batchId") BigInteger batchId,
@Param("modified") Date modified
);
/**
* 查询已经提交但尚未完成恢复调度的运行批次。
*
* @param now 当前时间
* @param limit 最大批次数
* @return 待恢复批次
*/
@Select("SELECT * FROM tb_document_import_batch "
+ "WHERE status='RUNNING' AND recovery_pending=1 "
+ "AND (recovery_token IS NULL OR recovery_lease_until <= #{now}) "
+ "ORDER BY modified, id LIMIT #{limit}")
List<DocumentImportBatch> selectRecoveryPendingBatches(
@Param("now") Date now,
@Param("limit") int limit
);
/**
* 使用租约令牌领取一个批次恢复待办。
*
* @param batchId 批次 ID
* @param recoveryToken 恢复调度令牌
* @param recoveryLeaseUntil 租约到期时间
* @param modified 领取时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch SET "
+ "recovery_token=#{recoveryToken}, "
+ "recovery_lease_until=#{recoveryLeaseUntil}, "
+ "modified=#{modified}, version=version + 1 "
+ "WHERE id=#{batchId} AND status='RUNNING' "
+ "AND recovery_pending=1 AND (recovery_token IS NULL "
+ "OR recovery_lease_until <= #{modified})")
int claimRecoveryPending(
@Param("batchId") BigInteger batchId,
@Param("recoveryToken") String recoveryToken,
@Param("recoveryLeaseUntil") Date recoveryLeaseUntil,
@Param("modified") Date modified
);
/**
* 续期当前持有者的批次恢复租约。
*
* @param batchId 批次 ID
* @param recoveryToken 恢复调度令牌
* @param recoveryLeaseUntil 新租约到期时间
* @param modified 续租时间
* @return 当前令牌仍有效且续租成功时返回 1
*/
@Update("UPDATE tb_document_import_batch SET "
+ "recovery_lease_until=#{recoveryLeaseUntil}, "
+ "modified=#{modified}, version=version + 1 "
+ "WHERE id=#{batchId} AND status='RUNNING' "
+ "AND recovery_pending=1 AND recovery_token=#{recoveryToken} "
+ "AND recovery_lease_until > #{modified}")
int renewRecoveryPendingLease(
@Param("batchId") BigInteger batchId,
@Param("recoveryToken") String recoveryToken,
@Param("recoveryLeaseUntil") Date recoveryLeaseUntil,
@Param("modified") Date modified
);
/**
* 按恢复令牌读取当前持有者领取后的精确恢复参数。
*
* @param batchId 批次 ID
* @param recoveryToken 恢复调度令牌
* @return 当前令牌持有的批次；令牌失效时返回空
*/
@Select("SELECT * FROM tb_document_import_batch "
+ "WHERE id=#{batchId} AND status='RUNNING' "
+ "AND recovery_pending=1 AND recovery_token=#{recoveryToken}")
DocumentImportBatch selectClaimedRecovery(
@Param("batchId") BigInteger batchId,
@Param("recoveryToken") String recoveryToken
);
/**
* 原子完成当前恢复调度并按真实计数收口批次状态。
*
* <p>终态汇总与令牌清理在同一条 SQL 中完成，避免进程在两次写入
* 之间退出后留下无恢复待办的运行批次。</p>
*
* @param batchId 批次 ID
* @param recoveryToken 恢复调度令牌
* @param modified 修改时间
* @return 当前令牌和租约仍有效且收尾成功时返回 1
*/
@Update("UPDATE tb_document_import_batch SET "
+ "status=CASE WHEN "
+ "completed_count + failed_count + skipped_count + cancelled_count "
+ ">= total_count AND processing_count=0 AND pending_count=0 "
+ "THEN CASE WHEN failed_count > 0 THEN 'PARTIAL_SUCCEEDED' "
+ "ELSE 'COMPLETED' END ELSE 'RUNNING' END, "
+ "finished_at=CASE WHEN "
+ "completed_count + failed_count + skipped_count + cancelled_count "
+ ">= total_count AND processing_count=0 AND pending_count=0 "
+ "THEN #{modified} ELSE NULL END, "
+ "recovery_pending=0, recovery_file_keys_json=NULL, "
+ "recovery_token=NULL, recovery_lease_until=NULL, "
+ "modified=#{modified}, version=version + 1 "
+ "WHERE id=#{batchId} AND import_mode='AUTO' AND status='RUNNING' "
+ "AND recovery_pending=1 AND recovery_token=#{recoveryToken} "
+ "AND recovery_lease_until > #{modified}")
int finalizeRecoveryPending(
@Param("batchId") BigInteger batchId,
@Param("recoveryToken") String recoveryToken,
@Param("modified") Date modified
);
/**
* 原子领取管理端人工继续权并清除上一轮中断信息。
*
* @param batchId 批次 ID
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch SET status='RUNNING', "
+ "finished_at=NULL, interrupt_code=NULL, interrupt_message=NULL, "
+ "interrupted_at=NULL, recovery_pending=1, "
+ "recovery_file_keys_json=NULL, "
+ "recovery_token=NULL, recovery_lease_until=NULL, "
+ "modified=#{modified}, version=version + 1 "
+ "WHERE id=#{batchId} "
+ "AND status IN ('PARTIAL_SUCCEEDED','INTERRUPTED')")
int claimContinue(
@Param("batchId") BigInteger batchId,
@Param("modified") Date modified
);
/**
* 原子领取 Public API 重试代次。
*
@@ -80,11 +324,16 @@ public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatc
* @param callerType 调用者类型
* @param callerId 调用者 ID
* @param expectedGeneration 预期重试代次
* @param recoveryFileKeysJson 本轮选择恢复的文件键 JSON；为空表示全部
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch SET "
+ "status='RUNNING', finished_at=NULL, "
+ "interrupt_code=NULL, interrupt_message=NULL, interrupted_at=NULL, "
+ "recovery_pending=1, "
+ "recovery_file_keys_json=#{recoveryFileKeysJson}, "
+ "recovery_token=NULL, recovery_lease_until=NULL, "
+ "retry_generation=retry_generation + 1, "
+ "version=version + 1, modified=#{modified} "
+ "WHERE id=#{batchId} AND caller_type=#{callerType} AND caller_id=#{callerId} "
@@ -95,6 +344,7 @@ public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatc
@Param("callerType") String callerType,
@Param("callerId") BigInteger callerId,
@Param("expectedGeneration") int expectedGeneration,
@Param("recoveryFileKeysJson") String recoveryFileKeysJson,
@Param("modified") Date modified
);

92
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportTaskMapper.java
View File

@@ -30,7 +30,9 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
+ "PARTITION BY task.phase, COALESCE(task.batch_id, task.id) "
+ "ORDER BY task.created, task.id) AS lane_row "
+ "FROM tb_document_import_task task "
+ "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore}"
+ "LEFT JOIN tb_document_import_batch batch ON batch.id=task.batch_id "
+ "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore} "
+ "AND (task.batch_id IS NULL OR batch.status='RUNNING')"
+ ") ranked ON ranked.id=task.id "
+ "ORDER BY ranked.lane_row, task.created, task.id LIMIT #{limit}")
List<DocumentImportTask> selectPendingFairly(
@@ -50,7 +52,11 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
@Update("UPDATE tb_document_import_task SET modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='PENDING' "
+ "AND modified <= #{redispatchBefore}")
+ "AND modified <= #{redispatchBefore} "
+ "AND (batch_id IS NULL OR EXISTS ("
+ "SELECT 1 FROM tb_document_import_batch batch "
+ "WHERE batch.id=tb_document_import_task.batch_id "
+ "AND batch.status='RUNNING'))")
int touchPendingForDispatch(
@Param("id") BigInteger id,
@Param("redispatchBefore") Date redispatchBefore,
@@ -93,7 +99,11 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
+ "started_at=COALESCE(started_at, #{now}), finished_at=NULL, "
+ "error_summary=NULL, failure_code=NULL, modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='PENDING'")
+ "WHERE id=#{id} AND status='PENDING' "
+ "AND (batch_id IS NULL OR EXISTS ("
+ "SELECT 1 FROM tb_document_import_batch batch "
+ "WHERE batch.id=tb_document_import_task.batch_id "
+ "AND batch.status='RUNNING'))")
int claimPending(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@@ -116,7 +126,11 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
+ "modified=#{now}, modified_by=#{operatorId}, "
+ "version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='RUNNING' "
+ "AND execution_token=#{executionToken}")
+ "AND execution_token=#{executionToken} "
+ "AND (batch_id IS NULL OR EXISTS ("
+ "SELECT 1 FROM tb_document_import_batch batch "
+ "WHERE batch.id=tb_document_import_task.batch_id "
+ "AND batch.status='RUNNING'))")
int renewLease(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@@ -142,7 +156,11 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
+ "lease_until=NULL, finished_at=#{now}, modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='RUNNING' "
+ "AND execution_token=#{executionToken}")
+ "AND execution_token=#{executionToken} "
+ "AND (batch_id IS NULL OR EXISTS ("
+ "SELECT 1 FROM tb_document_import_batch batch "
+ "WHERE batch.id=tb_document_import_task.batch_id "
+ "AND batch.status='RUNNING'))")
int finishOwned(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@@ -182,4 +200,68 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
@Param("legacyCutoff") Date legacyCutoff,
@Param("operatorId") BigInteger operatorId
);
/**
* 将中断批次中所有活跃任务强制收口并撤销执行令牌。
*
* @param batchId 批次 ID
* @param errorSummary 用户可见错误摘要
* @param failureCode 稳定失败码
* @param now 中断时间
* @param operatorId 操作人 ID
* @return 更新任务数量
*/
@Update("UPDATE tb_document_import_task task "
+ "INNER JOIN tb_document_import_batch batch ON batch.id=task.batch_id "
+ "SET task.status='FAILED', task.error_summary=#{errorSummary}, "
+ "task.failure_code=#{failureCode}, task.execution_token=NULL, "
+ "task.lease_until=NULL, task.finished_at=#{now}, "
+ "task.modified=#{now}, task.modified_by=#{operatorId}, "
+ "task.version=COALESCE(task.version, 0) + 1 "
+ "WHERE task.batch_id=#{batchId} AND batch.status='INTERRUPTED' "
+ "AND task.status IN ('PENDING','RUNNING')")
int interruptActiveTasks(
@Param("batchId") BigInteger batchId,
@Param("errorSummary") String errorSummary,
@Param("failureCode") String failureCode,
@Param("now") Date now,
@Param("operatorId") BigInteger operatorId
);
/**
* 将中断任务关联文档同步为对应阶段失败，供文档列表展示。
*
* @param batchId 批次 ID
* @param errorSummary 用户可见错误摘要
* @param failureCode 稳定失败码
* @param now 中断时间
* @param operatorId 操作人 ID
* @return 更新文档数量
*/
@Update("UPDATE tb_document document "
+ "INNER JOIN tb_document_import_task task "
+ "ON task.document_id=document.id "
+ "INNER JOIN tb_document_import_batch batch "
+ "ON batch.id=task.batch_id SET "
+ "document.process_status=CASE task.phase "
+ "WHEN 'PARSE' THEN 'PARSE_FAILED' "
+ "WHEN 'SPLIT' THEN 'SPLIT_FAILED' "
+ "ELSE 'INDEX_FAILED' END, "
+ "document.progress_percent=0, "
+ "document.last_task_error=#{errorSummary}, "
+ "document.options=JSON_SET("
+ "CASE WHEN JSON_VALID(document.options) "
+ "THEN document.options ELSE JSON_OBJECT() END, "
+ "'$.\"task.errorCode\"', #{failureCode}), "
+ "document.task_modified_at=#{now}, document.modified=#{now}, "
+ "document.modified_by=#{operatorId} "
+ "WHERE task.batch_id=#{batchId} AND batch.status='INTERRUPTED' "
+ "AND task.status IN ('PENDING','RUNNING')")
int interruptActiveDocuments(
@Param("batchId") BigInteger batchId,
@Param("errorSummary") String errorSummary,
@Param("failureCode") String failureCode,
@Param("now") Date now,
@Param("operatorId") BigInteger operatorId
);
}

332
easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java
View File

@@ -1,8 +1,10 @@
package tech.easyflow.ai.documentimport.task;
import com.mybatisflex.core.query.QueryWrapper;
import org.apache.ibatis.annotations.Update;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
@@ -29,10 +31,13 @@ import tech.easyflow.common.filestorage.FileStorageWriteResult;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BooleanSupplier;
/**
* {@link DocumentImportBatchAppService} 批次启动与重复策略回归测试。
@@ -577,6 +582,7 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
Mockito.nullable(String.class),
Mockito.any(Date.class)
)).thenReturn(1);
@@ -601,6 +607,7 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.eq(0),
Mockito.eq("[\"file-key\"]"),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
@@ -633,6 +640,7 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
Mockito.nullable(String.class),
Mockito.any(Date.class)
)).thenReturn(1);
@@ -666,6 +674,7 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.eq(0),
Mockito.isNull(),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
@@ -693,9 +702,8 @@ public class DocumentImportBatchAppServiceTest {
failed.setRetryable(false);
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(failed));
Mockito.when(context.batchMapper.updateByQuery(
Mockito.any(DocumentImportBatch.class),
Mockito.any(QueryWrapper.class)
Mockito.when(context.batchMapper.claimContinue(
Mockito.eq(batch.getId()), Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.batchTracker.toStatusResponse(batch))
.thenAnswer(invocation -> {
@@ -719,15 +727,261 @@ public class DocumentImportBatchAppServiceTest {
Assert.assertEquals(DocumentImportBatchStatus.RUNNING.name(),
response.getStatus());
Mockito.verify(context.batchMapper).claimContinue(
Mockito.eq(batch.getId()), Mockito.any(Date.class));
Mockito.verify(context.batchTracker, Mockito.never())
.refreshBatch(batch.getId());
Mockito.verify(context.taskAppService).retryBatchFailures(
batch.getId(),
Set.of(failed.getClientFileKey())
Mockito.eq(batch.getId()),
Mockito.eq(Set.of()),
Mockito.any(BooleanSupplier.class)
);
Mockito.verify(context.lockHandle).release();
}
/**
* 验证继续后的恢复调度再次异常时通过统一熔断器收口整个批次。
*/
@Test
public void continueRecoveryFailureShouldInterruptWholeBatch() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setImportMode(DocumentImportMode.AUTO.name());
batch.setStatus(DocumentImportBatchStatus.INTERRUPTED.name());
DocumentImportBatchItem failed = uploadedItem(
BigInteger.valueOf(34),
batch.getId()
);
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(failed));
Mockito.when(context.batchMapper.claimContinue(
Mockito.eq(batch.getId()), Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.batchTracker.toStatusResponse(batch))
.thenReturn(new DocumentImportBatchDtos.StatusResponse());
IllegalStateException recoveryError =
new IllegalStateException("Redis unavailable");
Mockito.doThrow(recoveryError)
.when(context.taskAppService)
.retryBatchFailures(
Mockito.eq(batch.getId()),
Mockito.eq(Set.of()),
Mockito.any(BooleanSupplier.class)
);
beginTransactionSynchronization();
try {
context.service.continueBatch(
batch.getKnowledgeId(), batch.getId()
);
} finally {
completeTransactionSynchronization(
TransactionSynchronization.STATUS_COMMITTED
);
}
Mockito.verify(context.circuitBreaker)
.interruptRecoveryBatch(
Mockito.eq(batch.getId()),
Mockito.anyString(),
Mockito.same(recoveryError)
);
Mockito.verify(context.batchMapper, Mockito.never())
.finalizeRecoveryPending(
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class));
}
/**
* 验证领取后恢复令牌已经失效时旧持有者直接退出且不触发批次熔断。
*/
@Test
public void staleRecoveryOwnerShouldStopWithoutInterruptingBatch() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
batch.setRecoveryPending(true);
Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
Mockito.any(Date.class), Mockito.anyInt()
)).thenReturn(List.of(batch));
Mockito.when(context.batchMapper.selectClaimedRecovery(
Mockito.eq(batch.getId()), Mockito.anyString()
)).thenReturn(null);
int recovered = context.service.recoverPendingBatchRetries();
Assert.assertEquals(1, recovered);
Mockito.verifyNoInteractions(context.circuitBreaker);
Mockito.verify(context.taskAppService, Mockito.never())
.retryBatchFailures(
Mockito.any(), Mockito.anySet(),
Mockito.any(BooleanSupplier.class));
}
/**
* 验证进程错过提交后回调时，调度器可从持久恢复待办重放任务创建。
*/
@Test
public void pendingRecoveryShouldReplayAndClearDurableMarker() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
batch.setRecoveryPending(true);
batch.setRecoveryFileKeysJson("[\"stale-file\"]");
DocumentImportBatch claimedBatch = new DocumentImportBatch();
claimedBatch.setId(batch.getId());
claimedBatch.setRecoveryFileKeysJson("[\"selected-file\"]");
Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
Mockito.any(Date.class), Mockito.anyInt()
)).thenReturn(List.of(batch));
Mockito.when(context.batchMapper.selectClaimedRecovery(
Mockito.eq(batch.getId()), Mockito.anyString()
)).thenReturn(claimedBatch);
int recovered = context.service.recoverPendingBatchRetries();
Assert.assertEquals(1, recovered);
Mockito.verify(context.taskAppService)
.retryBatchFailures(
Mockito.eq(batch.getId()),
Mockito.eq(Set.of("selected-file")),
Mockito.any(BooleanSupplier.class)
);
ArgumentCaptor<String> recoveryToken =
ArgumentCaptor.forClass(String.class);
Mockito.verify(context.batchMapper).claimRecoveryPending(
Mockito.eq(batch.getId()),
recoveryToken.capture(),
Mockito.any(Date.class),
Mockito.any(Date.class)
);
Mockito.verify(context.batchMapper).selectClaimedRecovery(
batch.getId(), recoveryToken.getValue());
Mockito.verify(context.batchMapper)
.finalizeRecoveryPending(
Mockito.eq(batch.getId()),
Mockito.eq(recoveryToken.getValue()),
Mockito.any(Date.class));
}
/**
* 验证文件恢复期间租约失效后旧持有者不会清理持久恢复待办。
*/
@Test
public void lostRecoveryLeaseShouldKeepDurableMarker() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
batch.setRecoveryPending(true);
Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
Mockito.any(Date.class), Mockito.anyInt()
)).thenReturn(List.of(batch));
Mockito.when(context.taskAppService.retryBatchFailures(
Mockito.eq(batch.getId()), Mockito.anySet(),
Mockito.any(BooleanSupplier.class)
)).thenReturn(false);
Assert.assertEquals(1, context.service.recoverPendingBatchRetries());
Mockito.verify(context.batchMapper, Mockito.never())
.finalizeRecoveryPending(
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class));
Mockito.verifyNoInteractions(context.circuitBreaker);
}
/**
* 验证长时间恢复会在半租期续租，并在令牌失效后停止旧实例。
*/
@Test
public void recoveryLeaseGuardShouldRenewAndStopWhenTokenIsLost() {
TestContext context = createContext();
BigInteger batchId = BigInteger.valueOf(84);
String recoveryToken = "recovery-token";
AtomicLong now = new AtomicLong(1_000L);
Mockito.when(context.batchMapper.renewRecoveryPendingLease(
Mockito.eq(batchId),
Mockito.eq(recoveryToken),
Mockito.any(Date.class),
Mockito.any(Date.class)
)).thenReturn(1, 0);
BooleanSupplier guard = context.service.createRecoveryLeaseGuard(
batchId,
recoveryToken,
121_000L,
now::get
);
Assert.assertTrue(guard.getAsBoolean());
Mockito.verify(context.batchMapper, Mockito.never())
.renewRecoveryPendingLease(
Mockito.any(),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.any(Date.class)
);
now.set(61_000L);
Assert.assertTrue(guard.getAsBoolean());
now.set(121_000L);
Assert.assertFalse(guard.getAsBoolean());
Mockito.verify(context.batchMapper, Mockito.times(2))
.renewRecoveryPendingLease(
Mockito.eq(batchId),
Mockito.eq(recoveryToken),
Mockito.any(Date.class),
Mockito.any(Date.class)
);
}
/**
* 验证恢复租约续期 SQL 同时校验批次状态、令牌和未过期租约。
*
* @throws Exception Mapper 方法不存在时抛出
*/
@Test
public void recoveryLeaseRenewalSqlShouldFenceCurrentOwner()
throws Exception {
Method method = DocumentImportBatchMapper.class.getMethod(
"renewRecoveryPendingLease",
BigInteger.class,
String.class,
Date.class,
Date.class
);
Update update = method.getAnnotation(Update.class);
String sql = String.join(" ", update.value());
Assert.assertTrue(sql.contains("status='RUNNING'"));
Assert.assertTrue(sql.contains("recovery_pending=1"));
Assert.assertTrue(sql.contains("recovery_token=#{recoveryToken}"));
Assert.assertTrue(sql.contains("recovery_lease_until > #{modified}"));
Method finalizeMethod = DocumentImportBatchMapper.class.getMethod(
"finalizeRecoveryPending",
BigInteger.class,
String.class,
Date.class
);
String finalizeSql = String.join(" ",
finalizeMethod.getAnnotation(Update.class).value());
Assert.assertTrue(finalizeSql.contains(
"status=CASE WHEN"));
Assert.assertTrue(finalizeSql.contains("recovery_pending=0"));
Assert.assertTrue(finalizeSql.contains(
"recovery_token=#{recoveryToken}"));
Assert.assertTrue(finalizeSql.contains(
"recovery_lease_until > #{modified}"));
}
/**
* 验证运行中的任务拒绝再次领取，调用方可继续查询原 taskId。
*/
@@ -761,6 +1015,50 @@ public class DocumentImportBatchAppServiceTest {
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
Mockito.nullable(String.class),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
}
/**
* 验证 Public API 不会在同一知识库已有运行批次时恢复旧批次。
*/
@Test
public void retryShouldRejectWhenAnotherAutoBatchIsRunning() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setImportMode(DocumentImportMode.AUTO.name());
batch.setStatus(DocumentImportBatchStatus.INTERRUPTED.name());
Mockito.when(context.batchService.count(Mockito.any(QueryWrapper.class)))
.thenReturn(1L);
beginTransactionSynchronization();
try {
context.service.retryOwnedBatch(
batch.getId(),
publicCaller(),
Set.of()
);
Assert.fail("Expected active automatic batch rejection");
} catch (BusinessException expected) {
Assert.assertTrue(
expected.getMessage().contains("当前知识库已有自动导入批次")
);
} finally {
completeTransactionSynchronization(
TransactionSynchronization.STATUS_ROLLED_BACK
);
}
Mockito.verify(context.batchMapper, Mockito.never()).claimRetry(
Mockito.any(),
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
Mockito.nullable(String.class),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
@@ -814,6 +1112,8 @@ public class DocumentImportBatchAppServiceTest {
Mockito.mock(DocumentImportBatchItemMapper.class);
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
DocumentImportBatchCircuitBreaker circuitBreaker =
Mockito.mock(DocumentImportBatchCircuitBreaker.class);
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
Mockito.when(redisLockExecutor.tryAcquire(
Mockito.anyString(), Mockito.any(), Mockito.any()
@@ -829,6 +1129,21 @@ public class DocumentImportBatchAppServiceTest {
Mockito.when(batchMapper.selectOwnedForUpdate(
Mockito.any(), Mockito.anyString(), Mockito.any()
)).thenReturn(batch);
Mockito.when(batchMapper.claimRecoveryPending(
Mockito.any(),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(batchMapper.selectClaimedRecovery(
Mockito.any(), Mockito.anyString()
)).thenReturn(batch);
Mockito.when(taskAppService.retryBatchFailures(
Mockito.any(), Mockito.anySet(), Mockito.any(BooleanSupplier.class)
)).thenReturn(true);
Mockito.when(batchMapper.finalizeRecoveryPending(
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class)
)).thenReturn(1);
DocumentImportBatchAppService service = new DocumentImportBatchAppService(
batchService,
@@ -839,11 +1154,13 @@ public class DocumentImportBatchAppServiceTest {
batchMapper,
itemMapper,
documentMapper,
redisLockExecutor
redisLockExecutor,
circuitBreaker
);
return new TestContext(
service, batchService, itemService, batchTracker,
taskAppService, batchMapper, itemMapper, documentMapper, lockHandle
taskAppService, batchMapper, itemMapper, documentMapper,
circuitBreaker, lockHandle
);
}
@@ -954,6 +1271,7 @@ public class DocumentImportBatchAppServiceTest {
DocumentImportBatchMapper batchMapper,
DocumentImportBatchItemMapper itemMapper,
DocumentMapper documentMapper,
DocumentImportBatchCircuitBreaker circuitBreaker,
RedisLockExecutor.LockHandle lockHandle
) {
}

297
easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchCircuitBreakerTest.java Normal file
View File

@@ -0,0 +1,297 @@
package tech.easyflow.ai.documentimport.task;
import org.apache.ibatis.annotations.Update;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.jdbc.BadSqlGrammarException;
import org.springframework.data.redis.RedisSystemException;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.entity.DocumentImportTask;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.enums.DocumentImportMode;
import tech.easyflow.ai.enums.DocumentImportTaskPhase;
import tech.easyflow.ai.enums.DocumentImportTaskStatus;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
import java.math.BigInteger;
import java.lang.reflect.Method;
import java.sql.SQLException;
import java.util.Date;
/**
* {@link DocumentImportBatchCircuitBreaker} 批次熔断回归测试。
*/
public class DocumentImportBatchCircuitBreakerTest {
/**
* 验证 Redis 超时会强制中断批次、撤销任务并刷新真实计数。
*/
@Test
public void shouldInterruptAutomaticBatchOnRedisFailure() {
BigInteger taskId = BigInteger.valueOf(11);
BigInteger batchId = BigInteger.valueOf(12);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setBatchId(batchId);
task.setPhase(DocumentImportTaskPhase.PARSE.name());
task.setStatus(DocumentImportTaskStatus.PENDING.name());
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(batchId);
batch.setImportMode(DocumentImportMode.AUTO.name());
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
DocumentImportBatchMapper batchMapper =
Mockito.mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper =
Mockito.mock(DocumentImportBatchItemMapper.class);
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
Mockito.when(batchMapper.interruptRunningBatchForActiveTask(
Mockito.eq(batchId),
Mockito.eq(taskId),
Mockito.eq("redis_unavailable"),
Mockito.anyString(),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(taskMapper.interruptActiveDocuments(
Mockito.eq(batchId), Mockito.anyString(),
Mockito.eq("redis_unavailable"), Mockito.any(Date.class),
Mockito.eq(BigInteger.ZERO)
)).thenReturn(1);
Mockito.when(taskMapper.interruptActiveTasks(
Mockito.eq(batchId), Mockito.anyString(),
Mockito.eq("redis_unavailable"), Mockito.any(Date.class),
Mockito.eq(BigInteger.ZERO)
)).thenReturn(2);
Mockito.when(itemMapper.interruptActiveItems(
Mockito.eq(batchId), Mockito.anyString(),
Mockito.eq("redis_unavailable"), Mockito.any(Date.class)
)).thenReturn(2);
DocumentImportBatchCircuitBreaker circuitBreaker =
new DocumentImportBatchCircuitBreaker(
batchMapper, itemMapper, taskMapper);
boolean interrupted = circuitBreaker.interruptTaskBatch(
taskId,
new RedisSystemException(
"Redis command timed out", new IllegalStateException("timeout"))
);
Assert.assertTrue(interrupted);
Mockito.verify(batchMapper).refreshCountersFromItems(
Mockito.eq(batchId), Mockito.any(Date.class));
}
/**
* 验证无批次任务不会误触发批次熔断。
*/
@Test
public void shouldLeaveStandaloneTaskForDeferredRetry() {
BigInteger taskId = BigInteger.valueOf(21);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
DocumentImportBatchMapper batchMapper =
Mockito.mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper =
Mockito.mock(DocumentImportBatchItemMapper.class);
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
DocumentImportBatchCircuitBreaker circuitBreaker =
new DocumentImportBatchCircuitBreaker(
batchMapper, itemMapper, taskMapper);
Assert.assertFalse(circuitBreaker.interruptTaskBatch(
taskId, new IllegalStateException("temporary failure")));
Mockito.verify(batchMapper, Mockito.never()).interruptRunningBatch(
Mockito.any(), Mockito.anyString(), Mockito.anyString(),
Mockito.any(Date.class));
}
/**
* 验证旧任务已结束后，迟到消费者异常不会中断新一轮运行批次。
*/
@Test
public void shouldIgnoreLateFailureFromFinishedTask() {
BigInteger taskId = BigInteger.valueOf(31);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setBatchId(BigInteger.valueOf(32));
task.setStatus(DocumentImportTaskStatus.FAILED.name());
DocumentImportBatchMapper batchMapper =
Mockito.mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper =
Mockito.mock(DocumentImportBatchItemMapper.class);
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
DocumentImportBatchCircuitBreaker circuitBreaker =
new DocumentImportBatchCircuitBreaker(
batchMapper, itemMapper, taskMapper);
Assert.assertTrue(circuitBreaker.interruptTaskBatch(
taskId, new IllegalStateException("迟到异常")));
Mockito.verify(batchMapper, Mockito.never())
.interruptRunningBatchForActiveTask(
Mockito.any(), Mockito.any(), Mockito.anyString(),
Mockito.anyString(), Mockito.any(Date.class));
Mockito.verify(batchMapper, Mockito.never()).selectOneById(Mockito.any());
}
/**
* 验证恢复令牌失效后旧持有者无法中断仍由新持有者运行的批次。
*/
@Test
public void shouldRejectInterruptionFromStaleRecoveryOwner() {
BigInteger batchId = BigInteger.valueOf(41);
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(batchId);
batch.setImportMode(DocumentImportMode.AUTO.name());
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
DocumentImportBatchMapper batchMapper =
Mockito.mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper =
Mockito.mock(DocumentImportBatchItemMapper.class);
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
Mockito.when(batchMapper.interruptOwnedRecoveryBatch(
Mockito.eq(batchId),
Mockito.eq("stale-token"),
Mockito.anyString(),
Mockito.anyString(),
Mockito.any(Date.class)
)).thenReturn(0);
DocumentImportBatchCircuitBreaker circuitBreaker =
new DocumentImportBatchCircuitBreaker(
batchMapper, itemMapper, taskMapper);
Assert.assertFalse(circuitBreaker.interruptRecoveryBatch(
batchId,
"stale-token",
new IllegalStateException("旧持有者异常")
));
Mockito.verifyNoInteractions(itemMapper);
Mockito.verify(taskMapper, Mockito.never()).interruptActiveTasks(
Mockito.any(), Mockito.anyString(), Mockito.anyString(),
Mockito.any(Date.class), Mockito.any());
}
/**
* 验证 SQL 语法错误保留为通用系统异常，避免伪装成数据库不可用。
*/
@Test
public void shouldKeepBadSqlGrammarAsInfrastructureFailure() {
BigInteger batchId = BigInteger.valueOf(51);
DocumentImportBatch batch = runningAutoBatch(batchId);
DocumentImportBatchMapper batchMapper =
Mockito.mock(DocumentImportBatchMapper.class);
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
Mockito.when(batchMapper.interruptRunningBatch(
Mockito.eq(batchId),
Mockito.eq("document_import_infrastructure_failure"),
Mockito.anyString(),
Mockito.any(Date.class)
)).thenReturn(1);
DocumentImportBatchCircuitBreaker circuitBreaker =
new DocumentImportBatchCircuitBreaker(
batchMapper,
Mockito.mock(DocumentImportBatchItemMapper.class),
Mockito.mock(DocumentImportTaskMapper.class)
);
Assert.assertTrue(circuitBreaker.interruptBatch(
batchId,
new BadSqlGrammarException(
"query", "SELECT broken", new SQLException("syntax", "42000"))
));
}
/**
* 验证明确的数据库资源故障仍归类为数据库不可用。
*/
@Test
public void shouldClassifyDatabaseResourceFailureAsUnavailable() {
BigInteger batchId = BigInteger.valueOf(61);
DocumentImportBatch batch = runningAutoBatch(batchId);
DocumentImportBatchMapper batchMapper =
Mockito.mock(DocumentImportBatchMapper.class);
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
Mockito.when(batchMapper.interruptRunningBatch(
Mockito.eq(batchId),
Mockito.eq("database_unavailable"),
Mockito.anyString(),
Mockito.any(Date.class)
)).thenReturn(1);
DocumentImportBatchCircuitBreaker circuitBreaker =
new DocumentImportBatchCircuitBreaker(
batchMapper,
Mockito.mock(DocumentImportBatchItemMapper.class),
Mockito.mock(DocumentImportTaskMapper.class)
);
Assert.assertTrue(circuitBreaker.interruptBatch(
batchId,
new DataAccessResourceFailureException("connection unavailable")
));
}
/**
* 验证恢复与任务熔断 SQL 都包含对应所有权围栏。
*
* @throws Exception Mapper 方法不存在时抛出
*/
@Test
public void interruptionSqlShouldFenceRecoveryAndActiveTask()
throws Exception {
Method recoveryMethod = DocumentImportBatchMapper.class.getMethod(
"interruptOwnedRecoveryBatch",
BigInteger.class,
String.class,
String.class,
String.class,
Date.class
);
String recoverySql = String.join(" ",
recoveryMethod.getAnnotation(Update.class).value());
Assert.assertTrue(recoverySql.contains("recovery_pending=1"));
Assert.assertTrue(recoverySql.contains(
"recovery_token=#{recoveryToken}"));
Assert.assertTrue(recoverySql.contains(
"recovery_lease_until > #{modified}"));
Method taskMethod = DocumentImportBatchMapper.class.getMethod(
"interruptRunningBatchForActiveTask",
BigInteger.class,
BigInteger.class,
String.class,
String.class,
Date.class
);
String taskSql = String.join(" ",
taskMethod.getAnnotation(Update.class).value());
Assert.assertTrue(taskSql.contains("task.id=#{taskId}"));
Assert.assertTrue(taskSql.contains(
"task.status IN ('PENDING','RUNNING')"));
}
/**
* 创建运行中的自动导入批次。
*
* @param batchId 批次 ID
* @return 测试批次
*/
private DocumentImportBatch runningAutoBatch(BigInteger batchId) {
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(batchId);
batch.setImportMode(DocumentImportMode.AUTO.name());
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
return batch;
}
}

45
easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java
View File

@@ -75,6 +75,9 @@ public class DocumentImportBatchTrackerTest {
batch.setCompletedCount(1);
batch.setFailedCount(1);
batch.setRetryableFailedCount(0);
batch.setInterruptCode("redis_unavailable");
batch.setInterruptMessage("缓存与消息服务异常");
batch.setInterruptedAt(new Date());
when(batchService.getById(batch.getId())).thenReturn(batch);
DocumentImportBatchTracker tracker =
@@ -84,6 +87,9 @@ public class DocumentImportBatchTrackerTest {
assertEquals(DocumentImportBatchStatus.INTERRUPTED.name(), response.getStatus());
assertEquals(100, response.getProgressPercent().intValue());
assertEquals(1, response.getRetryableFailedCount().intValue());
assertEquals("redis_unavailable", response.getInterruptCode());
assertEquals("缓存与消息服务异常", response.getInterruptMessage());
assertEquals(batch.getInterruptedAt(), response.getInterruptedAt());
}
/**
@@ -139,6 +145,45 @@ public class DocumentImportBatchTrackerTest {
);
}
/**
* 验证无文档失败项绑定后按真实文件状态刷新批次计数。
*/
@Test
public void shouldBindRecoveredDocumentAndRefreshCounters() {
DocumentImportBatchService batchService =
mock(DocumentImportBatchService.class);
DocumentImportBatchItemService itemService =
mock(DocumentImportBatchItemService.class);
DocumentImportBatchMapper batchMapper =
mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper =
mock(DocumentImportBatchItemMapper.class);
DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1);
batch.setPendingCount(1);
DocumentImportBatchItem item = new DocumentImportBatchItem();
item.setId(BigInteger.TEN);
item.setBatchId(batch.getId());
item.setStatus(DocumentImportBatchItemStatus.FAILED.name());
item.setRetryable(true);
BigInteger documentId = BigInteger.valueOf(99);
when(itemService.getById(item.getId())).thenReturn(item);
when(itemMapper.bindFailedDocument(
eq(item.getId()), eq(documentId), any(Date.class)
)).thenReturn(1);
when(batchService.getById(batch.getId())).thenReturn(batch);
DocumentImportBatchTracker tracker =
new DocumentImportBatchTracker(
batchService, itemService, batchMapper, itemMapper);
tracker.bindDocument(item.getId(), documentId);
verify(batchMapper).refreshCountersFromItems(
eq(batch.getId()), any(Date.class));
verify(batchMapper, never()).adjustCounters(
any(), anyInt(), anyInt(), anyInt(), anyInt(), anyInt(),
anyInt(), anyInt(), anyInt(), any(Date.class));
}
/**
* 验证迟到任务不能把已完成文件重新改为处理中。
*/

36
easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitorTest.java Normal file
View File

@@ -0,0 +1,36 @@
package tech.easyflow.ai.documentimport.task;
import org.junit.Test;
import org.mockito.Mockito;
/**
* {@link DocumentImportPendingTaskMonitor} 调度退避回归测试。
*
* @author Codex
* @since 2026-08-07
*/
public class DocumentImportPendingTaskMonitorTest {
/**
* 验证任务投递异常后立即进入冷却期，避免短周期重复扫描。
*/
@Test
public void dispatchFailureShouldEnterCooldown() {
KnowledgeDocumentImportTaskAppService appService =
Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
DocumentImportBatchAppService batchAppService =
Mockito.mock(DocumentImportBatchAppService.class);
Mockito.doThrow(new IllegalStateException("Redis unavailable"))
.when(appService)
.dispatchPendingTasks();
DocumentImportPendingTaskMonitor monitor =
new DocumentImportPendingTaskMonitor(appService, batchAppService);
monitor.dispatchPendingTasks();
monitor.dispatchPendingTasks();
Mockito.verify(batchAppService, Mockito.times(1))
.recoverPendingBatchRetries();
Mockito.verify(appService, Mockito.times(1)).dispatchPendingTasks();
}
}

587
easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java
View File

@@ -14,11 +14,15 @@ import com.easyagents.rag.ingestion.RagIngestionService;
import com.easyagents.rag.ingestion.model.AnalysisResult;
import com.easyagents.rag.ingestion.model.StrategyConfig;
import com.easyagents.search.engine.service.DocumentSearcher;
import com.mybatisflex.core.query.QueryWrapper;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.data.redis.RedisSystemException;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
import tech.easyflow.ai.document.model.DocumentParseArtifacts;
@@ -28,20 +32,26 @@ import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.DocumentImportKeys;
import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.entity.DocumentImportTask;
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.enums.DocumentImportMode;
import tech.easyflow.ai.enums.DocumentImportTaskStatus;
import tech.easyflow.ai.enums.DocumentImportTaskPhase;
import tech.easyflow.ai.enums.DocumentProcessStatus;
import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
import tech.easyflow.ai.service.DocumentImportTaskService;
import tech.easyflow.ai.service.DocumentCollectionService;
import tech.easyflow.ai.service.DocumentService;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.io.ByteArrayInputStream;
import java.lang.reflect.Constructor;
@@ -59,6 +69,7 @@ import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -192,6 +203,277 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Mockito.verify(producer, Mockito.never()).send(Mockito.any());
}
/**
* 验证自动导入任务投递异常会触发批次熔断，且不会阻断本轮扫描。
*
* @throws Exception 反射注入异常
*/
@Test
public void dispatchPendingTasksShouldInterruptBatchWhenProducerFails()
throws Exception {
BigInteger taskId = BigInteger.valueOf(31);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setBatchId(BigInteger.valueOf(32));
task.setPhase(DocumentImportTaskPhase.PARSE.name());
task.setStatus(DocumentImportTaskStatus.PENDING.name());
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.selectPendingFairly(
Mockito.any(Date.class), Mockito.anyInt()))
.thenReturn(List.of(task));
Mockito.when(taskMapper.touchPendingForDispatch(
Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
Mockito.any(BigInteger.class)))
.thenReturn(1);
DocumentImportParseTaskProducer producer =
Mockito.mock(DocumentImportParseTaskProducer.class);
IllegalStateException failure =
new IllegalStateException("Redis command timed out");
Mockito.doThrow(failure).when(producer).send(taskId);
DocumentImportBatchCircuitBreaker circuitBreaker =
Mockito.mock(DocumentImportBatchCircuitBreaker.class);
Mockito.when(circuitBreaker.interruptTaskBatch(taskId, failure))
.thenReturn(true);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "parseTaskProducer", producer);
setField(service, "documentImportBatchCircuitBreaker", circuitBreaker);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
service.dispatchPendingTasks();
Mockito.verify(circuitBreaker).interruptTaskBatch(taskId, failure);
}
/**
* 验证投递失败后若批次中断写入也失败，异常会传播给调度器触发退避。
*
* @throws Exception 反射注入异常
*/
@Test
public void dispatchPendingTasksShouldPropagateInterruptionWriteFailure()
throws Exception {
BigInteger taskId = BigInteger.valueOf(33);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setBatchId(BigInteger.valueOf(34));
task.setPhase(DocumentImportTaskPhase.PARSE.name());
task.setStatus(DocumentImportTaskStatus.PENDING.name());
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.selectPendingFairly(
Mockito.any(Date.class), Mockito.anyInt()))
.thenReturn(List.of(task));
Mockito.when(taskMapper.touchPendingForDispatch(
Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
Mockito.any(BigInteger.class)))
.thenReturn(1);
DocumentImportParseTaskProducer producer =
Mockito.mock(DocumentImportParseTaskProducer.class);
RedisSystemException dispatchFailure = new RedisSystemException(
"Redis command timed out",
new IllegalStateException("timeout")
);
Mockito.doThrow(dispatchFailure).when(producer).send(taskId);
DocumentImportBatchCircuitBreaker circuitBreaker =
Mockito.mock(DocumentImportBatchCircuitBreaker.class);
IllegalStateException interruptionFailure =
new IllegalStateException("database unavailable");
Mockito.when(circuitBreaker.interruptTaskBatch(taskId, dispatchFailure))
.thenThrow(interruptionFailure);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "parseTaskProducer", producer);
setField(service, "documentImportBatchCircuitBreaker", circuitBreaker);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
try {
service.dispatchPendingTasks();
Assert.fail("批次中断写入失败必须传播给外层调度器");
} catch (IllegalStateException error) {
Assert.assertSame(interruptionFailure, error.getCause());
Assert.assertEquals(1, error.getSuppressed().length);
Assert.assertSame(dispatchFailure, error.getSuppressed()[0]);
Assert.assertTrue(error.getMessage().contains("taskId=33"));
Assert.assertTrue(error.getMessage().contains("batchId=34"));
}
}
/**
* 验证无批次任务投递失败时保留任务并传播异常，避免两秒热循环。
*
* @throws Exception 反射注入异常
*/
@Test
public void dispatchPendingTasksShouldPropagateUnbatchedFailure()
throws Exception {
BigInteger taskId = BigInteger.valueOf(35);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setPhase(DocumentImportTaskPhase.PARSE.name());
task.setStatus(DocumentImportTaskStatus.PENDING.name());
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.selectPendingFairly(
Mockito.any(Date.class), Mockito.anyInt()))
.thenReturn(List.of(task));
Mockito.when(taskMapper.touchPendingForDispatch(
Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
Mockito.any(BigInteger.class)))
.thenReturn(1);
DocumentImportParseTaskProducer producer =
Mockito.mock(DocumentImportParseTaskProducer.class);
IllegalStateException dispatchFailure =
new IllegalStateException("Redis unavailable");
Mockito.doThrow(dispatchFailure).when(producer).send(taskId);
DocumentImportBatchCircuitBreaker circuitBreaker =
Mockito.mock(DocumentImportBatchCircuitBreaker.class);
Mockito.when(circuitBreaker.interruptTaskBatch(taskId, dispatchFailure))
.thenReturn(false);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "parseTaskProducer", producer);
setField(service, "documentImportBatchCircuitBreaker", circuitBreaker);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
try {
service.dispatchPendingTasks();
Assert.fail("无批次投递失败必须传播给外层调度器");
} catch (IllegalStateException error) {
Assert.assertSame(dispatchFailure, error.getCause());
Assert.assertTrue(error.getMessage().contains("taskId=35"));
Assert.assertTrue(error.getMessage().contains("batchId=null"));
}
}
/**
* 验证批次恢复遇到基础设施异常后向令牌持有者传播并停止后续重试。
*
* @throws Exception 反射注入异常
*/
@Test
public void retryBatchInfrastructureFailureShouldPropagateToOwner()
throws Exception {
BigInteger batchId = BigInteger.valueOf(91);
DocumentImportBatchItem first = new DocumentImportBatchItem();
first.setId(BigInteger.valueOf(92));
first.setBatchId(batchId);
DocumentImportBatchItem second = new DocumentImportBatchItem();
second.setId(BigInteger.valueOf(93));
second.setBatchId(batchId);
DocumentImportBatchItemService itemService =
Mockito.mock(DocumentImportBatchItemService.class);
Mockito.when(itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(first, second));
KnowledgeDocumentImportTaskAppService selfProxy =
Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
RedisSystemException failure = new RedisSystemException(
"Redis command timed out",
new IllegalStateException("timeout")
);
Mockito.doThrow(failure)
.when(selfProxy)
.retryBatchItemInNewTransaction(first.getId());
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportBatchItemService", itemService);
setField(service, "selfProxy", selfProxy);
try {
service.retryBatchFailures(batchId, Set.of());
Assert.fail("基础设施异常必须传播给恢复令牌持有者");
} catch (RedisSystemException error) {
Assert.assertSame(failure, error);
}
Mockito.verify(selfProxy, Mockito.never())
.retryBatchItemInNewTransaction(second.getId());
}
/**
* 验证单文件业务失败不会提前关闭批次并跳过后续失败项。
*
* @throws Exception 反射注入异常
*/
@Test
public void retryBatchBusinessFailureShouldContinueRemainingItems()
throws Exception {
BigInteger batchId = BigInteger.valueOf(96);
DocumentImportBatchItem first = new DocumentImportBatchItem();
first.setId(BigInteger.valueOf(97));
first.setBatchId(batchId);
first.setDocumentId(BigInteger.valueOf(98));
DocumentImportBatchItem second = new DocumentImportBatchItem();
second.setId(BigInteger.valueOf(99));
second.setBatchId(batchId);
second.setDocumentId(BigInteger.valueOf(100));
DocumentImportBatchItemService itemService =
Mockito.mock(DocumentImportBatchItemService.class);
Mockito.when(itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(first, second));
KnowledgeDocumentImportTaskAppService selfProxy =
Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
Mockito.doThrow(new BusinessException("格式不支持"))
.when(selfProxy)
.retryBatchItemInNewTransaction(first.getId());
DocumentImportBatchItemMapper itemMapper =
Mockito.mock(DocumentImportBatchItemMapper.class);
DocumentImportBatchTracker batchTracker =
Mockito.mock(DocumentImportBatchTracker.class);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportBatchItemService", itemService);
setField(service, "documentImportBatchItemMapper", itemMapper);
setField(service, "documentImportBatchTracker", batchTracker);
setField(service, "selfProxy", selfProxy);
Assert.assertTrue(service.retryBatchFailures(
batchId, Set.of(), () -> true));
org.mockito.InOrder order = Mockito.inOrder(
selfProxy, itemMapper);
order.verify(selfProxy).retryBatchItemInNewTransaction(first.getId());
order.verify(itemMapper).updateFailedRetryError(
Mockito.eq(first.getId()), Mockito.eq(batchId),
Mockito.eq("格式不支持"), Mockito.any(Date.class));
order.verify(selfProxy).retryBatchItemInNewTransaction(second.getId());
Mockito.verifyNoInteractions(batchTracker);
}
/**
* 验证恢复租约失效后旧实例停止重试剩余文件。
*
* @throws Exception 反射注入异常
*/
@Test
public void retryBatchFailuresShouldStopWhenRecoveryLeaseIsLost()
throws Exception {
BigInteger batchId = BigInteger.valueOf(94);
DocumentImportBatchItem failed = new DocumentImportBatchItem();
failed.setId(BigInteger.valueOf(95));
failed.setBatchId(batchId);
DocumentImportBatchItemService itemService =
Mockito.mock(DocumentImportBatchItemService.class);
Mockito.when(itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(failed));
KnowledgeDocumentImportTaskAppService selfProxy =
Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportBatchItemService", itemService);
setField(service, "selfProxy", selfProxy);
Assert.assertFalse(service.retryBatchFailures(
batchId, Set.of(), () -> false));
Mockito.verify(selfProxy, Mockito.never())
.retryBatchItemInNewTransaction(Mockito.any());
}
/**
* 验证重新投递资格更新包含待处理状态和过期时间边界，确保竞争更新仅一个成功。
*
@@ -213,6 +495,102 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Assert.assertTrue(sql.contains("status='PENDING'"));
Assert.assertTrue(sql.contains("modified <= #{redispatchBefore}"));
Assert.assertTrue(sql.contains("modified=#{now}"));
Assert.assertTrue(sql.contains("batch.status='RUNNING'"));
}
/**
* 验证任务扫描、领取、续租和完成 SQL 均包含批次运行状态门禁。
*/
@Test
public void activeTaskSqlShouldFenceInterruptedBatch() {
for (Method method : DocumentImportTaskMapper.class.getDeclaredMethods()) {
String name = method.getName();
if (!"selectPendingFairly".equals(name)
&& !"claimPending".equals(name)
&& !"renewLease".equals(name)
&& !"finishOwned".equals(name)) {
continue;
}
Select select = method.getAnnotation(Select.class);
Update update = method.getAnnotation(Update.class);
String sql = select == null
? String.join(" ", update.value())
: String.join(" ", select.value());
Assert.assertTrue(name + " 缺少批次运行状态门禁",
sql.contains("batch.status='RUNNING'"));
}
}
/**
* 验证失败项恢复同时锁定文件项与运行批次，阻止熔断后继续建任务。
*
* @throws Exception 映射方法不存在时抛出
*/
@Test
public void retryItemSqlShouldRequireRunningBatchLock()
throws Exception {
Method method = DocumentImportBatchItemMapper.class.getMethod(
"selectFailedForRetry",
BigInteger.class
);
Select select = method.getAnnotation(Select.class);
String sql = String.join(" ", select.value());
Assert.assertTrue(sql.contains("item.status='FAILED'"));
Assert.assertTrue(sql.contains("batch.status='RUNNING'"));
Assert.assertTrue(sql.contains("FOR UPDATE"));
}
/**
* 验证批次中断后，迟到 worker 不能重新打开文件项或覆盖中断原因。
*
* @throws Exception 映射方法不存在时抛出
*/
@Test
public void itemTransitionSqlShouldFenceInterruptedBatch()
throws Exception {
Method method = DocumentImportBatchItemMapper.class.getMethod(
"transitionStatus",
BigInteger.class,
String.class,
String.class,
String.class,
String.class,
String.class,
boolean.class,
int.class,
Date.class
);
Update update = method.getAnnotation(Update.class);
String sql = String.join(" ", update.value());
Assert.assertTrue(sql.contains(
"INNER JOIN tb_document_import_batch batch"));
Assert.assertTrue(sql.contains("batch.status<>'INTERRUPTED'"));
}
/**
* 验证批次中断同步文档状态时兼容历史无效 JSON 扩展字段。
*
* @throws Exception 映射方法不存在时抛出
*/
@Test
public void interruptDocumentSqlShouldHandleInvalidOptions()
throws Exception {
Method method = DocumentImportTaskMapper.class.getMethod(
"interruptActiveDocuments",
BigInteger.class,
String.class,
String.class,
Date.class,
BigInteger.class
);
Update update = method.getAnnotation(Update.class);
String sql = String.join(" ", update.value());
Assert.assertTrue(sql.contains("JSON_VALID(document.options)"));
Assert.assertTrue(sql.contains("ELSE JSON_OBJECT()"));
}
/**
@@ -423,6 +801,7 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
BigInteger batchId = BigInteger.valueOf(61);
BigInteger itemId = BigInteger.valueOf(62);
DocumentImportTask task = new DocumentImportTask();
task.setId(BigInteger.valueOf(60));
task.setBatchId(batchId);
task.setBatchItemId(itemId);
DocumentImportBatchItem item = new DocumentImportBatchItem();
@@ -431,10 +810,13 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
DocumentImportBatchTracker tracker =
Mockito.mock(DocumentImportBatchTracker.class);
DocumentImportBatchCircuitBreaker circuitBreaker =
Mockito.mock(DocumentImportBatchCircuitBreaker.class);
Mockito.when(tracker.requireItem(itemId)).thenReturn(item);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportBatchTracker", tracker);
setField(service, "documentImportBatchCircuitBreaker", circuitBreaker);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod(
@@ -446,13 +828,20 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
);
method.setAccessible(true);
method.invoke(
service,
task,
DocumentImportBatchItemStage.PARSE,
"任务执行中断，请继续批次",
"execution_interrupted"
);
TransactionSynchronizationManager.setActualTransactionActive(true);
TransactionSynchronizationManager.initSynchronization();
try {
method.invoke(
service,
task,
DocumentImportBatchItemStage.PARSE,
"任务执行中断，请继续批次",
"execution_interrupted"
);
} finally {
TransactionSynchronizationManager.clearSynchronization();
TransactionSynchronizationManager.setActualTransactionActive(false);
}
Mockito.verify(tracker).transitionItem(
itemId,
@@ -463,7 +852,12 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
0,
"execution_interrupted"
);
Mockito.verify(tracker).markInterrupted(batchId);
Mockito.verify(circuitBreaker).interruptBatchInCurrentTransaction(
Mockito.eq(batchId),
Mockito.eq("execution_interrupted"),
Mockito.eq("任务执行中断，请继续批次"),
Mockito.any(IllegalStateException.class)
);
}
/**
@@ -657,6 +1051,38 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Assert.assertEquals("invalid_parse_request", invalidRequest);
}
/**
* 验证 MinerU 不支持文件类型的响应经桥接异常包装后仍能安全归类。
*
* @throws Exception 反射调用异常
*/
@Test
public void resolveParseFailureShouldRecognizeWrappedUnsupportedFileType()
throws Exception {
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
Method codeMethod = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("resolveParseFailureCode", Throwable.class);
Method messageMethod = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("resolveParseFailureMessage", Throwable.class, String.class);
codeMethod.setAccessible(true);
messageMethod.setAccessible(true);
DocumentParseBridgeException error = DocumentParseBridgeException.taskFailed(
"异步解析任务失败",
new RuntimeException(
"MinerU request failed: path=/tasks, status=400, "
+ "body={\"detail\":\"Unsupported file type: doc\"}"
)
);
String code = (String) codeMethod.invoke(service, error);
String message = (String) messageMethod.invoke(service, error, code);
Assert.assertEquals("unsupported_document_source", code);
Assert.assertEquals("文档格式或来源不受解析服务支持，请检查文件后继续", message);
Assert.assertFalse(message.contains("doc"));
}
/**
* 验证未识别的系统异常仍会获得稳定错误码并允许后续人工重试。
*
@@ -677,27 +1103,144 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
}
/**
* 验证未知系统异常和临时源读取失败允许人工重试，确定性输入错误除外。
* 验证人工批次继续不受历史尝试次数限制。
*
* @throws Exception 反射调用异常
* @throws Exception 反射调用或依赖注入异常
*/
@Test
public void parseRetryabilityShouldDefaultToRecoverable()
public void retryBatchItemShouldIgnoreHistoricalAttemptCount()
throws Exception {
KnowledgeDocumentImportTaskAppService service = Mockito.spy(
new KnowledgeDocumentImportTaskAppService());
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
setField(service, "documentMapper", documentMapper);
DocumentImportBatchItem failed = new DocumentImportBatchItem();
failed.setId(BigInteger.valueOf(701));
failed.setKnowledgeId(BigInteger.valueOf(702));
failed.setDocumentId(BigInteger.valueOf(703));
failed.setStage(DocumentImportBatchItemStage.INDEX.name());
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
failed.setAttemptCount(99);
tech.easyflow.ai.entity.Document document =
new tech.easyflow.ai.entity.Document();
document.setId(failed.getDocumentId());
document.setProcessStatus(DocumentProcessStatus.INDEX_FAILED.name());
Mockito.when(documentMapper.selectOneById(failed.getDocumentId()))
.thenReturn(document);
Mockito.doReturn(null).when(service).retryFailedTask(
Mockito.any(DocumentImportDtos.TaskRetryRequest.class));
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("retryBatchItem", DocumentImportBatchItem.class);
method.setAccessible(true);
method.invoke(service, failed);
Mockito.verify(service).retryFailedTask(Mockito.argThat(request ->
failed.getKnowledgeId().equals(request.getKnowledgeId())
&& failed.getDocumentId().equals(request.getDocumentId())
));
}
/**
* 验证中断时形成的无文档失败项可以重新建档并创建解析任务。
*
* @throws Exception 反射调用或依赖注入异常
*/
@Test
public void retryBatchItemWithoutDocumentShouldRecreateImportTask()
throws Exception {
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("isRetryableParseFailure", String.class);
method.setAccessible(true);
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
DocumentImportTaskService taskService =
Mockito.mock(DocumentImportTaskService.class);
DocumentCollectionService knowledgeService =
Mockito.mock(DocumentCollectionService.class);
DocumentImportBatchTracker tracker =
Mockito.mock(DocumentImportBatchTracker.class);
DocumentImportParseTaskProducer producer =
Mockito.mock(DocumentImportParseTaskProducer.class);
BigInteger itemId = BigInteger.valueOf(711);
BigInteger batchId = BigInteger.valueOf(712);
BigInteger knowledgeId = BigInteger.valueOf(713);
BigInteger taskId = BigInteger.valueOf(714);
DocumentImportBatchItem failed = new DocumentImportBatchItem();
failed.setId(itemId);
failed.setBatchId(batchId);
failed.setKnowledgeId(knowledgeId);
failed.setFileName("recover.docx");
failed.setRelativePath("folder/recover.docx");
failed.setFilePath("/stored/recover.docx");
failed.setStage(DocumentImportBatchItemStage.UPLOAD.name());
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(batchId);
batch.setKnowledgeId(knowledgeId);
batch.setImportMode(DocumentImportMode.AUTO.name());
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
DocumentCollection knowledge = new DocumentCollection();
knowledge.setId(knowledgeId);
Mockito.when(tracker.requireBatch(batchId)).thenReturn(batch);
Mockito.when(knowledgeService.getById(knowledgeId))
.thenReturn(knowledge);
Mockito.when(taskService.save(Mockito.any(DocumentImportTask.class)))
.thenAnswer(invocation -> {
DocumentImportTask task = invocation.getArgument(0);
task.setId(taskId);
return true;
});
setField(service, "documentMapper", documentMapper);
setField(service, "documentImportTaskService", taskService);
setField(service, "knowledgeService", knowledgeService);
setField(service, "documentImportBatchTracker", tracker);
setField(service, "parseTaskProducer", producer);
Assert.assertTrue((Boolean) method.invoke(service, new Object[] {null}));
Assert.assertTrue((Boolean) method.invoke(
service, "document_source_unavailable"));
Assert.assertTrue((Boolean) method.invoke(service, "parse_failed"));
Assert.assertFalse((Boolean) method.invoke(
service, "unsupported_document_source"));
Assert.assertFalse((Boolean) method.invoke(
service, "invalid_parse_request"));
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("retryBatchItem", DocumentImportBatchItem.class);
method.setAccessible(true);
method.invoke(service, failed);
ArgumentCaptor<tech.easyflow.ai.entity.Document> documentCaptor =
ArgumentCaptor.forClass(tech.easyflow.ai.entity.Document.class);
Mockito.verify(documentMapper).insert(documentCaptor.capture());
tech.easyflow.ai.entity.Document document = documentCaptor.getValue();
Assert.assertNotNull(document.getId());
Assert.assertEquals(knowledgeId, document.getCollectionId());
Assert.assertEquals(DocumentProcessStatus.PARSING.name(),
document.getProcessStatus());
ArgumentCaptor<DocumentImportTask> taskCaptor =
ArgumentCaptor.forClass(DocumentImportTask.class);
Mockito.verify(taskService).save(taskCaptor.capture());
Assert.assertEquals(taskId, taskCaptor.getValue().getId());
Assert.assertEquals(batchId, taskCaptor.getValue().getBatchId());
Assert.assertEquals(itemId, taskCaptor.getValue().getBatchItemId());
Assert.assertEquals(DocumentImportTaskPhase.PARSE.name(),
taskCaptor.getValue().getPhase());
Mockito.verify(tracker).bindDocument(itemId, document.getId());
Mockito.verify(producer).send(taskId);
}
/**
* 验证无文档失败项绑定 SQL 同时校验失败状态、空文档和运行批次。
*
* @throws Exception Mapper 方法不存在时抛出
*/
@Test
public void bindFailedDocumentSqlShouldFenceRecoveryState()
throws Exception {
Method method = DocumentImportBatchItemMapper.class.getMethod(
"bindFailedDocument",
BigInteger.class,
BigInteger.class,
Date.class
);
Update update = method.getAnnotation(Update.class);
String sql = String.join(" ", update.value());
Assert.assertTrue(sql.contains("item.status='FAILED'"));
Assert.assertTrue(sql.contains("item.document_id IS NULL"));
Assert.assertTrue(sql.contains("batch.status='RUNNING'"));
Assert.assertTrue(sql.contains("item.status='PENDING'"));
}
/**