fix: 修复知识库失败批次继续与重分块

- 统一继续和重试逻辑,恢复全部失败明细

- 向量化失败时废弃旧快照并重新分块

- 增加 BGE-M3 分块硬上限与索引写入结果校验
This commit is contained in:
2026-08-07 12:39:27 +08:00
parent 7082041e6e
commit 7a64cfcaa4
9 changed files with 539 additions and 238 deletions

View File

@@ -16,6 +16,7 @@ import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.enums.DocumentImportMode;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
@@ -606,10 +607,10 @@ public class DocumentImportBatchAppServiceTest {
}
/**
* 验证修复前的可恢复失败项会在重试领取事务中恢复重试资格和批次计数
* 验证人工重试直接选择历史失败项,不受旧重试资格字段限制
*/
@Test
public void retryShouldRestoreLegacyRecoverableItem() {
public void retryShouldSelectLegacyFailedItemWithoutRetryableGate() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
@@ -627,17 +628,6 @@ public class DocumentImportBatchAppServiceTest {
failed.setErrorSummary("历史代码异常");
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(failed));
Mockito.when(context.taskAppService.isRecoverableBatchFailure(failed))
.thenReturn(true);
Mockito.when(context.batchTracker.transitionItem(
Mockito.eq(failed.getId()),
Mockito.eq(DocumentImportBatchItemStage.PARSE),
Mockito.eq(DocumentImportBatchItemStatus.FAILED),
Mockito.eq("历史代码异常"),
Mockito.eq(true),
Mockito.eq(0),
Mockito.eq("parse_failed")
)).thenReturn(true);
Mockito.when(context.batchMapper.claimRetry(
Mockito.any(),
Mockito.anyString(),
@@ -661,15 +651,15 @@ public class DocumentImportBatchAppServiceTest {
}
Assert.assertEquals(Integer.valueOf(1), result.getRetriedCount());
Assert.assertTrue(failed.getRetryable());
Mockito.verify(context.batchTracker).transitionItem(
failed.getId(),
DocumentImportBatchItemStage.PARSE,
DocumentImportBatchItemStatus.FAILED,
"历史代码异常",
true,
0,
"parse_failed"
Assert.assertFalse(failed.getRetryable());
Mockito.verify(context.batchTracker, Mockito.never()).transitionItem(
Mockito.any(),
Mockito.any(),
Mockito.any(),
Mockito.any(),
Mockito.anyBoolean(),
Mockito.anyInt(),
Mockito.any()
);
Mockito.verify(context.batchMapper).claimRetry(
Mockito.eq(batch.getId()),
@@ -681,6 +671,63 @@ public class DocumentImportBatchAppServiceTest {
Mockito.verify(context.lockHandle).release();
}
/**
* 验证管理端继续按真实失败项恢复,不依赖历史可重试失败计数。
*/
@Test
public void continueShouldResumeFailedItemsWhenCompatibilityCounterIsZero() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setImportMode(DocumentImportMode.AUTO.name());
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
batch.setFailedCount(1);
batch.setRetryableFailedCount(0);
DocumentImportBatchItem failed = uploadedItem(
BigInteger.valueOf(33),
batch.getId()
);
failed.setStage(DocumentImportBatchItemStage.INDEX.name());
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
failed.setRetryable(false);
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(failed));
Mockito.when(context.batchMapper.updateByQuery(
Mockito.any(DocumentImportBatch.class),
Mockito.any(QueryWrapper.class)
)).thenReturn(1);
Mockito.when(context.batchTracker.toStatusResponse(batch))
.thenAnswer(invocation -> {
DocumentImportBatchDtos.StatusResponse response =
new DocumentImportBatchDtos.StatusResponse();
response.setStatus(batch.getStatus());
return response;
});
DocumentImportBatchDtos.StatusResponse response;
beginTransactionSynchronization();
try {
response = context.service.continueBatch(
batch.getKnowledgeId(), batch.getId()
);
} finally {
completeTransactionSynchronization(
TransactionSynchronization.STATUS_COMMITTED
);
}
Assert.assertEquals(DocumentImportBatchStatus.RUNNING.name(),
response.getStatus());
Mockito.verify(context.batchTracker, Mockito.never())
.refreshBatch(batch.getId());
Mockito.verify(context.taskAppService).retryBatchFailures(
batch.getId(),
Set.of(failed.getClientFileKey())
);
Mockito.verify(context.lockHandle).release();
}
/**
* 验证运行中的任务拒绝再次领取,调用方可继续查询原 taskId。
*/

View File

@@ -63,10 +63,10 @@ public class DocumentImportBatchTrackerTest {
}
/**
* 验证中断批次在汇总失败项后仍保留可继续状态
* 验证兼容可重试计数与人工可继续的失败总数保持一致
*/
@Test
public void shouldPreserveInterruptedStatusUntilUserContinues() {
public void shouldExposeAllFailuresAsContinuable() {
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
@@ -74,7 +74,7 @@ public class DocumentImportBatchTrackerTest {
DocumentImportBatch batch = batch(DocumentImportBatchStatus.INTERRUPTED, 2);
batch.setCompletedCount(1);
batch.setFailedCount(1);
batch.setRetryableFailedCount(1);
batch.setRetryableFailedCount(0);
when(batchService.getById(batch.getId())).thenReturn(batch);
DocumentImportBatchTracker tracker =

View File

@@ -11,6 +11,9 @@ import tech.easyflow.ai.documentimport.ImportCallerContext;
import tech.easyflow.ai.documentimport.ImportCallerType;
import tech.easyflow.ai.documentimport.PublicDocumentImportDtos;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
@@ -318,6 +321,52 @@ public class KnowledgeImportBatchFacadeTest {
Assert.assertEquals(Integer.valueOf(2), response.getRetriedCount());
}
/**
* 验证 Public API 将所有失败文件统一标记为可重试,不受历史标记影响。
*/
@Test
public void statusShouldExposeEveryFailedItemAsRetryable() {
TestContext context = createContext();
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(BigInteger.ONE);
batch.setKnowledgeId(BigInteger.TWO);
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
Mockito.when(context.batchAppService.requireBatchForCaller(
BigInteger.ONE, context.caller
)).thenReturn(batch);
DocumentImportBatchDtos.StatusResponse internal =
new DocumentImportBatchDtos.StatusResponse();
internal.setTotalCount(1);
internal.setFailedCount(1);
internal.setRetryableFailedCount(1);
internal.setProgressPercent(100);
Mockito.when(context.batchTracker.toStatusResponse(batch))
.thenReturn(internal);
DocumentImportBatchItem failed = new DocumentImportBatchItem();
failed.setBatchId(batch.getId());
failed.setClientFileKey("failed.txt");
failed.setStage(DocumentImportBatchItemStage.INDEX.name());
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
failed.setRetryable(false);
Mockito.when(context.itemService.page(
Mockito.any(com.mybatisflex.core.paginate.Page.class),
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)
)).thenReturn(new com.mybatisflex.core.paginate.Page<>(
List.of(failed), 1, 1, 1
));
PublicDocumentImportDtos.StatusResponse response = context.facade.getStatus(
context.caller, batch.getId(), null, 1, 20
);
Assert.assertTrue(response.getCanRetry());
Assert.assertEquals(Integer.valueOf(1),
response.getCounts().getRetryableFailed());
Assert.assertTrue(response.getItems().getRecords().get(0).getRetryable());
}
/**
* 创建单文件元数据。
*
@@ -406,6 +455,7 @@ public class KnowledgeImportBatchFacadeTest {
return new TestContext(
facade,
batchAppService,
batchTracker,
batchService,
itemService,
batchMapper,
@@ -422,6 +472,7 @@ public class KnowledgeImportBatchFacadeTest {
private record TestContext(
KnowledgeImportBatchFacade facade,
DocumentImportBatchAppService batchAppService,
DocumentImportBatchTracker batchTracker,
DocumentImportBatchService batchService,
DocumentImportBatchItemService itemService,
DocumentImportBatchMapper batchMapper,