发布 v1.10 #5
@@ -441,9 +441,14 @@ public class DocumentImportBatchAppService {
|
|||||||
throw new BusinessException("当前批次无需继续");
|
throw new BusinessException("当前批次无需继续");
|
||||||
}
|
}
|
||||||
assertNoOtherActiveAutoBatch(batch);
|
assertNoOtherActiveAutoBatch(batch);
|
||||||
if (valueOrZero(batch.getRetryableFailedCount()) <= 0) {
|
List<DocumentImportBatchItem> failedItems =
|
||||||
throw new BusinessException("当前批次没有可继续的失败项");
|
listFailedItems(batchId, Set.of());
|
||||||
|
if (failedItems.isEmpty()) {
|
||||||
|
throw new BusinessException("当前批次没有失败项");
|
||||||
}
|
}
|
||||||
|
Set<String> selectedKeys = failedItems.stream()
|
||||||
|
.map(DocumentImportBatchItem::getClientFileKey)
|
||||||
|
.collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new));
|
||||||
Date now = new Date();
|
Date now = new Date();
|
||||||
DocumentImportBatch update = new DocumentImportBatch();
|
DocumentImportBatch update = new DocumentImportBatch();
|
||||||
update.setStatus(DocumentImportBatchStatus.RUNNING.name());
|
update.setStatus(DocumentImportBatchStatus.RUNNING.name());
|
||||||
@@ -459,8 +464,11 @@ public class DocumentImportBatchAppService {
|
|||||||
if (claimed <= 0) {
|
if (claimed <= 0) {
|
||||||
throw new BusinessException("批次状态已变化,请刷新后重试");
|
throw new BusinessException("批次状态已变化,请刷新后重试");
|
||||||
}
|
}
|
||||||
runAfterCommit(() -> resumeBatchFailures(batchId));
|
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
|
||||||
return batchTracker.refreshBatch(batchId);
|
batch.setFinishedAt(null);
|
||||||
|
batch.setModified(now);
|
||||||
|
runAfterCommit(() -> resumeBatchFailures(batchId, selectedKeys));
|
||||||
|
return batchTracker.toStatusResponse(batch);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -732,7 +740,7 @@ public class DocumentImportBatchAppService {
|
|||||||
*
|
*
|
||||||
* @param taskId 批次任务 ID
|
* @param taskId 批次任务 ID
|
||||||
* @param caller 调用者上下文
|
* @param caller 调用者上下文
|
||||||
* @param fileKeys 指定文件键;为空时重试全部可恢复失败项
|
* @param fileKeys 指定文件键;为空时重试全部失败项
|
||||||
* @return 稳定的重试响应快照
|
* @return 稳定的重试响应快照
|
||||||
*/
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
@@ -759,11 +767,10 @@ public class DocumentImportBatchAppService {
|
|||||||
}
|
}
|
||||||
Set<String> requestedFileKeys = fileKeys == null ? Set.of() : fileKeys;
|
Set<String> requestedFileKeys = fileKeys == null ? Set.of() : fileKeys;
|
||||||
List<DocumentImportBatchItem> retryItems =
|
List<DocumentImportBatchItem> retryItems =
|
||||||
listRetryItems(batch.getId(), requestedFileKeys);
|
listFailedItems(batch.getId(), requestedFileKeys);
|
||||||
if (retryItems.isEmpty()) {
|
if (retryItems.isEmpty()) {
|
||||||
throw new BusinessException(409, 40905, "当前任务没有可重试的失败文件");
|
throw new BusinessException(409, 40905, "当前任务没有失败文件");
|
||||||
}
|
}
|
||||||
restoreLegacyRetryability(retryItems);
|
|
||||||
Date now = new Date();
|
Date now = new Date();
|
||||||
int expectedGeneration = valueOrZero(batch.getRetryGeneration());
|
int expectedGeneration = valueOrZero(batch.getRetryGeneration());
|
||||||
int claimedGeneration = expectedGeneration + 1;
|
int claimedGeneration = expectedGeneration + 1;
|
||||||
@@ -1140,15 +1147,6 @@ public class DocumentImportBatchAppService {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 在批次状态提交后启动失败项重试;启动器异常时恢复为可继续状态。
|
|
||||||
*
|
|
||||||
* @param batchId 批次 ID
|
|
||||||
*/
|
|
||||||
private void resumeBatchFailures(BigInteger batchId) {
|
|
||||||
resumeBatchFailures(batchId, Set.of());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 在批次状态提交后启动选定失败项重试。
|
* 在批次状态提交后启动选定失败项重试。
|
||||||
*
|
*
|
||||||
@@ -1165,14 +1163,14 @@ public class DocumentImportBatchAppService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 查询并校验本次重试选中的可恢复失败项。
|
* 查询并校验本次人工重试选中的失败项。
|
||||||
*
|
*
|
||||||
* @param batchId 批次 ID
|
* @param batchId 批次 ID
|
||||||
* @param fileKeys 指定文件键
|
* @param fileKeys 指定文件键
|
||||||
* @return 可恢复失败项
|
* @return 失败项
|
||||||
*/
|
*/
|
||||||
private List<DocumentImportBatchItem> listRetryItems(BigInteger batchId,
|
private List<DocumentImportBatchItem> listFailedItems(BigInteger batchId,
|
||||||
Set<String> fileKeys) {
|
Set<String> fileKeys) {
|
||||||
QueryWrapper wrapper = QueryWrapper.create()
|
QueryWrapper wrapper = QueryWrapper.create()
|
||||||
.eq(DocumentImportBatchItem::getBatchId, batchId)
|
.eq(DocumentImportBatchItem::getBatchId, batchId)
|
||||||
.eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name());
|
.eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name());
|
||||||
@@ -1185,64 +1183,13 @@ public class DocumentImportBatchAppService {
|
|||||||
throw new BusinessException("fileKeys 包含空值或重复值");
|
throw new BusinessException("fileKeys 包含空值或重复值");
|
||||||
}
|
}
|
||||||
wrapper.in(DocumentImportBatchItem::getClientFileKey, normalized);
|
wrapper.in(DocumentImportBatchItem::getClientFileKey, normalized);
|
||||||
List<DocumentImportBatchItem> selected = itemService.list(wrapper).stream()
|
List<DocumentImportBatchItem> selected = itemService.list(wrapper);
|
||||||
.filter(this::isRetryCandidate)
|
|
||||||
.toList();
|
|
||||||
if (selected.size() != normalized.size()) {
|
if (selected.size() != normalized.size()) {
|
||||||
throw new BusinessException(409, 40906, "部分文件当前不可重试");
|
throw new BusinessException(409, 40906, "部分文件当前不是失败状态");
|
||||||
}
|
}
|
||||||
return selected;
|
return selected;
|
||||||
}
|
}
|
||||||
return itemService.list(wrapper).stream()
|
return itemService.list(wrapper);
|
||||||
.filter(this::isRetryCandidate)
|
|
||||||
.toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断失败项是否可以进入本次人工重试。
|
|
||||||
*
|
|
||||||
* @param item 批次失败项
|
|
||||||
* @return 是否可以重试
|
|
||||||
*/
|
|
||||||
private boolean isRetryCandidate(DocumentImportBatchItem item) {
|
|
||||||
return Boolean.TRUE.equals(item.getRetryable())
|
|
||||||
|| taskAppService.isRecoverableBatchFailure(item);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在领取重试前恢复旧失败项的持久化重试资格及批次计数。
|
|
||||||
*
|
|
||||||
* @param retryItems 本次重试项
|
|
||||||
*/
|
|
||||||
private void restoreLegacyRetryability(
|
|
||||||
List<DocumentImportBatchItem> retryItems) {
|
|
||||||
for (DocumentImportBatchItem item : retryItems) {
|
|
||||||
if (Boolean.TRUE.equals(item.getRetryable())) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
DocumentImportBatchItemStage stage;
|
|
||||||
try {
|
|
||||||
stage = DocumentImportBatchItemStage.valueOf(item.getStage());
|
|
||||||
} catch (IllegalArgumentException | NullPointerException error) {
|
|
||||||
throw new BusinessException(409, 40906, "部分文件当前不可重试");
|
|
||||||
}
|
|
||||||
if (!batchTracker.transitionItem(
|
|
||||||
item.getId(),
|
|
||||||
stage,
|
|
||||||
DocumentImportBatchItemStatus.FAILED,
|
|
||||||
item.getErrorSummary(),
|
|
||||||
true,
|
|
||||||
0,
|
|
||||||
item.getFailureCode()
|
|
||||||
)) {
|
|
||||||
throw new BusinessException(
|
|
||||||
409,
|
|
||||||
40903,
|
|
||||||
"文件状态已变化,请刷新任务状态"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
item.setRetryable(true);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -401,7 +401,8 @@ public class DocumentImportBatchTracker {
|
|||||||
response.setPendingCount(valueOrZero(batch.getPendingCount()));
|
response.setPendingCount(valueOrZero(batch.getPendingCount()));
|
||||||
response.setSkippedCount(valueOrZero(batch.getSkippedCount()));
|
response.setSkippedCount(valueOrZero(batch.getSkippedCount()));
|
||||||
response.setCancelledCount(valueOrZero(batch.getCancelledCount()));
|
response.setCancelledCount(valueOrZero(batch.getCancelledCount()));
|
||||||
response.setRetryableFailedCount(valueOrZero(batch.getRetryableFailedCount()));
|
// 人工继续统一覆盖全部失败项,兼容字段返回相同数量,避免旧计数影响前端判断。
|
||||||
|
response.setRetryableFailedCount(valueOrZero(batch.getFailedCount()));
|
||||||
int total = Math.max(1, valueOrZero(batch.getTotalCount()));
|
int total = Math.max(1, valueOrZero(batch.getTotalCount()));
|
||||||
int terminalCount = valueOrZero(batch.getCompletedCount())
|
int terminalCount = valueOrZero(batch.getCompletedCount())
|
||||||
+ valueOrZero(batch.getFailedCount())
|
+ valueOrZero(batch.getFailedCount())
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import com.easyagents.core.model.embedding.EmbeddingOptions;
|
|||||||
import com.easyagents.core.store.DocumentStore;
|
import com.easyagents.core.store.DocumentStore;
|
||||||
import com.easyagents.core.store.StoreOptions;
|
import com.easyagents.core.store.StoreOptions;
|
||||||
import com.easyagents.core.store.StoreResult;
|
import com.easyagents.core.store.StoreResult;
|
||||||
|
import com.easyagents.rag.core.BgeM3ChunkSafety;
|
||||||
import com.easyagents.rag.core.RagChunk;
|
import com.easyagents.rag.core.RagChunk;
|
||||||
import com.easyagents.rag.core.RagDefaults;
|
import com.easyagents.rag.core.RagDefaults;
|
||||||
import com.easyagents.rag.core.RagStrategyCodes;
|
import com.easyagents.rag.core.RagStrategyCodes;
|
||||||
@@ -129,6 +130,8 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
private static final String TASK_ERROR_EXECUTION_INTERRUPTED = "execution_interrupted";
|
private static final String TASK_ERROR_EXECUTION_INTERRUPTED = "execution_interrupted";
|
||||||
private static final String TASK_ERROR_SPLIT_FAILED = "split_failed";
|
private static final String TASK_ERROR_SPLIT_FAILED = "split_failed";
|
||||||
private static final String TASK_ERROR_INDEX_FAILED = "index_failed";
|
private static final String TASK_ERROR_INDEX_FAILED = "index_failed";
|
||||||
|
private static final String DEFAULT_INDEX_FAILURE_MESSAGE = "分块或向量化失败,请重试";
|
||||||
|
private static final String ROLLBACK_FAILURE_SUFFIX = ";外部索引回滚未完成,请联系管理员处理";
|
||||||
private static final Pattern HTTP_SERVER_ERROR_PATTERN = Pattern.compile("\\bstatus=5\\d{2}\\b");
|
private static final Pattern HTTP_SERVER_ERROR_PATTERN = Pattern.compile("\\bstatus=5\\d{2}\\b");
|
||||||
private static final Pattern MARKDOWN_IMAGE_PATTERN = Pattern.compile("!\\[(?:[^\\]]*)\\]\\(([^)]+)\\)");
|
private static final Pattern MARKDOWN_IMAGE_PATTERN = Pattern.compile("!\\[(?:[^\\]]*)\\]\\(([^)]+)\\)");
|
||||||
private final FlexIDKeyGenerator flexIdKeyGenerator = new FlexIDKeyGenerator();
|
private final FlexIDKeyGenerator flexIdKeyGenerator = new FlexIDKeyGenerator();
|
||||||
@@ -316,8 +319,8 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
buildDocumentPayload(document));
|
buildDocumentPayload(document));
|
||||||
documentImportBatchTracker.bindDocument(item.getId(), document.getId());
|
documentImportBatchTracker.bindDocument(item.getId(), document.getId());
|
||||||
taskIds.add(task.getId());
|
taskIds.add(task.getId());
|
||||||
documentImportTaskStatusStreamService.publishAfterCommit(document.getId());
|
|
||||||
}
|
}
|
||||||
|
// 启动接口返回后列表会统一刷新;批量建档阶段不逐文件推送,避免提交后形成 SSE 风暴。
|
||||||
int initialDispatchCount = Math.min(
|
int initialDispatchCount = Math.min(
|
||||||
taskIds.size(),
|
taskIds.size(),
|
||||||
Math.max(1, bulkProperties.getPerBatchParseMaxRunning())
|
Math.max(1, bulkProperties.getPerBatchParseMaxRunning())
|
||||||
@@ -339,13 +342,12 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
* 重试批次中选定的失败文件,单个文件重试失败不会中止其他文件。
|
* 重试批次中选定的失败文件,单个文件重试失败不会中止其他文件。
|
||||||
*
|
*
|
||||||
* @param batchId 批次 ID
|
* @param batchId 批次 ID
|
||||||
* @param fileKeys 指定文件键;为空时重试全部可恢复失败项
|
* @param fileKeys 指定文件键;为空时重试全部失败项
|
||||||
*/
|
*/
|
||||||
public void retryBatchFailures(BigInteger batchId, Set<String> fileKeys) {
|
public void retryBatchFailures(BigInteger batchId, Set<String> fileKeys) {
|
||||||
QueryWrapper query = QueryWrapper.create()
|
QueryWrapper query = QueryWrapper.create()
|
||||||
.eq(DocumentImportBatchItem::getBatchId, batchId)
|
.eq(DocumentImportBatchItem::getBatchId, batchId)
|
||||||
.eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name())
|
.eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name());
|
||||||
.eq(DocumentImportBatchItem::getRetryable, true);
|
|
||||||
if (fileKeys != null && !fileKeys.isEmpty()) {
|
if (fileKeys != null && !fileKeys.isEmpty()) {
|
||||||
query.in(DocumentImportBatchItem::getClientFileKey, fileKeys);
|
query.in(DocumentImportBatchItem::getClientFileKey, fileKeys);
|
||||||
}
|
}
|
||||||
@@ -827,6 +829,15 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
*/
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(DocumentImportDtos.TaskRetryRequest request) {
|
public Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(DocumentImportDtos.TaskRetryRequest request) {
|
||||||
|
DocumentCollection knowledge = assertDocumentCollection(request.getKnowledgeId());
|
||||||
|
tech.easyflow.ai.entity.Document document =
|
||||||
|
requireDocumentForKnowledge(request.getDocumentId(), knowledge.getId());
|
||||||
|
BigInteger batchId = optionAsBigInteger(
|
||||||
|
document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID);
|
||||||
|
if (DocumentProcessStatus.INDEX_FAILED.name().equals(document.getProcessStatus())
|
||||||
|
&& isAutomaticBatch(batchId)) {
|
||||||
|
return retryFailedTask(request);
|
||||||
|
}
|
||||||
DocumentImportDtos.TaskStartIndexRequest startRequest = new DocumentImportDtos.TaskStartIndexRequest();
|
DocumentImportDtos.TaskStartIndexRequest startRequest = new DocumentImportDtos.TaskStartIndexRequest();
|
||||||
startRequest.setKnowledgeId(request.getKnowledgeId());
|
startRequest.setKnowledgeId(request.getKnowledgeId());
|
||||||
startRequest.setDocumentId(request.getDocumentId());
|
startRequest.setDocumentId(request.getDocumentId());
|
||||||
@@ -849,21 +860,25 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
if (DocumentProcessStatus.PARSE_FAILED.name().equals(status)) {
|
if (DocumentProcessStatus.PARSE_FAILED.name().equals(status)) {
|
||||||
return retryParseTask(request);
|
return retryParseTask(request);
|
||||||
}
|
}
|
||||||
if (DocumentProcessStatus.INDEX_FAILED.name().equals(status)) {
|
|
||||||
return retryIndexTask(request);
|
|
||||||
}
|
|
||||||
if (!DocumentProcessStatus.SPLIT_FAILED.name().equals(status)) {
|
|
||||||
throw new BusinessException("当前文档无需重试");
|
|
||||||
}
|
|
||||||
if (!claimDocumentSplitRetry(document)) {
|
|
||||||
throw new BusinessException("文档状态已变化,请刷新后重试");
|
|
||||||
}
|
|
||||||
BigInteger batchId = optionAsBigInteger(
|
BigInteger batchId = optionAsBigInteger(
|
||||||
document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID);
|
document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID);
|
||||||
BigInteger batchItemId = optionAsBigInteger(
|
BigInteger batchItemId = optionAsBigInteger(
|
||||||
document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID);
|
document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID);
|
||||||
|
if (DocumentProcessStatus.INDEX_FAILED.name().equals(status)
|
||||||
|
&& !isAutomaticBatch(batchId)) {
|
||||||
|
return retryIndexTask(request);
|
||||||
|
}
|
||||||
|
if (!DocumentProcessStatus.SPLIT_FAILED.name().equals(status)
|
||||||
|
&& !DocumentProcessStatus.INDEX_FAILED.name().equals(status)) {
|
||||||
|
throw new BusinessException("当前文档无需重试");
|
||||||
|
}
|
||||||
|
if (!claimDocumentSplitRetry(document, status)) {
|
||||||
|
throw new BusinessException("文档状态已变化,请刷新后重试");
|
||||||
|
}
|
||||||
|
String staleSnapshotPath = invalidateChunkSnapshot(document);
|
||||||
DocumentImportTask splitTask =
|
DocumentImportTask splitTask =
|
||||||
enqueueAutomaticSplit(batchId, batchItemId, document);
|
enqueueAutomaticSplit(batchId, batchItemId, document);
|
||||||
|
deleteChunkSnapshotAfterCommit(staleSnapshotPath);
|
||||||
DocumentImportDtos.TaskStartIndexResponse response = new DocumentImportDtos.TaskStartIndexResponse();
|
DocumentImportDtos.TaskStartIndexResponse response = new DocumentImportDtos.TaskStartIndexResponse();
|
||||||
response.setTaskId(splitTask.getId());
|
response.setTaskId(splitTask.getId());
|
||||||
response.setProcessStatus(DocumentProcessStatus.SPLITTING.name());
|
response.setProcessStatus(DocumentProcessStatus.SPLITTING.name());
|
||||||
@@ -874,19 +889,6 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
if (item.getDocumentId() == null) {
|
if (item.getDocumentId() == null) {
|
||||||
throw new BusinessException("失败文件尚未生成文档");
|
throw new BusinessException("失败文件尚未生成文档");
|
||||||
}
|
}
|
||||||
int completedRetries = item.getAttemptCount() == null ? 0 : item.getAttemptCount();
|
|
||||||
if (completedRetries >= Math.max(1, bulkProperties.getMaxTaskAttempts()) - 1) {
|
|
||||||
documentImportBatchTracker.transitionItem(
|
|
||||||
item.getId(),
|
|
||||||
DocumentImportBatchItemStage.valueOf(item.getStage()),
|
|
||||||
DocumentImportBatchItemStatus.FAILED,
|
|
||||||
item.getErrorSummary(),
|
|
||||||
false,
|
|
||||||
0,
|
|
||||||
item.getFailureCode()
|
|
||||||
);
|
|
||||||
throw new BusinessException("该文件已达到最大重试次数");
|
|
||||||
}
|
|
||||||
tech.easyflow.ai.entity.Document document = requireDocument(item.getDocumentId());
|
tech.easyflow.ai.entity.Document document = requireDocument(item.getDocumentId());
|
||||||
DocumentImportDtos.TaskRetryRequest request = new DocumentImportDtos.TaskRetryRequest();
|
DocumentImportDtos.TaskRetryRequest request = new DocumentImportDtos.TaskRetryRequest();
|
||||||
request.setKnowledgeId(item.getKnowledgeId());
|
request.setKnowledgeId(item.getKnowledgeId());
|
||||||
@@ -1232,10 +1234,14 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
clearPersistedChunks(document.getId());
|
clearPersistedChunks(document.getId());
|
||||||
|
boolean rollbackSucceeded = true;
|
||||||
if (storeContext != null && !storedChunks.isEmpty()) {
|
if (storeContext != null && !storedChunks.isEmpty()) {
|
||||||
rollbackStoredChunks(taskId, document.getId(), storeContext, storedChunks);
|
rollbackSucceeded = rollbackStoredChunks(
|
||||||
|
taskId, document.getId(), storeContext, storedChunks);
|
||||||
}
|
}
|
||||||
markIndexFailed(task, document, "分块或向量化失败,请重试");
|
markIndexFailed(
|
||||||
|
task, document,
|
||||||
|
resolveIndexFailureMessage(e, rollbackSucceeded));
|
||||||
} finally {
|
} finally {
|
||||||
closeStoreContext(storeContext);
|
closeStoreContext(storeContext);
|
||||||
}
|
}
|
||||||
@@ -1441,8 +1447,7 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
task.getBatchItemId(),
|
task.getBatchItemId(),
|
||||||
DocumentImportBatchItemStage.SPLIT,
|
DocumentImportBatchItemStage.SPLIT,
|
||||||
errorMessage,
|
errorMessage,
|
||||||
failureCode,
|
failureCode
|
||||||
canRetryBatchItem(task.getBatchItemId())
|
|
||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -1578,8 +1583,7 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
task.getBatchItemId(),
|
task.getBatchItemId(),
|
||||||
stage,
|
stage,
|
||||||
errorMessage,
|
errorMessage,
|
||||||
failureCode,
|
failureCode);
|
||||||
canRetryBatchItem(task.getBatchItemId()));
|
|
||||||
if (task.getBatchId() != null) {
|
if (task.getBatchId() != null) {
|
||||||
documentImportBatchTracker.markInterrupted(task.getBatchId());
|
documentImportBatchTracker.markInterrupted(task.getBatchId());
|
||||||
}
|
}
|
||||||
@@ -1714,9 +1718,7 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
task.getBatchItemId(),
|
task.getBatchItemId(),
|
||||||
DocumentImportBatchItemStage.PARSE,
|
DocumentImportBatchItemStage.PARSE,
|
||||||
errorMessage,
|
errorMessage,
|
||||||
errorCode,
|
errorCode);
|
||||||
isRetryableParseFailure(errorCode)
|
|
||||||
&& canRetryBatchItem(task.getBatchItemId()));
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2177,16 +2179,55 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
null);
|
null);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markIndexFailed(DocumentImportTask task,
|
/**
|
||||||
tech.easyflow.ai.entity.Document document,
|
* 使用通用索引失败码标记任务失败。
|
||||||
String errorMessage) {
|
*
|
||||||
|
* @param task 索引任务
|
||||||
|
* @param document 文档
|
||||||
|
* @param errorMessage 用户可见错误信息
|
||||||
|
*/
|
||||||
|
private void markIndexFailed(
|
||||||
|
DocumentImportTask task,
|
||||||
|
tech.easyflow.ai.entity.Document document,
|
||||||
|
String errorMessage) {
|
||||||
|
markIndexFailed(
|
||||||
|
task, document, errorMessage, TASK_ERROR_INDEX_FAILED);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将索引任务标记为失败并保留阶段特定失败码。
|
||||||
|
*
|
||||||
|
* @param task 索引任务
|
||||||
|
* @param document 文档
|
||||||
|
* @param errorMessage 用户可见错误信息
|
||||||
|
* @param failureCode 稳定失败码
|
||||||
|
*/
|
||||||
|
private void markIndexFailed(
|
||||||
|
DocumentImportTask task,
|
||||||
|
tech.easyflow.ai.entity.Document document,
|
||||||
|
String errorMessage,
|
||||||
|
String failureCode) {
|
||||||
KnowledgeDocumentImportTaskAppService executor = selfProxy == null ? this : selfProxy;
|
KnowledgeDocumentImportTaskAppService executor = selfProxy == null ? this : selfProxy;
|
||||||
if (!executor.failIndexTask(task, document, errorMessage)) {
|
if (!executor.failIndexTask(task, document, errorMessage, failureCode)) {
|
||||||
LOG.warn("向量化任务所有权已失效,忽略迟到失败: taskId={}, documentId={}",
|
LOG.warn("向量化任务所有权已失效,忽略迟到失败: taskId={}, documentId={}",
|
||||||
task.getId(), document.getId());
|
task.getId(), document.getId());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 生成可持久化的索引失败原因。
|
||||||
|
*
|
||||||
|
* @param error 原始异常
|
||||||
|
* @param rollbackSucceeded 外部索引是否完整回滚
|
||||||
|
* @return 对用户安全且可执行的失败原因
|
||||||
|
*/
|
||||||
|
private String resolveIndexFailureMessage(Exception error, boolean rollbackSucceeded) {
|
||||||
|
String message = error instanceof BusinessException && StringUtil.hasText(error.getMessage())
|
||||||
|
? error.getMessage()
|
||||||
|
: DEFAULT_INDEX_FAILURE_MESSAGE;
|
||||||
|
return rollbackSucceeded ? message : message + ROLLBACK_FAILURE_SUFFIX;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 原子写入向量化任务、文档与批次项失败状态。
|
* 原子写入向量化任务、文档与批次项失败状态。
|
||||||
*
|
*
|
||||||
@@ -2196,13 +2237,35 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
* @return 是否仍持有任务终态写入权
|
* @return 是否仍持有任务终态写入权
|
||||||
*/
|
*/
|
||||||
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
|
public boolean failIndexTask(
|
||||||
|
DocumentImportTask task,
|
||||||
|
tech.easyflow.ai.entity.Document document,
|
||||||
|
String errorMessage) {
|
||||||
|
return failIndexTask(
|
||||||
|
task, document, errorMessage, TASK_ERROR_INDEX_FAILED);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 原子写入带阶段失败码的向量化任务、文档与批次项失败状态。
|
||||||
|
*
|
||||||
|
* @param task 向量化任务
|
||||||
|
* @param document 文档实体
|
||||||
|
* @param errorMessage 用户可见错误信息
|
||||||
|
* @param failureCode 稳定失败码
|
||||||
|
* @return 是否仍持有任务终态写入权
|
||||||
|
*/
|
||||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
public boolean failIndexTask(DocumentImportTask task,
|
public boolean failIndexTask(DocumentImportTask task,
|
||||||
tech.easyflow.ai.entity.Document document,
|
tech.easyflow.ai.entity.Document document,
|
||||||
String errorMessage) {
|
String errorMessage,
|
||||||
|
String failureCode) {
|
||||||
|
String resolvedFailureCode = StringUtil.hasText(failureCode)
|
||||||
|
? failureCode
|
||||||
|
: TASK_ERROR_INDEX_FAILED;
|
||||||
Date now = new Date();
|
Date now = new Date();
|
||||||
if (!finishTask(
|
if (!finishTask(
|
||||||
task, now, DocumentImportTaskStatus.FAILED,
|
task, now, DocumentImportTaskStatus.FAILED,
|
||||||
errorMessage, TASK_ERROR_INDEX_FAILED)) {
|
errorMessage, resolvedFailureCode)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
tech.easyflow.ai.entity.Document current = requireDocument(document.getId());
|
tech.easyflow.ai.entity.Document current = requireDocument(document.getId());
|
||||||
@@ -2211,7 +2274,7 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
current.setCompletedChunks(0);
|
current.setCompletedChunks(0);
|
||||||
current.setFailedChunks(defaultInt(current.getTotalChunks()));
|
current.setFailedChunks(defaultInt(current.getTotalChunks()));
|
||||||
current.setProgressPercent(0);
|
current.setProgressPercent(0);
|
||||||
setDocumentTaskError(current, errorMessage, TASK_ERROR_INDEX_FAILED);
|
setDocumentTaskError(current, errorMessage, resolvedFailureCode);
|
||||||
persistDocumentTaskState(current, now);
|
persistDocumentTaskState(current, now);
|
||||||
LOG.warn("文档向量化任务失败: taskId={}, documentId={}, processStatus={}, completedChunks={}, totalChunks={}, error={}",
|
LOG.warn("文档向量化任务失败: taskId={}, documentId={}, processStatus={}, completedChunks={}, totalChunks={}, error={}",
|
||||||
task.getId(),
|
task.getId(),
|
||||||
@@ -2225,8 +2288,7 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
task.getBatchItemId(),
|
task.getBatchItemId(),
|
||||||
DocumentImportBatchItemStage.INDEX,
|
DocumentImportBatchItemStage.INDEX,
|
||||||
errorMessage,
|
errorMessage,
|
||||||
TASK_ERROR_INDEX_FAILED,
|
resolvedFailureCode);
|
||||||
canRetryBatchItem(task.getBatchItemId()));
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2394,12 +2456,14 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 按分块失败状态原子领取重试权。
|
* 按指定失败状态原子领取重新分块重试权。
|
||||||
*
|
*
|
||||||
* @param document 文档实体
|
* @param document 文档实体
|
||||||
|
* @param expectedStatus 领取前预期的失败状态
|
||||||
* @return 是否领取成功
|
* @return 是否领取成功
|
||||||
*/
|
*/
|
||||||
private boolean claimDocumentSplitRetry(tech.easyflow.ai.entity.Document document) {
|
private boolean claimDocumentSplitRetry(tech.easyflow.ai.entity.Document document,
|
||||||
|
String expectedStatus) {
|
||||||
Date now = new Date();
|
Date now = new Date();
|
||||||
tech.easyflow.ai.entity.Document update = new tech.easyflow.ai.entity.Document();
|
tech.easyflow.ai.entity.Document update = new tech.easyflow.ai.entity.Document();
|
||||||
update.setProcessStatus(DocumentProcessStatus.SPLITTING.name());
|
update.setProcessStatus(DocumentProcessStatus.SPLITTING.name());
|
||||||
@@ -2411,7 +2475,7 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
QueryWrapper.create()
|
QueryWrapper.create()
|
||||||
.eq(tech.easyflow.ai.entity.Document::getId, document.getId())
|
.eq(tech.easyflow.ai.entity.Document::getId, document.getId())
|
||||||
.eq(tech.easyflow.ai.entity.Document::getProcessStatus,
|
.eq(tech.easyflow.ai.entity.Document::getProcessStatus,
|
||||||
DocumentProcessStatus.SPLIT_FAILED.name()));
|
expectedStatus));
|
||||||
if (updated > 0) {
|
if (updated > 0) {
|
||||||
document.setProcessStatus(DocumentProcessStatus.SPLITTING.name());
|
document.setProcessStatus(DocumentProcessStatus.SPLITTING.name());
|
||||||
return true;
|
return true;
|
||||||
@@ -2419,6 +2483,28 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 移除文档对旧分块快照的引用,确保重试只能使用新分块结果。
|
||||||
|
*
|
||||||
|
* @param document 待重新分块的文档
|
||||||
|
* @return 失效的旧快照路径
|
||||||
|
*/
|
||||||
|
private String invalidateChunkSnapshot(tech.easyflow.ai.entity.Document document) {
|
||||||
|
if (document.getOptions() == null || document.getOptions().isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String snapshotPath = optionAsString(
|
||||||
|
document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH);
|
||||||
|
if (!StringUtil.hasText(snapshotPath)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
Map<String, Object> options =
|
||||||
|
new LinkedHashMap<String, Object>(document.getOptions());
|
||||||
|
options.remove(DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH);
|
||||||
|
document.setOptions(options);
|
||||||
|
return snapshotPath;
|
||||||
|
}
|
||||||
|
|
||||||
private void updateDocumentIndexProgress(BigInteger documentId, int totalChunks, int completedChunks) {
|
private void updateDocumentIndexProgress(BigInteger documentId, int totalChunks, int completedChunks) {
|
||||||
int progressPercent = Math.min(100, totalChunks <= 0 ? 0 : (completedChunks * 100) / totalChunks);
|
int progressPercent = Math.min(100, totalChunks <= 0 ? 0 : (completedChunks * 100) / totalChunks);
|
||||||
tech.easyflow.ai.entity.Document document = requireDocument(documentId);
|
tech.easyflow.ai.entity.Document document = requireDocument(documentId);
|
||||||
@@ -2736,16 +2822,18 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
String sourceFormat,
|
String sourceFormat,
|
||||||
StrategyConfig strategyConfig,
|
StrategyConfig strategyConfig,
|
||||||
Map<String, Object> parseArtifactSummary) {
|
Map<String, Object> parseArtifactSummary) {
|
||||||
|
List<DocumentChunk> chunks;
|
||||||
if ("pptx".equals(sourceFormat)) {
|
if ("pptx".equals(sourceFormat)) {
|
||||||
return buildPptxDocumentChunks(document, parseArtifactSummary);
|
chunks = buildPptxDocumentChunks(document, parseArtifactSummary);
|
||||||
}
|
} else if ("xlsx".equals(sourceFormat)) {
|
||||||
if ("xlsx".equals(sourceFormat)) {
|
|
||||||
int rowsPerChunk = strategyConfig == null || strategyConfig.getRowsPerChunk() == null
|
int rowsPerChunk = strategyConfig == null || strategyConfig.getRowsPerChunk() == null
|
||||||
? 10
|
? 10
|
||||||
: Math.max(1, strategyConfig.getRowsPerChunk());
|
: Math.max(1, strategyConfig.getRowsPerChunk());
|
||||||
return buildXlsxDocumentChunks(document, parseArtifactSummary, rowsPerChunk);
|
chunks = buildXlsxDocumentChunks(document, parseArtifactSummary, rowsPerChunk);
|
||||||
|
} else {
|
||||||
|
chunks = new ArrayList<DocumentChunk>();
|
||||||
}
|
}
|
||||||
return new ArrayList<DocumentChunk>();
|
return enforceDocumentChunkHardLimit(chunks);
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
@@ -2916,6 +3004,94 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
return chunk;
|
return chunk;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 对 Office 分块应用统一的 BGE-M3 上下文硬限制。
|
||||||
|
*
|
||||||
|
* @param chunks Office 构建器生成的原始分块
|
||||||
|
* @return 内容均处于安全预算内且排序连续的分块
|
||||||
|
*/
|
||||||
|
private List<DocumentChunk> enforceDocumentChunkHardLimit(
|
||||||
|
List<DocumentChunk> chunks) {
|
||||||
|
List<DocumentChunk> safeChunks = new ArrayList<DocumentChunk>();
|
||||||
|
if (chunks == null || chunks.isEmpty()) {
|
||||||
|
return safeChunks;
|
||||||
|
}
|
||||||
|
int sorting = 1;
|
||||||
|
for (DocumentChunk chunk : chunks) {
|
||||||
|
String content = chunk == null || chunk.getContent() == null
|
||||||
|
? ""
|
||||||
|
: chunk.getContent();
|
||||||
|
List<BgeM3ChunkSafety.ChunkRange> parts =
|
||||||
|
BgeM3ChunkSafety.splitToHardLimit(content);
|
||||||
|
if (parts.isEmpty()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (int index = 0; index < parts.size(); index++) {
|
||||||
|
BgeM3ChunkSafety.ChunkRange part = parts.get(index);
|
||||||
|
String partContent =
|
||||||
|
content.substring(part.getStart(), part.getEnd()).trim();
|
||||||
|
if (!StringUtil.hasText(partContent)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
DocumentChunk safeChunk = parts.size() == 1
|
||||||
|
? chunk
|
||||||
|
: copyDocumentChunkPart(chunk);
|
||||||
|
safeChunk.setSorting(sorting++);
|
||||||
|
safeChunk.setContent(partContent);
|
||||||
|
Map<String, Object> options = copyOptions(safeChunk.getOptions());
|
||||||
|
options.put("charCount", partContent.length());
|
||||||
|
options.put("tokenEstimate", Math.max(
|
||||||
|
1, BgeM3ChunkSafety.estimateContentTokens(partContent)));
|
||||||
|
options.put("partNo", index + 1);
|
||||||
|
options.put("partTotal", parts.size());
|
||||||
|
if (parts.size() > 1) {
|
||||||
|
options.put("hardSplit", Boolean.TRUE);
|
||||||
|
options.put(
|
||||||
|
"hardSplitTokenLimit",
|
||||||
|
RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT);
|
||||||
|
boundSplitRenderMarkdown(options, partContent);
|
||||||
|
}
|
||||||
|
safeChunk.setOptions(options);
|
||||||
|
safeChunks.add(safeChunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return safeChunks;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 复制一个待继续切分的文档块,并为子块生成独立主键。
|
||||||
|
*
|
||||||
|
* @param source 原始分块
|
||||||
|
* @return 不共享可变元数据的子块
|
||||||
|
*/
|
||||||
|
private DocumentChunk copyDocumentChunkPart(DocumentChunk source) {
|
||||||
|
DocumentChunk copy = new DocumentChunk();
|
||||||
|
copy.setId(generateId(copy));
|
||||||
|
copy.setDocumentId(source.getDocumentId());
|
||||||
|
copy.setDocumentCollectionId(source.getDocumentCollectionId());
|
||||||
|
copy.setOptions(copyOptions(source.getOptions()));
|
||||||
|
return copy;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 避免强制子块继续复制整页超长渲染正文。
|
||||||
|
*
|
||||||
|
* @param options 子块元数据
|
||||||
|
* @param partContent 当前子块正文
|
||||||
|
*/
|
||||||
|
private void boundSplitRenderMarkdown(
|
||||||
|
Map<String, Object> options,
|
||||||
|
String partContent) {
|
||||||
|
Object renderMarkdown =
|
||||||
|
options.get(DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN);
|
||||||
|
if (renderMarkdown instanceof String
|
||||||
|
&& !BgeM3ChunkSafety.isWithinHardLimit((String) renderMarkdown)) {
|
||||||
|
options.put(
|
||||||
|
DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN,
|
||||||
|
partContent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private String buildPptxChunkRenderMarkdown(int slideIndex,
|
private String buildPptxChunkRenderMarkdown(int slideIndex,
|
||||||
String title,
|
String title,
|
||||||
String imageName,
|
String imageName,
|
||||||
@@ -3328,6 +3504,8 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
config.setChunkSize(asInteger(rawSnapshot.get("chunkSize"), RagDefaults.CHUNK_SIZE));
|
config.setChunkSize(asInteger(rawSnapshot.get("chunkSize"), RagDefaults.CHUNK_SIZE));
|
||||||
config.setOverlapSize(asInteger(rawSnapshot.get("overlapSize"), RagDefaults.OVERLAP_SIZE));
|
config.setOverlapSize(asInteger(rawSnapshot.get("overlapSize"), RagDefaults.OVERLAP_SIZE));
|
||||||
config.setRegex(asString(rawSnapshot.get("regex")));
|
config.setRegex(asString(rawSnapshot.get("regex")));
|
||||||
|
config.setRetainRegexMatch(
|
||||||
|
asBoolean(rawSnapshot.get("retainRegexMatch"), false));
|
||||||
config.setRowsPerChunk(asInteger(rawSnapshot.get("rowsPerChunk"), config.getRowsPerChunk()));
|
config.setRowsPerChunk(asInteger(rawSnapshot.get("rowsPerChunk"), config.getRowsPerChunk()));
|
||||||
config.setMdSplitterLevel(asInteger(rawSnapshot.get("mdSplitterLevel"), RagDefaults.MD_SPLITTER_LEVEL));
|
config.setMdSplitterLevel(asInteger(rawSnapshot.get("mdSplitterLevel"), RagDefaults.MD_SPLITTER_LEVEL));
|
||||||
return config;
|
return config;
|
||||||
@@ -3534,69 +3712,6 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断批次项是否仍未达到最大尝试次数。
|
|
||||||
*
|
|
||||||
* @param itemId 批次文件项 ID
|
|
||||||
* @return 是否允许继续重试
|
|
||||||
*/
|
|
||||||
private boolean canRetryBatchItem(BigInteger itemId) {
|
|
||||||
if (itemId == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
DocumentImportBatchItem item =
|
|
||||||
documentImportBatchTracker.requireItem(itemId);
|
|
||||||
int completedRetries =
|
|
||||||
item.getAttemptCount() == null ? 0 : item.getAttemptCount();
|
|
||||||
return completedRetries
|
|
||||||
< Math.max(1, bulkProperties.getMaxTaskAttempts()) - 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断历史批次失败项能否按当前失败分类恢复重试资格。
|
|
||||||
*
|
|
||||||
* <p>用于兼容修复前已经写入 {@code retryable=false} 的失败项。
|
|
||||||
* 新失败项仍在写入终态时持久化重试资格,避免正常路径重复计算。</p>
|
|
||||||
*
|
|
||||||
* @param item 批次失败项
|
|
||||||
* @return 是否可以恢复重试资格
|
|
||||||
*/
|
|
||||||
public boolean isRecoverableBatchFailure(DocumentImportBatchItem item) {
|
|
||||||
if (item == null
|
|
||||||
|| !DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
|
|
||||||
|| (item.getAttemptCount() != null
|
|
||||||
&& item.getAttemptCount()
|
|
||||||
>= Math.max(1, bulkProperties.getMaxTaskAttempts()) - 1)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
DocumentImportBatchItemStage stage;
|
|
||||||
try {
|
|
||||||
stage = DocumentImportBatchItemStage.valueOf(item.getStage());
|
|
||||||
} catch (IllegalArgumentException | NullPointerException error) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return switch (stage) {
|
|
||||||
case PARSE -> isRetryableParseFailure(item.getFailureCode());
|
|
||||||
case SPLIT, INDEX -> true;
|
|
||||||
case UPLOAD, DONE -> false;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断解析失败是否属于可恢复故障。
|
|
||||||
*
|
|
||||||
* <p>只有已明确识别的输入或请求问题禁止重试。文档源暂时不可用、
|
|
||||||
* 解析服务异常和未识别的系统错误允许人工重试,并继续受最大尝试
|
|
||||||
* 次数约束,便于服务恢复或代码修复后沿用原任务继续处理。</p>
|
|
||||||
*
|
|
||||||
* @param failureCode 稳定失败码
|
|
||||||
* @return 是否可恢复
|
|
||||||
*/
|
|
||||||
private boolean isRetryableParseFailure(String failureCode) {
|
|
||||||
return !TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE.equals(failureCode)
|
|
||||||
&& !TASK_ERROR_INVALID_PARSE_REQUEST.equals(failureCode);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void updateBatchItem(BigInteger itemId,
|
private void updateBatchItem(BigInteger itemId,
|
||||||
DocumentImportBatchItemStage stage,
|
DocumentImportBatchItemStage stage,
|
||||||
DocumentImportBatchItemStatus status,
|
DocumentImportBatchItemStatus status,
|
||||||
@@ -3613,14 +3728,12 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
* @param stage 失败阶段
|
* @param stage 失败阶段
|
||||||
* @param errorSummary 错误摘要
|
* @param errorSummary 错误摘要
|
||||||
* @param failureCode 稳定失败码
|
* @param failureCode 稳定失败码
|
||||||
* @param retryable 是否允许重试
|
|
||||||
*/
|
*/
|
||||||
private void updateBatchItemFailure(
|
private void updateBatchItemFailure(
|
||||||
BigInteger itemId,
|
BigInteger itemId,
|
||||||
DocumentImportBatchItemStage stage,
|
DocumentImportBatchItemStage stage,
|
||||||
String errorSummary,
|
String errorSummary,
|
||||||
String failureCode,
|
String failureCode) {
|
||||||
boolean retryable) {
|
|
||||||
if (itemId == null) {
|
if (itemId == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -3629,7 +3742,7 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
stage,
|
stage,
|
||||||
DocumentImportBatchItemStatus.FAILED,
|
DocumentImportBatchItemStatus.FAILED,
|
||||||
errorSummary,
|
errorSummary,
|
||||||
retryable,
|
true,
|
||||||
0,
|
0,
|
||||||
failureCode
|
failureCode
|
||||||
);
|
);
|
||||||
@@ -3694,6 +3807,7 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
private void storeDocumentChunks(StoreExecutionContext storeContext, List<DocumentChunk> documentChunks) {
|
private void storeDocumentChunks(StoreExecutionContext storeContext, List<DocumentChunk> documentChunks) {
|
||||||
List<Document> documents = new ArrayList<Document>();
|
List<Document> documents = new ArrayList<Document>();
|
||||||
for (DocumentChunk chunk : documentChunks) {
|
for (DocumentChunk chunk : documentChunks) {
|
||||||
|
assertEmbeddingChunkWithinHardLimit(chunk);
|
||||||
Document storeDocument = new Document();
|
Document storeDocument = new Document();
|
||||||
storeDocument.setId(chunk.getId());
|
storeDocument.setId(chunk.getId());
|
||||||
storeDocument.setContent(chunk.getContent());
|
storeDocument.setContent(chunk.getContent());
|
||||||
@@ -3705,10 +3819,25 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
if (result == null || !result.isSuccess()) {
|
if (result == null || !result.isSuccess()) {
|
||||||
throw new BusinessException("向量化写入失败");
|
throw new BusinessException("向量化写入失败");
|
||||||
}
|
}
|
||||||
if (storeContext.searcher != null) {
|
if (storeContext.searcher != null && !storeContext.searcher.addDocuments(documents)) {
|
||||||
for (Document storeDocument : documents) {
|
throw new BusinessException("关键词索引写入失败");
|
||||||
storeContext.searcher.addDocument(storeDocument);
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在调用向量模型前校验最终分块预算,阻止遗漏入口发送超长正文。
|
||||||
|
*
|
||||||
|
* @param chunk 待向量化分块
|
||||||
|
* @throws BusinessException 分块超过 BGE-M3 安全上限
|
||||||
|
*/
|
||||||
|
private void assertEmbeddingChunkWithinHardLimit(DocumentChunk chunk) {
|
||||||
|
String content = chunk == null ? null : chunk.getContent();
|
||||||
|
int tokenEstimate = BgeM3ChunkSafety.estimateContentTokens(content);
|
||||||
|
if (tokenEstimate > RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT) {
|
||||||
|
BigInteger chunkId = chunk == null ? null : chunk.getId();
|
||||||
|
throw new BusinessException(
|
||||||
|
"分块内容超过向量模型上下文上限,请重新分块:chunkId="
|
||||||
|
+ chunkId + ",tokenEstimate=" + tokenEstimate);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3720,10 +3849,19 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void rollbackStoredChunks(BigInteger taskId,
|
/**
|
||||||
BigInteger documentId,
|
* 回滚已写入的向量和关键词索引。
|
||||||
StoreExecutionContext storeContext,
|
*
|
||||||
List<DocumentChunk> documentChunks) {
|
* @param taskId 任务 ID
|
||||||
|
* @param documentId 文档 ID
|
||||||
|
* @param storeContext 外部存储上下文
|
||||||
|
* @param documentChunks 待回滚分块
|
||||||
|
* @return 两类外部索引均回滚成功时返回 {@code true}
|
||||||
|
*/
|
||||||
|
private boolean rollbackStoredChunks(BigInteger taskId,
|
||||||
|
BigInteger documentId,
|
||||||
|
StoreExecutionContext storeContext,
|
||||||
|
List<DocumentChunk> documentChunks) {
|
||||||
try {
|
try {
|
||||||
Set<BigInteger> uniqueIds = new LinkedHashSet<BigInteger>();
|
Set<BigInteger> uniqueIds = new LinkedHashSet<BigInteger>();
|
||||||
for (DocumentChunk chunk : documentChunks) {
|
for (DocumentChunk chunk : documentChunks) {
|
||||||
@@ -3732,6 +3870,9 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
List<BigInteger> ids = new ArrayList<BigInteger>(uniqueIds);
|
List<BigInteger> ids = new ArrayList<BigInteger>(uniqueIds);
|
||||||
|
if (ids.isEmpty()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
LOG.warn("开始回滚文档向量化外部索引: taskId={}, documentId={}, knowledgeId={}, chunkCount={}",
|
LOG.warn("开始回滚文档向量化外部索引: taskId={}, documentId={}, knowledgeId={}, chunkCount={}",
|
||||||
taskId,
|
taskId,
|
||||||
documentId,
|
documentId,
|
||||||
@@ -3742,18 +3883,18 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
String failReason = deleteResult == null ? "未返回结果" : deleteResult.getFailReason();
|
String failReason = deleteResult == null ? "未返回结果" : deleteResult.getFailReason();
|
||||||
throw new IllegalStateException("向量存储回滚失败: " + failReason);
|
throw new IllegalStateException("向量存储回滚失败: " + failReason);
|
||||||
}
|
}
|
||||||
if (storeContext.searcher != null) {
|
if (storeContext.searcher != null && !storeContext.searcher.deleteDocuments(ids)) {
|
||||||
for (BigInteger id : ids) {
|
throw new IllegalStateException("关键词索引回滚失败");
|
||||||
storeContext.searcher.deleteDocument(id);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
LOG.warn("文档向量化外部索引回滚完成: taskId={}, documentId={}, knowledgeId={}, chunkCount={}",
|
LOG.warn("文档向量化外部索引回滚完成: taskId={}, documentId={}, knowledgeId={}, chunkCount={}",
|
||||||
taskId,
|
taskId,
|
||||||
documentId,
|
documentId,
|
||||||
storeContext == null || storeContext.knowledge == null ? null : storeContext.knowledge.getId(),
|
storeContext == null || storeContext.knowledge == null ? null : storeContext.knowledge.getId(),
|
||||||
ids.size());
|
ids.size());
|
||||||
|
return true;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
LOG.error("回滚文档向量数据失败", e);
|
LOG.error("回滚文档向量数据失败", e);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4203,6 +4344,18 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在重试事务提交后删除已经失效的旧分块快照。
|
||||||
|
*
|
||||||
|
* @param snapshotPath 旧分块快照路径
|
||||||
|
*/
|
||||||
|
private void deleteChunkSnapshotAfterCommit(String snapshotPath) {
|
||||||
|
if (!StringUtil.hasText(snapshotPath)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
runAfterCommit(() -> deleteChunkSnapshotAfterCompletion(snapshotPath));
|
||||||
|
}
|
||||||
|
|
||||||
private StrategyConfig resolveStrategyConfig(DocumentCollection knowledge,
|
private StrategyConfig resolveStrategyConfig(DocumentCollection knowledge,
|
||||||
StrategyConfig requestConfig,
|
StrategyConfig requestConfig,
|
||||||
AnalysisResult analysisResult) {
|
AnalysisResult analysisResult) {
|
||||||
@@ -4247,9 +4400,27 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
if (config.getMdSplitterLevel() == null || config.getMdSplitterLevel() <= 0) {
|
if (config.getMdSplitterLevel() == null || config.getMdSplitterLevel() <= 0) {
|
||||||
config.setMdSplitterLevel(RagDefaults.MD_SPLITTER_LEVEL);
|
config.setMdSplitterLevel(RagDefaults.MD_SPLITTER_LEVEL);
|
||||||
}
|
}
|
||||||
|
if (config.getRetainRegexMatch() == null) {
|
||||||
|
config.setRetainRegexMatch(Boolean.FALSE);
|
||||||
|
}
|
||||||
|
validateStrategyOverlap(config);
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验长度分块参数能够保证分块游标持续前进。
|
||||||
|
*
|
||||||
|
* @param config 最终生效的分块策略
|
||||||
|
* @throws BusinessException 重叠大小不小于分块大小
|
||||||
|
*/
|
||||||
|
private void validateStrategyOverlap(StrategyConfig config) {
|
||||||
|
if (config.getOverlapSize() != null
|
||||||
|
&& config.getChunkSize() != null
|
||||||
|
&& config.getOverlapSize() >= config.getChunkSize()) {
|
||||||
|
throw new BusinessException("分块重叠大小必须小于分块大小");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 任务已被看门狗或其他执行者收口时抛出的内部异常。
|
* 任务已被看门狗或其他执行者收口时抛出的内部异常。
|
||||||
*/
|
*/
|
||||||
@@ -4285,6 +4456,10 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
config.setChunkSize(asInteger(rawProfile.get("chunkSize"), config.getChunkSize()));
|
config.setChunkSize(asInteger(rawProfile.get("chunkSize"), config.getChunkSize()));
|
||||||
config.setOverlapSize(asInteger(rawProfile.get("overlapSize"), config.getOverlapSize()));
|
config.setOverlapSize(asInteger(rawProfile.get("overlapSize"), config.getOverlapSize()));
|
||||||
config.setRegex(asString(rawProfile.get("regex")));
|
config.setRegex(asString(rawProfile.get("regex")));
|
||||||
|
if (rawProfile.containsKey("retainRegexMatch")) {
|
||||||
|
config.setRetainRegexMatch(
|
||||||
|
asBoolean(rawProfile.get("retainRegexMatch"), false));
|
||||||
|
}
|
||||||
config.setRowsPerChunk(asInteger(rawProfile.get("rowsPerChunk"), config.getRowsPerChunk()));
|
config.setRowsPerChunk(asInteger(rawProfile.get("rowsPerChunk"), config.getRowsPerChunk()));
|
||||||
config.setMdSplitterLevel(asInteger(rawProfile.get("mdSplitterLevel"), config.getMdSplitterLevel()));
|
config.setMdSplitterLevel(asInteger(rawProfile.get("mdSplitterLevel"), config.getMdSplitterLevel()));
|
||||||
return config;
|
return config;
|
||||||
@@ -4306,6 +4481,9 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
if (StringUtil.hasText(source.getRegex())) {
|
if (StringUtil.hasText(source.getRegex())) {
|
||||||
target.setRegex(source.getRegex());
|
target.setRegex(source.getRegex());
|
||||||
}
|
}
|
||||||
|
if (source.getRetainRegexMatch() != null) {
|
||||||
|
target.setRetainRegexMatch(source.getRetainRegexMatch());
|
||||||
|
}
|
||||||
if (source.getRowsPerChunk() != null) {
|
if (source.getRowsPerChunk() != null) {
|
||||||
target.setRowsPerChunk(source.getRowsPerChunk());
|
target.setRowsPerChunk(source.getRowsPerChunk());
|
||||||
}
|
}
|
||||||
@@ -4320,6 +4498,8 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
map.put("chunkSize", strategyConfig.getChunkSize());
|
map.put("chunkSize", strategyConfig.getChunkSize());
|
||||||
map.put("overlapSize", strategyConfig.getOverlapSize());
|
map.put("overlapSize", strategyConfig.getOverlapSize());
|
||||||
map.put("regex", strategyConfig.getRegex());
|
map.put("regex", strategyConfig.getRegex());
|
||||||
|
map.put("retainRegexMatch", Boolean.TRUE.equals(
|
||||||
|
strategyConfig.getRetainRegexMatch()));
|
||||||
map.put("rowsPerChunk", strategyConfig.getRowsPerChunk());
|
map.put("rowsPerChunk", strategyConfig.getRowsPerChunk());
|
||||||
map.put("mdSplitterLevel", strategyConfig.getMdSplitterLevel());
|
map.put("mdSplitterLevel", strategyConfig.getMdSplitterLevel());
|
||||||
return map;
|
return map;
|
||||||
|
|||||||
@@ -764,9 +764,9 @@ public class KnowledgeImportBatchFacade {
|
|||||||
counts.setPending(internal.getPendingCount());
|
counts.setPending(internal.getPendingCount());
|
||||||
counts.setFailed(internal.getFailedCount());
|
counts.setFailed(internal.getFailedCount());
|
||||||
counts.setSkipped(internal.getSkippedCount());
|
counts.setSkipped(internal.getSkippedCount());
|
||||||
counts.setRetryableFailed(internal.getRetryableFailedCount());
|
counts.setRetryableFailed(internal.getFailedCount());
|
||||||
response.setCounts(counts);
|
response.setCounts(counts);
|
||||||
response.setCanRetry(internal.getRetryableFailedCount() > 0
|
response.setCanRetry(internal.getFailedCount() > 0
|
||||||
&& (DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name().equals(batch.getStatus())
|
&& (DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name().equals(batch.getStatus())
|
||||||
|| DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())));
|
|| DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())));
|
||||||
PublicDocumentImportDtos.ItemPage items =
|
PublicDocumentImportDtos.ItemPage items =
|
||||||
@@ -795,7 +795,10 @@ public class KnowledgeImportBatchFacade {
|
|||||||
record.setStage(item.getStage());
|
record.setStage(item.getStage());
|
||||||
record.setStatus(item.getStatus());
|
record.setStatus(item.getStatus());
|
||||||
record.setAttemptCount(valueOrZero(item.getAttemptCount()));
|
record.setAttemptCount(valueOrZero(item.getAttemptCount()));
|
||||||
record.setRetryable(Boolean.TRUE.equals(item.getRetryable()));
|
// 人工重试按实际失败状态选择,兼容字段不再受历史 retryable 标记影响。
|
||||||
|
record.setRetryable(
|
||||||
|
DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
|
||||||
|
);
|
||||||
if (StringUtil.hasText(item.getFailureCode())
|
if (StringUtil.hasText(item.getFailureCode())
|
||||||
|| StringUtil.hasText(item.getErrorSummary())) {
|
|| StringUtil.hasText(item.getErrorSummary())) {
|
||||||
PublicDocumentImportDtos.ItemError error =
|
PublicDocumentImportDtos.ItemError error =
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
|||||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
||||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||||
|
import tech.easyflow.ai.enums.DocumentImportMode;
|
||||||
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
|
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
@@ -606,10 +607,10 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证修复前的可恢复失败项会在重试领取事务中恢复重试资格和批次计数。
|
* 验证人工重试直接选择历史失败项,不受旧重试资格字段限制。
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void retryShouldRestoreLegacyRecoverableItem() {
|
public void retryShouldSelectLegacyFailedItemWithoutRetryableGate() {
|
||||||
TestContext context = createContext();
|
TestContext context = createContext();
|
||||||
DocumentImportBatch batch = context.batchService.getOne(
|
DocumentImportBatch batch = context.batchService.getOne(
|
||||||
QueryWrapper.create()
|
QueryWrapper.create()
|
||||||
@@ -627,17 +628,6 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
failed.setErrorSummary("历史代码异常");
|
failed.setErrorSummary("历史代码异常");
|
||||||
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
||||||
.thenReturn(List.of(failed));
|
.thenReturn(List.of(failed));
|
||||||
Mockito.when(context.taskAppService.isRecoverableBatchFailure(failed))
|
|
||||||
.thenReturn(true);
|
|
||||||
Mockito.when(context.batchTracker.transitionItem(
|
|
||||||
Mockito.eq(failed.getId()),
|
|
||||||
Mockito.eq(DocumentImportBatchItemStage.PARSE),
|
|
||||||
Mockito.eq(DocumentImportBatchItemStatus.FAILED),
|
|
||||||
Mockito.eq("历史代码异常"),
|
|
||||||
Mockito.eq(true),
|
|
||||||
Mockito.eq(0),
|
|
||||||
Mockito.eq("parse_failed")
|
|
||||||
)).thenReturn(true);
|
|
||||||
Mockito.when(context.batchMapper.claimRetry(
|
Mockito.when(context.batchMapper.claimRetry(
|
||||||
Mockito.any(),
|
Mockito.any(),
|
||||||
Mockito.anyString(),
|
Mockito.anyString(),
|
||||||
@@ -661,15 +651,15 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Assert.assertEquals(Integer.valueOf(1), result.getRetriedCount());
|
Assert.assertEquals(Integer.valueOf(1), result.getRetriedCount());
|
||||||
Assert.assertTrue(failed.getRetryable());
|
Assert.assertFalse(failed.getRetryable());
|
||||||
Mockito.verify(context.batchTracker).transitionItem(
|
Mockito.verify(context.batchTracker, Mockito.never()).transitionItem(
|
||||||
failed.getId(),
|
Mockito.any(),
|
||||||
DocumentImportBatchItemStage.PARSE,
|
Mockito.any(),
|
||||||
DocumentImportBatchItemStatus.FAILED,
|
Mockito.any(),
|
||||||
"历史代码异常",
|
Mockito.any(),
|
||||||
true,
|
Mockito.anyBoolean(),
|
||||||
0,
|
Mockito.anyInt(),
|
||||||
"parse_failed"
|
Mockito.any()
|
||||||
);
|
);
|
||||||
Mockito.verify(context.batchMapper).claimRetry(
|
Mockito.verify(context.batchMapper).claimRetry(
|
||||||
Mockito.eq(batch.getId()),
|
Mockito.eq(batch.getId()),
|
||||||
@@ -681,6 +671,63 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
Mockito.verify(context.lockHandle).release();
|
Mockito.verify(context.lockHandle).release();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证管理端继续按真实失败项恢复,不依赖历史可重试失败计数。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void continueShouldResumeFailedItemsWhenCompatibilityCounterIsZero() {
|
||||||
|
TestContext context = createContext();
|
||||||
|
DocumentImportBatch batch = context.batchService.getOne(
|
||||||
|
QueryWrapper.create()
|
||||||
|
);
|
||||||
|
batch.setImportMode(DocumentImportMode.AUTO.name());
|
||||||
|
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
|
||||||
|
batch.setFailedCount(1);
|
||||||
|
batch.setRetryableFailedCount(0);
|
||||||
|
DocumentImportBatchItem failed = uploadedItem(
|
||||||
|
BigInteger.valueOf(33),
|
||||||
|
batch.getId()
|
||||||
|
);
|
||||||
|
failed.setStage(DocumentImportBatchItemStage.INDEX.name());
|
||||||
|
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||||
|
failed.setRetryable(false);
|
||||||
|
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
||||||
|
.thenReturn(List.of(failed));
|
||||||
|
Mockito.when(context.batchMapper.updateByQuery(
|
||||||
|
Mockito.any(DocumentImportBatch.class),
|
||||||
|
Mockito.any(QueryWrapper.class)
|
||||||
|
)).thenReturn(1);
|
||||||
|
Mockito.when(context.batchTracker.toStatusResponse(batch))
|
||||||
|
.thenAnswer(invocation -> {
|
||||||
|
DocumentImportBatchDtos.StatusResponse response =
|
||||||
|
new DocumentImportBatchDtos.StatusResponse();
|
||||||
|
response.setStatus(batch.getStatus());
|
||||||
|
return response;
|
||||||
|
});
|
||||||
|
|
||||||
|
DocumentImportBatchDtos.StatusResponse response;
|
||||||
|
beginTransactionSynchronization();
|
||||||
|
try {
|
||||||
|
response = context.service.continueBatch(
|
||||||
|
batch.getKnowledgeId(), batch.getId()
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
completeTransactionSynchronization(
|
||||||
|
TransactionSynchronization.STATUS_COMMITTED
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.assertEquals(DocumentImportBatchStatus.RUNNING.name(),
|
||||||
|
response.getStatus());
|
||||||
|
Mockito.verify(context.batchTracker, Mockito.never())
|
||||||
|
.refreshBatch(batch.getId());
|
||||||
|
Mockito.verify(context.taskAppService).retryBatchFailures(
|
||||||
|
batch.getId(),
|
||||||
|
Set.of(failed.getClientFileKey())
|
||||||
|
);
|
||||||
|
Mockito.verify(context.lockHandle).release();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证运行中的任务拒绝再次领取,调用方可继续查询原 taskId。
|
* 验证运行中的任务拒绝再次领取,调用方可继续查询原 taskId。
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -63,10 +63,10 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证中断批次在汇总失败项后仍保留可继续状态。
|
* 验证兼容可重试计数与人工可继续的失败总数保持一致。
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void shouldPreserveInterruptedStatusUntilUserContinues() {
|
public void shouldExposeAllFailuresAsContinuable() {
|
||||||
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
|
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
|
||||||
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
|
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
|
||||||
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
|
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
|
||||||
@@ -74,7 +74,7 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
DocumentImportBatch batch = batch(DocumentImportBatchStatus.INTERRUPTED, 2);
|
DocumentImportBatch batch = batch(DocumentImportBatchStatus.INTERRUPTED, 2);
|
||||||
batch.setCompletedCount(1);
|
batch.setCompletedCount(1);
|
||||||
batch.setFailedCount(1);
|
batch.setFailedCount(1);
|
||||||
batch.setRetryableFailedCount(1);
|
batch.setRetryableFailedCount(0);
|
||||||
when(batchService.getById(batch.getId())).thenReturn(batch);
|
when(batchService.getById(batch.getId())).thenReturn(batch);
|
||||||
|
|
||||||
DocumentImportBatchTracker tracker =
|
DocumentImportBatchTracker tracker =
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ import tech.easyflow.ai.documentimport.ImportCallerContext;
|
|||||||
import tech.easyflow.ai.documentimport.ImportCallerType;
|
import tech.easyflow.ai.documentimport.ImportCallerType;
|
||||||
import tech.easyflow.ai.documentimport.PublicDocumentImportDtos;
|
import tech.easyflow.ai.documentimport.PublicDocumentImportDtos;
|
||||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||||
|
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||||
|
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
||||||
|
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||||
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
||||||
@@ -318,6 +321,52 @@ public class KnowledgeImportBatchFacadeTest {
|
|||||||
Assert.assertEquals(Integer.valueOf(2), response.getRetriedCount());
|
Assert.assertEquals(Integer.valueOf(2), response.getRetriedCount());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证 Public API 将所有失败文件统一标记为可重试,不受历史标记影响。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void statusShouldExposeEveryFailedItemAsRetryable() {
|
||||||
|
TestContext context = createContext();
|
||||||
|
DocumentImportBatch batch = new DocumentImportBatch();
|
||||||
|
batch.setId(BigInteger.ONE);
|
||||||
|
batch.setKnowledgeId(BigInteger.TWO);
|
||||||
|
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
|
||||||
|
Mockito.when(context.batchAppService.requireBatchForCaller(
|
||||||
|
BigInteger.ONE, context.caller
|
||||||
|
)).thenReturn(batch);
|
||||||
|
|
||||||
|
DocumentImportBatchDtos.StatusResponse internal =
|
||||||
|
new DocumentImportBatchDtos.StatusResponse();
|
||||||
|
internal.setTotalCount(1);
|
||||||
|
internal.setFailedCount(1);
|
||||||
|
internal.setRetryableFailedCount(1);
|
||||||
|
internal.setProgressPercent(100);
|
||||||
|
Mockito.when(context.batchTracker.toStatusResponse(batch))
|
||||||
|
.thenReturn(internal);
|
||||||
|
|
||||||
|
DocumentImportBatchItem failed = new DocumentImportBatchItem();
|
||||||
|
failed.setBatchId(batch.getId());
|
||||||
|
failed.setClientFileKey("failed.txt");
|
||||||
|
failed.setStage(DocumentImportBatchItemStage.INDEX.name());
|
||||||
|
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||||
|
failed.setRetryable(false);
|
||||||
|
Mockito.when(context.itemService.page(
|
||||||
|
Mockito.any(com.mybatisflex.core.paginate.Page.class),
|
||||||
|
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)
|
||||||
|
)).thenReturn(new com.mybatisflex.core.paginate.Page<>(
|
||||||
|
List.of(failed), 1, 1, 1
|
||||||
|
));
|
||||||
|
|
||||||
|
PublicDocumentImportDtos.StatusResponse response = context.facade.getStatus(
|
||||||
|
context.caller, batch.getId(), null, 1, 20
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertTrue(response.getCanRetry());
|
||||||
|
Assert.assertEquals(Integer.valueOf(1),
|
||||||
|
response.getCounts().getRetryableFailed());
|
||||||
|
Assert.assertTrue(response.getItems().getRecords().get(0).getRetryable());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建单文件元数据。
|
* 创建单文件元数据。
|
||||||
*
|
*
|
||||||
@@ -406,6 +455,7 @@ public class KnowledgeImportBatchFacadeTest {
|
|||||||
return new TestContext(
|
return new TestContext(
|
||||||
facade,
|
facade,
|
||||||
batchAppService,
|
batchAppService,
|
||||||
|
batchTracker,
|
||||||
batchService,
|
batchService,
|
||||||
itemService,
|
itemService,
|
||||||
batchMapper,
|
batchMapper,
|
||||||
@@ -422,6 +472,7 @@ public class KnowledgeImportBatchFacadeTest {
|
|||||||
private record TestContext(
|
private record TestContext(
|
||||||
KnowledgeImportBatchFacade facade,
|
KnowledgeImportBatchFacade facade,
|
||||||
DocumentImportBatchAppService batchAppService,
|
DocumentImportBatchAppService batchAppService,
|
||||||
|
DocumentImportBatchTracker batchTracker,
|
||||||
DocumentImportBatchService batchService,
|
DocumentImportBatchService batchService,
|
||||||
DocumentImportBatchItemService itemService,
|
DocumentImportBatchItemService itemService,
|
||||||
DocumentImportBatchMapper batchMapper,
|
DocumentImportBatchMapper batchMapper,
|
||||||
|
|||||||
@@ -79,4 +79,78 @@ describe('documentImportBatchStatus', () => {
|
|||||||
);
|
);
|
||||||
wrapper.unmount();
|
wrapper.unmount();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('部分失败批次按 failedCount 继续并发送统一批次参数', async () => {
|
||||||
|
const failedBatch = {
|
||||||
|
batchId: 'batch-1',
|
||||||
|
completedCount: 1,
|
||||||
|
failedCount: 1,
|
||||||
|
importMode: 'AUTO',
|
||||||
|
pendingCount: 0,
|
||||||
|
processingCount: 0,
|
||||||
|
progressPercent: 100,
|
||||||
|
retryableFailedCount: 0,
|
||||||
|
skippedCount: 0,
|
||||||
|
status: 'PARTIAL_SUCCEEDED',
|
||||||
|
totalCount: 2,
|
||||||
|
};
|
||||||
|
apiMocks.get.mockResolvedValue({
|
||||||
|
data: failedBatch,
|
||||||
|
errorCode: 0,
|
||||||
|
});
|
||||||
|
apiMocks.post.mockResolvedValue({
|
||||||
|
data: {
|
||||||
|
...failedBatch,
|
||||||
|
failedCount: 0,
|
||||||
|
pendingCount: 1,
|
||||||
|
progressPercent: 50,
|
||||||
|
status: 'RUNNING',
|
||||||
|
},
|
||||||
|
errorCode: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
const wrapper = mount(DocumentImportBatchStatus, {
|
||||||
|
props: {
|
||||||
|
knowledgeId: 'knowledge-1',
|
||||||
|
manageable: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await flushPromises();
|
||||||
|
|
||||||
|
const continueButton = wrapper.find('.batch-status__continue');
|
||||||
|
expect(continueButton.exists()).toBe(true);
|
||||||
|
await continueButton.trigger('click');
|
||||||
|
await flushPromises();
|
||||||
|
|
||||||
|
expect(apiMocks.post).toHaveBeenCalledWith(
|
||||||
|
'/api/v1/document/import/batch/continue',
|
||||||
|
{
|
||||||
|
batchId: 'batch-1',
|
||||||
|
knowledgeId: 'knowledge-1',
|
||||||
|
},
|
||||||
|
);
|
||||||
|
wrapper.unmount();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('没有失败项时不使用旧兼容计数显示继续按钮', async () => {
|
||||||
|
apiMocks.get.mockResolvedValue({
|
||||||
|
data: {
|
||||||
|
...createBatch('COMPLETED', 2),
|
||||||
|
retryableFailedCount: 1,
|
||||||
|
status: 'PARTIAL_SUCCEEDED',
|
||||||
|
},
|
||||||
|
errorCode: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
const wrapper = mount(DocumentImportBatchStatus, {
|
||||||
|
props: {
|
||||||
|
knowledgeId: 'knowledge-1',
|
||||||
|
manageable: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await flushPromises();
|
||||||
|
|
||||||
|
expect(wrapper.find('.batch-status__continue').exists()).toBe(false);
|
||||||
|
wrapper.unmount();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ interface BatchStatus {
|
|||||||
pendingCount: number;
|
pendingCount: number;
|
||||||
processingCount: number;
|
processingCount: number;
|
||||||
progressPercent: number;
|
progressPercent: number;
|
||||||
retryableFailedCount?: number;
|
|
||||||
skippedCount: number;
|
skippedCount: number;
|
||||||
status:
|
status:
|
||||||
| 'CANCELLED'
|
| 'CANCELLED'
|
||||||
@@ -53,8 +52,7 @@ const canContinue = computed(
|
|||||||
props.manageable &&
|
props.manageable &&
|
||||||
(batch.value?.status === 'INTERRUPTED' ||
|
(batch.value?.status === 'INTERRUPTED' ||
|
||||||
batch.value?.status === 'PARTIAL_SUCCEEDED') &&
|
batch.value?.status === 'PARTIAL_SUCCEEDED') &&
|
||||||
(Number(batch.value?.retryableFailedCount || 0) > 0 ||
|
Number(batch.value?.failedCount || 0) > 0,
|
||||||
Number(batch.value?.failedCount || 0) > 0),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const processedCount = computed(
|
const processedCount = computed(
|
||||||
|
|||||||
Reference in New Issue
Block a user