发布 v1.10 #5
@@ -12,6 +12,7 @@ import com.easyagents.core.model.embedding.EmbeddingOptions;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.rag.core.BgeM3ChunkSafety;
|
||||
import com.easyagents.rag.core.RagChunk;
|
||||
import com.easyagents.rag.core.RagDefaults;
|
||||
import com.easyagents.rag.core.RagStrategyCodes;
|
||||
@@ -73,6 +74,7 @@ import static tech.easyflow.ai.entity.table.DocumentTableDef.DOCUMENT;
|
||||
public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> implements DocumentService {
|
||||
protected Logger Log = LoggerFactory.getLogger(DocumentServiceImpl.class);
|
||||
private static final String SOURCE_RANGES_KEY = "sourceRanges";
|
||||
private static final String ROLLBACK_FAILURE_SUFFIX = ";外部索引回滚未完成,请联系管理员处理";
|
||||
|
||||
@Resource
|
||||
private DocumentMapper documentMapper;
|
||||
@@ -392,9 +394,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
return Result.ok();
|
||||
} catch (Exception e) {
|
||||
cleanupPersistedDocument(document);
|
||||
rollbackStoredChunks(storeContext, validChunks);
|
||||
boolean rollbackSucceeded = rollbackStoredChunks(storeContext, validChunks);
|
||||
Log.error("保存文档失败: documentId={}, title={}", document.getId(), document.getTitle(), e);
|
||||
throw new BusinessException("保存失败:" + e.getMessage());
|
||||
throw new BusinessException(buildStoreFailureMessage("保存失败:", e, rollbackSucceeded));
|
||||
} finally {
|
||||
closeStoreContext(storeContext);
|
||||
}
|
||||
@@ -541,8 +543,8 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
updateKnowledgeAfterStore(storeContext);
|
||||
} catch (Exception e) {
|
||||
cleanupPersistedDocument(document);
|
||||
rollbackStoredChunks(storeContext, session.getDocumentChunks());
|
||||
throw new BusinessException("提交导入失败:" + e.getMessage());
|
||||
boolean rollbackSucceeded = rollbackStoredChunks(storeContext, session.getDocumentChunks());
|
||||
throw new BusinessException(buildStoreFailureMessage("提交导入失败:", e, rollbackSucceeded));
|
||||
} finally {
|
||||
closeStoreContext(storeContext);
|
||||
}
|
||||
@@ -897,6 +899,7 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
private void storeDocumentChunks(StoreExecutionContext storeContext, List<DocumentChunk> documentChunks) {
|
||||
List<com.easyagents.core.document.Document> documents = new ArrayList<>();
|
||||
for (DocumentChunk item : documentChunks) {
|
||||
assertEmbeddingChunkWithinHardLimit(item);
|
||||
com.easyagents.core.document.Document document = new com.easyagents.core.document.Document();
|
||||
document.setId(item.getId());
|
||||
document.setContent(item.getContent());
|
||||
@@ -921,30 +924,82 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
throw new BusinessException("DocumentStore.store failed");
|
||||
}
|
||||
|
||||
if (storeContext.searcher != null) {
|
||||
for (com.easyagents.core.document.Document document : documents) {
|
||||
storeContext.searcher.addDocument(document);
|
||||
}
|
||||
if (storeContext.searcher != null && !storeContext.searcher.addDocuments(documents)) {
|
||||
throw new BusinessException("关键词索引写入失败");
|
||||
}
|
||||
}
|
||||
|
||||
private void rollbackStoredChunks(StoreExecutionContext storeContext, List<DocumentChunk> documentChunks) {
|
||||
/**
|
||||
* 校验最终分块没有超过 BGE-M3 向量上下文安全预算。
|
||||
*
|
||||
* @param chunk 待向量化分块
|
||||
* @throws BusinessException 分块超过上下文安全上限
|
||||
*/
|
||||
private void assertEmbeddingChunkWithinHardLimit(DocumentChunk chunk) {
|
||||
String content = chunk == null ? null : chunk.getContent();
|
||||
int tokenEstimate = BgeM3ChunkSafety.estimateContentTokens(content);
|
||||
if (tokenEstimate > RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT) {
|
||||
BigInteger chunkId = chunk == null ? null : chunk.getId();
|
||||
throw new BusinessException(
|
||||
"分块内容超过向量模型上下文上限,请重新分块:chunkId="
|
||||
+ chunkId + ",tokenEstimate=" + tokenEstimate);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 回滚已写入的向量和关键词索引。
|
||||
*
|
||||
* @param storeContext 外部存储上下文
|
||||
* @param documentChunks 待回滚分块
|
||||
* @return 两类外部索引均回滚成功时返回 {@code true}
|
||||
*/
|
||||
private boolean rollbackStoredChunks(StoreExecutionContext storeContext,
|
||||
List<DocumentChunk> documentChunks) {
|
||||
try {
|
||||
List<BigInteger> chunkIds = new ArrayList<>();
|
||||
Set<BigInteger> uniqueIds = new LinkedHashSet<>();
|
||||
for (DocumentChunk chunk : documentChunks) {
|
||||
chunkIds.add(chunk.getId());
|
||||
}
|
||||
storeContext.documentStore.delete(chunkIds, storeContext.options);
|
||||
if (storeContext.searcher != null) {
|
||||
for (BigInteger chunkId : chunkIds) {
|
||||
storeContext.searcher.deleteDocument(chunkId);
|
||||
if (chunk != null && chunk.getId() != null) {
|
||||
uniqueIds.add(chunk.getId());
|
||||
}
|
||||
}
|
||||
List<BigInteger> chunkIds = new ArrayList<>(uniqueIds);
|
||||
if (chunkIds.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
StoreResult deleteResult = storeContext.documentStore.delete(chunkIds, storeContext.options);
|
||||
if (deleteResult == null || !deleteResult.isSuccess()) {
|
||||
String failReason = deleteResult == null ? "未返回结果" : deleteResult.getFailReason();
|
||||
throw new IllegalStateException("向量存储回滚失败: " + failReason);
|
||||
}
|
||||
if (storeContext.searcher != null
|
||||
&& !storeContext.searcher.deleteDocuments(chunkIds)) {
|
||||
throw new IllegalStateException("关键词索引回滚失败");
|
||||
}
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
Log.error("回滚向量文档失败: knowledgeId={}", storeContext.knowledge.getId(), e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成文档写入失败信息,并显式提示未完成的外部索引回滚。
|
||||
*
|
||||
* @param prefix 业务场景前缀
|
||||
* @param error 原始异常
|
||||
* @param rollbackSucceeded 外部索引是否完整回滚
|
||||
* @return 用户可见失败信息
|
||||
*/
|
||||
private String buildStoreFailureMessage(String prefix,
|
||||
Exception error,
|
||||
boolean rollbackSucceeded) {
|
||||
String detail = error != null && StringUtil.hasText(error.getMessage())
|
||||
? error.getMessage()
|
||||
: "未知错误";
|
||||
String message = prefix + detail;
|
||||
return rollbackSucceeded ? message : message + ROLLBACK_FAILURE_SUFFIX;
|
||||
}
|
||||
|
||||
private void updateKnowledgeAfterStore(StoreExecutionContext storeContext) {
|
||||
DocumentCollection documentCollection = new DocumentCollection();
|
||||
documentCollection.setId(storeContext.knowledge.getId());
|
||||
|
||||
@@ -7,6 +7,7 @@ import com.easyagents.core.model.embedding.EmbeddingModel;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.rag.core.BgeM3ChunkSafety;
|
||||
import com.easyagents.rag.core.RagChunk;
|
||||
import com.easyagents.rag.core.RagDefaults;
|
||||
import com.easyagents.rag.core.RagStrategyCodes;
|
||||
@@ -22,6 +23,8 @@ import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.data.redis.RedisSystemException;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||
@@ -56,6 +59,7 @@ import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Proxy;
|
||||
import java.math.BigInteger;
|
||||
@@ -1387,6 +1391,298 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证自动导入向量化失败后从分块阶段重试,并废弃旧分块快照。
|
||||
*
|
||||
* @throws Exception 反射注入异常
|
||||
*/
|
||||
@Test
|
||||
public void retryIndexTaskShouldResplitAutoBatchAfterIndexFailure()
|
||||
throws Exception {
|
||||
BigInteger knowledgeId = BigInteger.valueOf(101);
|
||||
BigInteger documentId = BigInteger.valueOf(102);
|
||||
BigInteger batchId = BigInteger.valueOf(103);
|
||||
BigInteger batchItemId = BigInteger.valueOf(104);
|
||||
BigInteger splitTaskId = BigInteger.valueOf(105);
|
||||
String staleSnapshotPath = "document-import/chunks/stale.snapshot";
|
||||
|
||||
DocumentCollection knowledge = new DocumentCollection();
|
||||
knowledge.setId(knowledgeId);
|
||||
tech.easyflow.ai.entity.Document document =
|
||||
new tech.easyflow.ai.entity.Document();
|
||||
document.setId(documentId);
|
||||
document.setCollectionId(knowledgeId);
|
||||
document.setProcessStatus(DocumentProcessStatus.INDEX_FAILED.name());
|
||||
Map<String, Object> options = new LinkedHashMap<String, Object>();
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID,
|
||||
batchId.toString());
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID,
|
||||
batchItemId.toString());
|
||||
options.put(DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH,
|
||||
staleSnapshotPath);
|
||||
document.setOptions(options);
|
||||
|
||||
DocumentImportBatch batch = new DocumentImportBatch();
|
||||
batch.setId(batchId);
|
||||
batch.setRequestedStrategyJson("{\"strategyCode\":\"AUTO\"}");
|
||||
|
||||
DocumentCollectionService knowledgeService =
|
||||
Mockito.mock(DocumentCollectionService.class);
|
||||
Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge);
|
||||
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
|
||||
Mockito.when(documentMapper.selectOneById(documentId)).thenReturn(document);
|
||||
Mockito.when(documentMapper.updateByQuery(
|
||||
Mockito.any(tech.easyflow.ai.entity.Document.class),
|
||||
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
|
||||
.thenReturn(1);
|
||||
DocumentImportBatchTracker tracker =
|
||||
Mockito.mock(DocumentImportBatchTracker.class);
|
||||
Mockito.when(tracker.isAutoBatch(batchId)).thenReturn(true);
|
||||
Mockito.when(tracker.requireBatch(batchId)).thenReturn(batch);
|
||||
DocumentImportTaskService taskService =
|
||||
Mockito.mock(DocumentImportTaskService.class);
|
||||
AtomicReference<DocumentImportTask> createdTask =
|
||||
new AtomicReference<DocumentImportTask>();
|
||||
Mockito.doAnswer(invocation -> {
|
||||
DocumentImportTask task = invocation.getArgument(0);
|
||||
task.setId(splitTaskId);
|
||||
createdTask.set(task);
|
||||
return true;
|
||||
}).when(taskService).save(Mockito.any(DocumentImportTask.class));
|
||||
DocumentImportSplitTaskProducer splitTaskProducer =
|
||||
Mockito.mock(DocumentImportSplitTaskProducer.class);
|
||||
DocumentImportSnapshotCleanupService cleanupService =
|
||||
Mockito.mock(DocumentImportSnapshotCleanupService.class);
|
||||
ThreadPoolTaskExecutor taskExecutor =
|
||||
Mockito.mock(ThreadPoolTaskExecutor.class);
|
||||
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
setField(service, "knowledgeService", knowledgeService);
|
||||
setField(service, "documentMapper", documentMapper);
|
||||
setField(service, "documentImportBatchTracker", tracker);
|
||||
setField(service, "documentImportTaskService", taskService);
|
||||
setField(service, "splitTaskProducer", splitTaskProducer);
|
||||
setField(service, "snapshotCleanupService", cleanupService);
|
||||
setField(service, "documentImportTaskExecutor", taskExecutor);
|
||||
setField(service, "documentImportTaskStatusStreamService",
|
||||
new NoopTaskStatusStreamService());
|
||||
|
||||
DocumentImportDtos.TaskRetryRequest request =
|
||||
new DocumentImportDtos.TaskRetryRequest();
|
||||
request.setKnowledgeId(knowledgeId);
|
||||
request.setDocumentId(documentId);
|
||||
DocumentImportDtos.TaskStartIndexResponse response =
|
||||
service.retryIndexTask(request).getData();
|
||||
|
||||
Assert.assertNotNull(response);
|
||||
Assert.assertEquals(splitTaskId, response.getTaskId());
|
||||
Assert.assertEquals(DocumentProcessStatus.SPLITTING.name(),
|
||||
response.getProcessStatus());
|
||||
Assert.assertFalse(document.getOptions().containsKey(
|
||||
DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH));
|
||||
Assert.assertNotNull(createdTask.get());
|
||||
Assert.assertEquals(DocumentImportTaskPhase.SPLIT.name(),
|
||||
createdTask.get().getPhase());
|
||||
Assert.assertEquals(batch.getRequestedStrategyJson(),
|
||||
createdTask.get().getPayloadJson().get("strategyConfigJson"));
|
||||
Mockito.verify(splitTaskProducer).send(splitTaskId);
|
||||
Mockito.verify(cleanupService).scheduleChunkSnapshot(
|
||||
staleSnapshotPath);
|
||||
Mockito.verify(documentMapper).update(
|
||||
Mockito.argThat(updated -> updated.getOptions() != null
|
||||
&& !updated.getOptions().containsKey(
|
||||
DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH)),
|
||||
Mockito.eq(false));
|
||||
Mockito.verify(tracker).updateItem(
|
||||
batchItemId,
|
||||
DocumentImportBatchItemStage.SPLIT,
|
||||
DocumentImportBatchItemStatus.PENDING,
|
||||
null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证旧分块快照只会在事务提交后登记可靠清理。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void deleteChunkSnapshotShouldWaitForTransactionCommit()
|
||||
throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
DocumentImportSnapshotCleanupService cleanupService =
|
||||
Mockito.mock(DocumentImportSnapshotCleanupService.class);
|
||||
setField(service, "snapshotCleanupService", cleanupService);
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class
|
||||
.getDeclaredMethod("deleteChunkSnapshotAfterCommit", String.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
TransactionSynchronizationManager.setActualTransactionActive(true);
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
try {
|
||||
method.invoke(service, "document-import/chunks/commit.snapshot");
|
||||
Mockito.verifyNoInteractions(cleanupService);
|
||||
|
||||
List<TransactionSynchronization> synchronizations =
|
||||
TransactionSynchronizationManager.getSynchronizations();
|
||||
for (TransactionSynchronization synchronization : synchronizations) {
|
||||
synchronization.afterCommit();
|
||||
synchronization.afterCompletion(
|
||||
TransactionSynchronization.STATUS_COMMITTED);
|
||||
}
|
||||
Mockito.verify(cleanupService).scheduleChunkSnapshot(
|
||||
"document-import/chunks/commit.snapshot");
|
||||
} finally {
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
TransactionSynchronizationManager.setActualTransactionActive(false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证事务回滚时保留旧分块快照。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void deleteChunkSnapshotShouldKeepSnapshotAfterRollback()
|
||||
throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
DocumentImportSnapshotCleanupService cleanupService =
|
||||
Mockito.mock(DocumentImportSnapshotCleanupService.class);
|
||||
setField(service, "snapshotCleanupService", cleanupService);
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class
|
||||
.getDeclaredMethod("deleteChunkSnapshotAfterCommit", String.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
TransactionSynchronizationManager.setActualTransactionActive(true);
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
try {
|
||||
method.invoke(service, "document-import/chunks/rollback.snapshot");
|
||||
for (TransactionSynchronization synchronization
|
||||
: TransactionSynchronizationManager.getSynchronizations()) {
|
||||
synchronization.afterCompletion(
|
||||
TransactionSynchronization.STATUS_ROLLED_BACK);
|
||||
}
|
||||
Mockito.verifyNoInteractions(cleanupService);
|
||||
} finally {
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
TransactionSynchronizationManager.setActualTransactionActive(false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 Office 超长正文也会应用统一的 BGE-M3 硬限制。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void officeChunksShouldUseSharedBgeM3HardLimit()
|
||||
throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
DocumentChunk source = new DocumentChunk();
|
||||
source.setId(BigInteger.valueOf(501));
|
||||
source.setDocumentId(BigInteger.valueOf(502));
|
||||
source.setDocumentCollectionId(BigInteger.valueOf(503));
|
||||
source.setContent("长".repeat(
|
||||
RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT + 100));
|
||||
Map<String, Object> options = new LinkedHashMap<String, Object>();
|
||||
options.put(
|
||||
DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN,
|
||||
source.getContent());
|
||||
source.setOptions(options);
|
||||
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class
|
||||
.getDeclaredMethod("enforceDocumentChunkHardLimit", List.class);
|
||||
method.setAccessible(true);
|
||||
@SuppressWarnings("unchecked")
|
||||
List<DocumentChunk> result =
|
||||
(List<DocumentChunk>) method.invoke(service, List.of(source));
|
||||
|
||||
Assert.assertTrue(result.size() > 1);
|
||||
Assert.assertTrue(result.stream().allMatch(
|
||||
chunk -> BgeM3ChunkSafety.isWithinHardLimit(chunk.getContent())));
|
||||
Assert.assertTrue(result.stream().allMatch(
|
||||
chunk -> BgeM3ChunkSafety.isWithinHardLimit(String.valueOf(
|
||||
chunk.getOptions().get(
|
||||
DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN)))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证服务端拒绝无法推进游标的分块重叠参数。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void strategyOverlapShouldBeSmallerThanChunkSize()
|
||||
throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
StrategyConfig config = StrategyConfig.defaults();
|
||||
config.setChunkSize(128);
|
||||
config.setOverlapSize(128);
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class
|
||||
.getDeclaredMethod("validateStrategyOverlap", StrategyConfig.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
try {
|
||||
method.invoke(service, config);
|
||||
Assert.fail("重叠大小等于分块大小时应拒绝请求");
|
||||
} catch (InvocationTargetException expected) {
|
||||
Assert.assertTrue(expected.getCause() instanceof BusinessException);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 AUTO 空结果使用默认长度与重叠参数回退自然段长度拆分。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void autoEmptyChunksShouldFallbackWithDefaultParagraphSettings()
|
||||
throws Exception {
|
||||
assertAutoParagraphFallback(RagDefaults.CHUNK_SIZE, RagDefaults.OVERLAP_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证手动预览选择 AUTO 时,兜底保留页面传入的长度与重叠参数。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void autoEmptyChunksShouldFallbackWithRequestedParagraphSettings()
|
||||
throws Exception {
|
||||
assertAutoParagraphFallback(768, 192);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证显式选择非 AUTO 策略时不触发兜底。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void explicitStrategyEmptyChunksShouldNotFallback() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
RagIngestionService ragIngestionService = Mockito.mock(RagIngestionService.class);
|
||||
setField(service, "ragIngestionService", ragIngestionService);
|
||||
AnalysisResult analysis = Mockito.mock(AnalysisResult.class);
|
||||
StrategyConfig requestedStrategy = StrategyConfig.defaults();
|
||||
requestedStrategy.setStrategyCode(RagStrategyCodes.OUTLINE_SECTION);
|
||||
StrategyConfig effectiveStrategy = requestedStrategy.copy();
|
||||
Mockito.when(ragIngestionService.split(analysis, effectiveStrategy))
|
||||
.thenReturn(List.of());
|
||||
|
||||
List<RagChunk> chunks = invokeAutoParagraphFallback(
|
||||
service, analysis, requestedStrategy, effectiveStrategy);
|
||||
|
||||
Assert.assertTrue(chunks.isEmpty());
|
||||
Assert.assertEquals(RagStrategyCodes.OUTLINE_SECTION, effectiveStrategy.getStrategyCode());
|
||||
Mockito.verify(ragIngestionService).split(analysis, effectiveStrategy);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证看门狗已经收口任务后,迟到执行者不能覆盖文档终态。
|
||||
*
|
||||
@@ -1726,12 +2022,13 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
List.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
method.invoke(service,
|
||||
Object rollbackResult = method.invoke(service,
|
||||
BigInteger.valueOf(701),
|
||||
BigInteger.valueOf(702),
|
||||
context,
|
||||
List.of(first, duplicate));
|
||||
|
||||
Assert.assertEquals(Boolean.FALSE, rollbackResult);
|
||||
@SuppressWarnings("rawtypes")
|
||||
ArgumentCaptor<Collection> idsCaptor = ArgumentCaptor.forClass(Collection.class);
|
||||
Mockito.verify(documentStore).delete(idsCaptor.capture(), Mockito.same(storeOptions));
|
||||
@@ -1741,51 +2038,171 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 AUTO 空结果使用默认长度与重叠参数回退自然段长度拆分。
|
||||
* 验证关键词索引删除失败会被回滚结果显式报告。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void autoEmptyChunksShouldFallbackWithDefaultParagraphSettings()
|
||||
throws Exception {
|
||||
assertAutoParagraphFallback(RagDefaults.CHUNK_SIZE, RagDefaults.OVERLAP_SIZE);
|
||||
public void rollbackStoredChunksShouldReportKeywordDeleteFailure() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
|
||||
DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class);
|
||||
StoreOptions storeOptions = StoreOptions.ofCollectionName("knowledge-test");
|
||||
Mockito.when(documentStore.delete(Mockito.anyCollection(), Mockito.same(storeOptions)))
|
||||
.thenReturn(StoreResult.success());
|
||||
Mockito.when(searcher.deleteDocuments(Mockito.anyCollection())).thenReturn(false);
|
||||
|
||||
DocumentCollection knowledge = new DocumentCollection();
|
||||
knowledge.setId(BigInteger.valueOf(751));
|
||||
Class<?> contextClass = Class.forName(
|
||||
KnowledgeDocumentImportTaskAppService.class.getName() + "$StoreExecutionContext"
|
||||
);
|
||||
Constructor<?> constructor = contextClass.getDeclaredConstructor(
|
||||
DocumentCollection.class,
|
||||
EmbeddingModel.class,
|
||||
DocumentStore.class,
|
||||
StoreOptions.class,
|
||||
DocumentSearcher.class
|
||||
);
|
||||
constructor.setAccessible(true);
|
||||
Object context = constructor.newInstance(knowledge, null, documentStore, storeOptions, searcher);
|
||||
|
||||
DocumentChunk chunk = new DocumentChunk();
|
||||
chunk.setId(BigInteger.valueOf(752));
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"rollbackStoredChunks",
|
||||
BigInteger.class,
|
||||
BigInteger.class,
|
||||
contextClass,
|
||||
List.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
Object rollbackResult = method.invoke(service,
|
||||
BigInteger.valueOf(753),
|
||||
BigInteger.valueOf(754),
|
||||
context,
|
||||
List.of(chunk));
|
||||
|
||||
Assert.assertEquals(Boolean.FALSE, rollbackResult);
|
||||
Mockito.verify(searcher).deleteDocuments(List.of(chunk.getId()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证手动预览选择 AUTO 时,兜底保留页面传入的长度与重叠参数。
|
||||
* 验证索引业务异常会保留准确原因,并在回滚失败时追加处置提示。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void autoEmptyChunksShouldFallbackWithRequestedParagraphSettings()
|
||||
throws Exception {
|
||||
assertAutoParagraphFallback(768, 192);
|
||||
public void resolveIndexFailureMessageShouldKeepBusinessReason() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"resolveIndexFailureMessage",
|
||||
Exception.class,
|
||||
boolean.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
String message = (String) method.invoke(
|
||||
service,
|
||||
new BusinessException("关键词索引写入失败"),
|
||||
false
|
||||
);
|
||||
|
||||
Assert.assertEquals(
|
||||
"关键词索引写入失败;外部索引回滚未完成,请联系管理员处理",
|
||||
message
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证显式选择非 AUTO 策略时不触发兜底。
|
||||
* 验证关键词索引批量写入失败会终止导入,避免任务误报完成。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void explicitStrategyEmptyChunksShouldNotFallback() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service =
|
||||
new KnowledgeDocumentImportTaskAppService();
|
||||
RagIngestionService ragIngestionService = Mockito.mock(RagIngestionService.class);
|
||||
setField(service, "ragIngestionService", ragIngestionService);
|
||||
AnalysisResult analysis = Mockito.mock(AnalysisResult.class);
|
||||
StrategyConfig requestedStrategy = StrategyConfig.defaults();
|
||||
requestedStrategy.setStrategyCode(RagStrategyCodes.OUTLINE_SECTION);
|
||||
StrategyConfig effectiveStrategy = requestedStrategy.copy();
|
||||
Mockito.when(ragIngestionService.split(analysis, effectiveStrategy))
|
||||
.thenReturn(List.of());
|
||||
public void storeDocumentChunksShouldPropagateKeywordIndexFailure() throws Exception {
|
||||
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
|
||||
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
|
||||
DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class);
|
||||
StoreOptions storeOptions = StoreOptions.ofCollectionName("knowledge-test");
|
||||
Mockito.when(documentStore.store(Mockito.anyList(), Mockito.same(storeOptions)))
|
||||
.thenReturn(StoreResult.success());
|
||||
Mockito.when(searcher.addDocuments(Mockito.anyList())).thenReturn(false);
|
||||
|
||||
List<RagChunk> chunks = invokeAutoParagraphFallback(
|
||||
service, analysis, requestedStrategy, effectiveStrategy);
|
||||
DocumentCollection knowledge = new DocumentCollection();
|
||||
knowledge.setId(BigInteger.valueOf(801));
|
||||
Class<?> contextClass = Class.forName(
|
||||
KnowledgeDocumentImportTaskAppService.class.getName() + "$StoreExecutionContext"
|
||||
);
|
||||
Constructor<?> constructor = contextClass.getDeclaredConstructor(
|
||||
DocumentCollection.class,
|
||||
EmbeddingModel.class,
|
||||
DocumentStore.class,
|
||||
StoreOptions.class,
|
||||
DocumentSearcher.class
|
||||
);
|
||||
constructor.setAccessible(true);
|
||||
Object context = constructor.newInstance(knowledge, null, documentStore, storeOptions, searcher);
|
||||
|
||||
Assert.assertTrue(chunks.isEmpty());
|
||||
Assert.assertEquals(RagStrategyCodes.OUTLINE_SECTION, effectiveStrategy.getStrategyCode());
|
||||
Mockito.verify(ragIngestionService).split(analysis, effectiveStrategy);
|
||||
DocumentChunk chunk = new DocumentChunk();
|
||||
chunk.setId(BigInteger.valueOf(802));
|
||||
chunk.setContent("关键词索引失败测试");
|
||||
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||
"storeDocumentChunks",
|
||||
contextClass,
|
||||
List.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
try {
|
||||
method.invoke(service, context, List.of(chunk));
|
||||
Assert.fail("关键词索引失败时应终止导入");
|
||||
} catch (InvocationTargetException expected) {
|
||||
Assert.assertTrue(expected.getCause() instanceof BusinessException);
|
||||
Assert.assertEquals("关键词索引写入失败", expected.getCause().getMessage());
|
||||
}
|
||||
Mockito.verify(searcher).addDocuments(Mockito.anyList());
|
||||
}
|
||||
|
||||
private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument,
|
||||
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef) {
|
||||
return (DocumentMapper) Proxy.newProxyInstance(
|
||||
DocumentMapper.class.getClassLoader(),
|
||||
new Class<?>[]{DocumentMapper.class},
|
||||
(proxy, method, args) -> {
|
||||
if ("selectOneById".equals(method.getName())) {
|
||||
return persistedDocument;
|
||||
}
|
||||
if ("update".equals(method.getName())) {
|
||||
updatedDocumentRef.set((tech.easyflow.ai.entity.Document) args[0]);
|
||||
return 1;
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private static FileStorageService mockFileStorageService(AtomicReference<String> savedPrePathRef,
|
||||
AtomicReference<String> savedFilenameRef) {
|
||||
return (FileStorageService) Proxy.newProxyInstance(
|
||||
FileStorageService.class.getClassLoader(),
|
||||
new Class<?>[]{FileStorageService.class},
|
||||
(proxy, method, args) -> {
|
||||
if ("save".equals(method.getName()) && args != null && args.length == 2 && args[0] instanceof MultipartFile file) {
|
||||
savedPrePathRef.set((String) args[1]);
|
||||
savedFilenameRef.set(file.getOriginalFilename());
|
||||
return "http://localhost:39000/easyflow/attachment/" + args[1] + "/" + file.getOriginalFilename();
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private static void setField(Object target, String fieldName, Object value) throws Exception {
|
||||
Field field = KnowledgeDocumentImportTaskAppService.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
|
||||
private static void assertAutoParagraphFallback(int chunkSize, int overlapSize) throws Exception {
|
||||
@@ -1842,46 +2259,6 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
||||
service, analysis, requestedStrategy, effectiveStrategy);
|
||||
}
|
||||
|
||||
private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument,
|
||||
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef) {
|
||||
return (DocumentMapper) Proxy.newProxyInstance(
|
||||
DocumentMapper.class.getClassLoader(),
|
||||
new Class<?>[]{DocumentMapper.class},
|
||||
(proxy, method, args) -> {
|
||||
if ("selectOneById".equals(method.getName())) {
|
||||
return persistedDocument;
|
||||
}
|
||||
if ("update".equals(method.getName())) {
|
||||
updatedDocumentRef.set((tech.easyflow.ai.entity.Document) args[0]);
|
||||
return 1;
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private static FileStorageService mockFileStorageService(AtomicReference<String> savedPrePathRef,
|
||||
AtomicReference<String> savedFilenameRef) {
|
||||
return (FileStorageService) Proxy.newProxyInstance(
|
||||
FileStorageService.class.getClassLoader(),
|
||||
new Class<?>[]{FileStorageService.class},
|
||||
(proxy, method, args) -> {
|
||||
if ("save".equals(method.getName()) && args != null && args.length == 2 && args[0] instanceof MultipartFile file) {
|
||||
savedPrePathRef.set((String) args[1]);
|
||||
savedFilenameRef.set(file.getOriginalFilename());
|
||||
return "http://localhost:39000/easyflow/attachment/" + args[1] + "/" + file.getOriginalFilename();
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private static void setField(Object target, String fieldName, Object value) throws Exception {
|
||||
Field field = KnowledgeDocumentImportTaskAppService.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
|
||||
private static Object defaultValue(Class<?> returnType) {
|
||||
if (returnType == boolean.class) {
|
||||
return false;
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package tech.easyflow.ai.service.impl;
|
||||
|
||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.StoreOptions;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.ai.entity.DocumentChunk;
|
||||
import tech.easyflow.ai.entity.DocumentCollection;
|
||||
import tech.easyflow.ai.entity.Model;
|
||||
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.lang.reflect.Method;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* {@link DocumentServiceImpl} 外部索引回滚回归测试。
|
||||
*/
|
||||
public class DocumentServiceImplRollbackTest {
|
||||
|
||||
/**
|
||||
* 验证手动导入回滚会检查关键词索引批量删除结果。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void rollbackStoredChunksShouldReportKeywordDeleteFailure() throws Exception {
|
||||
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
|
||||
DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class);
|
||||
StoreOptions options = StoreOptions.ofCollectionName("knowledge-test");
|
||||
Mockito.when(documentStore.delete(Mockito.anyCollection(), Mockito.same(options)))
|
||||
.thenReturn(StoreResult.success());
|
||||
Mockito.when(searcher.deleteDocuments(Mockito.anyCollection())).thenReturn(false);
|
||||
|
||||
DocumentCollection knowledge = new DocumentCollection();
|
||||
knowledge.setId(BigInteger.valueOf(901));
|
||||
Class<?> contextClass = Class.forName(
|
||||
DocumentServiceImpl.class.getName() + "$StoreExecutionContext"
|
||||
);
|
||||
Constructor<?> constructor = contextClass.getDeclaredConstructor(
|
||||
DocumentCollection.class,
|
||||
Model.class,
|
||||
EmbeddingModel.class,
|
||||
DocumentStore.class,
|
||||
StoreOptions.class,
|
||||
DocumentSearcher.class
|
||||
);
|
||||
constructor.setAccessible(true);
|
||||
Object context = constructor.newInstance(
|
||||
knowledge, null, null, documentStore, options, searcher);
|
||||
|
||||
DocumentChunk chunk = new DocumentChunk();
|
||||
chunk.setId(BigInteger.valueOf(902));
|
||||
Method method = DocumentServiceImpl.class.getDeclaredMethod(
|
||||
"rollbackStoredChunks",
|
||||
contextClass,
|
||||
List.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
Object rollbackResult = method.invoke(service, context, List.of(chunk));
|
||||
|
||||
Assert.assertEquals(Boolean.FALSE, rollbackResult);
|
||||
Mockito.verify(searcher).deleteDocuments(List.of(chunk.getId()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证手动导入会在外部索引回滚失败时追加明确处置提示。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void buildStoreFailureMessageShouldIncludeRollbackFailure() throws Exception {
|
||||
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||
Method method = DocumentServiceImpl.class.getDeclaredMethod(
|
||||
"buildStoreFailureMessage",
|
||||
String.class,
|
||||
Exception.class,
|
||||
boolean.class
|
||||
);
|
||||
method.setAccessible(true);
|
||||
|
||||
String message = (String) method.invoke(
|
||||
service,
|
||||
"提交导入失败:",
|
||||
new IllegalStateException("关键词索引写入失败"),
|
||||
false
|
||||
);
|
||||
|
||||
Assert.assertEquals(
|
||||
"提交导入失败:关键词索引写入失败;外部索引回滚未完成,请联系管理员处理",
|
||||
message
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package tech.easyflow.ai.service.impl;
|
||||
|
||||
import com.easyagents.core.store.DocumentStore;
|
||||
import com.easyagents.core.store.StoreResult;
|
||||
import com.easyagents.rag.core.RagDefaults;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
@@ -9,6 +10,7 @@ import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.ai.config.SearcherFactory;
|
||||
import tech.easyflow.ai.entity.Document;
|
||||
import tech.easyflow.ai.entity.DocumentChunk;
|
||||
import tech.easyflow.ai.entity.DocumentCollection;
|
||||
import tech.easyflow.ai.entity.Model;
|
||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||
@@ -20,6 +22,8 @@ import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@@ -304,6 +308,33 @@ public class DocumentServiceImplTest {
|
||||
Mockito.verifyNoInteractions(storageService);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证旧版向量化入口同样拒绝超过 BGE-M3 上下文预算的分块。
|
||||
*
|
||||
* @throws Exception 反射调用异常
|
||||
*/
|
||||
@Test
|
||||
public void embeddingGuardShouldRejectOversizedChunk()
|
||||
throws Exception {
|
||||
DocumentServiceImpl service = new DocumentServiceImpl();
|
||||
DocumentChunk chunk = new DocumentChunk();
|
||||
chunk.setId(BigInteger.valueOf(8801));
|
||||
chunk.setContent("长".repeat(
|
||||
RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT + 1));
|
||||
Method method = DocumentServiceImpl.class.getDeclaredMethod(
|
||||
"assertEmbeddingChunkWithinHardLimit", DocumentChunk.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
try {
|
||||
method.invoke(service, chunk);
|
||||
Assert.fail("超过 BGE-M3 上下文预算的分块应被拒绝");
|
||||
} catch (InvocationTargetException expected) {
|
||||
Assert.assertTrue(expected.getCause() instanceof BusinessException);
|
||||
Assert.assertTrue(expected.getCause().getMessage().contains(
|
||||
"tokenEstimate"));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过反射注入测试依赖。
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user