发布 v1.10 #5
@@ -2547,7 +2547,8 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
}
|
}
|
||||||
AnalysisResult analysis = ragIngestionService.analyze(document.getContent(), normalizeSourceFormat(document));
|
AnalysisResult analysis = ragIngestionService.analyze(document.getContent(), normalizeSourceFormat(document));
|
||||||
StrategyConfig strategyConfig = resolveStrategyConfig(knowledge, requestedStrategy, analysis);
|
StrategyConfig strategyConfig = resolveStrategyConfig(knowledge, requestedStrategy, analysis);
|
||||||
List<RagChunk> previewChunks = ragIngestionService.split(analysis, strategyConfig);
|
List<RagChunk> previewChunks =
|
||||||
|
splitWithAutoParagraphFallback(analysis, requestedStrategy, strategyConfig);
|
||||||
if (previewChunks == null || previewChunks.isEmpty()) {
|
if (previewChunks == null || previewChunks.isEmpty()) {
|
||||||
throw new BusinessException("未生成有效分块,请调整策略后重试");
|
throw new BusinessException("未生成有效分块,请调整策略后重试");
|
||||||
}
|
}
|
||||||
@@ -2568,6 +2569,42 @@ public class KnowledgeDocumentImportTaskAppService {
|
|||||||
return session;
|
return session;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 执行分块,并在 AUTO 推荐策略未产生分块时回退到自然段长度拆分。
|
||||||
|
*
|
||||||
|
* @param analysis 文档分析结果
|
||||||
|
* @param requestedStrategy 用户请求的策略配置
|
||||||
|
* @param effectiveStrategy 解析后的实际策略配置
|
||||||
|
* @return 分块结果
|
||||||
|
*/
|
||||||
|
private List<RagChunk> splitWithAutoParagraphFallback(AnalysisResult analysis,
|
||||||
|
StrategyConfig requestedStrategy,
|
||||||
|
StrategyConfig effectiveStrategy) {
|
||||||
|
List<RagChunk> chunks = ragIngestionService.split(analysis, effectiveStrategy);
|
||||||
|
if (chunks != null && !chunks.isEmpty()) {
|
||||||
|
return chunks;
|
||||||
|
}
|
||||||
|
if (requestedStrategy == null
|
||||||
|
|| !RagStrategyCodes.AUTO.equals(requestedStrategy.getStrategyCode())
|
||||||
|
|| RagStrategyCodes.PARAGRAPH_LENGTH.equals(effectiveStrategy.getStrategyCode())) {
|
||||||
|
return chunks;
|
||||||
|
}
|
||||||
|
|
||||||
|
String recommendedStrategyCode = effectiveStrategy.getStrategyCode();
|
||||||
|
StrategyConfig fallbackStrategy = effectiveStrategy.copy();
|
||||||
|
fallbackStrategy.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH);
|
||||||
|
List<RagChunk> fallbackChunks = ragIngestionService.split(analysis, fallbackStrategy);
|
||||||
|
if (fallbackChunks != null && !fallbackChunks.isEmpty()) {
|
||||||
|
effectiveStrategy.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH);
|
||||||
|
LOG.warn(
|
||||||
|
"AUTO 推荐策略未生成分块,已回退自然段长度拆分: recommendedStrategy={}, chunkSize={}, overlapSize={}",
|
||||||
|
recommendedStrategyCode,
|
||||||
|
fallbackStrategy.getChunkSize(),
|
||||||
|
fallbackStrategy.getOverlapSize());
|
||||||
|
}
|
||||||
|
return fallbackChunks;
|
||||||
|
}
|
||||||
|
|
||||||
private DocumentImportDtos.PreviewSession buildOfficePreviewSession(DocumentCollection knowledge,
|
private DocumentImportDtos.PreviewSession buildOfficePreviewSession(DocumentCollection knowledge,
|
||||||
tech.easyflow.ai.entity.Document document,
|
tech.easyflow.ai.entity.Document document,
|
||||||
StrategyConfig requestedStrategy,
|
StrategyConfig requestedStrategy,
|
||||||
|
|||||||
@@ -7,6 +7,11 @@ import com.easyagents.core.model.embedding.EmbeddingModel;
|
|||||||
import com.easyagents.core.store.DocumentStore;
|
import com.easyagents.core.store.DocumentStore;
|
||||||
import com.easyagents.core.store.StoreOptions;
|
import com.easyagents.core.store.StoreOptions;
|
||||||
import com.easyagents.core.store.StoreResult;
|
import com.easyagents.core.store.StoreResult;
|
||||||
|
import com.easyagents.rag.core.RagChunk;
|
||||||
|
import com.easyagents.rag.core.RagDefaults;
|
||||||
|
import com.easyagents.rag.core.RagStrategyCodes;
|
||||||
|
import com.easyagents.rag.ingestion.RagIngestionService;
|
||||||
|
import com.easyagents.rag.ingestion.model.AnalysisResult;
|
||||||
import com.easyagents.rag.ingestion.model.StrategyConfig;
|
import com.easyagents.rag.ingestion.model.StrategyConfig;
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
import org.apache.ibatis.annotations.Update;
|
import org.apache.ibatis.annotations.Update;
|
||||||
@@ -1151,6 +1156,108 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
|
|||||||
Mockito.verifyNoInteractions(searcher);
|
Mockito.verifyNoInteractions(searcher);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证 AUTO 空结果使用默认长度与重叠参数回退自然段长度拆分。
|
||||||
|
*
|
||||||
|
* @throws Exception 反射调用异常
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void autoEmptyChunksShouldFallbackWithDefaultParagraphSettings()
|
||||||
|
throws Exception {
|
||||||
|
assertAutoParagraphFallback(RagDefaults.CHUNK_SIZE, RagDefaults.OVERLAP_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证手动预览选择 AUTO 时,兜底保留页面传入的长度与重叠参数。
|
||||||
|
*
|
||||||
|
* @throws Exception 反射调用异常
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void autoEmptyChunksShouldFallbackWithRequestedParagraphSettings()
|
||||||
|
throws Exception {
|
||||||
|
assertAutoParagraphFallback(768, 192);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证显式选择非 AUTO 策略时不触发兜底。
|
||||||
|
*
|
||||||
|
* @throws Exception 反射调用异常
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void explicitStrategyEmptyChunksShouldNotFallback() throws Exception {
|
||||||
|
KnowledgeDocumentImportTaskAppService service =
|
||||||
|
new KnowledgeDocumentImportTaskAppService();
|
||||||
|
RagIngestionService ragIngestionService = Mockito.mock(RagIngestionService.class);
|
||||||
|
setField(service, "ragIngestionService", ragIngestionService);
|
||||||
|
AnalysisResult analysis = Mockito.mock(AnalysisResult.class);
|
||||||
|
StrategyConfig requestedStrategy = StrategyConfig.defaults();
|
||||||
|
requestedStrategy.setStrategyCode(RagStrategyCodes.OUTLINE_SECTION);
|
||||||
|
StrategyConfig effectiveStrategy = requestedStrategy.copy();
|
||||||
|
Mockito.when(ragIngestionService.split(analysis, effectiveStrategy))
|
||||||
|
.thenReturn(List.of());
|
||||||
|
|
||||||
|
List<RagChunk> chunks = invokeAutoParagraphFallback(
|
||||||
|
service, analysis, requestedStrategy, effectiveStrategy);
|
||||||
|
|
||||||
|
Assert.assertTrue(chunks.isEmpty());
|
||||||
|
Assert.assertEquals(RagStrategyCodes.OUTLINE_SECTION, effectiveStrategy.getStrategyCode());
|
||||||
|
Mockito.verify(ragIngestionService).split(analysis, effectiveStrategy);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void assertAutoParagraphFallback(int chunkSize, int overlapSize) throws Exception {
|
||||||
|
KnowledgeDocumentImportTaskAppService service =
|
||||||
|
new KnowledgeDocumentImportTaskAppService();
|
||||||
|
RagIngestionService ragIngestionService = Mockito.mock(RagIngestionService.class);
|
||||||
|
setField(service, "ragIngestionService", ragIngestionService);
|
||||||
|
AnalysisResult analysis = Mockito.mock(AnalysisResult.class);
|
||||||
|
StrategyConfig requestedStrategy = StrategyConfig.defaults();
|
||||||
|
requestedStrategy.setChunkSize(chunkSize);
|
||||||
|
requestedStrategy.setOverlapSize(overlapSize);
|
||||||
|
StrategyConfig effectiveStrategy = requestedStrategy.copy();
|
||||||
|
effectiveStrategy.setStrategyCode(RagStrategyCodes.OUTLINE_SECTION);
|
||||||
|
RagChunk fallbackChunk = new RagChunk();
|
||||||
|
List<String> invokedStrategies = new ArrayList<String>();
|
||||||
|
|
||||||
|
Mockito.when(ragIngestionService.split(
|
||||||
|
Mockito.eq(analysis), Mockito.any(StrategyConfig.class)))
|
||||||
|
.thenAnswer(invocation -> {
|
||||||
|
StrategyConfig config = invocation.getArgument(1);
|
||||||
|
invokedStrategies.add(
|
||||||
|
config.getStrategyCode() + ":" + config.getChunkSize() + ":" + config.getOverlapSize());
|
||||||
|
return invokedStrategies.size() == 1
|
||||||
|
? List.of()
|
||||||
|
: List.of(fallbackChunk);
|
||||||
|
});
|
||||||
|
|
||||||
|
List<RagChunk> chunks = invokeAutoParagraphFallback(
|
||||||
|
service, analysis, requestedStrategy, effectiveStrategy);
|
||||||
|
|
||||||
|
Assert.assertEquals(List.of(fallbackChunk), chunks);
|
||||||
|
Assert.assertEquals(
|
||||||
|
List.of(
|
||||||
|
RagStrategyCodes.OUTLINE_SECTION + ":" + chunkSize + ":" + overlapSize,
|
||||||
|
RagStrategyCodes.PARAGRAPH_LENGTH + ":" + chunkSize + ":" + overlapSize),
|
||||||
|
invokedStrategies);
|
||||||
|
Assert.assertEquals(
|
||||||
|
RagStrategyCodes.PARAGRAPH_LENGTH, effectiveStrategy.getStrategyCode());
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
private static List<RagChunk> invokeAutoParagraphFallback(
|
||||||
|
KnowledgeDocumentImportTaskAppService service,
|
||||||
|
AnalysisResult analysis,
|
||||||
|
StrategyConfig requestedStrategy,
|
||||||
|
StrategyConfig effectiveStrategy) throws Exception {
|
||||||
|
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
|
||||||
|
"splitWithAutoParagraphFallback",
|
||||||
|
AnalysisResult.class,
|
||||||
|
StrategyConfig.class,
|
||||||
|
StrategyConfig.class);
|
||||||
|
method.setAccessible(true);
|
||||||
|
return (List<RagChunk>) method.invoke(
|
||||||
|
service, analysis, requestedStrategy, effectiveStrategy);
|
||||||
|
}
|
||||||
|
|
||||||
private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument,
|
private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument,
|
||||||
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef) {
|
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef) {
|
||||||
return (DocumentMapper) Proxy.newProxyInstance(
|
return (DocumentMapper) Proxy.newProxyInstance(
|
||||||
|
|||||||
Reference in New Issue
Block a user