feat: 完善统一文档解析与分块原文映射
- 兼容 MinerU docx 嵌套压缩工件与数组模型输出 - 补充异步解析日志与 sourceRanges 原文区间映射
This commit is contained in:
@@ -12,6 +12,7 @@ import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class RagIngestionPipelineTest {
|
||||
|
||||
@@ -58,6 +59,7 @@ public class RagIngestionPipelineTest {
|
||||
Assert.assertEquals(3, chunks.size());
|
||||
Assert.assertEquals("第1章 总则", chunks.get(0).getSourceLabel());
|
||||
Assert.assertEquals(2, chunks.get(1).getHeadingPath().size());
|
||||
assertHasValidSourceRanges(analysis, chunks.get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -76,5 +78,38 @@ public class RagIngestionPipelineTest {
|
||||
Assert.assertEquals(RagChunkTypes.QA_PAIR, chunks.get(0).getChunkType());
|
||||
Assert.assertTrue(chunks.get(0).getContent().contains("问题"));
|
||||
Assert.assertTrue(chunks.get(1).getAnswer().contains("系统配置"));
|
||||
assertHasValidSourceRanges(analysis, chunks.get(0));
|
||||
Assert.assertEquals(2, ((List<?>) chunks.get(0).getOptions().get("sourceRanges")).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldSplitParagraphDocumentWithSourceRanges() {
|
||||
String content = "第一段内容用于测试原文映射。\n第二段内容继续补充,便于生成多个分块。\n第三段内容再长一点,确保范围映射稳定。";
|
||||
AnalysisResult analysis = recommender.recommend(analyzer.analyze(content, "txt"));
|
||||
StrategyConfig config = StrategyConfig.defaults();
|
||||
config.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH);
|
||||
config.setChunkSize(18);
|
||||
config.setOverlapSize(4);
|
||||
|
||||
List<RagChunk> chunks = registry.split(analysis, config);
|
||||
|
||||
Assert.assertTrue(chunks.size() > 1);
|
||||
assertHasValidSourceRanges(analysis, chunks.get(0));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void assertHasValidSourceRanges(AnalysisResult analysis, RagChunk chunk) {
|
||||
Object rawRanges = chunk.getOptions().get("sourceRanges");
|
||||
Assert.assertTrue(rawRanges instanceof List);
|
||||
List<Map<String, Object>> ranges = (List<Map<String, Object>>) rawRanges;
|
||||
Assert.assertFalse(ranges.isEmpty());
|
||||
int normalizedLength = analysis.getNormalizedContent().length();
|
||||
for (Map<String, Object> range : ranges) {
|
||||
int start = ((Number) range.get("start")).intValue();
|
||||
int end = ((Number) range.get("end")).intValue();
|
||||
Assert.assertTrue(start >= 0);
|
||||
Assert.assertTrue(end > start);
|
||||
Assert.assertTrue(end <= normalizedLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user