feat: 完善统一文档解析与分块原文映射

- 兼容 MinerU docx 嵌套压缩工件与数组模型输出

- 补充异步解析日志与 sourceRanges 原文区间映射
This commit is contained in:
2026-04-15 19:27:22 +08:00
parent 0c7b362173
commit 547d4f6ee0
6 changed files with 427 additions and 77 deletions

View File

@@ -12,6 +12,7 @@ import org.junit.Assert;
import org.junit.Test;
import java.util.List;
import java.util.Map;
public class RagIngestionPipelineTest {
@@ -58,6 +59,7 @@ public class RagIngestionPipelineTest {
Assert.assertEquals(3, chunks.size());
Assert.assertEquals("第1章 总则", chunks.get(0).getSourceLabel());
Assert.assertEquals(2, chunks.get(1).getHeadingPath().size());
assertHasValidSourceRanges(analysis, chunks.get(0));
}
@Test
@@ -76,5 +78,38 @@ public class RagIngestionPipelineTest {
Assert.assertEquals(RagChunkTypes.QA_PAIR, chunks.get(0).getChunkType());
Assert.assertTrue(chunks.get(0).getContent().contains("问题"));
Assert.assertTrue(chunks.get(1).getAnswer().contains("系统配置"));
assertHasValidSourceRanges(analysis, chunks.get(0));
Assert.assertEquals(2, ((List<?>) chunks.get(0).getOptions().get("sourceRanges")).size());
}
@Test
public void shouldSplitParagraphDocumentWithSourceRanges() {
String content = "第一段内容用于测试原文映射。\n第二段内容继续补充便于生成多个分块。\n第三段内容再长一点确保范围映射稳定。";
AnalysisResult analysis = recommender.recommend(analyzer.analyze(content, "txt"));
StrategyConfig config = StrategyConfig.defaults();
config.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH);
config.setChunkSize(18);
config.setOverlapSize(4);
List<RagChunk> chunks = registry.split(analysis, config);
Assert.assertTrue(chunks.size() > 1);
assertHasValidSourceRanges(analysis, chunks.get(0));
}
@SuppressWarnings("unchecked")
private void assertHasValidSourceRanges(AnalysisResult analysis, RagChunk chunk) {
Object rawRanges = chunk.getOptions().get("sourceRanges");
Assert.assertTrue(rawRanges instanceof List);
List<Map<String, Object>> ranges = (List<Map<String, Object>>) rawRanges;
Assert.assertFalse(ranges.isEmpty());
int normalizedLength = analysis.getNormalizedContent().length();
for (Map<String, Object> range : ranges) {
int start = ((Number) range.get("start")).intValue();
int end = ((Number) range.get("end")).intValue();
Assert.assertTrue(start >= 0);
Assert.assertTrue(end > start);
Assert.assertTrue(end <= normalizedLength);
}
}
}