EasyFlow-Tech-Dev/Easy-Agents
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复知识库索引并发与向量化边界

Browse Source 
- 复用 Lucene 和 Elasticsearch 客户端并支持有界批量写入与删除

- 记录脱敏后的 Embedding 失败请求与完整响应

- 为 BGE-M3 分块统一增加上下文硬上限
This commit is contained in:
陈子默
2026-08-07 12:38:09 +08:00
parent bdb69a2250
commit f13e24751a
12 changed files with 1472 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
easy-agents-rag/easy-agents-rag-ingestion/src/test/java/com/easyagents/rag/ingestion/RagIngestionPipelineTest.java
View File

@@ -2,6 +2,7 @@ package com.easyagents.rag.ingestion;
import com.easyagents.rag.core.RagChunk;
import com.easyagents.rag.core.RagChunkTypes;
import com.easyagents.rag.core.RagDefaults;
import com.easyagents.rag.core.RagStrategyCodes;
import com.easyagents.rag.ingestion.analysis.DocumentStructureAnalyzer;
import com.easyagents.rag.ingestion.chunk.RagSplitStrategyRegistry;
@@ -97,6 +98,157 @@ public class RagIngestionPipelineTest {
assertHasValidSourceRanges(analysis, chunks.get(0));
}
@Test
public void shouldEnforceBgeM3HardLimitForAutoQaTxt() {
StringBuilder qa = new StringBuilder("问：自动导入为什么失败？\n答：");
int i = 0;
while (qa.length() < 77946) {
qa.append((char) ('一' + i % 20));
if (i > 0 && i % 700 == 0) {
qa.append('。');
}
i++;
}
qa.setLength(77946);
AnalysisResult analysis = recommender.recommend(analyzer.analyze(qa.toString(), "txt"));
StrategyConfig config = StrategyConfig.defaults();
config.setStrategyCode(RagStrategyCodes.AUTO);
List<RagChunk> chunks = registry.split(analysis, config);
Assert.assertEquals(RagStrategyCodes.QA_PAIR, analysis.getRecommendedStrategyCode());
Assert.assertEquals(77946, analysis.getNormalizedContent().length());
Assert.assertTrue(chunks.size() > 1);
assertWithinBgeM3HardLimit(chunks);
Assert.assertTrue(chunks.stream().allMatch(
chunk -> chunk.getTokenEstimate().intValue() <= RagDefaults.CHUNK_SIZE));
Assert.assertTrue(chunks.stream().allMatch(
chunk -> chunk.getAnswer() != null
&& chunk.getAnswer().length() < analysis.getNormalizedContent().length()));
assertHasValidSourceRanges(analysis, chunks.get(0));
}
@Test
public void shouldEnforceBgeM3HardLimitForCustomRegexChunk() {
StringBuilder content = new StringBuilder();
for (int i = 0; i < 20000; i++) {
content.append((char) ('甲' + i % 16));
if (i > 0 && i % 997 == 0) {
content.append('；');
}
}
AnalysisResult analysis = recommender.recommend(analyzer.analyze(content.toString(), "txt"));
StrategyConfig config = StrategyConfig.defaults();
config.setStrategyCode(RagStrategyCodes.CUSTOM_REGEX);
config.setRegex("\\|NEVER_MATCH\\|");
List<RagChunk> chunks = registry.split(analysis, config);
Assert.assertTrue(chunks.size() > 1);
assertWithinBgeM3HardLimit(chunks);
assertHasValidSourceRanges(analysis, chunks.get(0));
}
/**
* Verifies that custom-regex matches remain discarded when the new option is omitted.
*/
@Test
public void shouldDiscardCustomRegexMatchByDefault() {
AnalysisResult analysis = new AnalysisResult();
analysis.setNormalizedContent(
"<QUESTION>第一个问题及其答案内容。<QUESTION>第二个问题及其答案内容。");
StrategyConfig config = StrategyConfig.defaults();
config.setStrategyCode(RagStrategyCodes.CUSTOM_REGEX);
config.setRegex("<QUESTION>");
List<RagChunk> chunks = registry.split(analysis, config);
Assert.assertEquals(2, chunks.size());
Assert.assertEquals("第一个问题及其答案内容。", chunks.get(0).getContent());
Assert.assertEquals("第二个问题及其答案内容。", chunks.get(1).getContent());
}
/**
* Verifies that an enabled option retains each match at the beginning of the next chunk.
*/
@Test
public void shouldRetainCustomRegexMatchInNextChunk() {
AnalysisResult analysis = new AnalysisResult();
analysis.setNormalizedContent(
"<QUESTION>第一个问题及其答案内容。<QUESTION>第二个问题及其答案内容。");
StrategyConfig config = StrategyConfig.defaults();
config.setStrategyCode(RagStrategyCodes.CUSTOM_REGEX);
config.setRegex("<QUESTION>");
config.setRetainRegexMatch(Boolean.TRUE);
List<RagChunk> chunks = registry.split(analysis, config);
Assert.assertEquals(2, chunks.size());
Assert.assertEquals(
"<QUESTION>第一个问题及其答案内容。",
chunks.get(0).getContent());
Assert.assertEquals(
"<QUESTION>第二个问题及其答案内容。",
chunks.get(1).getContent());
assertHasValidSourceRanges(analysis, chunks.get(1));
}
@Test
public void shouldUseChineseAwareTokenEstimate() {
String content = "知识库自动导入向量化失败，需要检查分块大小和模型上下文窗口。";
AnalysisResult analysis = recommender.recommend(analyzer.analyze(content, "txt"));
StrategyConfig config = StrategyConfig.defaults();
config.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH);
List<RagChunk> chunks = registry.split(analysis, config);
Assert.assertEquals(Integer.valueOf(content.codePointCount(0, content.length())),
chunks.get(0).getTokenEstimate());
}
@Test
public void shouldRejectOverlapEqualToChunkSize() {
AnalysisResult analysis = recommender.recommend(
analyzer.analyze("知识库分块参数需要保证游标持续向前推进。", "txt"));
StrategyConfig config = StrategyConfig.defaults();
config.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH);
config.setChunkSize(128);
config.setOverlapSize(128);
try {
registry.split(analysis, config);
Assert.fail("overlapSize 等于 chunkSize 时应拒绝分块");
} catch (IllegalArgumentException expected) {
Assert.assertTrue(expected.getMessage().contains("overlapSize"));
}
}
@Test
public void shouldRejectOverlapGreaterThanChunkSize() {
AnalysisResult analysis = recommender.recommend(
analyzer.analyze("知识库分块参数需要保证游标持续向前推进。", "txt"));
StrategyConfig config = StrategyConfig.defaults();
config.setStrategyCode(RagStrategyCodes.PARAGRAPH_LENGTH);
config.setChunkSize(128);
config.setOverlapSize(512);
try {
registry.split(analysis, config);
Assert.fail("overlapSize 大于 chunkSize 时应拒绝分块");
} catch (IllegalArgumentException expected) {
Assert.assertTrue(expected.getMessage().contains("overlapSize"));
}
}
private void assertWithinBgeM3HardLimit(List<RagChunk> chunks) {
for (RagChunk chunk : chunks) {
Assert.assertNotNull(chunk.getTokenEstimate());
Assert.assertTrue(
"chunk token estimate exceeds BGE-M3 hard limit: " + chunk.getTokenEstimate(),
chunk.getTokenEstimate().intValue() <= RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT);
}
}
@SuppressWarnings("unchecked")
private void assertHasValidSourceRanges(AnalysisResult analysis, RagChunk chunk) {
Object rawRanges = chunk.getOptions().get("sourceRanges");