feat: 完善文档分块参数配置
- 支持自定义正则匹配内容保留开关及配置持久化 - 约束分块重叠长度始终小于分块长度 - 补充中英文交互文案
This commit is contained in:
@@ -720,6 +720,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
if (config.getMdSplitterLevel() == null || config.getMdSplitterLevel() <= 0) {
|
||||
config.setMdSplitterLevel(RagDefaults.MD_SPLITTER_LEVEL);
|
||||
}
|
||||
if (config.getRetainRegexMatch() == null) {
|
||||
config.setRetainRegexMatch(Boolean.FALSE);
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
@@ -742,6 +745,10 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
config.setChunkSize(asInteger(rawProfile.get("chunkSize"), config.getChunkSize()));
|
||||
config.setOverlapSize(asInteger(rawProfile.get("overlapSize"), config.getOverlapSize()));
|
||||
config.setRegex(asString(rawProfile.get("regex")));
|
||||
if (rawProfile.containsKey("retainRegexMatch")) {
|
||||
config.setRetainRegexMatch(
|
||||
asBoolean(rawProfile.get("retainRegexMatch"), false));
|
||||
}
|
||||
config.setRowsPerChunk(asInteger(rawProfile.get("rowsPerChunk"), config.getRowsPerChunk()));
|
||||
config.setMdSplitterLevel(asInteger(rawProfile.get("mdSplitterLevel"), config.getMdSplitterLevel()));
|
||||
return config;
|
||||
@@ -763,6 +770,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
if (StringUtil.hasText(source.getRegex())) {
|
||||
target.setRegex(source.getRegex());
|
||||
}
|
||||
if (source.getRetainRegexMatch() != null) {
|
||||
target.setRetainRegexMatch(source.getRetainRegexMatch());
|
||||
}
|
||||
if (source.getRowsPerChunk() != null) {
|
||||
target.setRowsPerChunk(source.getRowsPerChunk());
|
||||
}
|
||||
@@ -777,6 +787,8 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
||||
map.put("chunkSize", strategyConfig.getChunkSize());
|
||||
map.put("overlapSize", strategyConfig.getOverlapSize());
|
||||
map.put("regex", strategyConfig.getRegex());
|
||||
map.put("retainRegexMatch", Boolean.TRUE.equals(
|
||||
strategyConfig.getRetainRegexMatch()));
|
||||
map.put("rowsPerChunk", strategyConfig.getRowsPerChunk());
|
||||
map.put("mdSplitterLevel", strategyConfig.getMdSplitterLevel());
|
||||
return map;
|
||||
|
||||
Reference in New Issue
Block a user