diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java index 598cf8c4..e8a29d41 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java @@ -720,6 +720,9 @@ public class DocumentServiceImpl extends ServiceImpl i if (config.getMdSplitterLevel() == null || config.getMdSplitterLevel() <= 0) { config.setMdSplitterLevel(RagDefaults.MD_SPLITTER_LEVEL); } + if (config.getRetainRegexMatch() == null) { + config.setRetainRegexMatch(Boolean.FALSE); + } return config; } @@ -742,6 +745,10 @@ public class DocumentServiceImpl extends ServiceImpl i config.setChunkSize(asInteger(rawProfile.get("chunkSize"), config.getChunkSize())); config.setOverlapSize(asInteger(rawProfile.get("overlapSize"), config.getOverlapSize())); config.setRegex(asString(rawProfile.get("regex"))); + if (rawProfile.containsKey("retainRegexMatch")) { + config.setRetainRegexMatch( + asBoolean(rawProfile.get("retainRegexMatch"), false)); + } config.setRowsPerChunk(asInteger(rawProfile.get("rowsPerChunk"), config.getRowsPerChunk())); config.setMdSplitterLevel(asInteger(rawProfile.get("mdSplitterLevel"), config.getMdSplitterLevel())); return config; @@ -763,6 +770,9 @@ public class DocumentServiceImpl extends ServiceImpl i if (StringUtil.hasText(source.getRegex())) { target.setRegex(source.getRegex()); } + if (source.getRetainRegexMatch() != null) { + target.setRetainRegexMatch(source.getRetainRegexMatch()); + } if (source.getRowsPerChunk() != null) { target.setRowsPerChunk(source.getRowsPerChunk()); } @@ -777,6 +787,8 @@ public class DocumentServiceImpl extends ServiceImpl i map.put("chunkSize", strategyConfig.getChunkSize()); map.put("overlapSize", strategyConfig.getOverlapSize()); map.put("regex", strategyConfig.getRegex()); + map.put("retainRegexMatch", Boolean.TRUE.equals( + strategyConfig.getRetainRegexMatch())); map.put("rowsPerChunk", strategyConfig.getRowsPerChunk()); map.put("mdSplitterLevel", strategyConfig.getMdSplitterLevel()); return map; diff --git a/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json b/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json index 46b68831..b3cabae6 100644 --- a/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json +++ b/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json @@ -159,6 +159,9 @@ "chunkSize": "SegmentLength", "overlapSize": "SegmentOverlap", "regex": "RegularExpression", + "regexMatchHandling": "Matched content", + "retainRegexMatch": "Keep in next chunk", + "discardRegexMatch": "Discard", "document": "Document", "simpleDocumentSplitter": "SimpleDocumentSplitter", "simpleTokenizeSplitter": "SimpleTokenizeSplitter", diff --git a/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json b/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json index 8771a0d3..4457b60e 100644 --- a/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json +++ b/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json @@ -159,6 +159,9 @@ "chunkSize": "分段长度", "overlapSize": "分段重叠", "regex": "正则表达式", + "regexMatchHandling": "匹配内容处理", + "retainRegexMatch": "保留到下一分块", + "discardRegexMatch": "不保留", "document": "文档", "simpleDocumentSplitter": "简单文档分割器", "simpleTokenizeSplitter": "简单分词器", diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/SegmenterDoc.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/SegmenterDoc.vue index 8cb79182..2121efea 100644 --- a/easyflow-ui-admin/app/src/views/ai/documentCollection/SegmenterDoc.vue +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/SegmenterDoc.vue @@ -13,6 +13,7 @@ import { ElOption, ElSelect, ElSlider, + ElSwitch, } from 'element-plus'; import { api } from '#/api/request'; @@ -88,6 +89,7 @@ const createDefaultFormState = () => ({ mdSplitterLevel: 2, overlapSize: 128, regex: '', + retainRegexMatch: false, rowsPerChunk: 10, strategyCode: 'AUTO', }); @@ -154,6 +156,10 @@ const showLengthSettings = (strategyCode?: string) => const showOverlapSettings = (strategyCode?: string) => ['AUTO', 'PARAGRAPH_LENGTH'].includes(strategyCode || ''); +const maxOverlapSize = computed(() => + Math.max(0, Number(formState.chunkSize || 0) - 1), +); + const clearPreviewTimer = () => { if (!previewDebounceTimer) { return; @@ -388,6 +394,15 @@ watch( { immediate: true }, ); +watch( + () => formState.chunkSize, + () => { + if (formState.overlapSize > maxOverlapSize.value) { + formState.overlapSize = maxOverlapSize.value; + } + }, +); + watch( formState, () => { @@ -456,7 +471,7 @@ watch( > @@ -484,6 +499,20 @@ watch( > + + + +