EasyFlow-Tech-Dev/Easy-Agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 下沉知识库检索编排能力

Browse Source 
- 新增 rag retrieval 核心协议、RRF 融合与相关度归一化

- 支持关键词检索按 knowledgeId 过滤并补充 ES/Lucene 单测

- 扩展 KnowledgeNode 检索模式与 Milvus 检索参数透传
This commit is contained in:
陈子默
2026-04-05 20:22:59 +08:00
parent 941995d1b8
commit f57544daa2
28 changed files with 1309 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
easy-agents-search-engine/easy-agents-search-engine-lucene/pom.xml
View File

@@ -51,5 +51,10 @@
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-search-engine-service</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

31
easy-agents-search-engine/easy-agents-search-engine-lucene/src/main/java/com/easyagents/search/engine/lucene/LuceneSearcher.java
View File

@@ -17,6 +17,8 @@ package com.easyagents.search.engine.lucene;
import com.easyagents.core.document.Document;
import com.easyagents.search.engine.service.DocumentSearcher;
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
import com.easyagents.search.engine.service.KeywordSearchRequest;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
@@ -78,7 +80,7 @@ public class LuceneSearcher implements DocumentSearcher {
if (document.getTitle() != null) {
luceneDoc.add(new TextField("title", document.getTitle(), Field.Store.YES));
}
appendKnowledgeId(document, luceneDoc);
indexWriter.addDocument(luceneDoc);
indexWriter.commit();
@@ -127,7 +129,7 @@ public class LuceneSearcher implements DocumentSearcher {
if (document.getTitle() != null) {
luceneDoc.add(new TextField("title", document.getTitle(), Field.Store.YES));
}
appendKnowledgeId(document, luceneDoc);
indexWriter.updateDocument(term, luceneDoc);
indexWriter.commit();
return true;
@@ -140,18 +142,21 @@ public class LuceneSearcher implements DocumentSearcher {
}
@Override
public List<Document> searchDocuments(String keyword, int count) {
public List<Document> searchDocuments(KeywordSearchRequest request) {
List<Document> results = new ArrayList<>();
try (IndexReader reader = DirectoryReader.open(directory)) {
IndexSearcher searcher = new IndexSearcher(reader);
Query query = buildQuery(keyword);
TopDocs topDocs = searcher.search(query, count);
Query query = buildQuery(request);
TopDocs topDocs = searcher.search(query, request == null ? 10 : request.getCount());
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
org.apache.lucene.document.Document doc = searcher.doc(scoreDoc.doc);
Document resultDoc = new Document();
resultDoc.setId(doc.get("id"));
resultDoc.setContent(doc.get("content"));
resultDoc.setTitle(doc.get("title"));
if (doc.get(KeywordSearchMetadataKeys.KNOWLEDGE_ID) != null) {
resultDoc.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, doc.get(KeywordSearchMetadataKeys.KNOWLEDGE_ID));
}
resultDoc.setScore((double) scoreDoc.score);
@@ -164,9 +169,10 @@ public class LuceneSearcher implements DocumentSearcher {
return results;
}
private static Query buildQuery(String keyword) {
Query buildQuery(KeywordSearchRequest request) {
try {
Analyzer analyzer = createAnalyzer();
String keyword = request == null ? null : request.getKeyword();
QueryParser titleQueryParser = new QueryParser("title", analyzer);
Query titleQuery = titleQueryParser.parse(keyword);
@@ -179,6 +185,9 @@ public class LuceneSearcher implements DocumentSearcher {
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(titleBooleanClause)
.add(contentBooleanClause);
if (request != null && request.getKnowledgeId() != null && !request.getKnowledgeId().trim().isEmpty()) {
builder.add(new TermQuery(new Term(KeywordSearchMetadataKeys.KNOWLEDGE_ID, request.getKnowledgeId().trim())), BooleanClause.Occur.MUST);
}
return builder.build();
} catch (ParseException e) {
LOG.error(e.toString(), e);
@@ -200,6 +209,16 @@ public class LuceneSearcher implements DocumentSearcher {
return new JcsegAnalyzer(ISegment.Type.NLP, config, DictionaryFactory.createSingletonDictionary(config));
}
private void appendKnowledgeId(Document document, org.apache.lucene.document.Document luceneDoc) {
if (document == null || document.getMetadataMap() == null) {
return;
}
Object knowledgeId = document.getMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID);
if (knowledgeId != null) {
luceneDoc.add(new StringField(KeywordSearchMetadataKeys.KNOWLEDGE_ID, String.valueOf(knowledgeId), Field.Store.YES));
}
}
public void close(IndexWriter indexWriter) {
try {
if (indexWriter != null) {

45
easy-agents-search-engine/easy-agents-search-engine-lucene/src/test/java/com/easyagents/search/engine/lucene/LuceneSearcherTest.java Normal file
View File

@@ -0,0 +1,45 @@
package com.easyagents.search.engine.lucene;
import com.easyagents.core.document.Document;
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
import com.easyagents.search.engine.service.KeywordSearchRequest;
import org.junit.Assert;
import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
public class LuceneSearcherTest {
@Test
public void shouldFilterByKnowledgeIdAndSearchTitleAndContent() throws Exception {
Path tempDir = Files.createTempDirectory("lucene-searcher-test");
LuceneConfig config = new LuceneConfig();
config.setIndexDirPath(tempDir.toString());
LuceneSearcher searcher = new LuceneSearcher(config);
Document first = new Document();
first.setId("1");
first.setTitle("客服标题");
first.setContent("这里没有关键字");
first.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "100");
Document second = new Document();
second.setId("2");
second.setTitle("别的知识库");
second.setContent("客服内容");
second.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "200");
Assert.assertTrue(searcher.addDocument(first));
Assert.assertTrue(searcher.addDocument(second));
KeywordSearchRequest request = KeywordSearchRequest.of("客服", 10);
request.setKnowledgeId("100");
List<Document> results = searcher.searchDocuments(request);
Assert.assertEquals(1, results.size());
Assert.assertEquals("1", String.valueOf(results.get(0).getId()));
Assert.assertEquals("100", String.valueOf(results.get(0).getMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID)));
}
}