EasyFlow-Tech-Dev/Easy-Agents
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复知识库索引并发与向量化边界

Browse Source 
- 复用 Lucene 和 Elasticsearch 客户端并支持有界批量写入与删除

- 记录脱敏后的 Embedding 失败请求与完整响应

- 为 BGE-M3 分块统一增加上下文硬上限
This commit is contained in:
陈子默
2026-08-07 12:38:09 +08:00
parent bdb69a2250
commit f13e24751a
12 changed files with 1472 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
easy-agents-search-engine/easy-agents-search-engine-lucene/src/main/java/com/easyagents/search/engine/lucene/LuceneSearcher.java
View File

@@ -29,7 +29,7 @@ import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.jetbrains.annotations.NotNull;
import org.apache.lucene.util.IOUtils;
import org.lionsoul.jcseg.ISegment;
import org.lionsoul.jcseg.analyzer.JcsegAnalyzer;
import org.lionsoul.jcseg.dic.DictionaryFactory;
@@ -40,17 +40,38 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
public class LuceneSearcher implements DocumentSearcher {
/**
* 基于 Lucene 的本地关键词搜索器。
*
* <p>每个实例长期复用一个线程安全的 {@link IndexWriter}，避免并发任务重复抢占
* 同一索引目录的 {@code write.lock}。</p>
*/
public class LuceneSearcher implements DocumentSearcher, AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(LuceneSearcher.class);
private Directory directory;
private final Directory directory;
private final Analyzer analyzer;
private final IndexWriter indexWriter;
/**
* 创建 Lucene 搜索器并打开索引写入器。
*
* @param config Lucene 配置
* @throws IllegalStateException 索引目录或写入器初始化失败时抛出
*/
public LuceneSearcher(LuceneConfig config) {
Objects.requireNonNull(config, "LuceneConfig 不能为 null");
Directory openedDirectory = null;
Analyzer openedAnalyzer = null;
IndexWriter openedWriter = null;
try {
String indexDirPath = config.getIndexDirPath(); // 索引目录路径
File indexDir = new File(indexDirPath);
@@ -58,89 +79,123 @@ public class LuceneSearcher implements DocumentSearcher {
throw new IllegalStateException("can not mkdirs for path: " + indexDirPath);
}
this.directory = FSDirectory.open(indexDir.toPath());
} catch (IOException e) {
openedDirectory = FSDirectory.open(indexDir.toPath());
openedAnalyzer = createAnalyzer();
openedWriter = new IndexWriter(openedDirectory, new IndexWriterConfig(openedAnalyzer));
} catch (IOException | RuntimeException e) {
IOUtils.closeWhileHandlingException(openedWriter, openedAnalyzer, openedDirectory);
LOG.error("初始化 Lucene 索引失败", e);
throw new RuntimeException(e);
throw new IllegalStateException("初始化 Lucene 索引失败", e);
}
this.directory = openedDirectory;
this.analyzer = openedAnalyzer;
this.indexWriter = openedWriter;
}
/**
* 以文档 ID 为键写入或覆盖单个文档。
*
* @param document 待写入文档
* @return 写入成功时返回 {@code true}
*/
@Override
public boolean addDocument(Document document) {
if (document == null || document.getContent() == null) return false;
return addDocuments(Collections.singletonList(document));
}
IndexWriter indexWriter = null;
try {
indexWriter = createIndexWriter();
org.apache.lucene.document.Document luceneDoc = new org.apache.lucene.document.Document();
luceneDoc.add(new StringField("id", document.getId().toString(), Field.Store.YES));
luceneDoc.add(new TextField("content", document.getContent(), Field.Store.YES));
if (document.getTitle() != null) {
luceneDoc.add(new TextField("title", document.getTitle(), Field.Store.YES));
/**
* 批量写入或覆盖文档，并在批次结束后统一提交。
*
* @param documents 待写入文档
* @return 全部文档写入成功时返回 {@code true}
*/
@Override
public boolean addDocuments(List<Document> documents) {
if (documents == null || documents.isEmpty()) {
return false;
}
List<org.apache.lucene.document.Document> luceneDocuments = new ArrayList<>(documents.size());
for (Document document : documents) {
if (document == null || document.getId() == null || document.getContent() == null) {
return false;
}
luceneDocuments.add(toLuceneDocument(document));
}
try {
for (org.apache.lucene.document.Document luceneDocument : luceneDocuments) {
String documentId = luceneDocument.get("id");
// updateDocument 以 ID 执行 upsert，确保失败重试不会生成重复索引。
indexWriter.updateDocument(new Term("id", documentId), luceneDocument);
}
appendKnowledgeId(document, luceneDoc);
indexWriter.addDocument(luceneDoc);
indexWriter.commit();
return true;
} catch (Exception e) {
LOG.error("添加文档失败", e);
LOG.error("批量添加文档失败: count={}", documents.size(), e);
return false;
} finally {
close(indexWriter);
}
}
/**
* 删除指定文档。
*
* @param id 文档 ID
* @return 删除成功时返回 {@code true}
*/
@Override
public boolean deleteDocument(Object id) {
if (id == null) return false;
return deleteDocuments(Collections.singletonList(id));
}
IndexWriter indexWriter = null;
/**
* 批量删除指定文档，并在批次结束后统一提交。
*
* @param ids 文档 ID 集合
* @return 全部文档删除成功时返回 {@code true}
*/
@Override
public boolean deleteDocuments(Collection<?> ids) {
if (ids == null || ids.isEmpty()) {
return false;
}
Set<String> uniqueIds = new LinkedHashSet<>();
for (Object id : ids) {
if (id == null) {
return false;
}
uniqueIds.add(id.toString());
}
try {
indexWriter = createIndexWriter();
Term term = new Term("id", id.toString());
indexWriter.deleteDocuments(term);
Term[] terms = new Term[uniqueIds.size()];
int index = 0;
for (String id : uniqueIds) {
terms[index++] = new Term("id", id);
}
indexWriter.deleteDocuments(terms);
indexWriter.commit();
return true;
} catch (IOException e) {
LOG.error("删除文档失败", e);
LOG.error("批量删除文档失败: count={}", uniqueIds.size(), e);
return false;
} finally {
close(indexWriter);
}
}
/**
* 以文档 ID 为键更新文档。
*
* @param document 待更新文档
* @return 更新成功时返回 {@code true}
*/
@Override
public boolean updateDocument(Document document) {
if (document == null || document.getId() == null) return false;
IndexWriter indexWriter = null;
try {
indexWriter = createIndexWriter();
Term term = new Term("id", document.getId().toString());
org.apache.lucene.document.Document luceneDoc = new org.apache.lucene.document.Document();
luceneDoc.add(new StringField("id", document.getId().toString(), Field.Store.YES));
luceneDoc.add(new TextField("content", document.getContent(), Field.Store.YES));
if (document.getTitle() != null) {
luceneDoc.add(new TextField("title", document.getTitle(), Field.Store.YES));
}
appendKnowledgeId(document, luceneDoc);
indexWriter.updateDocument(term, luceneDoc);
indexWriter.commit();
return true;
} catch (IOException e) {
LOG.error("更新文档失败", e);
return false;
} finally {
close(indexWriter);
}
return addDocument(document);
}
/**
* 按请求条件搜索文档。
*
* @param request 搜索请求
* @return 命中文档
*/
@Override
public List<Document> searchDocuments(KeywordSearchRequest request) {
List<Document> results = new ArrayList<>();
@@ -171,7 +226,6 @@ public class LuceneSearcher implements DocumentSearcher {
Query buildQuery(KeywordSearchRequest request) {
try {
Analyzer analyzer = createAnalyzer();
String keyword = request == null ? null : request.getKeyword();
QueryParser titleQueryParser = new QueryParser("title", analyzer);
@@ -195,20 +249,22 @@ public class LuceneSearcher implements DocumentSearcher {
return null;
}
@NotNull
private IndexWriter createIndexWriter() throws IOException {
Analyzer analyzer = createAnalyzer();
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer);
return new IndexWriter(directory, indexWriterConfig);
}
private static Analyzer createAnalyzer() {
SegmenterConfig config = new SegmenterConfig(true);
return new JcsegAnalyzer(ISegment.Type.NLP, config, DictionaryFactory.createSingletonDictionary(config));
}
private org.apache.lucene.document.Document toLuceneDocument(Document document) {
org.apache.lucene.document.Document luceneDoc = new org.apache.lucene.document.Document();
luceneDoc.add(new StringField("id", document.getId().toString(), Field.Store.YES));
luceneDoc.add(new TextField("content", document.getContent(), Field.Store.YES));
if (document.getTitle() != null) {
luceneDoc.add(new TextField("title", document.getTitle(), Field.Store.YES));
}
appendKnowledgeId(document, luceneDoc);
return luceneDoc;
}
private void appendKnowledgeId(Document document, org.apache.lucene.document.Document luceneDoc) {
if (document == null || document.getMetadataMap() == null) {
return;
@@ -219,13 +275,13 @@ public class LuceneSearcher implements DocumentSearcher {
}
}
public void close(IndexWriter indexWriter) {
try {
if (indexWriter != null) {
indexWriter.close();
}
} catch (IOException e) {
LOG.error("关闭 Lucene 失败", e);
}
/**
* 关闭写入器、分词器和索引目录。
*
* @throws IOException 任一 Lucene 资源关闭失败时抛出
*/
@Override
public void close() throws IOException {
IOUtils.close(indexWriter, analyzer, directory);
}
}

145
easy-agents-search-engine/easy-agents-search-engine-lucene/src/test/java/com/easyagents/search/engine/lucene/LuceneSearcherTest.java
View File

@@ -8,38 +8,145 @@ import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* {@link LuceneSearcher} 回归测试。
*/
public class LuceneSearcherTest {
/**
* 验证知识库过滤同时覆盖标题和正文。
*
* @throws Exception 临时目录或 Lucene 资源操作失败时抛出
*/
@Test
public void shouldFilterByKnowledgeIdAndSearchTitleAndContent() throws Exception {
Path tempDir = Files.createTempDirectory("lucene-searcher-test");
LuceneConfig config = new LuceneConfig();
config.setIndexDirPath(tempDir.toString());
LuceneSearcher searcher = new LuceneSearcher(config);
try (LuceneSearcher searcher = new LuceneSearcher(config)) {
Document first = new Document();
first.setId("1");
first.setTitle("客服标题");
first.setContent("这里没有关键字");
first.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "100");
Document first = new Document();
first.setId("1");
first.setTitle("客服标题");
first.setContent("这里没有关键字");
first.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "100");
Document second = new Document();
second.setId("2");
second.setTitle("别的知识库");
second.setContent("客服内容");
second.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "200");
Document second = new Document();
second.setId("2");
second.setTitle("别的知识库");
second.setContent("客服内容");
second.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "200");
Assert.assertTrue(searcher.addDocument(first));
Assert.assertTrue(searcher.addDocument(second));
Assert.assertTrue(searcher.addDocument(first));
Assert.assertTrue(searcher.addDocument(second));
KeywordSearchRequest request = KeywordSearchRequest.of("客服", 10);
request.setKnowledgeId("100");
List<Document> results = searcher.searchDocuments(request);
KeywordSearchRequest request = KeywordSearchRequest.of("客服", 10);
request.setKnowledgeId("100");
List<Document> results = searcher.searchDocuments(request);
Assert.assertEquals(1, results.size());
Assert.assertEquals("1", String.valueOf(results.get(0).getId()));
Assert.assertEquals("100", String.valueOf(
results.get(0).getMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID)));
}
}
Assert.assertEquals(1, results.size());
Assert.assertEquals("1", String.valueOf(results.get(0).getId()));
Assert.assertEquals("100", String.valueOf(results.get(0).getMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID)));
/**
* 验证多个导入线程可共享同一个 IndexWriter 完成批量写入。
*
* @throws Exception 并发执行或 Lucene 资源操作失败时抛出
*/
@Test
public void shouldWriteConcurrentBatchesWithoutLockConflict() throws Exception {
Path tempDir = Files.createTempDirectory("lucene-searcher-concurrent-test");
LuceneConfig config = new LuceneConfig();
config.setIndexDirPath(tempDir.toString());
int workerCount = 4;
int documentsPerWorker = 20;
ExecutorService executor = Executors.newFixedThreadPool(workerCount);
try (LuceneSearcher searcher = new LuceneSearcher(config)) {
CountDownLatch startSignal = new CountDownLatch(1);
List<Future<Boolean>> futures = new ArrayList<>();
for (int worker = 0; worker < workerCount; worker++) {
int currentWorker = worker;
futures.add(executor.submit(() -> {
startSignal.await();
List<Document> documents = new ArrayList<>();
for (int offset = 0; offset < documentsPerWorker; offset++) {
Document document = new Document();
document.setId(currentWorker + "-" + offset);
document.setContent("concurrent indexing content");
document.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, "concurrent-kb");
documents.add(document);
}
return searcher.addDocuments(documents);
}));
}
startSignal.countDown();
for (Future<Boolean> future : futures) {
Assert.assertTrue(future.get());
}
int expectedCount = workerCount * documentsPerWorker;
KeywordSearchRequest request = KeywordSearchRequest.of("concurrent", expectedCount);
request.setKnowledgeId("concurrent-kb");
Assert.assertEquals(expectedCount, searcher.searchDocuments(request).size());
} finally {
executor.shutdownNow();
}
}
/**
* 验证相同文档 ID 重试写入时执行覆盖而不会保留重复旧索引。
*
* @throws Exception 临时目录或 Lucene 资源操作失败时抛出
*/
@Test
public void shouldUpsertDocumentByIdOnRetry() throws Exception {
Path tempDir = Files.createTempDirectory("lucene-searcher-upsert-test");
LuceneConfig config = new LuceneConfig();
config.setIndexDirPath(tempDir.toString());
try (LuceneSearcher searcher = new LuceneSearcher(config)) {
Document document = new Document();
document.setId("retry-id");
document.setContent("oldkeyword");
Assert.assertTrue(searcher.addDocument(document));
document.setContent("newkeyword");
Assert.assertTrue(searcher.addDocument(document));
Assert.assertTrue(searcher.searchDocuments("oldkeyword", 10).isEmpty());
Assert.assertEquals(1, searcher.searchDocuments("newkeyword", 10).size());
}
}
/**
* 验证多个文档可在一次提交中批量删除。
*
* @throws Exception 临时目录或 Lucene 资源操作失败时抛出
*/
@Test
public void shouldDeleteDocumentsInSingleBatch() throws Exception {
Path tempDir = Files.createTempDirectory("lucene-searcher-delete-test");
LuceneConfig config = new LuceneConfig();
config.setIndexDirPath(tempDir.toString());
try (LuceneSearcher searcher = new LuceneSearcher(config)) {
Document first = new Document();
first.setId("delete-first");
first.setContent("batchdelete");
Document second = new Document();
second.setId("delete-second");
second.setContent("batchdelete");
Assert.assertTrue(searcher.addDocuments(List.of(first, second)));
Assert.assertTrue(searcher.deleteDocuments(List.of(first.getId(), second.getId())));
Assert.assertTrue(searcher.searchDocuments("batchdelete", 10).isEmpty());
}
}
}