feat: 重构知识库文档导入任务化流程

- 新增上传建单、异步解析、分块处理与异步向量化闭环 - 收口分享页权限、完成态检索过滤与 SSE 局部状态刷新
2026-04-15 19:27:22 +08:00
parent a41b50959e
commit 2689adfa40
56 changed files with 6376 additions and 1060 deletions
--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java
@@ -0,0 +1,148 @@
+package tech.easyflow.ai.documentimport.task;
+
+import org.junit.Assert;
+import org.junit.Test;
+import tech.easyflow.ai.entity.DocumentImportTask;
+import tech.easyflow.ai.enums.DocumentImportTaskStatus;
+import tech.easyflow.ai.enums.DocumentProcessStatus;
+import tech.easyflow.ai.mapper.DocumentMapper;
+import tech.easyflow.ai.service.DocumentImportTaskService;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.math.BigInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * {@link KnowledgeDocumentImportTaskAppService} 回归测试。
+ *
+ * @author Codex
+ * @since 2026-04-15
+ */
+public class KnowledgeDocumentImportTaskAppServiceTest {
+
+    /**
+     * 验证向量化失败会按整文档失败语义重置进度，并刷新任务错误信息。
+     *
+     * @throws Exception 反射调用异常
+     */
+    @Test
+    public void markIndexFailedShouldResetProgressAndPersistLatestError() throws Exception {
+        BigInteger documentId = BigInteger.valueOf(10);
+        BigInteger knowledgeId = BigInteger.valueOf(20);
+
+        tech.easyflow.ai.entity.Document persistedDocument = new tech.easyflow.ai.entity.Document();
+        persistedDocument.setId(documentId);
+        persistedDocument.setCollectionId(knowledgeId);
+        persistedDocument.setProcessStatus(DocumentProcessStatus.INDEXING.name());
+        persistedDocument.setTotalChunks(8);
+        persistedDocument.setCompletedChunks(5);
+        persistedDocument.setFailedChunks(1);
+        persistedDocument.setProgressPercent(63);
+        persistedDocument.setLastTaskError("旧错误");
+
+        AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef = new AtomicReference<tech.easyflow.ai.entity.Document>();
+        AtomicReference<DocumentImportTask> updatedTaskRef = new AtomicReference<DocumentImportTask>();
+
+        KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
+        setField(service, "documentMapper", mockDocumentMapper(persistedDocument, updatedDocumentRef));
+        setField(service, "documentImportTaskService", mockDocumentImportTaskService(updatedTaskRef));
+        setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService());
+
+        DocumentImportTask task = new DocumentImportTask();
+        task.setId(BigInteger.valueOf(30));
+        task.setDocumentId(documentId);
+        task.setKnowledgeId(knowledgeId);
+        task.setStatus(DocumentImportTaskStatus.RUNNING.name());
+        task.setErrorSummary("旧错误");
+
+        tech.easyflow.ai.entity.Document inputDocument = new tech.easyflow.ai.entity.Document();
+        inputDocument.setId(documentId);
+        inputDocument.setCollectionId(knowledgeId);
+
+        Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
+            "markIndexFailed",
+            DocumentImportTask.class,
+            tech.easyflow.ai.entity.Document.class,
+            String.class
+        );
+        method.setAccessible(true);
+        method.invoke(service, task, inputDocument, "新错误");
+
+        tech.easyflow.ai.entity.Document updatedDocument = updatedDocumentRef.get();
+        Assert.assertNotNull(updatedDocument);
+        Assert.assertEquals(DocumentProcessStatus.INDEX_FAILED.name(), updatedDocument.getProcessStatus());
+        Assert.assertEquals(Integer.valueOf(0), updatedDocument.getCompletedChunks());
+        Assert.assertEquals(Integer.valueOf(8), updatedDocument.getFailedChunks());
+        Assert.assertEquals(Integer.valueOf(0), updatedDocument.getProgressPercent());
+        Assert.assertEquals("新错误", updatedDocument.getLastTaskError());
+
+        DocumentImportTask updatedTask = updatedTaskRef.get();
+        Assert.assertNotNull(updatedTask);
+        Assert.assertEquals(DocumentImportTaskStatus.FAILED.name(), updatedTask.getStatus());
+        Assert.assertEquals("新错误", updatedTask.getErrorSummary());
+    }
+
+    private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument,
+                                                     AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef) {
+        return (DocumentMapper) Proxy.newProxyInstance(
+            DocumentMapper.class.getClassLoader(),
+            new Class<?>[]{DocumentMapper.class},
+            (proxy, method, args) -> {
+                if ("selectOneById".equals(method.getName())) {
+                    return persistedDocument;
+                }
+                if ("update".equals(method.getName())) {
+                    updatedDocumentRef.set((tech.easyflow.ai.entity.Document) args[0]);
+                    return 1;
+                }
+                return defaultValue(method.getReturnType());
+            }
+        );
+    }
+
+    private static DocumentImportTaskService mockDocumentImportTaskService(AtomicReference<DocumentImportTask> updatedTaskRef) {
+        return (DocumentImportTaskService) Proxy.newProxyInstance(
+            DocumentImportTaskService.class.getClassLoader(),
+            new Class<?>[]{DocumentImportTaskService.class},
+            (proxy, method, args) -> {
+                if ("updateById".equals(method.getName())) {
+                    updatedTaskRef.set((DocumentImportTask) args[0]);
+                    return true;
+                }
+                return defaultValue(method.getReturnType());
+            }
+        );
+    }
+
+    private static void setField(Object target, String fieldName, Object value) throws Exception {
+        Field field = KnowledgeDocumentImportTaskAppService.class.getDeclaredField(fieldName);
+        field.setAccessible(true);
+        field.set(target, value);
+    }
+
+    private static Object defaultValue(Class<?> returnType) {
+        if (returnType == boolean.class) {
+            return false;
+        }
+        if (returnType == int.class) {
+            return 0;
+        }
+        if (returnType == long.class) {
+            return 0L;
+        }
+        return null;
+    }
+
+    /**
+     * 测试用 SSE 推送桩，避免依赖线程池和真实推送。
+     */
+    private static class NoopTaskStatusStreamService extends DocumentImportTaskStatusStreamService {
+
+        @Override
+        public void publishAfterCommit(BigInteger documentId) {
+            // no-op
+        }
+    }
+}
--- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImplTest.java
+++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImplTest.java
@@ -0,0 +1,238 @@
+package tech.easyflow.ai.service.impl;
+
+import com.easyagents.core.document.Document;
+import com.easyagents.search.engine.service.DocumentSearcher;
+import com.easyagents.search.engine.service.KeywordSearchRequest;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.ObjectProvider;
+import tech.easyflow.ai.config.SearcherFactory;
+import tech.easyflow.ai.enums.DocumentProcessStatus;
+import tech.easyflow.ai.mapper.DocumentChunkMapper;
+import tech.easyflow.ai.mapper.DocumentMapper;
+
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.lang.reflect.Proxy;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static tech.easyflow.ai.entity.DocumentCollection.KEY_DOC_RECALL_MAX_NUM;
+import static tech.easyflow.ai.entity.DocumentCollection.KEY_SIMILARITY_THRESHOLD;
+
+/**
+ * {@link DocumentCollectionServiceImpl} 回归测试。
+ *
+ * @author Codex
+ * @since 2026-04-15
+ */
+public class DocumentCollectionServiceImplTest {
+
+    /**
+     * 验证检索结果会在重排前过滤掉未完成文档，避免高分进行中文档挤占最终名额。
+     *
+     * @throws Exception 反射注入异常
+     */
+    @Test
+    public void searchShouldFilterNonCompletedChunksBeforeFinalTopK() throws Exception {
+        BigInteger knowledgeId = BigInteger.ONE;
+        BigInteger completedDocumentId = BigInteger.valueOf(101);
+        BigInteger indexingDocumentId = BigInteger.valueOf(102);
+        BigInteger completedChunkId = BigInteger.valueOf(1001);
+        BigInteger indexingChunkId = BigInteger.valueOf(1002);
+
+        tech.easyflow.ai.entity.DocumentCollection collection = new tech.easyflow.ai.entity.DocumentCollection();
+        collection.setId(knowledgeId);
+        collection.setCollectionType(tech.easyflow.ai.entity.DocumentCollection.TYPE_DOCUMENT);
+        collection.setOptions(new HashMap<String, Object>() {{
+            put(KEY_DOC_RECALL_MAX_NUM, 1);
+            put(KEY_SIMILARITY_THRESHOLD, BigDecimal.ZERO);
+        }});
+
+        tech.easyflow.ai.entity.DocumentChunk completedChunk = new tech.easyflow.ai.entity.DocumentChunk();
+        completedChunk.setId(completedChunkId);
+        completedChunk.setDocumentId(completedDocumentId);
+        completedChunk.setDocumentCollectionId(knowledgeId);
+        completedChunk.setContent("completed chunk");
+
+        tech.easyflow.ai.entity.DocumentChunk indexingChunk = new tech.easyflow.ai.entity.DocumentChunk();
+        indexingChunk.setId(indexingChunkId);
+        indexingChunk.setDocumentId(indexingDocumentId);
+        indexingChunk.setDocumentCollectionId(knowledgeId);
+        indexingChunk.setContent("indexing chunk");
+
+        tech.easyflow.ai.entity.Document completedDocument = new tech.easyflow.ai.entity.Document();
+        completedDocument.setId(completedDocumentId);
+        completedDocument.setCollectionId(knowledgeId);
+        completedDocument.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
+        completedDocument.setTitle("completed");
+
+        TestKeywordSearcher searcher = new TestKeywordSearcher(List.of(
+            buildHit(indexingChunkId, 0.99D),
+            buildHit(completedChunkId, 0.75D)
+        ));
+
+        DocumentCollectionServiceImpl service = new TestDocumentCollectionService(collection);
+        setField(service, "searcherFactory", new SearcherFactory(new StaticObjectProvider<DocumentSearcher>(searcher)));
+        setField(service, "documentChunkMapper", mockDocumentChunkMapper(completedChunk, indexingChunk));
+        setField(service, "documentMapper", mockDocumentMapper(completedDocument));
+
+        tech.easyflow.ai.rag.KnowledgeRetrievalRequest request = new tech.easyflow.ai.rag.KnowledgeRetrievalRequest();
+        request.setKnowledgeId(knowledgeId);
+        request.setQuery("test-query");
+        request.setRetrievalMode(com.easyagents.rag.retrieval.RetrievalMode.KEYWORD);
+
+        List<Document> result = service.search(request);
+
+        Assert.assertEquals("内部关键词召回应扩容到业务 topK 的 5 倍", 5, searcher.lastRequestCount);
+        Assert.assertEquals("知识库过滤后只应保留完成态文档", 1, result.size());
+        Assert.assertEquals(completedChunkId, result.get(0).getId());
+        Assert.assertEquals("completed chunk", result.get(0).getContent());
+        Assert.assertEquals(String.valueOf(knowledgeId), searcher.lastKnowledgeId);
+    }
+
+    private static Document buildHit(BigInteger id, double score) {
+        Document document = new Document();
+        document.setId(id);
+        document.setScore(score);
+        document.setContent("raw-hit-" + id);
+        return document;
+    }
+
+    private static DocumentChunkMapper mockDocumentChunkMapper(tech.easyflow.ai.entity.DocumentChunk... chunks) {
+        Map<String, tech.easyflow.ai.entity.DocumentChunk> chunkMap = new HashMap<String, tech.easyflow.ai.entity.DocumentChunk>();
+        for (tech.easyflow.ai.entity.DocumentChunk chunk : chunks) {
+            chunkMap.put(String.valueOf(chunk.getId()), chunk);
+        }
+        return (DocumentChunkMapper) Proxy.newProxyInstance(
+            DocumentChunkMapper.class.getClassLoader(),
+            new Class<?>[]{DocumentChunkMapper.class},
+            (proxy, method, args) -> {
+                if ("selectListByQuery".equals(method.getName())) {
+                    return List.copyOf(chunkMap.values());
+                }
+                return defaultValue(method.getReturnType());
+            }
+        );
+    }
+
+    private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document completedDocument) {
+        return (DocumentMapper) Proxy.newProxyInstance(
+            DocumentMapper.class.getClassLoader(),
+            new Class<?>[]{DocumentMapper.class},
+            (proxy, method, args) -> {
+                if ("selectListByQuery".equals(method.getName())) {
+                    return List.of(completedDocument);
+                }
+                return defaultValue(method.getReturnType());
+            }
+        );
+    }
+
+    private static void setField(Object target, String fieldName, Object value) throws Exception {
+        Field field = DocumentCollectionServiceImpl.class.getDeclaredField(fieldName);
+        field.setAccessible(true);
+        field.set(target, value);
+    }
+
+    private static Object defaultValue(Class<?> returnType) {
+        if (returnType == boolean.class) {
+            return false;
+        }
+        if (returnType == int.class) {
+            return 0;
+        }
+        if (returnType == long.class) {
+            return 0L;
+        }
+        return null;
+    }
+
+    /**
+     * 固定返回测试知识库实体，避免依赖数据库。
+     */
+    private static class TestDocumentCollectionService extends DocumentCollectionServiceImpl {
+
+        private final tech.easyflow.ai.entity.DocumentCollection collection;
+
+        private TestDocumentCollectionService(tech.easyflow.ai.entity.DocumentCollection collection) {
+            this.collection = collection;
+        }
+
+        @Override
+        public tech.easyflow.ai.entity.DocumentCollection getById(Serializable id) {
+            return collection;
+        }
+    }
+
+    /**
+     * 记录关键词检索请求参数的搜索器桩实现。
+     */
+    private static class TestKeywordSearcher implements DocumentSearcher {
+
+        private final List<Document> documents;
+        private int lastRequestCount;
+        private String lastKnowledgeId;
+
+        private TestKeywordSearcher(List<Document> documents) {
+            this.documents = documents;
+        }
+
+        @Override
+        public boolean addDocument(Document document) {
+            return true;
+        }
+
+        @Override
+        public boolean deleteDocument(Object id) {
+            return true;
+        }
+
+        @Override
+        public boolean updateDocument(Document document) {
+            return true;
+        }
+
+        @Override
+        public List<Document> searchDocuments(KeywordSearchRequest request) {
+            this.lastRequestCount = request.getCount();
+            this.lastKnowledgeId = request.getKnowledgeId();
+            return documents;
+        }
+    }
+
+    /**
+     * 最小 ObjectProvider 实现，仅服务搜索器工厂测试注入。
+     */
+    private static class StaticObjectProvider<T> implements ObjectProvider<T> {
+
+        private final T value;
+
+        private StaticObjectProvider(T value) {
+            this.value = value;
+        }
+
+        @Override
+        public T getObject(Object... args) {
+            return value;
+        }
+
+        @Override
+        public T getIfAvailable() {
+            return value;
+        }
+
+        @Override
+        public T getIfUnique() {
+            return value;
+        }
+
+        @Override
+        public T getObject() {
+            return value;
+        }
+    }
+}