feat: 异步同步知识库分块检索索引

This commit is contained in:
2026-09-04 11:32:52 +08:00
parent c4799760cf
commit cc7f0c1a43
38 changed files with 2628 additions and 231 deletions

View File

@@ -1,23 +1,18 @@
package tech.easyflow.admin.controller.ai;
import cn.dev33.satoken.annotation.SaCheckPermission;
import com.easyagents.core.model.embedding.EmbeddingModel;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
import tech.easyflow.ai.dto.DocumentChunkSyncRetryRequest;
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
import tech.easyflow.ai.dto.DocumentChunkSyncStatusRequest;
import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.entity.Model;
import tech.easyflow.ai.service.DocumentChunkService;
import tech.easyflow.ai.service.DocumentCollectionService;
import tech.easyflow.ai.service.ModelService;
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
import tech.easyflow.common.annotation.UsePermission;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.web.controller.BaseCurdController;
import tech.easyflow.common.web.controller.BaseController;
import tech.easyflow.common.web.jsonbody.JsonBody;
import com.easyagents.core.document.Document;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.GetMapping;
@@ -28,12 +23,8 @@ import tech.easyflow.system.enums.ResourceAction;
import tech.easyflow.system.enums.ResourceLookup;
import tech.easyflow.system.permission.resource.RequireResourceAccess;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 控制层。
@@ -44,19 +35,12 @@ import java.util.Map;
@RestController
@RequestMapping("/api/v1/documentChunk")
@UsePermission(moduleName = "/api/v1/documentCollection")
public class DocumentChunkController extends BaseCurdController<DocumentChunkService, DocumentChunk> {
public class DocumentChunkController extends BaseController {
@Resource
DocumentCollectionService documentCollectionService;
@Resource
ModelService modelService;
@Resource
DocumentChunkService documentChunkService;
private final DocumentChunkService documentChunkService;
public DocumentChunkController(DocumentChunkService service) {
super(service);
this.documentChunkService = service;
}
@GetMapping("page")
@@ -68,9 +52,30 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
idExpr = "#request.getParameter('documentId')",
denyMessage = "无权限访问知识库"
)
@Override
public Result<Page<DocumentChunk>> page(HttpServletRequest request, String sortKey, String sortType, Long pageNumber, Long pageSize) {
return super.page(request, sortKey, sortType, pageNumber, pageSize);
public Result<Page<DocumentChunk>> page(
HttpServletRequest request,
Long pageNumber,
Long pageSize
) {
String documentIdValue = request.getParameter("documentId");
if (documentIdValue == null || documentIdValue.isBlank()) {
return Result.<Page<DocumentChunk>>fail("documentId不能为空", null);
}
BigInteger documentId;
try {
documentId = new BigInteger(documentIdValue);
} catch (NumberFormatException e) {
return Result.<Page<DocumentChunk>>fail("documentId格式不正确", null);
}
long normalizedPageNumber = pageNumber == null || pageNumber < 1 ? 1 : pageNumber;
long normalizedPageSize = pageSize == null || pageSize < 1 ? 10 : pageSize;
QueryWrapper query = QueryWrapper.create()
.eq(DocumentChunk::getDocumentId, documentId)
.orderBy("sorting asc");
return Result.ok(documentChunkService.page(
new Page<>(normalizedPageNumber, normalizedPageSize),
query
));
}
@PostMapping("update")
@@ -79,43 +84,23 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
resource = CategoryResourceType.KNOWLEDGE,
action = ResourceAction.MANAGE,
lookup = ResourceLookup.DOCUMENT_CHUNK_ID,
idExpr = "#documentChunk.id",
idExpr = "#request.id",
denyMessage = "无权限管理知识库"
)
public Result<?> update(@JsonBody DocumentChunk documentChunk) {
boolean success = service.updateById(documentChunk);
if (success){
DocumentChunk record = documentChunkService.getById(documentChunk.getId());
DocumentCollection knowledge = documentCollectionService.getById(record.getDocumentCollectionId());
if (knowledge == null) {
return Result.fail(1, "知识库不存在");
}
DocumentStore documentStore = knowledge.toDocumentStore();
if (documentStore == null) {
return Result.fail(2, "知识库没有配置向量库");
}
try {
// 设置向量模型
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
if (model == null) {
return Result.fail(3, "知识库没有配置向量模型");
}
EmbeddingModel embeddingModel = model.toEmbeddingModel();
documentStore.setEmbeddingModel(embeddingModel);
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
Document document = Document.of(documentChunk.getContent());
document.setId(documentChunk.getId());
Map<String, Object> metadata = new HashMap<>();
metadata.put("keywords", documentChunk.getMetadataKeyWords());
metadata.put("questions", documentChunk.getMetadataQuestions());
document.setMetadataMap(metadata);
StoreResult result = documentStore.update(document, options); // 更新已有记录
return Result.ok(result);
} finally {
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
}
public Result<?> update(
@JsonBody(required = true, skipConvertError = false)
DocumentChunkContentUpdateRequest request
) {
DocumentChunk current = documentChunkService.getById(request.getId());
if (current == null) {
return Result.fail(1, "记录不存在");
}
return Result.ok(false);
DocumentChunk updated = documentChunkService.updateContent(
current.getDocumentCollectionId(),
current.getId(),
request.getContent()
);
return Result.ok(updated);
}
@PostMapping("removeChunk")
@@ -127,36 +112,58 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
idExpr = "#chunkId",
denyMessage = "无权限管理知识库"
)
public Result<?> remove(@JsonBody(value = "id", required = true) BigInteger chunkId) {
DocumentChunk docChunk = documentChunkService.getById(chunkId);
public Result<?> removeChunk(@JsonBody(value = "id", required = true) BigInteger chunkId) {
DocumentChunk docChunk = documentChunkService.getById(chunkId);
if (docChunk == null) {
return Result.fail(1, "记录不存在");
}
DocumentCollection knowledge = documentCollectionService.getById(docChunk.getDocumentCollectionId());
if (knowledge == null) {
return Result.fail(2, "知识库不存在");
}
DocumentStore documentStore = knowledge.toDocumentStore();
if (documentStore == null) {
return Result.fail(3, "知识库没有配置向量库");
}
try {
// 设置向量模型
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
if (model == null) {
return Result.fail(4, "知识库没有配置向量模型");
}
EmbeddingModel embeddingModel = model.toEmbeddingModel();
documentStore.setEmbeddingModel(embeddingModel);
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
List<BigInteger> deleteList = new ArrayList<>();
deleteList.add(chunkId);
documentStore.delete(deleteList, options);
documentChunkService.removeChunk(knowledge, chunkId);
return Result.ok(documentChunkService.deleteChunk(
docChunk.getDocumentCollectionId(),
chunkId
));
}
return super.remove(chunkId);
} finally {
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
@PostMapping("syncStatus")
@SaCheckPermission("/api/v1/documentCollection/query")
@RequireResourceAccess(
resource = CategoryResourceType.KNOWLEDGE,
action = ResourceAction.READ,
lookup = ResourceLookup.DOCUMENT_ID,
idExpr = "#request.documentId",
denyMessage = "无权限访问知识库"
)
public Result<List<DocumentChunkSyncStatus>> syncStatus(
@JsonBody(required = true, skipConvertError = false)
DocumentChunkSyncStatusRequest request
) {
return Result.ok(documentChunkService.listIndexSyncStatus(
null,
request.getDocumentId(),
request.getIds()
));
}
@PostMapping("retrySync")
@SaCheckPermission("/api/v1/documentCollection/save")
@RequireResourceAccess(
resource = CategoryResourceType.KNOWLEDGE,
action = ResourceAction.MANAGE,
lookup = ResourceLookup.DOCUMENT_CHUNK_ID,
idExpr = "#request.id",
denyMessage = "无权限管理知识库"
)
public Result<?> retrySync(
@JsonBody(required = true, skipConvertError = false)
DocumentChunkSyncRetryRequest request
) {
DocumentChunk current = documentChunkService.getById(request.getId());
if (current == null || request.getIndexSyncVersion() == null) {
return Result.fail(1, "记录不存在或同步版本缺失");
}
return Result.ok(documentChunkService.retryIndexSync(
current.getDocumentCollectionId(),
current.getId(),
request.getIndexSyncVersion()
));
}
}

View File

@@ -118,12 +118,10 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
List<Serializable> ids = Collections.singletonList(id);
Result<?> result = onRemoveBefore(ids);
if (result != null) return result;
boolean isSuccess = documentService.removeDoc(id);
if (!isSuccess){
return Result.ok(false);
boolean success = documentService.removeDoc(id);
if (success) {
onRemoveAfter(ids);
}
boolean success = service.removeById(id);
onRemoveAfter(ids);
return Result.ok(success);
}

View File

@@ -1,10 +1,6 @@
package tech.easyflow.admin.controller.ai;
import cn.hutool.core.io.IoUtil;
import com.easyagents.core.model.embedding.EmbeddingModel;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryColumn;
import com.mybatisflex.core.query.QueryWrapper;
@@ -22,6 +18,11 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
import tech.easyflow.ai.dto.DocumentChunkSyncRetryRequest;
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
import tech.easyflow.ai.dto.DocumentChunkSyncStatusRequest;
import tech.easyflow.ai.dto.KnowledgeShareLimitedConfigRequest;
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
import tech.easyflow.ai.entity.Document;
@@ -43,7 +44,6 @@ import tech.easyflow.ai.service.KnowledgeEmbeddingService;
import tech.easyflow.ai.service.KnowledgeShareAuditService;
import tech.easyflow.ai.service.KnowledgeShareService;
import tech.easyflow.ai.service.ModelService;
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
import tech.easyflow.ai.vo.FaqImportResultVo;
import tech.easyflow.ai.vo.KnowledgeShareAuthContext;
import tech.easyflow.ai.vo.KnowledgeShareViewDetail;
@@ -62,7 +62,6 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -505,43 +504,27 @@ public class ShareKnowledgeController {
@PostMapping("/documentChunk/update")
public Result<?> updateDocumentChunk(
@RequestParam String shareKey,
@JsonBody DocumentChunk documentChunk
@JsonBody(required = true, skipConvertError = false)
DocumentChunkContentUpdateRequest request
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
null,
KnowledgeShareActionScope.CONTENT_UPDATE.name()
);
DocumentChunk current = documentChunkService.getById(documentChunk.getId());
DocumentChunk current = documentChunkService.getById(request.getId());
if (current == null || current.getDocumentCollectionId() == null
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
throw new BusinessException("记录不存在");
}
boolean success = documentChunkService.updateById(documentChunk);
if (success) {
DocumentStore documentStore = context.getKnowledge().toDocumentStore();
if (documentStore == null) {
return Result.fail(2, "知识库没有配置向量库");
}
try {
Model model = modelService.getModelInstance(context.getKnowledge().getVectorEmbedModelId());
if (model == null) {
return Result.fail(3, "知识库没有配置向量模型");
}
EmbeddingModel embeddingModel = model.toEmbeddingModel();
documentStore.setEmbeddingModel(embeddingModel);
StoreOptions options = StoreOptions.ofCollectionName(context.getKnowledge().getVectorStoreCollection());
com.easyagents.core.document.Document doc = com.easyagents.core.document.Document.of(documentChunk.getContent());
doc.setId(documentChunk.getId());
StoreResult result = documentStore.update(doc, options);
audit(context, "更新分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", documentChunk.getId()));
return Result.ok(result);
} finally {
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
}
}
return Result.ok(false);
DocumentChunk updated = documentChunkService.updateContent(
context.getKnowledge().getId(),
current.getId(),
request.getContent()
);
audit(context, "更新分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", request.getId()));
return Result.ok(updated);
}
/**
@@ -562,25 +545,50 @@ public class ShareKnowledgeController {
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
return Result.fail(1, "记录不存在");
}
DocumentStore documentStore = context.getKnowledge().toDocumentStore();
if (documentStore == null) {
return Result.fail(2, "知识库没有配置向量库");
DocumentChunkDeleteResult removed = documentChunkService.deleteChunk(
context.getKnowledge().getId(),
chunkId
);
audit(context, "删除分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", chunkId));
return Result.ok(removed);
}
@PostMapping("/documentChunk/syncStatus")
public Result<List<DocumentChunkSyncStatus>> documentChunkSyncStatus(
@RequestParam String shareKey,
@JsonBody DocumentChunkSyncStatusRequest request
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey, null, KnowledgeShareActionScope.VIEW.name()
);
Document document = documentService.getById(request.getDocumentId());
if (document == null || document.getCollectionId() == null
|| document.getCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
throw new BusinessException("文档不存在");
}
try {
Model model = modelService.getModelInstance(context.getKnowledge().getVectorEmbedModelId());
if (model == null) {
return Result.fail(3, "知识库没有配置向量模型");
}
documentStore.setEmbeddingModel(model.toEmbeddingModel());
StoreOptions options = StoreOptions.ofCollectionName(context.getKnowledge().getVectorStoreCollection());
documentStore.delete(Collections.singletonList(chunkId), options);
documentChunkService.removeById(chunkId);
audit(context, "删除分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", chunkId));
return Result.ok(true);
} finally {
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
return Result.ok(documentChunkService.listIndexSyncStatus(
context.getKnowledge().getId(), request.getDocumentId(), request.getIds()
));
}
@PostMapping("/documentChunk/retrySync")
public Result<DocumentChunk> retryDocumentChunkSync(
@RequestParam String shareKey,
@JsonBody DocumentChunkSyncRetryRequest request
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey, null, KnowledgeShareActionScope.CONTENT_UPDATE.name()
);
if (request.getIndexSyncVersion() == null) {
throw new BusinessException("同步版本不能为空");
}
DocumentChunk chunk = documentChunkService.retryIndexSync(
context.getKnowledge().getId(), request.getId(), request.getIndexSyncVersion()
);
audit(context, "重试分享文档 Chunk 索引同步", "KNOWLEDGE_SHARE_URL_WRITE", true,
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", request.getId()));
return Result.ok(chunk);
}
/**

View File

@@ -0,0 +1,120 @@
package tech.easyflow.admin.controller.ai;
import org.testng.Assert;
import org.testng.annotations.Test;
import org.springframework.web.bind.annotation.PostMapping;
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.ai.service.DocumentChunkService;
import tech.easyflow.common.web.controller.BaseController;
import tech.easyflow.common.web.jsonbody.JsonBody;
import tech.easyflow.system.permission.resource.RequireResourceAccess;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Parameter;
import java.math.BigInteger;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 文档分块维护接口契约测试。
*/
public class DocumentChunkControllerContractTest {
@Test
public void updateEndpointsShouldOnlyAcceptChunkIdAndContent() throws Exception {
Method adminUpdate = DocumentChunkController.class.getDeclaredMethod(
"update",
DocumentChunkContentUpdateRequest.class
);
Method shareUpdate = ShareKnowledgeController.class.getDeclaredMethod(
"updateDocumentChunk",
String.class,
DocumentChunkContentUpdateRequest.class
);
Assert.assertNotNull(adminUpdate);
Assert.assertNotNull(shareUpdate);
assertStrictJsonBody(adminUpdate.getParameters()[0]);
assertStrictJsonBody(shareUpdate.getParameters()[1]);
Assert.assertEquals(
adminUpdate.getAnnotation(RequireResourceAccess.class).idExpr(),
"#request.id"
);
Set<String> fields = Arrays.stream(
DocumentChunkContentUpdateRequest.class.getDeclaredFields()
)
.map(Field::getName)
.collect(Collectors.toSet());
Assert.assertEquals(Set.of("id", "content"), fields);
Assert.assertEquals(
DocumentChunkController.class.getMethod(
"update",
DocumentChunkContentUpdateRequest.class
).getDeclaringClass(),
DocumentChunkController.class
);
Assert.assertEquals(
Arrays.stream(DocumentChunkController.class.getDeclaredMethods())
.filter(method -> method.getName().equals("update"))
.filter(method -> !method.isBridge() && !method.isSynthetic())
.count(),
1L
);
}
@Test
public void controllerShouldNotExposeGenericWriteEndpoints() {
Assert.assertEquals(DocumentChunkController.class.getSuperclass(), BaseController.class);
Set<String> postMappings = Arrays.stream(DocumentChunkController.class.getMethods())
.map(method -> method.getAnnotation(PostMapping.class))
.filter(annotation -> annotation != null)
.flatMap(annotation -> Arrays.stream(annotation.value()))
.collect(Collectors.toSet());
Assert.assertEquals(
postMappings,
Set.of("update", "removeChunk", "syncStatus", "retrySync")
);
Assert.assertFalse(postMappings.contains("save"));
Assert.assertFalse(postMappings.contains("remove"));
Assert.assertFalse(postMappings.contains("removeBatch"));
}
@Test
public void adminUpdateAndDeleteShouldUseUnifiedMaintenanceService() {
DocumentChunkService service = mock(DocumentChunkService.class);
DocumentChunkController controller = new DocumentChunkController(service);
DocumentChunk current = new DocumentChunk();
current.setId(BigInteger.ONE);
current.setDocumentCollectionId(BigInteger.TWO);
when(service.getById(BigInteger.ONE)).thenReturn(current);
DocumentChunkContentUpdateRequest request = new DocumentChunkContentUpdateRequest();
request.setId(BigInteger.ONE);
request.setContent("updated");
controller.update(request);
controller.removeChunk(BigInteger.ONE);
verify(service).updateContent(BigInteger.TWO, BigInteger.ONE, "updated");
verify(service).deleteChunk(BigInteger.TWO, BigInteger.ONE);
verify(service, never()).updateById(any(DocumentChunk.class));
verify(service, never()).removeById(any());
}
private static void assertStrictJsonBody(Parameter parameter) {
JsonBody jsonBody = parameter.getAnnotation(JsonBody.class);
Assert.assertNotNull(jsonBody);
Assert.assertTrue(jsonBody.required());
Assert.assertFalse(jsonBody.skipConvertError());
}
}