diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentController.java index 694ca72f..2d81c131 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentController.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentController.java @@ -11,8 +11,11 @@ import org.springframework.core.io.ClassPathResource; import org.springframework.http.MediaType; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; +import tech.easyflow.ai.documentimport.DocumentImportBatchDtos; import tech.easyflow.ai.documentimport.DocumentImportDtos; +import tech.easyflow.ai.documentimport.task.DocumentImportBatchAppService; import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService; import tech.easyflow.ai.entity.Document; import tech.easyflow.ai.entity.DocumentCollection; @@ -83,6 +86,9 @@ public class DocumentController extends BaseCurdController createImportBatch( + @JsonBody DocumentImportBatchDtos.CreateRequest request) { + if (request == null || request.getKnowledgeId() == null) { + throw new BusinessException("知识库id不能为空"); + } + getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库"); + return Result.ok(documentImportBatchAppService.createBatch(request)); + } + + /** + * 上传一个批次文件。 + * + * @param batchId 批次 ID + * @param itemId 文件项 ID + * @param knowledgeId 知识库 ID + * @param file 上传文件 + * @return 文件项状态 + */ + @PostMapping(value = "import/batch/{batchId}/item/{itemId}/upload", + consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @SaCheckPermission("/api/v1/documentCollection/save") + public Result uploadImportBatchItem( + @PathVariable BigInteger batchId, + @PathVariable BigInteger itemId, + @RequestParam BigInteger knowledgeId, + @RequestPart("file") MultipartFile file) { + getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库"); + return Result.ok(documentImportBatchAppService.uploadItem( + knowledgeId, batchId, itemId, file)); + } + + /** + * 启动手动或自动批量导入。 + * + * @param request 启动请求 + * @return 批次状态 + */ + @PostMapping("import/batch/start") + @SaCheckPermission("/api/v1/documentCollection/save") + public Result startImportBatch( + @JsonBody DocumentImportBatchDtos.StartRequest request) { + if (request == null || request.getKnowledgeId() == null) { + throw new BusinessException("知识库id不能为空"); + } + getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库"); + return Result.ok(documentImportBatchAppService.startBatch(request)); + } + + /** + * 取消一个尚未启动的上传批次。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @return 空结果 + */ + @PostMapping("import/batch/cancel") + @SaCheckPermission("/api/v1/documentCollection/save") + public Result cancelImportBatch( + @JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId, + @JsonBody(value = "batchId", required = true) BigInteger batchId) { + getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库"); + documentImportBatchAppService.cancelBatch(knowledgeId, batchId); + return Result.ok(); + } + + /** + * 查询批次状态。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @return 批次状态 + */ + @GetMapping("import/batch/status") + @SaCheckPermission("/api/v1/documentCollection/query") + public Result getImportBatchStatus( + @RequestParam BigInteger knowledgeId, + @RequestParam BigInteger batchId) { + getDocumentCollection(knowledgeId.toString(), ResourceAction.READ, "无权限访问知识库"); + return Result.ok(documentImportBatchAppService.getBatchStatus(knowledgeId, batchId)); + } + + /** + * 查询知识库最近一个自动导入批次。 + * + * @param knowledgeId 知识库 ID + * @return 最近批次状态 + */ + @GetMapping("import/batch/current") + @SaCheckPermission("/api/v1/documentCollection/query") + public Result getCurrentImportBatch( + @RequestParam BigInteger knowledgeId) { + getDocumentCollection(knowledgeId.toString(), ResourceAction.READ, "无权限访问知识库"); + return Result.ok(documentImportBatchAppService.getLatestAutoBatch(knowledgeId)); + } + + /** + * 继续中断或部分失败的自动导入批次。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @return 继续后的批次状态 + */ + @PostMapping("import/batch/continue") + @SaCheckPermission("/api/v1/documentCollection/save") + public Result continueImportBatch( + @JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId, + @JsonBody(value = "batchId", required = true) BigInteger batchId) { + getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库"); + return Result.ok(documentImportBatchAppService.continueBatch(knowledgeId, batchId)); + } + + /** + * 根据解析、分块或向量化失败阶段统一重试。 + * + * @param request 重试请求 + * @return 重试任务状态 + */ + @PostMapping("import/task/retry") + @SaCheckPermission("/api/v1/documentCollection/save") + public Result retryImportTask( + @JsonBody DocumentImportDtos.TaskRetryRequest request) { + if (request == null || request.getKnowledgeId() == null || request.getDocumentId() == null) { + throw new BusinessException("重试信息不完整"); + } + getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库"); + return documentService.retryFailedTask(request); + } + /** * 更新 entity * diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/system/SysApiKeyController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/system/SysApiKeyController.java index 6f9fa8e8..3042b4b6 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/system/SysApiKeyController.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/system/SysApiKeyController.java @@ -6,18 +6,21 @@ import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.table.TableInfo; import com.mybatisflex.core.table.TableInfoFactory; import jakarta.servlet.http.HttpServletRequest; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import tech.easyflow.ai.service.KnowledgeSharePermissionService; import tech.easyflow.ai.service.WorkflowApiPermissionService; +import tech.easyflow.ai.enums.KnowledgeApiPermissionScope; import tech.easyflow.common.domain.Result; import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.satoken.util.SaTokenUtil; import tech.easyflow.common.util.IdUtil; import tech.easyflow.common.vo.PkVo; import tech.easyflow.common.web.controller.BaseCurdController; +import tech.easyflow.common.web.jsonbody.JsonBody; import tech.easyflow.system.entity.SysApiKey; import tech.easyflow.system.entity.SysApiKeyResourceMapping; import tech.easyflow.system.service.SysApiKeyResourceMappingService; @@ -29,6 +32,7 @@ import java.time.LocalDate; import java.time.ZoneId; import java.util.Date; import java.util.List; +import java.util.Set; /** * 控制层。 @@ -83,12 +87,57 @@ public class SysApiKeyController extends BaseCurdController权限开关不映射数据库列，权限更新请求可能只包含主键与权限字段。 + * 此时跳过主表更新，避免 MyBatis-Flex 生成空的 {@code SET} 子句。

+ * + * @param entity 待更新的访问令牌 + * @return 更新结果 + */ + @Override + @PostMapping("/update") + @Transactional(rollbackFor = Exception.class) + public Result update(@JsonBody SysApiKey entity) { + if (entity == null || entity.getId() == null) { + return Result.fail("访问令牌 ID 不能为空"); + } + if (!hasPersistentUpdateFields(entity) && !hasPermissionUpdateFields(entity)) { + return Result.fail("没有可更新的访问令牌字段"); + } + if (hasNewKnowledgePermissionFields(entity) + && !hasCompleteKnowledgePermissionFields(entity)) { + return Result.fail("知识库读取、导入、维护权限必须同时提交"); + } + if (service.getById(entity.getId()) == null) { + return Result.fail("访问令牌不存在"); + } + Result beforeResult = onSaveOrUpdateBefore(entity, false); + if (beforeResult != null) { + return beforeResult; + } + if (hasPersistentUpdateFields(entity)) { + service.updateById(entity); + } + onSaveOrUpdateAfter(entity, false); + return Result.ok(); + } + @Override protected void onSaveOrUpdateAfter(SysApiKey entity, boolean isSave) { if (entity.getPermissionIds() != null) { sysApiKeyResourceMappingService.authInterface(entity); } - if (entity.getKnowledgeShareEnabled() != null) { + if (hasNewKnowledgePermissionFields(entity)) { + knowledgeSharePermissionService.replaceApiPermissions( + entity.getId(), + Boolean.TRUE.equals(entity.getKnowledgeReadEnabled()), + Boolean.TRUE.equals(entity.getKnowledgeImportEnabled()), + Boolean.TRUE.equals(entity.getKnowledgeMaintenanceEnabled()) + ); + } else if (entity.getKnowledgeShareEnabled() != null) { + // 兼容旧客户端：开启旧总开关只授予读取和导入，维护权限保持关闭。 knowledgeSharePermissionService.replaceApiShareEnabled(entity.getId(), entity.getKnowledgeShareEnabled()); } if (entity.getWorkflowApiEnabled() != null) { @@ -130,11 +179,18 @@ public class SysApiKeyController extends BaseCurdController resourceIds = sysApiKeyResourceMappingService.listAs(interfaceWrapper, BigInteger.class); entity.setPermissionIds(resourceIds); - QueryWrapper knowledgeWrapper = QueryWrapper.create() - .select(SysApiKeyResourceMapping::getId) - .eq(SysApiKeyResourceMapping::getApiKeyId, entity.getId()) - .eq(SysApiKeyResourceMapping::getResourceType, "KNOWLEDGE"); - entity.setKnowledgeShareEnabled(sysApiKeyResourceMappingService.count(knowledgeWrapper) > 0); + Set knowledgeScopes = + knowledgeSharePermissionService.getApiPermissionScopes(entity.getId()); + boolean readEnabled = + knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); + boolean importEnabled = + knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); + boolean maintenanceEnabled = + knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name()); + entity.setKnowledgeReadEnabled(readEnabled); + entity.setKnowledgeImportEnabled(importEnabled); + entity.setKnowledgeMaintenanceEnabled(maintenanceEnabled); + entity.setKnowledgeShareEnabled(readEnabled || importEnabled || maintenanceEnabled); QueryWrapper workflowWrapper = QueryWrapper.create() .select(SysApiKeyResourceMapping::getId) @@ -142,4 +198,57 @@ public class SysApiKeyController extends BaseCurdController 0); } + + /** + * 判断请求是否提交了任一新版知识库权限字段。 + * + * @param entity 访问令牌 + * @return 是否提交新版字段 + */ + private boolean hasNewKnowledgePermissionFields(SysApiKey entity) { + return entity.getKnowledgeReadEnabled() != null + || entity.getKnowledgeImportEnabled() != null + || entity.getKnowledgeMaintenanceEnabled() != null; + } + + /** + * 判断请求是否完整提交三个新版知识库权限字段。 + * + * @param entity 访问令牌 + * @return 三个字段是否均已提交 + */ + private boolean hasCompleteKnowledgePermissionFields(SysApiKey entity) { + return entity.getKnowledgeReadEnabled() != null + && entity.getKnowledgeImportEnabled() != null + && entity.getKnowledgeMaintenanceEnabled() != null; + } + + /** + * 判断请求是否包含主表可持久化字段。 + * + * @param entity 访问令牌 + * @return 是否需要更新访问令牌主表 + */ + private boolean hasPersistentUpdateFields(SysApiKey entity) { + return entity.getApiKey() != null + || entity.getCreated() != null + || entity.getStatus() != null + || entity.getDeptId() != null + || entity.getTenantId() != null + || entity.getExpiredAt() != null + || entity.getCreatedBy() != null; + } + + /** + * 判断请求是否包含任一非主表权限字段。 + * + * @param entity 访问令牌 + * @return 是否需要更新权限映射 + */ + private boolean hasPermissionUpdateFields(SysApiKey entity) { + return entity.getPermissionIds() != null + || entity.getKnowledgeShareEnabled() != null + || hasNewKnowledgePermissionFields(entity) + || entity.getWorkflowApiEnabled() != null; + } } diff --git a/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/controller/system/SysApiKeyControllerTest.java b/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/controller/system/SysApiKeyControllerTest.java new file mode 100644 index 00000000..ed1b0777 --- /dev/null +++ b/easyflow-api/easyflow-api-admin/src/test/java/tech/easyflow/admin/controller/system/SysApiKeyControllerTest.java @@ -0,0 +1,142 @@ +package tech.easyflow.admin.controller.system; + +import org.testng.annotations.Test; +import tech.easyflow.ai.service.KnowledgeSharePermissionService; +import tech.easyflow.ai.service.WorkflowApiPermissionService; +import tech.easyflow.common.domain.Result; +import tech.easyflow.system.entity.SysApiKey; +import tech.easyflow.system.service.SysApiKeyResourceMappingService; +import tech.easyflow.system.service.SysApiKeyService; + +import java.lang.reflect.Field; +import java.math.BigInteger; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; + +/** + * {@link SysApiKeyController} 更新访问令牌测试。 + */ +public class SysApiKeyControllerTest { + + /** + * 验证只更新权限时不会执行缺少主表更新字段的 SQL。 + */ + @Test + public void updateShouldSkipMainTableForPermissionOnlyRequest() { + BigInteger apiKeyId = BigInteger.valueOf(100); + SysApiKeyService apiKeyService = mock(SysApiKeyService.class); + KnowledgeSharePermissionService knowledgePermissionService = + mock(KnowledgeSharePermissionService.class); + SysApiKeyController controller = controller(apiKeyService, knowledgePermissionService); + SysApiKey existing = new SysApiKey(); + existing.setId(apiKeyId); + when(apiKeyService.getById(apiKeyId)).thenReturn(existing); + + SysApiKey request = new SysApiKey(); + request.setId(apiKeyId); + request.setKnowledgeReadEnabled(true); + request.setKnowledgeImportEnabled(false); + request.setKnowledgeMaintenanceEnabled(false); + + Result result = controller.update(request); + + assertEquals(result.getErrorCode(), 0); + verify(apiKeyService, never()).updateById(request); + verify(knowledgePermissionService).replaceApiPermissions(apiKeyId, true, false, false); + } + + /** + * 验证基础字段与权限同时更新时，两类数据都被保存。 + */ + @Test + public void updateShouldPersistMainTableAndPermissionsTogether() { + BigInteger apiKeyId = BigInteger.valueOf(101); + SysApiKeyService apiKeyService = mock(SysApiKeyService.class); + KnowledgeSharePermissionService knowledgePermissionService = + mock(KnowledgeSharePermissionService.class); + SysApiKeyController controller = controller(apiKeyService, knowledgePermissionService); + SysApiKey existing = new SysApiKey(); + existing.setId(apiKeyId); + when(apiKeyService.getById(apiKeyId)).thenReturn(existing); + + SysApiKey request = new SysApiKey(); + request.setId(apiKeyId); + request.setStatus(1); + request.setKnowledgeReadEnabled(true); + request.setKnowledgeImportEnabled(true); + request.setKnowledgeMaintenanceEnabled(true); + + Result result = controller.update(request); + + assertEquals(result.getErrorCode(), 0); + verify(apiKeyService).updateById(request); + verify(knowledgePermissionService).replaceApiPermissions(apiKeyId, true, true, true); + } + + /** + * 验证新版知识库权限缺少字段时拒绝更新，避免遗漏字段被隐式关闭。 + */ + @Test + public void updateShouldRejectPartialKnowledgePermissions() { + SysApiKeyService apiKeyService = mock(SysApiKeyService.class); + KnowledgeSharePermissionService knowledgePermissionService = + mock(KnowledgeSharePermissionService.class); + SysApiKeyController controller = controller(apiKeyService, knowledgePermissionService); + SysApiKey request = new SysApiKey(); + request.setId(BigInteger.valueOf(102)); + request.setKnowledgeReadEnabled(true); + + Result result = controller.update(request); + + assertNotEquals(result.getErrorCode(), 0); + verifyNoInteractions(apiKeyService, knowledgePermissionService); + } + + /** + * 创建注入模拟依赖的控制器。 + * + * @param apiKeyService 访问令牌服务 + * @param knowledgePermissionService 知识库权限服务 + * @return 测试控制器 + */ + private SysApiKeyController controller( + SysApiKeyService apiKeyService, + KnowledgeSharePermissionService knowledgePermissionService + ) { + SysApiKeyController controller = new SysApiKeyController(apiKeyService); + setField(controller, "sysApiKeyResourceMappingService", mock(SysApiKeyResourceMappingService.class)); + setField(controller, "knowledgeSharePermissionService", knowledgePermissionService); + setField(controller, "workflowApiPermissionService", mock(WorkflowApiPermissionService.class)); + return controller; + } + + /** + * 通过反射注入控制器字段。 + * + * @param target 目标对象 + * @param fieldName 字段名 + * @param value 字段值 + */ + private void setField(Object target, String fieldName, Object value) { + Class current = target.getClass(); + while (current != null) { + try { + Field field = current.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + return; + } catch (NoSuchFieldException ignored) { + current = current.getSuperclass(); + } catch (IllegalAccessException e) { + throw new IllegalStateException("设置测试字段失败: " + fieldName, e); + } + } + throw new IllegalArgumentException("未找到字段: " + fieldName); + } +} diff --git a/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/controller/PublicKnowledgeDocumentImportController.java b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/controller/PublicKnowledgeDocumentImportController.java new file mode 100644 index 00000000..b8cfa521 --- /dev/null +++ b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/controller/PublicKnowledgeDocumentImportController.java @@ -0,0 +1,326 @@ +package tech.easyflow.publicapi.controller; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.Part; +import org.springframework.http.InvalidMediaTypeException; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.ai.documentimport.ImportCallerContext; +import tech.easyflow.ai.documentimport.ImportCallerType; +import tech.easyflow.ai.documentimport.PublicDocumentImportDtos; +import tech.easyflow.ai.documentimport.task.KnowledgeImportBatchFacade; +import tech.easyflow.ai.entity.DocumentCollection; +import tech.easyflow.ai.enums.KnowledgeApiPermissionScope; +import tech.easyflow.ai.service.DocumentCollectionService; +import tech.easyflow.ai.service.KnowledgeShareAuditService; +import tech.easyflow.ai.service.KnowledgeSharePermissionService; +import tech.easyflow.common.domain.Result; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.common.web.jsonbody.JsonBody; +import tech.easyflow.publicapi.interceptor.PublicApiInterceptor; +import tech.easyflow.system.entity.SysApiKey; +import tech.easyflow.system.service.SysApiKeyService; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 知识库文档 Public API 批量异步导入接口。 + * + * @author Codex + * @since 2026-08-02 + */ +@RestController +@RequestMapping( + value = "/public-api/knowledge-share/document/import/batch", + produces = MediaType.APPLICATION_JSON_VALUE +) +public class PublicKnowledgeDocumentImportController { + + private static final long MAX_METADATA_BYTES = 1024L * 1024L; + + private final SysApiKeyService sysApiKeyService; + private final KnowledgeSharePermissionService permissionService; + private final KnowledgeShareAuditService auditService; + private final DocumentCollectionService documentCollectionService; + private final KnowledgeImportBatchFacade importFacade; + + /** + * 创建 Public API 批量导入控制器。 + * + * @param sysApiKeyService 访问令牌服务 + * @param permissionService 知识库权限服务 + * @param auditService 审计服务 + * @param documentCollectionService 知识库服务 + * @param importFacade 批量导入门面 + */ + public PublicKnowledgeDocumentImportController( + SysApiKeyService sysApiKeyService, + KnowledgeSharePermissionService permissionService, + KnowledgeShareAuditService auditService, + DocumentCollectionService documentCollectionService, + KnowledgeImportBatchFacade importFacade) { + this.sysApiKeyService = sysApiKeyService; + this.permissionService = permissionService; + this.auditService = auditService; + this.documentCollectionService = documentCollectionService; + this.importFacade = importFacade; + } + + /** + * 接收多文件并创建异步导入任务。 + * + * @param apiKey 访问令牌 + * @param metadataPart JSON 元数据 Part + * @param files 多个文件 Part + * @param servletRequest Servlet 请求 + * @return HTTP 202 异步任务响应 + */ + @PostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity> submit( + @RequestHeader("ApiKey") String apiKey, + @RequestPart("metadata") Part metadataPart, + @RequestPart("files") List files, + HttpServletRequest servletRequest) { + PublicDocumentImportDtos.BatchMetadata metadata = + parseMetadata(metadataPart); + SysApiKey token = resolveAuthenticatedApiKey(servletRequest, apiKey); + assertImportPermission( + token, + servletRequest.getRequestURI(), + metadata.getKnowledgeId() + ); + requireDocumentKnowledge(metadata.getKnowledgeId()); + ImportCallerContext caller = + new ImportCallerContext(ImportCallerType.PUBLIC_API, token.getId()); + PublicDocumentImportDtos.SubmitResponse response = + importFacade.submit(caller, metadata, files); + audit( + token, + "API批量导入文档", + servletRequest.getRequestURI(), + Map.of( + "knowledgeId", metadata.getKnowledgeId(), + "taskId", response.getTaskId(), + "totalCount", response.getTotalCount() + ) + ); + return ResponseEntity.status(HttpStatus.ACCEPTED).body(Result.ok(response)); + } + + /** + * 查询异步导入任务状态。 + * + * @param apiKey 访问令牌 + * @param taskId 批次任务 ID + * @param itemStatus 可选文件状态 + * @param pageNumber 页码 + * @param pageSize 每页数量 + * @param servletRequest Servlet 请求 + * @return 任务状态 + */ + @GetMapping("/status") + public Result status( + @RequestHeader("ApiKey") String apiKey, + @RequestParam BigInteger taskId, + @RequestParam(required = false) String itemStatus, + @RequestParam(defaultValue = "1") long pageNumber, + @RequestParam(defaultValue = "20") long pageSize, + HttpServletRequest servletRequest) { + SysApiKey token = resolveAuthenticatedApiKey(servletRequest, apiKey); + ImportCallerContext caller = + new ImportCallerContext(ImportCallerType.PUBLIC_API, token.getId()); + BigInteger knowledgeId = + importFacade.getOwnedKnowledgeId(caller, taskId); + assertImportPermission(token, servletRequest.getRequestURI(), knowledgeId); + PublicDocumentImportDtos.StatusResponse response = + importFacade.getStatus( + caller, + taskId, + itemStatus, + pageNumber, + pageSize + ); + return Result.ok(response); + } + + /** + * 对异常任务执行断点重试。 + * + * @param apiKey 访问令牌 + * @param request 重试请求 + * @param servletRequest Servlet 请求 + * @return 重试结果 + */ + @PostMapping(value = "/retry", consumes = MediaType.APPLICATION_JSON_VALUE) + public Result retry( + @RequestHeader("ApiKey") String apiKey, + @JsonBody PublicDocumentImportDtos.RetryRequest request, + HttpServletRequest servletRequest) { + SysApiKey token = resolveAuthenticatedApiKey(servletRequest, apiKey); + ImportCallerContext caller = + new ImportCallerContext(ImportCallerType.PUBLIC_API, token.getId()); + BigInteger taskId = request == null ? null : request.getTaskId(); + if (taskId == null) { + throw new BusinessException("taskId 不能为空"); + } + BigInteger knowledgeId = + importFacade.getOwnedKnowledgeId(caller, taskId); + assertImportPermission(token, servletRequest.getRequestURI(), knowledgeId); + PublicDocumentImportDtos.RetryResponse response = + importFacade.retry(caller, request); + audit( + token, + "API重试批量导入任务", + servletRequest.getRequestURI(), + Map.of( + "knowledgeId", knowledgeId, + "taskId", response.getTaskId(), + "retriedCount", response.getRetriedCount() + ) + ); + return Result.ok(response); + } + + /** + * 解析并校验 metadata JSON Part。 + * + * @param metadataPart metadata Part + * @return 批量元数据 + */ + private PublicDocumentImportDtos.BatchMetadata parseMetadata( + Part metadataPart) { + if (metadataPart == null || metadataPart.getSize() <= 0) { + throw new BusinessException("metadata 不能为空"); + } + if (metadataPart.getSize() > MAX_METADATA_BYTES) { + throw new BusinessException(413, 41304, "metadata 不能超过1MiB"); + } + String contentType = metadataPart.getContentType(); + try { + if (contentType == null + || !MediaType.APPLICATION_JSON.includes( + MediaType.parseMediaType(contentType))) { + throw new BusinessException(415, 41503, + "metadata Part 必须使用 application/json"); + } + } catch (InvalidMediaTypeException error) { + throw new BusinessException(415, 41503, + "metadata Part Content-Type 无效", error); + } + try { + String json = new String( + metadataPart.getInputStream().readAllBytes(), + StandardCharsets.UTF_8 + ); + PublicDocumentImportDtos.BatchMetadata metadata = + JSON.parseObject( + json, + PublicDocumentImportDtos.BatchMetadata.class + ); + if (metadata == null) { + throw new BusinessException("metadata 不能为空"); + } + return metadata; + } catch (JSONException error) { + throw new BusinessException(400, 40021, + "metadata JSON 格式无效", error); + } catch (java.io.IOException error) { + throw new BusinessException(500, 50023, + "读取 metadata 失败", error); + } + } + + /** + * 断言访问令牌具有知识导入权限。 + * + * @param token 访问令牌 + * @param requestUri 请求 URI + * @param knowledgeId 知识库 ID + */ + private void assertImportPermission(SysApiKey token, + String requestUri, + BigInteger knowledgeId) { + permissionService.assertApiShare( + token.getId(), + requestUri, + knowledgeId, + KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name() + ); + } + + /** + * 复用拦截器已经完成认证的访问令牌。 + * + * @param request Servlet 请求 + * @param apiKey 访问令牌明文 + * @return 已认证访问令牌 + */ + private SysApiKey resolveAuthenticatedApiKey( + HttpServletRequest request, + String apiKey) { + Object authenticated = + request.getAttribute( + PublicApiInterceptor.AUTHENTICATED_API_KEY_ATTRIBUTE + ); + if (authenticated instanceof SysApiKey token) { + return token; + } + // 兼容控制器单测和绕过 MVC 拦截器的内部直接调用。 + return sysApiKeyService.getSysApiKey(apiKey); + } + + /** + * 断言知识库存在且为文档类型。 + * + * @param knowledgeId 知识库 ID + */ + private void requireDocumentKnowledge(BigInteger knowledgeId) { + DocumentCollection knowledge = + knowledgeId == null ? null : documentCollectionService.getById(knowledgeId); + if (knowledge == null) { + throw new BusinessException(404, 404, "知识库不存在"); + } + if (!knowledge.isDocumentCollection()) { + throw new BusinessException("当前知识库类型不支持文档导入"); + } + } + + /** + * 记录不含令牌明文的 Public API 审计。 + * + * @param token 访问令牌 + * @param actionName 操作名称 + * @param actionUrl 操作 URI + * @param detail 业务详情 + */ + private void audit(SysApiKey token, + String actionName, + String actionUrl, + Map detail) { + Map payload = new HashMap<>(detail); + payload.put("apiKeyId", token.getId()); + payload.put("channel", "API"); + auditService.log( + null, + actionName, + "KNOWLEDGE_API_SHARE_WRITE", + actionUrl, + payload + ); + } +} diff --git a/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/controller/PublicKnowledgeShareController.java b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/controller/PublicKnowledgeShareController.java index 2ebd87b5..a459034e 100644 --- a/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/controller/PublicKnowledgeShareController.java +++ b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/controller/PublicKnowledgeShareController.java @@ -18,13 +18,12 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import tech.easyflow.ai.documentimport.DocumentImportDtos; -import tech.easyflow.ai.dto.KnowledgeSearchResultItem; import tech.easyflow.ai.entity.Document; import tech.easyflow.ai.entity.DocumentChunk; import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.FaqItem; import tech.easyflow.ai.entity.Model; -import tech.easyflow.ai.enums.KnowledgeShareActionScope; +import tech.easyflow.ai.enums.KnowledgeApiPermissionScope; import tech.easyflow.ai.rag.KnowledgeRetrievalModes; import tech.easyflow.ai.rag.KnowledgeRetrievalRequest; import tech.easyflow.ai.service.DocumentChunkService; @@ -42,6 +41,8 @@ import tech.easyflow.common.domain.Result; import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.common.web.jsonbody.JsonBody; +import tech.easyflow.publicapi.dto.PublicKnowledgeDetailResponse; +import tech.easyflow.publicapi.dto.PublicKnowledgeSearchResultItem; import tech.easyflow.system.entity.SysApiKey; import tech.easyflow.system.service.SysApiKeyService; @@ -89,28 +90,43 @@ public class PublicKnowledgeShareController { * 获取知识库详情。 */ @GetMapping("/detail") - public Result detail( + public Result detail( @RequestHeader("ApiKey") String apiKey, @RequestParam BigInteger knowledgeId, + @RequestParam(defaultValue = "1") int pageNumber, + @RequestParam(defaultValue = "50") int pageSize, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); + validateDocumentPage(pageNumber, pageSize); + DocumentCollection knowledge = documentCollectionService.getDetail(knowledgeId.toString()); + if (knowledge == null) { + throw new BusinessException("知识库不存在"); + } + Page documents = knowledge.isDocumentCollection() + ? documentService.getDocumentList( + knowledgeId.toString(), + pageSize, + pageNumber, + null + ) + : new Page<>(Collections.emptyList(), pageNumber, pageSize, 0L); audit(apiKey, "API读取知识库详情", "KNOWLEDGE_API_SHARE_ACCESS", request.getRequestURI(), Map.of("knowledgeId", knowledgeId)); - return Result.ok(documentCollectionService.getDetail(knowledgeId.toString())); + return Result.ok(new PublicKnowledgeDetailResponse(knowledge, documents)); } /** * 检索知识库。 */ @GetMapping("/search") - public Result> search( + public Result> search( @RequestHeader("ApiKey") String apiKey, @RequestParam BigInteger knowledgeId, @RequestParam String keyword, @RequestParam(required = false) String retrievalMode, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.SEARCH.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); KnowledgeRetrievalRequest retrievalRequest = new KnowledgeRetrievalRequest(); retrievalRequest.setKnowledgeId(knowledgeId); retrievalRequest.setQuery(keyword); @@ -128,14 +144,19 @@ public class PublicKnowledgeShareController { public Result> documentPage( @RequestHeader("ApiKey") String apiKey, @RequestParam BigInteger knowledgeId, - @RequestParam(required = false) String title, + @RequestParam(required = false) BigInteger documentId, @RequestParam(defaultValue = "10") int pageSize, @RequestParam(defaultValue = "1") int pageNumber, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); requireDocumentKnowledge(knowledgeId); - return Result.ok(documentService.getDocumentList(knowledgeId.toString(), pageSize, pageNumber, title)); + return Result.ok(documentService.getDocumentListById( + knowledgeId.toString(), + pageSize, + pageNumber, + documentId + )); } /** @@ -149,7 +170,7 @@ public class PublicKnowledgeShareController { HttpServletRequest request, HttpServletResponse response ) throws Exception { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); requireDocumentKnowledge(knowledgeId); Document document = requireDocument(documentId, knowledgeId); response.setContentType("application/octet-stream"); @@ -169,11 +190,11 @@ public class PublicKnowledgeShareController { @PostMapping("/document/remove") public Result removeDocument( @RequestHeader("ApiKey") String apiKey, - @RequestParam BigInteger knowledgeId, - @JsonBody("id") String id, + @JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId, + @JsonBody(value = "id", required = true) String id, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_DELETE.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name()); requireDocumentKnowledge(knowledgeId); requireDocument(new BigInteger(id), knowledgeId); audit(apiKey, "API删除文档", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "documentId", id)); @@ -189,7 +210,7 @@ public class PublicKnowledgeShareController { @JsonBody DocumentImportDtos.AnalyzeRequest request, HttpServletRequest servletRequest ) { - assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(request.getKnowledgeId()); audit(apiKey, "API分析文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); return documentService.analyzeImport(request); @@ -204,7 +225,7 @@ public class PublicKnowledgeShareController { @JsonBody DocumentImportDtos.PreviewRequest request, HttpServletRequest servletRequest ) { - assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(request.getKnowledgeId()); audit(apiKey, "API预览文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); return documentService.previewImport(request); @@ -219,7 +240,7 @@ public class PublicKnowledgeShareController { @JsonBody DocumentImportDtos.CommitRequest request, HttpServletRequest servletRequest ) { - assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(request.getKnowledgeId()); audit(apiKey, "API提交文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); return documentService.commitImport(request); @@ -231,7 +252,7 @@ public class PublicKnowledgeShareController { @JsonBody DocumentImportDtos.TaskCreateRequest request, HttpServletRequest servletRequest ) { - assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(request.getKnowledgeId()); audit(apiKey, "API创建文档导入任务", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); return documentService.createImportTask(request); @@ -244,7 +265,7 @@ public class PublicKnowledgeShareController { @RequestParam BigInteger taskId, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(knowledgeId); Result result = documentService.getImportTaskDetail(taskId); if (result.getData() == null || result.getData().getKnowledgeId() == null @@ -260,7 +281,7 @@ public class PublicKnowledgeShareController { @JsonBody DocumentImportDtos.PreviewRequest request, HttpServletRequest servletRequest ) { - assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(request.getKnowledgeId()); audit(apiKey, "API预览文档分块", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); return documentService.previewImportTask(request); @@ -272,7 +293,7 @@ public class PublicKnowledgeShareController { @JsonBody DocumentImportDtos.TaskStartIndexRequest request, HttpServletRequest servletRequest ) { - assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(request.getKnowledgeId()); audit(apiKey, "API启动文档向量化", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); return documentService.startIndexTask(request); @@ -284,7 +305,7 @@ public class PublicKnowledgeShareController { @JsonBody DocumentImportDtos.TaskRetryRequest request, HttpServletRequest servletRequest ) { - assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(request.getKnowledgeId()); audit(apiKey, "API重试文档解析", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); return documentService.retryParseTask(request); @@ -296,7 +317,7 @@ public class PublicKnowledgeShareController { @JsonBody DocumentImportDtos.TaskRetryRequest request, HttpServletRequest servletRequest ) { - assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireDocumentKnowledge(request.getKnowledgeId()); audit(apiKey, "API重试文档向量化", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); return documentService.retryIndexTask(request); @@ -314,7 +335,7 @@ public class PublicKnowledgeShareController { @RequestParam(defaultValue = "10") long pageSize, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); requireDocumentKnowledge(knowledgeId); requireDocument(documentId, knowledgeId); QueryWrapper wrapper = QueryWrapper.create() @@ -329,11 +350,11 @@ public class PublicKnowledgeShareController { @PostMapping("/documentChunk/update") public Result updateDocumentChunk( @RequestHeader("ApiKey") String apiKey, - @RequestParam BigInteger knowledgeId, + @JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId, @JsonBody DocumentChunk documentChunk, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_UPDATE.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name()); requireDocumentKnowledge(knowledgeId); DocumentChunk current = requireDocumentChunk(documentChunk.getId(), knowledgeId); boolean success = documentChunkService.updateById(documentChunk); @@ -369,11 +390,11 @@ public class PublicKnowledgeShareController { @PostMapping("/documentChunk/remove") public Result removeDocumentChunk( @RequestHeader("ApiKey") String apiKey, - @RequestParam BigInteger knowledgeId, - @JsonBody("id") BigInteger chunkId, + @JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId, + @JsonBody(value = "id", required = true) BigInteger chunkId, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_DELETE.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name()); requireDocumentKnowledge(knowledgeId); requireDocumentChunk(chunkId, knowledgeId); DocumentCollection knowledge = documentCollectionService.getById(knowledgeId); @@ -410,7 +431,7 @@ public class PublicKnowledgeShareController { @RequestParam(defaultValue = "10") long pageSize, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); requireFaqKnowledge(knowledgeId); faqCategoryService.ensureDefaultCategory(knowledgeId); QueryWrapper queryWrapper = QueryWrapper.create() @@ -447,7 +468,7 @@ public class PublicKnowledgeShareController { @RequestParam String id, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); requireFaqKnowledge(knowledgeId); FaqItem faqItem = requireFaq(new BigInteger(id), knowledgeId); return Result.ok(faqItem); @@ -462,7 +483,7 @@ public class PublicKnowledgeShareController { @JsonBody FaqItem entity, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), entity.getCollectionId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); + assertApiShare(apiKey, request.getRequestURI(), entity.getCollectionId(), KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name()); requireFaqKnowledge(entity.getCollectionId()); audit(apiKey, "API新增FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", entity.getCollectionId())); return Result.ok(faqItemService.saveFaqItem(entity)); @@ -478,7 +499,7 @@ public class PublicKnowledgeShareController { @JsonBody FaqItem entity, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_UPDATE.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name()); requireFaqKnowledge(knowledgeId); requireFaq(entity.getId(), knowledgeId); audit(apiKey, "API更新FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "faqId", entity.getId())); @@ -495,7 +516,7 @@ public class PublicKnowledgeShareController { @JsonBody("id") BigInteger id, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_DELETE.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name()); requireFaqKnowledge(knowledgeId); requireFaq(id, knowledgeId); audit(apiKey, "API删除FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "faqId", id)); @@ -512,7 +533,7 @@ public class PublicKnowledgeShareController { BigInteger collectionId, HttpServletRequest request ) { - assertApiShare(apiKey, request.getRequestURI(), collectionId, KnowledgeShareActionScope.IMPORT_EXPORT.name()); + assertApiShare(apiKey, request.getRequestURI(), collectionId, KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireFaqKnowledge(collectionId); audit(apiKey, "API导入FAQ Excel", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", collectionId)); return Result.ok(faqItemService.importFromExcel(collectionId, file)); @@ -528,7 +549,7 @@ public class PublicKnowledgeShareController { HttpServletRequest request, HttpServletResponse response ) throws Exception { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.IMPORT_EXPORT.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()); requireFaqKnowledge(knowledgeId); response.setContentType("application/octet-stream"); response.setHeader( @@ -550,7 +571,7 @@ public class PublicKnowledgeShareController { HttpServletRequest request, HttpServletResponse response ) throws Exception { - assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.IMPORT_EXPORT.name()); + assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name()); requireFaqKnowledge(knowledgeId); String fileName = "faq_export_" + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + ".xlsx"; response.setContentType("application/octet-stream"); @@ -610,6 +631,22 @@ public class PublicKnowledgeShareController { return knowledge; } + /** + * 校验详情接口中的文档分页参数。 + * + * @param pageNumber 页码 + * @param pageSize 每页条数 + * @throws BusinessException 页码小于 1 或每页条数不在 1 到 100 之间时抛出 + */ + private void validateDocumentPage(int pageNumber, int pageSize) { + if (pageNumber < 1) { + throw new BusinessException("pageNumber 必须大于等于 1"); + } + if (pageSize < 1 || pageSize > 100) { + throw new BusinessException("pageSize 必须在 1 到 100 之间"); + } + } + private Document requireDocument(BigInteger documentId, BigInteger knowledgeId) { Document document = documentService.getById(documentId); if (document == null || document.getCollectionId() == null || document.getCollectionId().compareTo(knowledgeId) != 0) { @@ -642,15 +679,38 @@ public class PublicKnowledgeShareController { knowledgeShareAuditService.log(null, actionName, actionType, actionUrl, payload); } - private List toKnowledgeSearchResult(List documents) { - List result = new java.util.ArrayList<>(); + private List toKnowledgeSearchResult( + List documents + ) { + List result = new java.util.ArrayList<>(); for (com.easyagents.core.document.Document document : documents) { - KnowledgeSearchResultItem item = new KnowledgeSearchResultItem(); + PublicKnowledgeSearchResultItem item = + new PublicKnowledgeSearchResultItem(); item.setContent(document.getContent()); + String resultType = + asString(document.getMetadata("resultType")); + item.setResultType(resultType); Object renderMarkdown = document.getMetadata("renderMarkdown"); item.setRenderMarkdown(renderMarkdown == null ? null : String.valueOf(renderMarkdown)); Object sourceFileName = document.getMetadata("sourceFileName"); - item.setSourceFileName(sourceFileName == null ? null : String.valueOf(sourceFileName)); + String documentName = + sourceFileName == null ? null : String.valueOf(sourceFileName); + item.setSourceFileName(documentName); + if (DocumentCollection.TYPE_FAQ.equalsIgnoreCase(resultType)) { + item.setFaqId(asBigInteger(document.getMetadata("faqId"))); + item.setQuestion(asString(document.getMetadata("question"))); + item.setAnswerText( + asString(document.getMetadata("answerText")) + ); + item.setCategoryId( + asBigInteger(document.getMetadata("categoryId")) + ); + } else { + item.setDocumentName(documentName); + item.setDocumentId( + asBigInteger(document.getMetadata("documentId")) + ); + } item.setScore(document.getScore()); Object hitSource = document.getMetadata("hitSource"); item.setHitSource(hitSource == null ? null : String.valueOf(hitSource)); @@ -661,6 +721,36 @@ public class PublicKnowledgeShareController { return result; } + /** + * 将检索元数据转换为字符串。 + * + * @param value 元数据值 + * @return 字符串值；原值为空时返回 {@code null} + */ + private String asString(Object value) { + return value == null ? null : String.valueOf(value); + } + + /** + * 将检索元数据转换为大整数 ID。 + * + * @param value 元数据值 + * @return 大整数 ID；原值为空或格式无效时返回 {@code null} + */ + private BigInteger asBigInteger(Object value) { + if (value == null) { + return null; + } + if (value instanceof BigInteger) { + return (BigInteger) value; + } + try { + return new BigInteger(String.valueOf(value)); + } catch (NumberFormatException ignored) { + return null; + } + } + private Double asDouble(Object value) { if (value == null) { return null; diff --git a/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/dto/PublicKnowledgeDetailResponse.java b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/dto/PublicKnowledgeDetailResponse.java new file mode 100644 index 00000000..dee4a6c1 --- /dev/null +++ b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/dto/PublicKnowledgeDetailResponse.java @@ -0,0 +1,217 @@ +package tech.easyflow.publicapi.dto; + +import cn.hutool.core.bean.BeanUtil; +import com.mybatisflex.core.paginate.Page; +import tech.easyflow.ai.entity.Document; +import tech.easyflow.ai.entity.DocumentCollection; + +import java.math.BigInteger; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +/** + * 公开知识库详情响应。 + * + *

知识库原有字段保持在响应顶层，文档摘要通过 {@code documents} 分页返回。

+ */ +public class PublicKnowledgeDetailResponse extends DocumentCollection { + + /** + * 已上传文档的分页摘要。 + */ + private final Page documents; + + /** + * 创建公开知识库详情响应。 + * + * @param knowledge 知识库基本信息 + * @param documentPage 文档实体分页；FAQ 知识库可传 {@code null} + */ + public PublicKnowledgeDetailResponse( + DocumentCollection knowledge, + Page documentPage + ) { + BeanUtil.copyProperties(knowledge, this); + this.documents = toDocumentSummaryPage(documentPage); + } + + /** + * 获取文档分页摘要。 + * + * @return 文档分页摘要 + */ + public Page getDocuments() { + return documents; + } + + /** + * 将文档实体分页映射为公开摘要分页。 + * + * @param source 文档实体分页 + * @return 不包含内部路径、正文和配置的公开摘要分页 + */ + private static Page toDocumentSummaryPage(Page source) { + if (source == null) { + return new Page<>(Collections.emptyList(), 1, 50, 0L); + } + List records = source.getRecords() == null + ? Collections.emptyList() + : source.getRecords().stream().map(DocumentSummary::new).toList(); + return new Page<>( + records, + source.getPageNumber(), + source.getPageSize(), + source.getTotalRow() + ); + } + + /** + * 公开文档摘要。 + */ + public static class DocumentSummary { + + /** + * 文档 ID。 + */ + private final BigInteger id; + + /** + * 文档标题。 + */ + private final String title; + + /** + * 文档类型。 + */ + private final String documentType; + + /** + * 文档内容类型。 + */ + private final String contentType; + + /** + * 文档处理状态。 + */ + private final String processStatus; + + /** + * 文档分块数。 + */ + private final long chunkCount; + + /** + * 处理进度百分比。 + */ + private final Integer progressPercent; + + /** + * 创建时间。 + */ + private final Date created; + + /** + * 最后修改时间。 + */ + private final Date modified; + + /** + * 从文档实体创建公开摘要。 + * + * @param document 文档实体 + */ + public DocumentSummary(Document document) { + this.id = document.getId(); + this.title = document.getTitle(); + this.documentType = document.getDocumentType(); + this.contentType = document.getContentType(); + this.processStatus = document.getProcessStatus(); + this.chunkCount = document.getDisplayChunkCount(); + this.progressPercent = document.getProgressPercent(); + this.created = document.getCreated(); + this.modified = document.getModified(); + } + + /** + * 获取文档 ID。 + * + * @return 文档 ID + */ + public BigInteger getId() { + return id; + } + + /** + * 获取文档标题。 + * + * @return 文档标题 + */ + public String getTitle() { + return title; + } + + /** + * 获取文档类型。 + * + * @return 文档类型 + */ + public String getDocumentType() { + return documentType; + } + + /** + * 获取文档内容类型。 + * + * @return 文档内容类型 + */ + public String getContentType() { + return contentType; + } + + /** + * 获取文档处理状态。 + * + * @return 文档处理状态 + */ + public String getProcessStatus() { + return processStatus; + } + + /** + * 获取文档分块数。 + * + * @return 文档分块数 + */ + public long getChunkCount() { + return chunkCount; + } + + /** + * 获取处理进度百分比。 + * + * @return 处理进度百分比 + */ + public Integer getProgressPercent() { + return progressPercent; + } + + /** + * 获取创建时间。 + * + * @return 创建时间 + */ + public Date getCreated() { + return created; + } + + /** + * 获取最后修改时间。 + * + * @return 最后修改时间 + */ + public Date getModified() { + return modified; + } + } +} diff --git a/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/dto/PublicKnowledgeSearchResultItem.java b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/dto/PublicKnowledgeSearchResultItem.java new file mode 100644 index 00000000..5062e02f --- /dev/null +++ b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/dto/PublicKnowledgeSearchResultItem.java @@ -0,0 +1,176 @@ +package tech.easyflow.publicapi.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import tech.easyflow.ai.dto.KnowledgeSearchResultItem; + +import java.math.BigInteger; + +/** + * 公开知识库检索结果。 + * + *

根据命中来源补充文档或 FAQ 标识，便于调用方继续查询对应详情。

+ */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PublicKnowledgeSearchResultItem extends KnowledgeSearchResultItem { + + /** + * 命中来源类型。 + */ + private String resultType; + + /** + * 来源文档 ID。 + */ + private BigInteger documentId; + + /** + * 来源文档名称。 + */ + private String documentName; + + /** + * 来源 FAQ ID。 + */ + private BigInteger faqId; + + /** + * 来源 FAQ 问题。 + */ + private String question; + + /** + * 来源 FAQ 纯文本答案。 + */ + private String answerText; + + /** + * 来源 FAQ 分类 ID。 + */ + private BigInteger categoryId; + + /** + * 获取命中来源类型。 + * + * @return {@code DOCUMENT} 或 {@code FAQ} + */ + public String getResultType() { + return resultType; + } + + /** + * 设置命中来源类型。 + * + * @param resultType 命中来源类型 + */ + public void setResultType(String resultType) { + this.resultType = resultType; + } + + /** + * 获取来源文档 ID。 + * + * @return 来源文档 ID + */ + public BigInteger getDocumentId() { + return documentId; + } + + /** + * 设置来源文档 ID。 + * + * @param documentId 来源文档 ID + */ + public void setDocumentId(BigInteger documentId) { + this.documentId = documentId; + } + + /** + * 获取来源文档名称。 + * + * @return 来源文档名称 + */ + public String getDocumentName() { + return documentName; + } + + /** + * 设置来源文档名称。 + * + * @param documentName 来源文档名称 + */ + public void setDocumentName(String documentName) { + this.documentName = documentName; + } + + /** + * 获取来源 FAQ ID。 + * + * @return 来源 FAQ ID + */ + public BigInteger getFaqId() { + return faqId; + } + + /** + * 设置来源 FAQ ID。 + * + * @param faqId 来源 FAQ ID + */ + public void setFaqId(BigInteger faqId) { + this.faqId = faqId; + } + + /** + * 获取来源 FAQ 问题。 + * + * @return 来源 FAQ 问题 + */ + public String getQuestion() { + return question; + } + + /** + * 设置来源 FAQ 问题。 + * + * @param question 来源 FAQ 问题 + */ + public void setQuestion(String question) { + this.question = question; + } + + /** + * 获取来源 FAQ 纯文本答案。 + * + * @return 来源 FAQ 纯文本答案 + */ + public String getAnswerText() { + return answerText; + } + + /** + * 设置来源 FAQ 纯文本答案。 + * + * @param answerText 来源 FAQ 纯文本答案 + */ + public void setAnswerText(String answerText) { + this.answerText = answerText; + } + + /** + * 获取来源 FAQ 分类 ID。 + * + * @return 来源 FAQ 分类 ID + */ + public BigInteger getCategoryId() { + return categoryId; + } + + /** + * 设置来源 FAQ 分类 ID。 + * + * @param categoryId 来源 FAQ 分类 ID + */ + public void setCategoryId(BigInteger categoryId) { + this.categoryId = categoryId; + } +} diff --git a/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/interceptor/PublicApiInterceptor.java b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/interceptor/PublicApiInterceptor.java index 24f001c2..7825c4ff 100644 --- a/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/interceptor/PublicApiInterceptor.java +++ b/easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/interceptor/PublicApiInterceptor.java @@ -9,11 +9,21 @@ import org.springframework.stereotype.Component; import org.springframework.web.servlet.HandlerInterceptor; import tech.easyflow.common.domain.Result; import tech.easyflow.common.util.ResponseUtil; +import tech.easyflow.system.entity.SysApiKey; import tech.easyflow.system.service.SysApiKeyService; +/** + * Public API 访问令牌与接口权限拦截器。 + */ @Component public class PublicApiInterceptor implements HandlerInterceptor { + /** + * 请求中已完成认证的访问令牌属性名，供后续资源级鉴权复用。 + */ + public static final String AUTHENTICATED_API_KEY_ATTRIBUTE = + PublicApiInterceptor.class.getName() + ".authenticatedApiKey"; + private static final Logger log = LoggerFactory.getLogger(PublicApiInterceptor.class); @Resource @@ -31,7 +41,12 @@ public class PublicApiInterceptor implements HandlerInterceptor { ResponseUtil.renderJson(response, failed); return false; } - sysApiKeyService.checkApikeyPermission(apiKey, requestURI); + SysApiKey authenticatedApiKey = + sysApiKeyService.checkApikeyPermission(apiKey, requestURI); + request.setAttribute( + AUTHENTICATED_API_KEY_ATTRIBUTE, + authenticatedApiKey + ); return true; } } diff --git a/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/controller/PublicKnowledgeDocumentImportControllerTest.java b/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/controller/PublicKnowledgeDocumentImportControllerTest.java new file mode 100644 index 00000000..ca06da62 --- /dev/null +++ b/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/controller/PublicKnowledgeDocumentImportControllerTest.java @@ -0,0 +1,64 @@ +package tech.easyflow.publicapi.controller; + +import jakarta.servlet.http.HttpServletRequest; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.ai.documentimport.PublicDocumentImportDtos; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.publicapi.interceptor.PublicApiInterceptor; +import tech.easyflow.system.entity.SysApiKey; + +import java.lang.reflect.Proxy; +import java.math.BigInteger; + +/** + * {@link PublicKnowledgeDocumentImportController} 请求边界测试。 + * + * @author Codex + * @since 2026-08-02 + */ +public class PublicKnowledgeDocumentImportControllerTest { + + /** + * 验证重试请求缺少 taskId 时返回明确参数错误。 + */ + @Test + public void retryShouldRejectMissingTaskId() { + SysApiKey token = new SysApiKey(); + token.setId(BigInteger.ONE); + HttpServletRequest servletRequest = (HttpServletRequest) Proxy.newProxyInstance( + HttpServletRequest.class.getClassLoader(), + new Class[]{HttpServletRequest.class}, + (instance, method, args) -> { + if ("getAttribute".equals(method.getName())) { + Assert.assertEquals( + PublicApiInterceptor.AUTHENTICATED_API_KEY_ATTRIBUTE, + args[0] + ); + return token; + } + throw new AssertionError( + "测试路径不应调用 HttpServletRequest." + method.getName() + ); + } + ); + PublicKnowledgeDocumentImportController controller = + new PublicKnowledgeDocumentImportController( + null, + null, + null, + null, + null + ); + PublicDocumentImportDtos.RetryRequest request = + new PublicDocumentImportDtos.RetryRequest(); + + try { + controller.retry("test-key", request, servletRequest); + Assert.fail("Expected missing taskId rejection"); + } catch (BusinessException expected) { + Assert.assertEquals(400, expected.getHttpStatus()); + Assert.assertTrue(expected.getMessage().contains("taskId 不能为空")); + } + } +} diff --git a/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/controller/PublicKnowledgeShareControllerContractTest.java b/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/controller/PublicKnowledgeShareControllerContractTest.java new file mode 100644 index 00000000..245aefa3 --- /dev/null +++ b/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/controller/PublicKnowledgeShareControllerContractTest.java @@ -0,0 +1,192 @@ +package tech.easyflow.publicapi.controller; + +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.http.HttpServletRequest; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.web.bind.annotation.RequestParam; +import tech.easyflow.ai.entity.DocumentChunk; +import tech.easyflow.ai.entity.DocumentCollection; +import tech.easyflow.common.web.jsonbody.JsonBody; +import tech.easyflow.publicapi.dto.PublicKnowledgeSearchResultItem; + +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; +import java.math.BigInteger; +import java.util.List; + +/** + * {@link PublicKnowledgeShareController} 公开接口参数契约测试。 + */ +public class PublicKnowledgeShareControllerContractTest { + + /** + * 验证文档删除接口的 knowledgeId 和 id 均从 JSON 请求体读取。 + * + * @throws Exception 反射失败 + */ + @Test + public void documentRemoveShouldReadIdentifiersFromJsonBody() + throws Exception { + Method method = PublicKnowledgeShareController.class.getDeclaredMethod( + "removeDocument", + String.class, + BigInteger.class, + String.class, + HttpServletRequest.class + ); + Parameter knowledgeId = method.getParameters()[1]; + Parameter documentId = method.getParameters()[2]; + + assertRequiredJsonField(knowledgeId, "knowledgeId"); + assertRequiredJsonField(documentId, "id"); + Assert.assertNull(knowledgeId.getAnnotation(RequestParam.class)); + } + + /** + * 验证文档分块更新接口从 JSON 请求体读取 knowledgeId。 + * + * @throws Exception 反射失败 + */ + @Test + public void documentChunkUpdateShouldReadKnowledgeIdFromJsonBody() + throws Exception { + Method method = PublicKnowledgeShareController.class.getDeclaredMethod( + "updateDocumentChunk", + String.class, + BigInteger.class, + DocumentChunk.class, + HttpServletRequest.class + ); + Parameter knowledgeId = method.getParameters()[1]; + Parameter chunk = method.getParameters()[2]; + + assertRequiredJsonField(knowledgeId, "knowledgeId"); + Assert.assertNull(knowledgeId.getAnnotation(RequestParam.class)); + JsonBody chunkBody = chunk.getAnnotation(JsonBody.class); + Assert.assertNotNull(chunkBody); + Assert.assertEquals("", chunkBody.value()); + } + + /** + * 验证文档分块删除接口从 JSON 请求体读取 knowledgeId 和 id。 + * + * @throws Exception 反射失败 + */ + @Test + public void documentChunkRemoveShouldReadIdentifiersFromJsonBody() + throws Exception { + Method method = PublicKnowledgeShareController.class.getDeclaredMethod( + "removeDocumentChunk", + String.class, + BigInteger.class, + BigInteger.class, + HttpServletRequest.class + ); + Parameter knowledgeId = method.getParameters()[1]; + Parameter chunkId = method.getParameters()[2]; + + assertRequiredJsonField(knowledgeId, "knowledgeId"); + assertRequiredJsonField(chunkId, "id"); + Assert.assertNull(knowledgeId.getAnnotation(RequestParam.class)); + } + + /** + * 验证文档检索结果包含来源文档信息且不混入 FAQ 字段。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void searchShouldExposeDocumentIdentity() throws Exception { + BigInteger documentId = BigInteger.valueOf(2002); + com.easyagents.core.document.Document hit = + com.easyagents.core.document.Document.of("命中文本"); + hit.addMetadata("resultType", DocumentCollection.TYPE_DOCUMENT); + hit.addMetadata("documentId", documentId); + hit.addMetadata("sourceFileName", "manual.pdf"); + List results = + mapSearchResult(hit); + + Assert.assertEquals(1, results.size()); + PublicKnowledgeSearchResultItem item = results.get(0); + Assert.assertEquals(DocumentCollection.TYPE_DOCUMENT, item.getResultType()); + Assert.assertEquals(documentId, item.getDocumentId()); + Assert.assertEquals("manual.pdf", item.getDocumentName()); + Assert.assertEquals("manual.pdf", item.getSourceFileName()); + Assert.assertNull(item.getFaqId()); + Assert.assertNull(item.getQuestion()); + String json = new ObjectMapper().writeValueAsString(item); + Assert.assertFalse(json.contains("\"faqId\"")); + Assert.assertFalse(json.contains("\"question\"")); + } + + /** + * 验证 FAQ 检索结果包含 FAQ 信息且不混入文档字段。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void searchShouldExposeFaqIdentity() throws Exception { + BigInteger faqId = BigInteger.valueOf(3003); + BigInteger categoryId = BigInteger.valueOf(4004); + com.easyagents.core.document.Document hit = + com.easyagents.core.document.Document.of("FAQ 命中文本"); + hit.addMetadata("resultType", DocumentCollection.TYPE_FAQ); + hit.addMetadata("faqId", faqId); + hit.addMetadata("question", "如何申请账号？"); + hit.addMetadata("answerText", "请联系管理员。"); + hit.addMetadata("categoryId", categoryId); + List results = + mapSearchResult(hit); + + Assert.assertEquals(1, results.size()); + PublicKnowledgeSearchResultItem item = results.get(0); + Assert.assertEquals(DocumentCollection.TYPE_FAQ, item.getResultType()); + Assert.assertEquals(faqId, item.getFaqId()); + Assert.assertEquals("如何申请账号？", item.getQuestion()); + Assert.assertEquals("请联系管理员。", item.getAnswerText()); + Assert.assertEquals(categoryId, item.getCategoryId()); + Assert.assertNull(item.getDocumentId()); + Assert.assertNull(item.getDocumentName()); + String json = new ObjectMapper().writeValueAsString(item); + Assert.assertFalse(json.contains("\"documentId\"")); + Assert.assertFalse(json.contains("\"documentName\"")); + Assert.assertFalse(json.contains("\"sourceFileName\"")); + } + + /** + * 调用公开检索结果映射。 + * + * @param hit 检索命中 + * @return 公开检索结果 + * @throws Exception 反射调用失败 + */ + @SuppressWarnings("unchecked") + private List mapSearchResult( + com.easyagents.core.document.Document hit + ) throws Exception { + Method method = PublicKnowledgeShareController.class + .getDeclaredMethod( + "toKnowledgeSearchResult", + List.class + ); + method.setAccessible(true); + return (List) method.invoke( + new PublicKnowledgeShareController(), + List.of(hit) + ); + } + + /** + * 断言参数绑定到指定的必填 JSON 字段。 + * + * @param parameter 方法参数 + * @param field JSON 字段名 + */ + private void assertRequiredJsonField(Parameter parameter, String field) { + JsonBody jsonBody = parameter.getAnnotation(JsonBody.class); + Assert.assertNotNull(jsonBody); + Assert.assertEquals(field, jsonBody.value()); + Assert.assertTrue(jsonBody.required()); + } +} diff --git a/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/dto/PublicKnowledgeDetailResponseTest.java b/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/dto/PublicKnowledgeDetailResponseTest.java new file mode 100644 index 00000000..dbba5f0f --- /dev/null +++ b/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/dto/PublicKnowledgeDetailResponseTest.java @@ -0,0 +1,78 @@ +package tech.easyflow.publicapi.dto; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.mybatisflex.core.paginate.Page; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.ai.entity.Document; +import tech.easyflow.ai.entity.DocumentCollection; + +import java.math.BigInteger; +import java.util.Collections; +import java.util.List; + +/** + * {@link PublicKnowledgeDetailResponse} 响应结构测试。 + */ +public class PublicKnowledgeDetailResponseTest { + + /** + * 验证知识库字段保持在顶层，文档只暴露公开摘要。 + * + */ + @Test + public void shouldKeepKnowledgeFieldsAtTopLevelAndExposeDocumentSummary() { + DocumentCollection knowledge = new DocumentCollection(); + knowledge.setId(BigInteger.valueOf(100)); + knowledge.setTitle("测试知识库"); + knowledge.setCollectionType(DocumentCollection.TYPE_DOCUMENT); + + Document document = new Document(); + document.setId(BigInteger.valueOf(200)); + document.setTitle("manual.pdf"); + document.setDocumentType("pdf"); + document.setContentType("application/pdf"); + document.setDocumentPath("private/path/manual.pdf"); + document.setContent("内部正文"); + document.setProcessStatus("INDEXED"); + document.setTotalChunks(12); + document.setProgressPercent(100); + Page source = new Page<>(List.of(document), 1, 10, 1L); + + PublicKnowledgeDetailResponse response = + new PublicKnowledgeDetailResponse(knowledge, source); + JSONObject json = JSON.parseObject(JSON.toJSONString(response)); + + Assert.assertEquals("100", json.getString("id")); + Assert.assertEquals("测试知识库", json.getString("title")); + Assert.assertFalse(json.containsKey("knowledge")); + JSONObject summary = json.getJSONObject("documents") + .getJSONArray("records") + .getJSONObject(0); + Assert.assertEquals("manual.pdf", summary.getString("title")); + Assert.assertEquals(12L, summary.getLongValue("chunkCount")); + Assert.assertFalse(summary.containsKey("documentPath")); + Assert.assertFalse(summary.containsKey("content")); + Assert.assertFalse(summary.containsKey("options")); + } + + /** + * 验证空文档分页保留调用方请求的分页参数。 + */ + @Test + public void shouldKeepRequestedPaginationForEmptyDocumentPage() { + DocumentCollection knowledge = new DocumentCollection(); + knowledge.setId(BigInteger.valueOf(300)); + knowledge.setCollectionType(DocumentCollection.TYPE_FAQ); + Page source = new Page<>(Collections.emptyList(), 3, 7, 0L); + + PublicKnowledgeDetailResponse response = + new PublicKnowledgeDetailResponse(knowledge, source); + + Assert.assertEquals(3L, response.getDocuments().getPageNumber()); + Assert.assertEquals(7L, response.getDocuments().getPageSize()); + Assert.assertEquals(0L, response.getDocuments().getTotalRow()); + Assert.assertTrue(response.getDocuments().getRecords().isEmpty()); + } +} diff --git a/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/interceptor/PublicApiInterceptorTest.java b/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/interceptor/PublicApiInterceptorTest.java index 53fcd352..d5fe4489 100644 --- a/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/interceptor/PublicApiInterceptorTest.java +++ b/easyflow-api/easyflow-api-public/src/test/java/tech/easyflow/publicapi/interceptor/PublicApiInterceptorTest.java @@ -4,11 +4,15 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import org.junit.Assert; import org.junit.Test; +import tech.easyflow.system.entity.SysApiKey; +import tech.easyflow.system.service.SysApiKeyService; import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Proxy; +import java.lang.reflect.Field; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * {@link PublicApiInterceptor} 鉴权响应测试。 @@ -66,6 +70,65 @@ public class PublicApiInterceptorTest { Assert.assertTrue(body.toString().contains("密钥不正确")); } + /** + * 验证通过接口权限校验的访问令牌会写入请求，供资源级鉴权复用。 + * + * @throws Exception 拦截器处理失败时抛出 + */ + @Test + public void shouldExposeAuthenticatedApiKeyToController() throws Exception { + SysApiKey authenticated = new SysApiKey(); + AtomicReference requestAttribute = new AtomicReference<>(); + HttpServletRequest request = proxy( + HttpServletRequest.class, + (instance, method, args) -> { + if ("getRequestURI".equals(method.getName())) { + return "/public-api/knowledge-share/detail"; + } + if ("getHeader".equals(method.getName())) { + return "test-key"; + } + if ("setAttribute".equals(method.getName())) { + Assert.assertEquals( + PublicApiInterceptor.AUTHENTICATED_API_KEY_ATTRIBUTE, + args[0]); + requestAttribute.set(args[1]); + return null; + } + throw new AssertionError( + "测试路径不应调用 HttpServletRequest." + + method.getName()); + }); + HttpServletResponse response = proxy( + HttpServletResponse.class, + (instance, method, args) -> { + throw new AssertionError( + "测试路径不应调用 HttpServletResponse." + + method.getName()); + }); + SysApiKeyService service = proxy( + SysApiKeyService.class, + (instance, method, args) -> { + if ("checkApikeyPermission".equals(method.getName())) { + return authenticated; + } + throw new AssertionError( + "测试路径不应调用 SysApiKeyService." + + method.getName()); + }); + PublicApiInterceptor interceptor = new PublicApiInterceptor(); + Field serviceField = PublicApiInterceptor.class + .getDeclaredField("sysApiKeyService"); + serviceField.setAccessible(true); + serviceField.set(interceptor, service); + + boolean allowed = + interceptor.preHandle(request, response, new Object()); + + Assert.assertTrue(allowed); + Assert.assertSame(authenticated, requestAttribute.get()); + } + /** * 创建接口代理。 * diff --git a/easyflow-commons/easyflow-common-file-storage/src/main/java/tech/easyflow/common/filestorage/impl/LocalFileStorageServiceImpl.java b/easyflow-commons/easyflow-common-file-storage/src/main/java/tech/easyflow/common/filestorage/impl/LocalFileStorageServiceImpl.java index 8b92e13b..ef92a88e 100644 --- a/easyflow-commons/easyflow-common-file-storage/src/main/java/tech/easyflow/common/filestorage/impl/LocalFileStorageServiceImpl.java +++ b/easyflow-commons/easyflow-common-file-storage/src/main/java/tech/easyflow/common/filestorage/impl/LocalFileStorageServiceImpl.java @@ -108,7 +108,7 @@ public class LocalFileStorageServiceImpl implements FileStorageService { } /** - * 删除旧版路径对应的本地文件。 + * 幂等删除旧版路径对应的本地文件。 * * @param path 文件路径 */ @@ -116,7 +116,7 @@ public class LocalFileStorageServiceImpl implements FileStorageService { public void delete(String path) { try { File file = getLocalFile(path); - Files.delete(file.toPath()); + Files.deleteIfExists(file.toPath()); } catch (IOException e) { LOG.error("删除本地文件出错: {}", path, e); throw new RuntimeException("删除本地文件出错：",e); diff --git a/easyflow-commons/easyflow-common-file-storage/src/test/java/tech/easyflow/common/filestorage/impl/LocalFileStorageServiceImplTest.java b/easyflow-commons/easyflow-common-file-storage/src/test/java/tech/easyflow/common/filestorage/impl/LocalFileStorageServiceImplTest.java index c35f2f51..f909c9ec 100644 --- a/easyflow-commons/easyflow-common-file-storage/src/test/java/tech/easyflow/common/filestorage/impl/LocalFileStorageServiceImplTest.java +++ b/easyflow-commons/easyflow-common-file-storage/src/test/java/tech/easyflow/common/filestorage/impl/LocalFileStorageServiceImplTest.java @@ -80,6 +80,22 @@ public class LocalFileStorageServiceImplTest { assertFalse(Files.exists(part)); } + /** + * 验证旧版路径删除在文件已不存在时仍可幂等成功。 + * + * @throws Exception 测试目录或反射配置失败 + */ + @Test + public void deleteMissingLegacyPathIsIdempotent() throws Exception { + File root = temporaryFolder.newFolder("legacy-delete-root"); + LocalFileStorageServiceImpl service = createService(root, "/files"); + + service.delete("/files/missing.bin"); + service.delete("/files/missing.bin"); + + assertFalse(Files.exists(root.toPath().resolve("missing.bin"))); + } + /** * 验证句柄路径中的符号链接不会被跟随到存储根目录外。 * diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/config/AiModuleConfig.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/config/AiModuleConfig.java index d19c127c..ba855d96 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/config/AiModuleConfig.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/config/AiModuleConfig.java @@ -5,11 +5,13 @@ import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; import tech.easyflow.ai.documentimport.task.DocumentImportParseMonitorProperties; +import tech.easyflow.ai.documentimport.task.DocumentImportBulkProperties; import tech.easyflow.ai.documentimport.task.DocumentImportStatusBroadcastProperties; @MapperScan("tech.easyflow.ai.mapper") @ComponentScan("tech.easyflow.ai") @EnableConfigurationProperties({ + DocumentImportBulkProperties.class, DocumentImportParseMonitorProperties.class, DocumentImportStatusBroadcastProperties.class, RagHealthProperties.class diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/DocumentParseBridgeService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/DocumentParseBridgeService.java index 88af40a2..3cee757d 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/DocumentParseBridgeService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/DocumentParseBridgeService.java @@ -54,6 +54,19 @@ public interface DocumentParseBridgeService { */ DocumentParsedResult queryResult(String taskId); + /** + * 按提交文档的源信息获取异步任务最终结果。 + * + *

源信息仅用于精确选择提交任务时使用的解析服务，不会重新读取文档内容。

+ * + * @param taskId 任务 ID + * @param source 提交任务时的文档源信息 + * @return 标准化解析结果 + */ + default DocumentParsedResult queryResult(String taskId, DocumentSourceRef source) { + return queryResult(taskId); + } + /** * 聚合查询异步任务信息。 * @@ -64,4 +77,17 @@ public interface DocumentParseBridgeService { * @return 聚合任务信息 */ DocumentParseTaskInfo queryTaskInfo(String taskId); + + /** + * 按提交文档的源信息聚合查询异步任务信息。 + * + *

源信息仅用于精确选择提交任务时使用的解析服务，不会重新读取文档内容。

+ * + * @param taskId 任务 ID + * @param source 提交任务时的文档源信息 + * @return 聚合任务信息 + */ + default DocumentParseTaskInfo queryTaskInfo(String taskId, DocumentSourceRef source) { + return queryTaskInfo(taskId); + } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java index 02af4fc5..2da2b084 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImpl.java @@ -150,12 +150,24 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic */ @Override public DocumentParsedResult queryResult(String taskId) { + return queryResult(taskId, null); + } + + /** + * {@inheritDoc} + */ + @Override + public DocumentParsedResult queryResult(String taskId, @Nullable DocumentSourceRef source) { if (!StringUtils.hasText(taskId)) { throw DocumentParseBridgeException.resultFetchFailed("taskId 不能为空"); } try { LOG.info("桥接服务开始获取异步解析结果: providerTaskId={}", taskId); - ParseResponse response = executeAgainstTaskService(taskId, service -> service.queryResult(taskId)); + ParseResponse response = executeAgainstTaskService( + taskId, + source, + service -> service.queryResult(taskId) + ); DocumentParsedResult result = parseResultMapper.map(extractSingleResult(response, true)); LOG.info("桥接服务获取异步解析结果完成: providerTaskId={}, preferredTextLength={}", taskId, resolveTextLength(result)); @@ -174,11 +186,23 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic */ @Override public DocumentParseTaskInfo queryTaskInfo(String taskId) { + return queryTaskInfo(taskId, null); + } + + /** + * {@inheritDoc} + */ + @Override + public DocumentParseTaskInfo queryTaskInfo(String taskId, @Nullable DocumentSourceRef source) { if (!StringUtils.hasText(taskId)) { throw DocumentParseBridgeException.taskFailed("taskId 不能为空"); } try { - ParseTaskInfo taskInfo = executeAgainstTaskService(taskId, service -> service.queryTaskInfo(taskId)); + ParseTaskInfo taskInfo = executeAgainstTaskService( + taskId, + source, + service -> service.queryTaskInfo(taskId) + ); DocumentParseTaskInfo mappedTaskInfo = parseResultMapper.map(taskInfo); LOG.info("桥接服务查询异步解析任务状态: providerTaskId={}, status={}, hasResult={}", taskId, @@ -223,6 +247,16 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic private DocumentParseService resolveService(LoadedDocumentSource loadedSource) { DocumentParseSourceType sourceType = DocumentParseSourceType.resolve(loadedSource.getFileName(), loadedSource.getContentType()); + return resolveService(sourceType); + } + + /** + * 按文档源类型选择解析服务。 + * + * @param sourceType 文档源类型 + * @return 对应解析服务 + */ + private DocumentParseService resolveService(DocumentParseSourceType sourceType) { switch (sourceType) { case PDF: return requireSpecificService(pdfDocumentParseService, defaultDocumentParseService, "PDF"); @@ -249,6 +283,28 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic throw DocumentParseBridgeException.serviceNotEnabled("未启用 " + sourceType + " 文档解析服务"); } + /** + * 在已知任务源信息时精确查询对应服务，缺少源信息时保留旧版兼容遍历。 + * + * @param taskId 任务 ID + * @param source 提交任务时的文档源信息 + * @param action 查询操作 + * @param 查询结果类型 + * @return 查询结果 + */ + private T executeAgainstTaskService(String taskId, + @Nullable DocumentSourceRef source, + Function action) { + if (source == null) { + return executeAgainstTaskService(taskId, action); + } + DocumentParseSourceType sourceType = DocumentParseSourceType.resolve( + source.getFileName(), + source.getContentType() + ); + return action.apply(resolveService(sourceType)); + } + private T executeAgainstTaskService(String taskId, Function action) { List services = availableServices(); if (services.isEmpty()) { diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchCreateContext.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchCreateContext.java new file mode 100644 index 00000000..1c392cd5 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchCreateContext.java @@ -0,0 +1,82 @@ +package tech.easyflow.ai.documentimport; + +/** + * 批量导入建单上下文。 + * + * @author Codex + * @since 2026-08-02 + */ +public final class DocumentImportBatchCreateContext { + + private final ImportCallerContext caller; + private final String idempotencyKeyHash; + private final String requestDigest; + private final String duplicatePolicy; + private final String requestedStrategyJson; + + /** + * 创建批量导入建单上下文。 + * + * @param caller 调用者上下文 + * @param idempotencyKeyHash 幂等键哈希 + * @param requestDigest 请求摘要 + * @param duplicatePolicy 重复文件策略 + * @param requestedStrategyJson 请求分块策略 JSON + */ + public DocumentImportBatchCreateContext(ImportCallerContext caller, + String idempotencyKeyHash, + String requestDigest, + String duplicatePolicy, + String requestedStrategyJson) { + this.caller = caller; + this.idempotencyKeyHash = idempotencyKeyHash; + this.requestDigest = requestDigest; + this.duplicatePolicy = duplicatePolicy; + this.requestedStrategyJson = requestedStrategyJson; + } + + /** + * 获取调用者上下文。 + * + * @return 调用者上下文 + */ + public ImportCallerContext getCaller() { + return caller; + } + + /** + * 获取幂等键哈希。 + * + * @return 幂等键哈希 + */ + public String getIdempotencyKeyHash() { + return idempotencyKeyHash; + } + + /** + * 获取请求摘要。 + * + * @return 请求摘要 + */ + public String getRequestDigest() { + return requestDigest; + } + + /** + * 获取重复文件策略。 + * + * @return 重复文件策略 + */ + public String getDuplicatePolicy() { + return duplicatePolicy; + } + + /** + * 获取请求分块策略 JSON。 + * + * @return 分块策略 JSON + */ + public String getRequestedStrategyJson() { + return requestedStrategyJson; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchDtos.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchDtos.java new file mode 100644 index 00000000..efd3f2bf --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchDtos.java @@ -0,0 +1,388 @@ +package tech.easyflow.ai.documentimport; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * 文档批量导入接口数据对象。 + * + * @author Codex + * @since 2026-07-31 + */ +public final class DocumentImportBatchDtos { + + private DocumentImportBatchDtos() { + } + + /** + * 客户端文件清单项。 + */ + public static class ManifestItem implements Serializable { + private String clientFileKey; + private String fileName; + private String relativePath; + private Long fileSize; + + public String getClientFileKey() { + return clientFileKey; + } + + public void setClientFileKey(String clientFileKey) { + this.clientFileKey = clientFileKey; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getRelativePath() { + return relativePath; + } + + public void setRelativePath(String relativePath) { + this.relativePath = relativePath; + } + + public Long getFileSize() { + return fileSize; + } + + public void setFileSize(Long fileSize) { + this.fileSize = fileSize; + } + } + + /** + * 创建批次请求。 + */ + public static class CreateRequest implements Serializable { + private BigInteger knowledgeId; + private List files = new ArrayList(); + + public BigInteger getKnowledgeId() { + return knowledgeId; + } + + public void setKnowledgeId(BigInteger knowledgeId) { + this.knowledgeId = knowledgeId; + } + + public List getFiles() { + return files; + } + + public void setFiles(List files) { + this.files = files; + } + } + + /** + * 服务端文件项。 + */ + public static class ItemResponse implements Serializable { + private BigInteger itemId; + private BigInteger documentId; + private String clientFileKey; + private String fileName; + private String relativePath; + private Long fileSize; + private String stage; + private String status; + private String errorSummary; + + public BigInteger getItemId() { + return itemId; + } + + public void setItemId(BigInteger itemId) { + this.itemId = itemId; + } + + public BigInteger getDocumentId() { + return documentId; + } + + public void setDocumentId(BigInteger documentId) { + this.documentId = documentId; + } + + public String getClientFileKey() { + return clientFileKey; + } + + public void setClientFileKey(String clientFileKey) { + this.clientFileKey = clientFileKey; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getRelativePath() { + return relativePath; + } + + public void setRelativePath(String relativePath) { + this.relativePath = relativePath; + } + + public Long getFileSize() { + return fileSize; + } + + public void setFileSize(Long fileSize) { + this.fileSize = fileSize; + } + + public String getStage() { + return stage; + } + + public void setStage(String stage) { + this.stage = stage; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getErrorSummary() { + return errorSummary; + } + + public void setErrorSummary(String errorSummary) { + this.errorSummary = errorSummary; + } + } + + /** + * 创建批次响应。 + */ + public static class CreateResponse implements Serializable { + private BigInteger batchId; + private Integer uploadConcurrency; + private List items = new ArrayList(); + + public BigInteger getBatchId() { + return batchId; + } + + public void setBatchId(BigInteger batchId) { + this.batchId = batchId; + } + + public Integer getUploadConcurrency() { + return uploadConcurrency; + } + + public void setUploadConcurrency(Integer uploadConcurrency) { + this.uploadConcurrency = uploadConcurrency; + } + + public List getItems() { + return items; + } + + public void setItems(List items) { + this.items = items; + } + } + + /** + * 启动批次请求。 + */ + public static class StartRequest implements Serializable { + private BigInteger knowledgeId; + private BigInteger batchId; + private String importMode; + private String duplicatePolicy; + + public BigInteger getKnowledgeId() { + return knowledgeId; + } + + public void setKnowledgeId(BigInteger knowledgeId) { + this.knowledgeId = knowledgeId; + } + + public BigInteger getBatchId() { + return batchId; + } + + public void setBatchId(BigInteger batchId) { + this.batchId = batchId; + } + + public String getImportMode() { + return importMode; + } + + public void setImportMode(String importMode) { + this.importMode = importMode; + } + + public String getDuplicatePolicy() { + return duplicatePolicy; + } + + public void setDuplicatePolicy(String duplicatePolicy) { + this.duplicatePolicy = duplicatePolicy; + } + } + + /** + * 批次状态响应。 + */ + public static class StatusResponse implements Serializable { + private BigInteger batchId; + private String importMode; + private String status; + private Integer totalCount; + private Long totalBytes; + private Integer completedCount; + private Integer processingCount; + private Integer failedCount; + private Integer pendingCount; + private Integer skippedCount; + private Integer cancelledCount; + private Integer retryableFailedCount; + private Integer progressPercent; + private Date startedAt; + private Date finishedAt; + + public BigInteger getBatchId() { + return batchId; + } + + public void setBatchId(BigInteger batchId) { + this.batchId = batchId; + } + + public String getImportMode() { + return importMode; + } + + public void setImportMode(String importMode) { + this.importMode = importMode; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Integer getTotalCount() { + return totalCount; + } + + public void setTotalCount(Integer totalCount) { + this.totalCount = totalCount; + } + + public Long getTotalBytes() { + return totalBytes; + } + + public void setTotalBytes(Long totalBytes) { + this.totalBytes = totalBytes; + } + + public Integer getCompletedCount() { + return completedCount; + } + + public void setCompletedCount(Integer completedCount) { + this.completedCount = completedCount; + } + + public Integer getProcessingCount() { + return processingCount; + } + + public void setProcessingCount(Integer processingCount) { + this.processingCount = processingCount; + } + + public Integer getFailedCount() { + return failedCount; + } + + public void setFailedCount(Integer failedCount) { + this.failedCount = failedCount; + } + + public Integer getPendingCount() { + return pendingCount; + } + + public void setPendingCount(Integer pendingCount) { + this.pendingCount = pendingCount; + } + + public Integer getSkippedCount() { + return skippedCount; + } + + public void setSkippedCount(Integer skippedCount) { + this.skippedCount = skippedCount; + } + + public Integer getCancelledCount() { + return cancelledCount; + } + + public void setCancelledCount(Integer cancelledCount) { + this.cancelledCount = cancelledCount; + } + + public Integer getRetryableFailedCount() { + return retryableFailedCount; + } + + public void setRetryableFailedCount(Integer retryableFailedCount) { + this.retryableFailedCount = retryableFailedCount; + } + + public Integer getProgressPercent() { + return progressPercent; + } + + public void setProgressPercent(Integer progressPercent) { + this.progressPercent = progressPercent; + } + + public Date getStartedAt() { + return startedAt; + } + + public void setStartedAt(Date startedAt) { + this.startedAt = startedAt; + } + + public Date getFinishedAt() { + return finishedAt; + } + + public void setFinishedAt(Date finishedAt) { + this.finishedAt = finishedAt; + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchRetryResult.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchRetryResult.java new file mode 100644 index 00000000..0debe5d1 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportBatchRetryResult.java @@ -0,0 +1,71 @@ +package tech.easyflow.ai.documentimport; + +import java.math.BigInteger; + +/** + * 文档批量导入重试的稳定响应快照。 + * + * @author Codex + * @since 2026-08-02 + */ +public class DocumentImportBatchRetryResult { + + private final BigInteger taskId; + private final String status; + private final Integer retryGeneration; + private final Integer retriedCount; + + /** + * 创建重试响应快照。 + * + * @param taskId 批次任务 ID + * @param status 领取重试时的任务状态 + * @param retryGeneration 重试代次 + * @param retriedCount 本次领取的文件数 + */ + public DocumentImportBatchRetryResult(BigInteger taskId, + String status, + Integer retryGeneration, + Integer retriedCount) { + this.taskId = taskId; + this.status = status; + this.retryGeneration = retryGeneration; + this.retriedCount = retriedCount; + } + + /** + * 获取批次任务 ID。 + * + * @return 批次任务 ID + */ + public BigInteger getTaskId() { + return taskId; + } + + /** + * 获取领取重试时的任务状态。 + * + * @return 任务状态 + */ + public String getStatus() { + return status; + } + + /** + * 获取重试代次。 + * + * @return 重试代次 + */ + public Integer getRetryGeneration() { + return retryGeneration; + } + + /** + * 获取本次领取的文件数。 + * + * @return 本次领取的文件数 + */ + public Integer getRetriedCount() { + return retriedCount; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportKeys.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportKeys.java index 1393773c..7377a1a6 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportKeys.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/DocumentImportKeys.java @@ -15,6 +15,7 @@ public final class DocumentImportKeys { public static final String KEY_DOCUMENT_STRATEGY_CODE = "splitter.strategyCode"; public static final String KEY_DOCUMENT_STRATEGY_LABEL = "splitter.strategyLabel"; public static final String KEY_DOCUMENT_STRATEGY_SNAPSHOT = "splitter.strategySnapshot"; + public static final String KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH = "splitter.chunkSnapshotPath"; public static final String KEY_DOCUMENT_ANALYSIS_SUMMARY = "splitter.analysisSummary"; public static final String KEY_DOCUMENT_SOURCE_FILE_EXT = "splitter.sourceFileExt"; public static final String KEY_DOCUMENT_PREVIEW_VERSION = "splitter.previewVersion"; @@ -30,6 +31,7 @@ public final class DocumentImportKeys { public static final String KEY_DOCUMENT_PARSE_PROCESSED_ITEMS = "parse.processedItems"; public static final String KEY_DOCUMENT_PARSE_TOTAL_ITEMS = "parse.totalItems"; public static final String KEY_DOCUMENT_PARSE_STATUS_MESSAGE = "parse.statusMessage"; + public static final String KEY_DOCUMENT_TASK_ERROR_CODE = "task.errorCode"; public static final String KEY_DOCUMENT_RENDER_MARKDOWN = "renderMarkdown"; public static final String KEY_DOCUMENT_PAGE_INDEX = "pageIndex"; public static final String KEY_DOCUMENT_SHEET_NAME = "sheetName"; @@ -37,4 +39,8 @@ public final class DocumentImportKeys { public static final String KEY_DOCUMENT_ROW_END = "rowEnd"; public static final String KEY_DOCUMENT_IMAGE_REFS = "imageRefs"; public static final String KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY = "parseArtifactSummary"; + public static final String KEY_DOCUMENT_IMPORT_MODE = "import.mode"; + public static final String KEY_DOCUMENT_IMPORT_BATCH_ID = "import.batchId"; + public static final String KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID = "import.batchItemId"; + public static final String KEY_DOCUMENT_IMPORT_RELATIVE_PATH = "import.relativePath"; } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/ImportCallerContext.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/ImportCallerContext.java new file mode 100644 index 00000000..039dbd76 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/ImportCallerContext.java @@ -0,0 +1,50 @@ +package tech.easyflow.ai.documentimport; + +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.math.BigInteger; + +/** + * 文档批量导入调用者上下文。 + * + * @author Codex + * @since 2026-08-02 + */ +public final class ImportCallerContext { + + private final ImportCallerType callerType; + private final BigInteger callerId; + + /** + * 创建调用者上下文。 + * + * @param callerType 调用者类型 + * @param callerId 调用者 ID + * @throws BusinessException 调用者信息不完整时抛出 + */ + public ImportCallerContext(ImportCallerType callerType, BigInteger callerId) { + if (callerType == null || callerId == null) { + throw new BusinessException("导入调用者信息不完整"); + } + this.callerType = callerType; + this.callerId = callerId; + } + + /** + * 获取调用者类型。 + * + * @return 调用者类型 + */ + public ImportCallerType getCallerType() { + return callerType; + } + + /** + * 获取调用者 ID。 + * + * @return 调用者 ID + */ + public BigInteger getCallerId() { + return callerId; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/ImportCallerType.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/ImportCallerType.java new file mode 100644 index 00000000..c39037ad --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/ImportCallerType.java @@ -0,0 +1,20 @@ +package tech.easyflow.ai.documentimport; + +/** + * 文档批量导入调用者类型。 + * + * @author Codex + * @since 2026-08-02 + */ +public enum ImportCallerType { + + /** + * 管理端登录用户。 + */ + ADMIN, + + /** + * Public API 访问令牌。 + */ + PUBLIC_API +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/PublicDocumentImportDtos.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/PublicDocumentImportDtos.java new file mode 100644 index 00000000..5280c7e1 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/PublicDocumentImportDtos.java @@ -0,0 +1,455 @@ +package tech.easyflow.ai.documentimport; + +import com.easyagents.rag.ingestion.model.StrategyConfig; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * 知识库 Public API 批量导入数据对象。 + * + * @author Codex + * @since 2026-08-02 + */ +public final class PublicDocumentImportDtos { + + private PublicDocumentImportDtos() { + } + + /** + * Multipart 元数据。 + */ + public static class BatchMetadata implements Serializable { + private BigInteger knowledgeId; + private StrategyConfig chunkStrategy = StrategyConfig.defaults(); + private String duplicatePolicy = "SKIP"; + private List files = new ArrayList<>(); + + public BigInteger getKnowledgeId() { + return knowledgeId; + } + + public void setKnowledgeId(BigInteger knowledgeId) { + this.knowledgeId = knowledgeId; + } + + public StrategyConfig getChunkStrategy() { + return chunkStrategy; + } + + public void setChunkStrategy(StrategyConfig chunkStrategy) { + this.chunkStrategy = chunkStrategy == null + ? StrategyConfig.defaults() + : chunkStrategy; + } + + public String getDuplicatePolicy() { + return duplicatePolicy; + } + + public void setDuplicatePolicy(String duplicatePolicy) { + this.duplicatePolicy = duplicatePolicy; + } + + public List getFiles() { + return files; + } + + public void setFiles(List files) { + this.files = files; + } + } + + /** + * 批量提交响应。 + */ + public static class SubmitResponse implements Serializable { + private BigInteger taskId; + private String status; + private Integer totalCount; + private Long totalBytes; + private Date createdAt; + + public BigInteger getTaskId() { + return taskId; + } + + public void setTaskId(BigInteger taskId) { + this.taskId = taskId; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Integer getTotalCount() { + return totalCount; + } + + public void setTotalCount(Integer totalCount) { + this.totalCount = totalCount; + } + + public Long getTotalBytes() { + return totalBytes; + } + + public void setTotalBytes(Long totalBytes) { + this.totalBytes = totalBytes; + } + + public Date getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(Date createdAt) { + this.createdAt = createdAt; + } + } + + /** + * 批次计数。 + */ + public static class Counts implements Serializable { + private Integer total; + private Integer completed; + private Integer processing; + private Integer pending; + private Integer failed; + private Integer skipped; + private Integer retryableFailed; + + public Integer getTotal() { + return total; + } + + public void setTotal(Integer total) { + this.total = total; + } + + public Integer getCompleted() { + return completed; + } + + public void setCompleted(Integer completed) { + this.completed = completed; + } + + public Integer getProcessing() { + return processing; + } + + public void setProcessing(Integer processing) { + this.processing = processing; + } + + public Integer getPending() { + return pending; + } + + public void setPending(Integer pending) { + this.pending = pending; + } + + public Integer getFailed() { + return failed; + } + + public void setFailed(Integer failed) { + this.failed = failed; + } + + public Integer getSkipped() { + return skipped; + } + + public void setSkipped(Integer skipped) { + this.skipped = skipped; + } + + public Integer getRetryableFailed() { + return retryableFailed; + } + + public void setRetryableFailed(Integer retryableFailed) { + this.retryableFailed = retryableFailed; + } + } + + /** + * 文件失败信息。 + */ + public static class ItemError implements Serializable { + private String code; + private String message; + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + } + + /** + * 批次文件状态记录。 + */ + public static class ItemRecord implements Serializable { + private String fileKey; + private String relativePath; + private BigInteger documentId; + private String stage; + private String status; + private Integer attemptCount; + private Boolean retryable; + private ItemError error; + + public String getFileKey() { + return fileKey; + } + + public void setFileKey(String fileKey) { + this.fileKey = fileKey; + } + + public String getRelativePath() { + return relativePath; + } + + public void setRelativePath(String relativePath) { + this.relativePath = relativePath; + } + + public BigInteger getDocumentId() { + return documentId; + } + + public void setDocumentId(BigInteger documentId) { + this.documentId = documentId; + } + + public String getStage() { + return stage; + } + + public void setStage(String stage) { + this.stage = stage; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Integer getAttemptCount() { + return attemptCount; + } + + public void setAttemptCount(Integer attemptCount) { + this.attemptCount = attemptCount; + } + + public Boolean getRetryable() { + return retryable; + } + + public void setRetryable(Boolean retryable) { + this.retryable = retryable; + } + + public ItemError getError() { + return error; + } + + public void setError(ItemError error) { + this.error = error; + } + } + + /** + * 批次文件状态分页。 + */ + public static class ItemPage implements Serializable { + private Long pageNumber; + private Long pageSize; + private Long total; + private List records = new ArrayList<>(); + + public Long getPageNumber() { + return pageNumber; + } + + public void setPageNumber(Long pageNumber) { + this.pageNumber = pageNumber; + } + + public Long getPageSize() { + return pageSize; + } + + public void setPageSize(Long pageSize) { + this.pageSize = pageSize; + } + + public Long getTotal() { + return total; + } + + public void setTotal(Long total) { + this.total = total; + } + + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } + } + + /** + * 批次状态响应。 + */ + public static class StatusResponse implements Serializable { + private BigInteger taskId; + private BigInteger knowledgeId; + private String status; + private Integer progressPercent; + private Counts counts; + private Boolean canRetry; + private ItemPage items; + + public BigInteger getTaskId() { + return taskId; + } + + public void setTaskId(BigInteger taskId) { + this.taskId = taskId; + } + + public BigInteger getKnowledgeId() { + return knowledgeId; + } + + public void setKnowledgeId(BigInteger knowledgeId) { + this.knowledgeId = knowledgeId; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Integer getProgressPercent() { + return progressPercent; + } + + public void setProgressPercent(Integer progressPercent) { + this.progressPercent = progressPercent; + } + + public Counts getCounts() { + return counts; + } + + public void setCounts(Counts counts) { + this.counts = counts; + } + + public Boolean getCanRetry() { + return canRetry; + } + + public void setCanRetry(Boolean canRetry) { + this.canRetry = canRetry; + } + + public ItemPage getItems() { + return items; + } + + public void setItems(ItemPage items) { + this.items = items; + } + } + + /** + * 异常任务重试请求。 + */ + public static class RetryRequest implements Serializable { + private BigInteger taskId; + private List fileKeys; + + public BigInteger getTaskId() { + return taskId; + } + + public void setTaskId(BigInteger taskId) { + this.taskId = taskId; + } + + public List getFileKeys() { + return fileKeys; + } + + public void setFileKeys(List fileKeys) { + this.fileKeys = fileKeys; + } + } + + /** + * 异常任务重试响应。 + */ + public static class RetryResponse implements Serializable { + private BigInteger taskId; + private String status; + private Integer retriedCount; + + public BigInteger getTaskId() { + return taskId; + } + + public void setTaskId(BigInteger taskId) { + this.taskId = taskId; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Integer getRetriedCount() { + return retriedCount; + } + + public void setRetriedCount(Integer retriedCount) { + this.retriedCount = retriedCount; + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppService.java new file mode 100644 index 00000000..d38ce287 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppService.java @@ -0,0 +1,1342 @@ +package tech.easyflow.ai.documentimport.task; + +import cn.dev33.satoken.stp.StpUtil; +import com.mybatisflex.core.query.QueryWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext; +import tech.easyflow.ai.documentimport.DocumentImportBatchDtos; +import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult; +import tech.easyflow.ai.documentimport.ImportCallerContext; +import tech.easyflow.ai.documentimport.ImportCallerType; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.entity.DocumentImportBatchItem; +import tech.easyflow.ai.enums.DocumentImportBatchItemStage; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; +import tech.easyflow.ai.enums.DocumentImportBatchStatus; +import tech.easyflow.ai.enums.DocumentImportMode; +import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; +import tech.easyflow.ai.mapper.DocumentMapper; +import tech.easyflow.ai.service.DocumentImportBatchItemService; +import tech.easyflow.ai.service.DocumentImportBatchService; +import tech.easyflow.common.cache.RedisLockExecutor; +import tech.easyflow.common.cache.RedisLockExecutor.LockHandle; +import tech.easyflow.common.filestorage.FileStorageService; +import tech.easyflow.common.filestorage.FileStorageWriteHandle; +import tech.easyflow.common.filestorage.FileStorageWriteResult; +import tech.easyflow.common.util.StringUtil; +import tech.easyflow.common.web.exceptions.BusinessException; + +import javax.annotation.Resource; +import java.math.BigInteger; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * 知识库文档批量导入应用服务。 + * + * @author Codex + * @since 2026-07-31 + */ +@Service +public class DocumentImportBatchAppService { + + private static final Logger LOG = LoggerFactory.getLogger(DocumentImportBatchAppService.class); + private static final Set SUPPORTED_EXTENSIONS = + Set.of("txt", "pdf", "docx", "md", "pptx", "xlsx"); + private static final Duration BATCH_MUTATION_LOCK_LEASE = Duration.ofMinutes(30); + + private final DocumentImportBatchService batchService; + private final DocumentImportBatchItemService itemService; + private final DocumentImportBatchTracker batchTracker; + private final DocumentImportBulkProperties properties; + private final KnowledgeDocumentImportTaskAppService taskAppService; + private final DocumentImportBatchMapper batchMapper; + private final DocumentImportBatchItemMapper itemMapper; + private final DocumentMapper documentMapper; + private final RedisLockExecutor redisLockExecutor; + + @Resource(name = "default") + private FileStorageService storageService; + + /** + * 创建批量导入应用服务。 + * + * @param batchService 批次服务 + * @param itemService 批次项服务 + * @param batchTracker 批次状态跟踪器 + * @param properties 容量与并发配置 + * @param taskAppService 文档任务服务 + * @param batchMapper 批次 Mapper + * @param itemMapper 批次项 Mapper + * @param documentMapper 文档 Mapper + * @param redisLockExecutor 分布式锁执行器 + */ + public DocumentImportBatchAppService(DocumentImportBatchService batchService, + DocumentImportBatchItemService itemService, + DocumentImportBatchTracker batchTracker, + DocumentImportBulkProperties properties, + KnowledgeDocumentImportTaskAppService taskAppService, + DocumentImportBatchMapper batchMapper, + DocumentImportBatchItemMapper itemMapper, + DocumentMapper documentMapper, + RedisLockExecutor redisLockExecutor) { + this.batchService = batchService; + this.itemService = itemService; + this.batchTracker = batchTracker; + this.properties = properties; + this.taskAppService = taskAppService; + this.batchMapper = batchMapper; + this.itemMapper = itemMapper; + this.documentMapper = documentMapper; + this.redisLockExecutor = redisLockExecutor; + } + + /** + * 根据客户端文件清单创建上传批次。 + * + * @param request 创建请求 + * @return 批次及文件项映射 + */ + @Transactional + public DocumentImportBatchDtos.CreateResponse createBatch(DocumentImportBatchDtos.CreateRequest request) { + BigInteger operatorId = resolveOperatorId(); + return createBatch( + request, + new DocumentImportBatchCreateContext( + new ImportCallerContext(ImportCallerType.ADMIN, operatorId), + null, + null, + "SKIP", + null + ) + ); + } + + /** + * 根据客户端文件清单和调用者上下文创建上传批次。 + * + * @param request 创建请求 + * @param createContext 建单上下文 + * @return 批次及文件项映射 + */ + @Transactional + public DocumentImportBatchDtos.CreateResponse createBatch( + DocumentImportBatchDtos.CreateRequest request, + DocumentImportBatchCreateContext createContext) { + if (request == null || request.getKnowledgeId() == null) { + throw new BusinessException("知识库id不能为空"); + } + if (createContext == null || createContext.getCaller() == null) { + throw new BusinessException("导入调用者信息不完整"); + } + List files = request.getFiles(); + if (files == null || files.isEmpty()) { + throw new BusinessException("请选择需要上传的文件"); + } + if (files.size() > properties.getMaxFileCount()) { + throw new BusinessException("单批次文件数不能超过" + properties.getMaxFileCount()); + } + long totalBytes = validateManifest(files); + Date now = new Date(); + BigInteger operatorId = resolveOperatorId(); + ImportCallerContext caller = createContext.getCaller(); + + DocumentImportBatch batch = new DocumentImportBatch(); + batch.setKnowledgeId(request.getKnowledgeId()); + batch.setCallerType(caller.getCallerType().name()); + batch.setCallerId(caller.getCallerId()); + batch.setIdempotencyKeyHash(createContext.getIdempotencyKeyHash()); + batch.setRequestDigest(createContext.getRequestDigest()); + batch.setDuplicatePolicy(createContext.getDuplicatePolicy()); + batch.setRequestedStrategyJson(createContext.getRequestedStrategyJson()); + batch.setRetryGeneration(0); + batch.setVersion(0); + batch.setStatus(DocumentImportBatchStatus.UPLOADING.name()); + batch.setTotalCount(files.size()); + batch.setTotalBytes(totalBytes); + batch.setCompletedCount(0); + batch.setProcessingCount(0); + batch.setFailedCount(0); + batch.setPendingCount(files.size()); + batch.setUploadedCount(0); + batch.setSkippedCount(0); + batch.setCancelledCount(0); + batch.setRetryableFailedCount(0); + batch.setCreated(now); + batch.setModified(now); + batch.setCreatedBy(operatorId); + batch.setModifiedBy(operatorId); + batchService.save(batch); + + List entities = new ArrayList(files.size()); + for (DocumentImportBatchDtos.ManifestItem file : files) { + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setBatchId(batch.getId()); + item.setKnowledgeId(batch.getKnowledgeId()); + item.setClientFileKey(file.getClientFileKey()); + item.setFileName(file.getFileName()); + item.setRelativePath(normalizeRelativePath(file.getRelativePath(), file.getFileName())); + item.setFileSize(file.getFileSize()); + item.setStage(DocumentImportBatchItemStage.UPLOAD.name()); + item.setStatus(DocumentImportBatchItemStatus.PENDING.name()); + item.setCleanupPending(false); + item.setRetryable(false); + item.setAttemptCount(0); + item.setCreated(now); + item.setModified(now); + item.setCreatedBy(operatorId); + item.setModifiedBy(operatorId); + entities.add(item); + } + itemService.saveBatch(entities); + + DocumentImportBatchDtos.CreateResponse response = new DocumentImportBatchDtos.CreateResponse(); + response.setBatchId(batch.getId()); + response.setUploadConcurrency(Math.max(1, properties.getUploadConcurrency())); + response.setItems(entities.stream().map(this::toItemResponse).toList()); + return response; + } + + /** + * 上传单个批次文件。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @param itemId 文件项 ID + * @param file 上传文件 + * @return 文件项状态 + */ + public DocumentImportBatchDtos.ItemResponse uploadItem(BigInteger knowledgeId, + BigInteger batchId, + BigInteger itemId, + MultipartFile file) { + return uploadItem(knowledgeId, batchId, itemId, file, null); + } + + /** + * 上传单个批次文件并校验调用者归属。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @param itemId 文件项 ID + * @param file 上传文件 + * @param caller 调用者上下文；为空时沿用管理端兼容校验 + * @return 文件项状态 + */ + public DocumentImportBatchDtos.ItemResponse uploadItem(BigInteger knowledgeId, + BigInteger batchId, + BigInteger itemId, + MultipartFile file, + ImportCallerContext caller) { + DocumentImportBatch batch = requireOwnedBatch(knowledgeId, batchId, caller); + if (!DocumentImportBatchStatus.UPLOADING.name().equals(batch.getStatus()) + && !DocumentImportBatchStatus.READY.name().equals(batch.getStatus())) { + throw new BusinessException("当前批次不允许继续上传"); + } + DocumentImportBatchItem item = batchTracker.requireItem(itemId); + if (!batchId.equals(item.getBatchId())) { + throw new BusinessException("导入文件不属于当前批次"); + } + validateUploadedFile(item, file); + if (DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus())) { + return toItemResponse(item); + } + if (itemMapper.claimUpload(batchId, itemId, knowledgeId, new Date()) <= 0) { + throw new BusinessException("文件正在上传或状态已变化，请稍后重试"); + } + + FileStorageWriteHandle writeHandle; + try { + writeHandle = storageService.prepareRecoverableWrite( + "knowledge-import/" + batchId + "/" + itemId, + buildRecoverableFileName(item) + ); + } catch (RuntimeException error) { + abortUploadBeforeWrite(itemId, null, error); + throw error; + } + String storageLocator = writeHandle.encodeLocator(); + try { + if (itemMapper.registerUploadWriteIntent( + itemId, + storageLocator, + new Date() + ) <= 0) { + throw new BusinessException("文件上传状态已变化，请重新提交"); + } + } catch (RuntimeException error) { + abortUploadBeforeWrite(itemId, storageLocator, error); + throw error; + } + + FileStorageWriteResult writeResult; + try { + writeResult = storageService.saveRecoverable(file, writeHandle); + if (!storageLocator.equals(writeResult.getLocator())) { + throw new IllegalStateException("文件存储恢复定位符不一致"); + } + } catch (RuntimeException error) { + markStoredObjectForCleanup(itemId, storageLocator, null, error); + throw error; + } + try { + if (!batchTracker.completeUpload( + itemId, + writeResult.getUrl(), + storageLocator + )) { + throw new BusinessException("上传批次已取消，请重新选择文件"); + } + } catch (RuntimeException error) { + DocumentImportBatchItem committed = resolveCommittedUpload( + itemId, + writeResult.getUrl(), + storageLocator + ); + if (committed != null) { + return toItemResponse(committed); + } + markStoredObjectForCleanup( + itemId, + storageLocator, + writeResult.getUrl(), + error + ); + throw error; + } + return toItemResponse(batchTracker.requireItem(itemId)); + } + + /** + * 启动手动或自动批量导入。 + * + * @param request 启动请求 + * @return 启动后的批次状态 + */ + @Transactional + public DocumentImportBatchDtos.StatusResponse startBatch(DocumentImportBatchDtos.StartRequest request) { + return startBatch(request, null); + } + + /** + * 启动批次并校验调用者归属。 + * + * @param request 启动请求 + * @param caller 调用者上下文；为空时沿用管理端兼容校验 + * @return 启动后的批次状态 + */ + @Transactional + public DocumentImportBatchDtos.StatusResponse startBatch( + DocumentImportBatchDtos.StartRequest request, + ImportCallerContext caller) { + if (request == null || request.getKnowledgeId() == null || request.getBatchId() == null) { + throw new BusinessException("批次信息不完整"); + } + DocumentImportBatch batch = + requireOwnedBatch(request.getKnowledgeId(), request.getBatchId(), caller); + DocumentImportMode mode = parseMode(request.getImportMode()); + DuplicatePolicy duplicatePolicy = parseDuplicatePolicy( + StringUtil.hasText(request.getDuplicatePolicy()) + ? request.getDuplicatePolicy() + : batch.getDuplicatePolicy() + ); + acquireBatchMutationLock(batch.getKnowledgeId()); + if (!DocumentImportBatchStatus.READY.name().equals(batch.getStatus())) { + throw new BusinessException("文件尚未全部上传完成"); + } + if (mode == DocumentImportMode.AUTO) { + assertNoOtherActiveAutoBatch(batch); + } + List items = itemService.list( + QueryWrapper.create().eq(DocumentImportBatchItem::getBatchId, batch.getId()) + ); + if (items.stream().anyMatch(item -> !DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus()))) { + throw new BusinessException("文件尚未全部上传完成"); + } + + markHistoricalDuplicates(items, batch.getId(), duplicatePolicy); + Date now = new Date(); + DocumentImportBatch update = new DocumentImportBatch(); + update.setImportMode(mode.name()); + update.setStatus(DocumentImportBatchStatus.RUNNING.name()); + update.setStartedAt(now); + update.setFinishedAt(null); + update.setModified(now); + int claimed = batchMapper.updateByQuery(update, + QueryWrapper.create() + .eq(DocumentImportBatch::getId, batch.getId()) + .eq(DocumentImportBatch::getStatus, DocumentImportBatchStatus.READY.name()) + .isNull(DocumentImportBatch::getImportMode)); + if (claimed <= 0) { + throw new BusinessException("导入批次已启动，请勿重复提交"); + } + batch.setImportMode(mode.name()); + batch.setStatus(DocumentImportBatchStatus.RUNNING.name()); + batch.setStartedAt(now); + batch.setFinishedAt(null); + taskAppService.createBatchImportTasks(batch, items); + return batchTracker.refreshBatch(batch.getId()); + } + + /** + * 查询指定批次状态。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @return 批次状态 + */ + public DocumentImportBatchDtos.StatusResponse getBatchStatus(BigInteger knowledgeId, BigInteger batchId) { + DocumentImportBatch batch = requireOwnedBatch(knowledgeId, batchId); + return batchTracker.toStatusResponse(batch); + } + + /** + * 查询知识库最近一个自动导入批次。 + * + * @param knowledgeId 知识库 ID + * @return 最近批次；不存在时返回 null + */ + public DocumentImportBatchDtos.StatusResponse getLatestAutoBatch(BigInteger knowledgeId) { + DocumentImportBatch batch = batchService.getOne( + QueryWrapper.create() + .eq(DocumentImportBatch::getKnowledgeId, knowledgeId) + .eq(DocumentImportBatch::getImportMode, DocumentImportMode.AUTO.name()) + .orderBy(DocumentImportBatch::getCreated, false) + .limit(1) + ); + return batch == null ? null : batchTracker.toStatusResponse(batch); + } + + /** + * 继续一个存在失败项或中断项的自动导入批次。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @return 继续后的批次状态 + */ + @Transactional + public DocumentImportBatchDtos.StatusResponse continueBatch(BigInteger knowledgeId, BigInteger batchId) { + DocumentImportBatch batch = requireOwnedBatch(knowledgeId, batchId); + acquireBatchMutationLock(knowledgeId); + if (!DocumentImportMode.AUTO.name().equals(batch.getImportMode())) { + throw new BusinessException("手动导入批次无需批量继续"); + } + if (!DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name().equals(batch.getStatus()) + && !DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())) { + throw new BusinessException("当前批次无需继续"); + } + assertNoOtherActiveAutoBatch(batch); + if (valueOrZero(batch.getRetryableFailedCount()) <= 0) { + throw new BusinessException("当前批次没有可继续的失败项"); + } + Date now = new Date(); + DocumentImportBatch update = new DocumentImportBatch(); + update.setStatus(DocumentImportBatchStatus.RUNNING.name()); + update.setFinishedAt(null); + update.setModified(now); + int claimed = batchMapper.updateByQuery(update, + QueryWrapper.create() + .eq(DocumentImportBatch::getId, batchId) + .in(DocumentImportBatch::getStatus, List.of( + DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name(), + DocumentImportBatchStatus.INTERRUPTED.name() + ))); + if (claimed <= 0) { + throw new BusinessException("批次状态已变化，请刷新后重试"); + } + runAfterCommit(() -> resumeBatchFailures(batchId)); + return batchTracker.refreshBatch(batchId); + } + + /** + * 取消尚未启动的上传批次，并清理未被文档引用的上传对象。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + */ + @Transactional + public void cancelBatch(BigInteger knowledgeId, BigInteger batchId) { + cancelBatch(knowledgeId, batchId, null); + } + + /** + * 取消尚未启动的上传批次并校验调用者归属。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @param caller 调用者上下文；为空时沿用管理端兼容校验 + */ + @Transactional + public void cancelBatch(BigInteger knowledgeId, + BigInteger batchId, + ImportCallerContext caller) { + DocumentImportBatch batch = requireOwnedBatch(knowledgeId, batchId, caller); + acquireBatchMutationLock(knowledgeId); + if (DocumentImportBatchStatus.CANCELLED.name().equals(batch.getStatus())) { + return; + } + if (!DocumentImportBatchStatus.UPLOADING.name().equals(batch.getStatus()) + && !DocumentImportBatchStatus.READY.name().equals(batch.getStatus())) { + throw new BusinessException("已启动的导入批次不能取消"); + } + Date now = new Date(); + DocumentImportBatch update = new DocumentImportBatch(); + update.setStatus(DocumentImportBatchStatus.CANCELLED.name()); + update.setFinishedAt(now); + update.setModified(now); + int cancelled = batchMapper.updateByQuery(update, + QueryWrapper.create() + .eq(DocumentImportBatch::getId, batchId) + .in(DocumentImportBatch::getStatus, List.of( + DocumentImportBatchStatus.UPLOADING.name(), + DocumentImportBatchStatus.READY.name() + ))); + if (cancelled <= 0) { + throw new BusinessException("批次状态已变化，请刷新后重试"); + } + cancelPendingItemsAndScheduleCleanup(batchId); + } + + /** + * 原子取消超过截止时间且仍无进展的未完成批次。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @param caller 调用者上下文 + * @param incompleteCutoff 最后进展截止时间 + * @return 是否成功领取取消权 + */ + @Transactional + public boolean cancelStaleBatch(BigInteger knowledgeId, + BigInteger batchId, + ImportCallerContext caller, + Date incompleteCutoff) { + if (knowledgeId == null || batchId == null || caller == null + || caller.getCallerType() == null || caller.getCallerId() == null + || incompleteCutoff == null) { + throw new BusinessException("超时批次取消信息不完整"); + } + acquireBatchMutationLock(knowledgeId); + Date now = new Date(); + int cancelled = batchMapper.claimStaleCancellation( + batchId, + knowledgeId, + caller.getCallerType().name(), + caller.getCallerId(), + incompleteCutoff, + now + ); + if (cancelled <= 0) { + return false; + } + cancelPendingItemsAndScheduleCleanup(batchId); + return true; + } + + /** + * 重试删除未完成绑定的上传对象。 + * + *

{@code cleanupPending} 是可索引的持久化清理状态；删除成功后通过 + * 定位符与路径条件更新清除，失败项由下一轮调度继续处理。

+ * + * @param limit 单次最大处理数量 + * @return 清理成功数量 + */ + public int cleanupCancelledStoredObjects(int limit) { + int boundedLimit = Math.max(1, Math.min(limit, 500)); + List items = itemService.list( + QueryWrapper.create() + .eq( + DocumentImportBatchItem::getCleanupPending, + true + ) + .orderBy(DocumentImportBatchItem::getModified, true) + .orderBy(DocumentImportBatchItem::getId, true) + .limit(boundedLimit) + ); + int cleaned = 0; + for (DocumentImportBatchItem item : items) { + if (cleanupStoredObject(item)) { + cleaned++; + } + } + return cleaned; + } + + /** + * 识别事务提交结果未知但数据库实际已完成的上传。 + * + * @param itemId 文件项 ID + * @param storedPath 已写入的对象路径 + * @param storageLocator 可恢复存储定位符 + * @return 已完成上传的最新文件项；未确认完成时返回 null + */ + private DocumentImportBatchItem resolveCommittedUpload(BigInteger itemId, + String storedPath, + String storageLocator) { + try { + DocumentImportBatchItem current = batchTracker.requireItem(itemId); + if (DocumentImportBatchItemStatus.UPLOADED.name().equals(current.getStatus()) + && storedPath.equals(current.getFilePath()) + && storageLocator.equals(current.getStorageLocator()) + && !Boolean.TRUE.equals(current.getCleanupPending())) { + return current; + } + } catch (RuntimeException lookupError) { + LOG.warn( + "上传提交结果未知且暂时无法读取文件项，保留恢复定位符等待回收: itemId={}", + itemId, + lookupError + ); + } + return null; + } + + /** + * 在物理写入前原子撤销上传领取，并处理写意图登记提交结果未知。 + * + * @param itemId 文件项 ID + * @param storageLocator 本次预期定位符，可为空 + * @param originalError 上传准备或登记异常 + */ + private void abortUploadBeforeWrite(BigInteger itemId, + String storageLocator, + RuntimeException originalError) { + try { + if (itemMapper.abortUploadBeforeWrite( + itemId, + storageLocator, + truncateUploadError(originalError.getMessage()), + new Date() + ) > 0) { + return; + } + DocumentImportBatchItem current = batchTracker.requireItem(itemId); + boolean recovered = DocumentImportBatchItemStatus.PENDING.name() + .equals(current.getStatus()) + && !StringUtil.hasText(current.getStorageLocator()) + && !Boolean.TRUE.equals(current.getCleanupPending()); + boolean cancelled = DocumentImportBatchItemStatus.CANCELLED.name() + .equals(current.getStatus()); + if (!recovered && !cancelled) { + LOG.error( + "物理写入前撤销上传状态未完成: itemId={}, status={}, cleanupPending={}", + itemId, + current.getStatus(), + current.getCleanupPending() + ); + } + } catch (RuntimeException recoveryError) { + LOG.error( + "物理写入前撤销上传状态失败: itemId={}", + itemId, + recoveryError + ); + } + } + + /** + * 将预先登记的写意图标记为清理待办并立即尝试回收。 + * + *

清理期间保留 {@code UPLOADING} 状态；对象回收成功后恢复为 + * {@code PENDING}，使瞬时存储或绑定异常可以重新上传。若批次并发 + * 取消，取消事务会将文件项迁移为 {@code CANCELLED}，清理逻辑会 + * 根据最新状态保留取消结果。

+ * + * @param itemId 文件项 ID + * @param storageLocator 可恢复存储定位符 + * @param storedPath 已写入的兼容对象路径，可为空 + * @param originalError 导致对象未绑定的原始异常 + */ + private void markStoredObjectForCleanup(BigInteger itemId, + String storageLocator, + String storedPath, + RuntimeException originalError) { + int marked; + try { + marked = itemMapper.markUploadCleanupPending( + itemId, + storageLocator, + new Date() + ); + } catch (RuntimeException markerError) { + LOG.error( + "标记上传对象等待清理失败，保留预写入定位符供超时回收: itemId={}", + itemId, + markerError + ); + return; + } + if (marked <= 0) { + LOG.warn( + "上传对象未领取清理权，保留预写入定位符供状态恢复: itemId={}, path={}", + itemId, + storedPath, + originalError + ); + return; + } + cleanupStoredObject(itemId); + } + + /** + * 将未启动文件项迁移为取消，并在事务提交后清理上传对象。 + * + * @param batchId 批次 ID + */ + private void cancelPendingItemsAndScheduleCleanup(BigInteger batchId) { + List items = itemService.list( + QueryWrapper.create() + .eq(DocumentImportBatchItem::getBatchId, batchId) + .in(DocumentImportBatchItem::getStatus, List.of( + DocumentImportBatchItemStatus.PENDING.name(), + DocumentImportBatchItemStatus.UPLOADING.name(), + DocumentImportBatchItemStatus.UPLOADED.name() + )) + ); + List cleanupItemIds = new ArrayList<>(); + for (DocumentImportBatchItem item : items) { + batchTracker.transitionItem(item.getId(), + DocumentImportBatchItemStage.UPLOAD, + DocumentImportBatchItemStatus.CANCELLED, + null, + false, + 0); + if (itemMapper.markCancelledCleanupPending( + item.getId(), + new Date() + ) > 0) { + cleanupItemIds.add(item.getId()); + } + } + runAfterCommit(() -> cleanupItemIds.forEach(this::cleanupStoredObject)); + } + + /** + * 原子领取并调度 Public API 批次重试。 + * + * @param taskId 批次任务 ID + * @param caller 调用者上下文 + * @param fileKeys 指定文件键；为空时重试全部可恢复失败项 + * @return 稳定的重试响应快照 + */ + @Transactional + public DocumentImportBatchRetryResult retryOwnedBatch( + BigInteger taskId, + ImportCallerContext caller, + Set fileKeys) { + if (taskId == null || caller == null) { + throw new BusinessException("重试任务信息不完整"); + } + DocumentImportBatch initial = requireBatchForCaller(taskId, caller); + acquireBatchMutationLock(initial.getKnowledgeId()); + DocumentImportBatch batch = batchMapper.selectOwnedForUpdate( + taskId, + caller.getCallerType().name(), + caller.getCallerId() + ); + if (batch == null) { + throw new BusinessException(404, 404, "导入任务不存在"); + } + if (!DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name().equals(batch.getStatus()) + && !DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())) { + throw new BusinessException(409, 40904, "当前任务状态不允许重试"); + } + Set requestedFileKeys = fileKeys == null ? Set.of() : fileKeys; + List retryItems = + listRetryItems(batch.getId(), requestedFileKeys); + if (retryItems.isEmpty()) { + throw new BusinessException(409, 40905, "当前任务没有可重试的失败文件"); + } + restoreLegacyRetryability(retryItems); + Date now = new Date(); + int expectedGeneration = valueOrZero(batch.getRetryGeneration()); + int claimedGeneration = expectedGeneration + 1; + int claimed = batchMapper.claimRetry( + batch.getId(), + caller.getCallerType().name(), + caller.getCallerId(), + expectedGeneration, + now + ); + if (claimed <= 0) { + throw new BusinessException(409, 40903, "任务已被其他请求重试，请刷新任务状态"); + } + Set selectedKeys = retryItems.stream() + .map(DocumentImportBatchItem::getClientFileKey) + .collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new)); + runAfterCommit(() -> resumeBatchFailures(batch.getId(), selectedKeys)); + return new DocumentImportBatchRetryResult( + batch.getId(), + DocumentImportBatchStatus.RUNNING.name(), + claimedGeneration, + retryItems.size() + ); + } + + /** + * 查询 Public API 调用者拥有的批次。 + * + * @param taskId 批次任务 ID + * @param caller 调用者上下文 + * @return 导入批次 + */ + public DocumentImportBatch requireBatchForCaller(BigInteger taskId, + ImportCallerContext caller) { + if (taskId == null || caller == null) { + throw new BusinessException(404, 404, "导入任务不存在"); + } + DocumentImportBatch batch = batchService.getOne( + QueryWrapper.create() + .eq(DocumentImportBatch::getId, taskId) + .eq(DocumentImportBatch::getCallerType, caller.getCallerType().name()) + .eq(DocumentImportBatch::getCallerId, caller.getCallerId()) + ); + if (batch == null) { + throw new BusinessException(404, 404, "导入任务不存在"); + } + return batch; + } + + private long validateManifest(List files) { + long totalBytes = 0L; + Set keys = new HashSet(); + for (DocumentImportBatchDtos.ManifestItem file : files) { + if (file == null || !StringUtil.hasText(file.getClientFileKey()) + || !StringUtil.hasText(file.getFileName()) || file.getFileSize() == null) { + throw new BusinessException("文件清单不完整"); + } + if (file.getClientFileKey().length() > 64 || !keys.add(file.getClientFileKey())) { + throw new BusinessException("文件清单包含重复或非法文件"); + } + if (file.getFileName().length() > 512) { + throw new BusinessException("文件名过长"); + } + if (file.getFileSize() <= 0 || file.getFileSize() > properties.getMaxFileSize().toBytes()) { + throw new BusinessException("单个文件不能超过100MB"); + } + assertSupportedExtension(file.getFileName()); + normalizeRelativePath(file.getRelativePath(), file.getFileName()); + totalBytes = Math.addExact(totalBytes, file.getFileSize()); + if (totalBytes > properties.getMaxTotalSize().toBytes()) { + throw new BusinessException("文件夹总大小不能超过1GB"); + } + } + return totalBytes; + } + + private void validateUploadedFile(DocumentImportBatchItem item, MultipartFile file) { + if (file == null || file.isEmpty()) { + throw new BusinessException("上传文件不能为空"); + } + if (!item.getFileName().equals(file.getOriginalFilename()) || item.getFileSize() != file.getSize()) { + throw new BusinessException("上传文件与文件清单不一致"); + } + if (file.getSize() > properties.getMaxFileSize().toBytes()) { + throw new BusinessException("单个文件不能超过100MB"); + } + } + + private void markHistoricalDuplicates(List items, + BigInteger currentBatchId, + DuplicatePolicy duplicatePolicy) { + if (duplicatePolicy == DuplicatePolicy.REIMPORT) { + return; + } + List keys = items.stream().map(DocumentImportBatchItem::getClientFileKey).toList(); + if (keys.isEmpty()) { + return; + } + List historical = itemService.list( + QueryWrapper.create() + .eq(DocumentImportBatchItem::getKnowledgeId, items.get(0).getKnowledgeId()) + .ne(DocumentImportBatchItem::getBatchId, currentBatchId) + .in(DocumentImportBatchItem::getClientFileKey, keys) + .eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.COMPLETED.name()) + .isNotNull(DocumentImportBatchItem::getDocumentId) + .orderBy(DocumentImportBatchItem::getCreated, false) + ); + Set historicalDocumentIds = historical.stream() + .map(DocumentImportBatchItem::getDocumentId) + .collect(java.util.stream.Collectors.toSet()); + Set existingDocumentIds = historicalDocumentIds.isEmpty() + ? Set.of() + : documentMapper.selectListByQuery( + QueryWrapper.create() + .select(tech.easyflow.ai.entity.Document::getId) + .eq(tech.easyflow.ai.entity.Document::getCollectionId, + items.get(0).getKnowledgeId()) + .in(tech.easyflow.ai.entity.Document::getId, historicalDocumentIds) + ).stream() + .map(tech.easyflow.ai.entity.Document::getId) + .collect(java.util.stream.Collectors.toSet()); + Map existingDocuments = new LinkedHashMap(); + for (DocumentImportBatchItem existing : historical) { + if (existingDocumentIds.contains(existing.getDocumentId())) { + existingDocuments.putIfAbsent(existing.getClientFileKey(), existing.getDocumentId()); + } + } + for (DocumentImportBatchItem item : items) { + BigInteger existingDocumentId = existingDocuments.get(item.getClientFileKey()); + if (existingDocumentId == null) { + continue; + } + if (duplicatePolicy == DuplicatePolicy.SKIP) { + item.setStage(DocumentImportBatchItemStage.DONE.name()); + item.setStatus(DocumentImportBatchItemStatus.SKIPPED.name()); + item.setErrorSummary("重复文件已跳过"); + item.setModified(new Date()); + batchTracker.transitionItem(item.getId(), + DocumentImportBatchItemStage.DONE, + DocumentImportBatchItemStatus.SKIPPED, + "重复文件已跳过", + false, + 0); + } else { + batchTracker.markReplacement(item.getId(), existingDocumentId); + item.setReplacedDocumentId(existingDocumentId); + } + } + } + + private void assertNoOtherActiveAutoBatch(DocumentImportBatch current) { + long activeCount = batchService.count( + QueryWrapper.create() + .eq(DocumentImportBatch::getKnowledgeId, current.getKnowledgeId()) + .eq(DocumentImportBatch::getImportMode, DocumentImportMode.AUTO.name()) + .ne(DocumentImportBatch::getId, current.getId()) + .in(DocumentImportBatch::getStatus, List.of( + DocumentImportBatchStatus.RUNNING.name() + )) + ); + if (activeCount > 0) { + throw new BusinessException("当前知识库已有自动导入批次，请完成后再试"); + } + } + + private DocumentImportBatch requireOwnedBatch(BigInteger knowledgeId, BigInteger batchId) { + return requireOwnedBatch(knowledgeId, batchId, null); + } + + /** + * 查询批次并校验知识库与调用者归属。 + * + *

Public API 归属不匹配统一返回任务不存在，避免泄露其他令牌的任务信息。

+ * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @param caller 调用者上下文；为空时仅校验知识库 + * @return 导入批次 + */ + public DocumentImportBatch requireOwnedBatch(BigInteger knowledgeId, + BigInteger batchId, + ImportCallerContext caller) { + if (knowledgeId == null || batchId == null) { + throw new BusinessException("批次信息不完整"); + } + QueryWrapper wrapper = QueryWrapper.create() + .eq(DocumentImportBatch::getId, batchId) + .eq(DocumentImportBatch::getKnowledgeId, knowledgeId); + if (caller != null) { + wrapper.eq(DocumentImportBatch::getCallerType, caller.getCallerType().name()) + .eq(DocumentImportBatch::getCallerId, caller.getCallerId()); + } + DocumentImportBatch batch = batchService.getOne(wrapper); + if (batch == null && caller != null) { + throw new BusinessException(404, 404, "导入任务不存在"); + } + if (batch == null) { + throw new BusinessException("导入批次不属于当前知识库"); + } + return batch; + } + + private DocumentImportMode parseMode(String value) { + try { + return DocumentImportMode.valueOf(String.valueOf(value).toUpperCase(Locale.ROOT)); + } catch (Exception error) { + throw new BusinessException("导入模式无效"); + } + } + + /** + * 解析重复文件处理策略，缺省跳过历史重复项。 + * + * @param value 请求值 + * @return 重复文件策略 + */ + private DuplicatePolicy parseDuplicatePolicy(String value) { + if (!StringUtil.hasText(value)) { + return DuplicatePolicy.SKIP; + } + try { + return DuplicatePolicy.valueOf(value.toUpperCase(Locale.ROOT)); + } catch (Exception error) { + throw new BusinessException("重复文件处理策略无效"); + } + } + + /** + * 获取知识库批次变更锁，并在当前事务完成后释放。 + * + * @param knowledgeId 知识库 ID + */ + private void acquireBatchMutationLock(BigInteger knowledgeId) { + LockHandle handle = redisLockExecutor.tryAcquire( + "easyflow:lock:document-import:batch:" + knowledgeId, + Duration.ZERO, + BATCH_MUTATION_LOCK_LEASE + ); + if (handle == null) { + throw new BusinessException("导入批次正在变更，请稍后重试"); + } + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + handle.release(); + throw new IllegalStateException("批次变更必须在事务中执行"); + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + handle.release(); + } + }); + } + + /** + * 事务提交后执行文件清理。 + * + * @param action 清理动作 + */ + private void runAfterCommit(Runnable action) { + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + action.run(); + return; + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + action.run(); + } + }); + } + + /** + * 读取最新文件项并清理其存储待办。 + * + * @param itemId 批次项 ID + * @return 是否删除成功并完成标记清理 + */ + private boolean cleanupStoredObject(BigInteger itemId) { + try { + return cleanupStoredObject(batchTracker.requireItem(itemId)); + } catch (RuntimeException lookupError) { + LOG.error( + "读取上传对象清理待办失败: itemId={}", + itemId, + lookupError + ); + return false; + } + } + + /** + * 按恢复定位符优先、兼容路径兜底的顺序幂等删除上传对象。 + * + * @param item 存储清理待办 + * @return 是否删除成功并完成标记清理 + */ + private boolean cleanupStoredObject(DocumentImportBatchItem item) { + if (item == null || item.getId() == null + || !Boolean.TRUE.equals(item.getCleanupPending())) { + return false; + } + try { + if (StringUtil.hasText(item.getStorageLocator())) { + storageService.deleteRecoverable( + FileStorageWriteHandle.decodeLocator(item.getStorageLocator()) + ); + } else if (StringUtil.hasText(item.getFilePath())) { + storageService.delete(item.getFilePath()); + } + } catch (RuntimeException error) { + LOG.error( + "清理上传对象失败: itemId={}, path={}", + item.getId(), + item.getFilePath(), + error + ); + return false; + } + int completed; + try { + if (DocumentImportBatchItemStatus.UPLOADING.name() + .equals(item.getStatus())) { + completed = itemMapper.completeUploadingStorageCleanup( + item.getId(), + item.getStorageLocator(), + item.getFilePath(), + "上次上传对象已回收，请重新上传", + new Date() + ); + } else if (DocumentImportBatchItemStatus.CANCELLED.name() + .equals(item.getStatus())) { + completed = itemMapper.completeCancelledStorageCleanup( + item.getId(), + item.getStorageLocator(), + item.getFilePath(), + new Date() + ); + } else { + return false; + } + } catch (RuntimeException error) { + LOG.error( + "上传对象已删除但原子恢复状态失败，保留清理待办等待重试: itemId={}, path={}", + item.getId(), + item.getFilePath(), + error + ); + return false; + } + if (completed > 0) { + return true; + } + try { + DocumentImportBatchItem current = batchTracker.requireItem(item.getId()); + if (!Boolean.TRUE.equals(current.getCleanupPending()) + && !StringUtil.hasText(current.getStorageLocator()) + && !StringUtil.hasText(current.getFilePath()) + && (DocumentImportBatchItemStatus.PENDING.name().equals(current.getStatus()) + || DocumentImportBatchItemStatus.CANCELLED.name().equals(current.getStatus()))) { + return true; + } + } catch (RuntimeException lookupError) { + LOG.warn( + "上传对象清理 CAS 未命中且最新状态读取失败: itemId={}", + item.getId(), + lookupError + ); + } + LOG.warn( + "上传对象已删除但清理状态未恢复，等待下一轮重试: itemId={}, path={}", + item.getId(), + item.getFilePath() + ); + return false; + } + + /** + * 在批次状态提交后启动失败项重试；启动器异常时恢复为可继续状态。 + * + * @param batchId 批次 ID + */ + private void resumeBatchFailures(BigInteger batchId) { + resumeBatchFailures(batchId, Set.of()); + } + + /** + * 在批次状态提交后启动选定失败项重试。 + * + * @param batchId 批次 ID + * @param fileKeys 指定文件键；为空时重试全部 + */ + private void resumeBatchFailures(BigInteger batchId, Set fileKeys) { + try { + taskAppService.retryBatchFailures(batchId, fileKeys); + } catch (RuntimeException error) { + LOG.error("批次失败项恢复调度异常: batchId={}", batchId, error); + batchTracker.markInterrupted(batchId); + } + } + + /** + * 查询并校验本次重试选中的可恢复失败项。 + * + * @param batchId 批次 ID + * @param fileKeys 指定文件键 + * @return 可恢复失败项 + */ + private List listRetryItems(BigInteger batchId, + Set fileKeys) { + QueryWrapper wrapper = QueryWrapper.create() + .eq(DocumentImportBatchItem::getBatchId, batchId) + .eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name()); + if (fileKeys != null && !fileKeys.isEmpty()) { + Set normalized = fileKeys.stream() + .filter(StringUtil::hasText) + .map(String::trim) + .collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new)); + if (normalized.size() != fileKeys.size()) { + throw new BusinessException("fileKeys 包含空值或重复值"); + } + wrapper.in(DocumentImportBatchItem::getClientFileKey, normalized); + List selected = itemService.list(wrapper).stream() + .filter(this::isRetryCandidate) + .toList(); + if (selected.size() != normalized.size()) { + throw new BusinessException(409, 40906, "部分文件当前不可重试"); + } + return selected; + } + return itemService.list(wrapper).stream() + .filter(this::isRetryCandidate) + .toList(); + } + + /** + * 判断失败项是否可以进入本次人工重试。 + * + * @param item 批次失败项 + * @return 是否可以重试 + */ + private boolean isRetryCandidate(DocumentImportBatchItem item) { + return Boolean.TRUE.equals(item.getRetryable()) + || taskAppService.isRecoverableBatchFailure(item); + } + + /** + * 在领取重试前恢复旧失败项的持久化重试资格及批次计数。 + * + * @param retryItems 本次重试项 + */ + private void restoreLegacyRetryability( + List retryItems) { + for (DocumentImportBatchItem item : retryItems) { + if (Boolean.TRUE.equals(item.getRetryable())) { + continue; + } + DocumentImportBatchItemStage stage; + try { + stage = DocumentImportBatchItemStage.valueOf(item.getStage()); + } catch (IllegalArgumentException | NullPointerException error) { + throw new BusinessException(409, 40906, "部分文件当前不可重试"); + } + if (!batchTracker.transitionItem( + item.getId(), + stage, + DocumentImportBatchItemStatus.FAILED, + item.getErrorSummary(), + true, + 0, + item.getFailureCode() + )) { + throw new BusinessException( + 409, + 40903, + "文件状态已变化，请刷新任务状态" + ); + } + item.setRetryable(true); + } + } + + /** + * 限制上传错误写入批次项的长度。 + * + * @param message 原始异常信息 + * @return 安全长度的错误摘要 + */ + private String truncateUploadError(String message) { + if (!StringUtil.hasText(message)) { + return "文件上传失败，请重试"; + } + return message.length() > 500 ? message.substring(0, 500) : message; + } + + /** + * 将可空计数转换为零。 + * + * @param value 可空计数 + * @return 非空计数 + */ + private int valueOrZero(Integer value) { + return value == null ? 0 : value; + } + + private void assertSupportedExtension(String fileName) { + int dotIndex = fileName == null ? -1 : fileName.lastIndexOf('.'); + String extension = dotIndex < 0 ? "" : fileName.substring(dotIndex + 1).toLowerCase(Locale.ROOT); + if (!SUPPORTED_EXTENSIONS.contains(extension)) { + throw new BusinessException("暂不支持该文件格式"); + } + } + + /** + * 为可恢复写入生成长度稳定且不会跨文件项冲突的物理文件名。 + * + * @param item 批次文件项 + * @return 由文件项 ID 与原扩展名组成的文件名 + */ + private String buildRecoverableFileName(DocumentImportBatchItem item) { + String fileName = item.getFileName(); + int dotIndex = fileName == null ? -1 : fileName.lastIndexOf('.'); + String extension = dotIndex < 0 + ? "bin" + : fileName.substring(dotIndex + 1).toLowerCase(Locale.ROOT); + return item.getId() + "." + extension; + } + + private String normalizeRelativePath(String relativePath, String fileName) { + String normalized = StringUtil.hasText(relativePath) ? relativePath.replace('\\', '/') : fileName; + if (normalized.length() > 1024 + || normalized.startsWith("/") + || normalized.split("/").length > 64) { + throw new BusinessException("文件相对路径无效"); + } + for (String segment : normalized.split("/")) { + if ("..".equals(segment)) { + throw new BusinessException("文件相对路径无效"); + } + } + return normalized; + } + + private DocumentImportBatchDtos.ItemResponse toItemResponse(DocumentImportBatchItem item) { + DocumentImportBatchDtos.ItemResponse response = new DocumentImportBatchDtos.ItemResponse(); + response.setItemId(item.getId()); + response.setDocumentId(item.getDocumentId()); + response.setClientFileKey(item.getClientFileKey()); + response.setFileName(item.getFileName()); + response.setRelativePath(item.getRelativePath()); + response.setFileSize(item.getFileSize()); + response.setStage(item.getStage()); + response.setStatus(item.getStatus()); + response.setErrorSummary(item.getErrorSummary()); + return response; + } + + private BigInteger resolveOperatorId() { + try { + return BigInteger.valueOf(StpUtil.getLoginIdAsLong()); + } catch (Exception ignored) { + return BigInteger.ZERO; + } + } + + /** + * 历史重复文件处理策略。 + */ + private enum DuplicatePolicy { + /** 跳过历史重复项。 */ + SKIP, + /** 新文档成功入库后覆盖历史文档。 */ + OVERWRITE, + /** 保留历史文档并重新导入一份。 */ + REIMPORT + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java new file mode 100644 index 00000000..206b5c3e --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTracker.java @@ -0,0 +1,498 @@ +package tech.easyflow.ai.documentimport.task; + +import com.mybatisflex.core.query.QueryWrapper; +import org.springframework.stereotype.Service; +import tech.easyflow.ai.documentimport.DocumentImportBatchDtos; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.entity.DocumentImportBatchItem; +import tech.easyflow.ai.enums.DocumentImportBatchItemStage; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; +import tech.easyflow.ai.enums.DocumentImportBatchStatus; +import tech.easyflow.ai.enums.DocumentImportMode; +import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; +import tech.easyflow.ai.service.DocumentImportBatchItemService; +import tech.easyflow.ai.service.DocumentImportBatchService; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.math.BigInteger; +import java.util.Date; + +/** + * 批量导入文件项与批次汇总状态跟踪器。 + * + * @author Codex + * @since 2026-07-31 + */ +@Service +public class DocumentImportBatchTracker { + + private final DocumentImportBatchService batchService; + private final DocumentImportBatchItemService itemService; + private final DocumentImportBatchMapper batchMapper; + private final DocumentImportBatchItemMapper itemMapper; + + /** + * 创建批次状态跟踪器。 + * + * @param batchService 批次服务 + * @param itemService 批次项服务 + * @param batchMapper 批次 Mapper + * @param itemMapper 批次项 Mapper + */ + public DocumentImportBatchTracker(DocumentImportBatchService batchService, + DocumentImportBatchItemService itemService, + DocumentImportBatchMapper batchMapper, + DocumentImportBatchItemMapper itemMapper) { + this.batchService = batchService; + this.itemService = itemService; + this.batchMapper = batchMapper; + this.itemMapper = itemMapper; + } + + /** + * 查询批次。 + * + * @param batchId 批次 ID + * @return 批次实体 + */ + public DocumentImportBatch requireBatch(BigInteger batchId) { + DocumentImportBatch batch = batchId == null ? null : batchService.getById(batchId); + if (batch == null) { + throw new BusinessException("导入批次不存在"); + } + return batch; + } + + /** + * 查询批次文件项。 + * + * @param itemId 文件项 ID + * @return 文件项实体 + */ + public DocumentImportBatchItem requireItem(BigInteger itemId) { + DocumentImportBatchItem item = itemId == null ? null : itemService.getById(itemId); + if (item == null) { + throw new BusinessException("导入文件不存在"); + } + return item; + } + + /** + * 判断指定批次是否采用自动导入。 + * + * @param batchId 批次 ID + * @return 是否自动导入 + */ + public boolean isAutoBatch(BigInteger batchId) { + if (batchId == null) { + return false; + } + return DocumentImportMode.AUTO.name().equals(requireBatch(batchId).getImportMode()); + } + + /** + * 更新文件项阶段与状态。 + * + * @param itemId 文件项 ID + * @param stage 处理阶段 + * @param status 处理状态 + * @param errorSummary 错误摘要 + */ + @org.springframework.transaction.annotation.Transactional + public boolean updateItem(BigInteger itemId, + DocumentImportBatchItemStage stage, + DocumentImportBatchItemStatus status, + String errorSummary) { + if (itemId == null) { + return false; + } + DocumentImportBatchItem current = requireItem(itemId); + int attemptDelta = (status == DocumentImportBatchItemStatus.PENDING + || status == DocumentImportBatchItemStatus.RUNNING) + && DocumentImportBatchItemStatus.FAILED.name().equals(current.getStatus()) + ? 1 + : 0; + return transitionItem(itemId, stage, status, errorSummary, + status == DocumentImportBatchItemStatus.FAILED, attemptDelta, null); + } + + /** + * 按预期旧状态原子迁移文件项，并同步更新批次计数。 + * + * @param itemId 文件项 ID + * @param stage 新阶段 + * @param status 新状态 + * @param errorSummary 错误摘要 + * @param retryable 是否允许批量重试 + * @param attemptDelta 重试次数增量 + * @return 状态迁移或同状态刷新成功时返回 {@code true} + */ + @org.springframework.transaction.annotation.Transactional + public boolean transitionItem(BigInteger itemId, + DocumentImportBatchItemStage stage, + DocumentImportBatchItemStatus status, + String errorSummary, + boolean retryable, + int attemptDelta) { + return transitionItem(itemId, stage, status, errorSummary, + retryable, attemptDelta, null); + } + + /** + * 按预期旧状态原子迁移文件项，并记录稳定失败码。 + * + * @param itemId 文件项 ID + * @param stage 新阶段 + * @param status 新状态 + * @param errorSummary 错误摘要 + * @param retryable 是否允许批量重试 + * @param attemptDelta 重试次数增量 + * @param failureCode 稳定失败码 + * @return 状态迁移或同状态刷新成功时返回 {@code true} + */ + @org.springframework.transaction.annotation.Transactional + public boolean transitionItem(BigInteger itemId, + DocumentImportBatchItemStage stage, + DocumentImportBatchItemStatus status, + String errorSummary, + boolean retryable, + int attemptDelta, + String failureCode) { + if (itemId == null) { + return false; + } + for (int attempt = 0; attempt < 3; attempt++) { + DocumentImportBatchItem current = requireItem(itemId); + DocumentImportBatchItemStatus currentStatus = + DocumentImportBatchItemStatus.valueOf(current.getStatus()); + if (!isAllowedTransition(currentStatus, status)) { + return false; + } + String expectedStatus = current.getStatus(); + Date now = new Date(); + int updated = itemMapper.transitionStatus( + itemId, + expectedStatus, + stage.name(), + status.name(), + errorSummary, + failureCode, + retryable, + Math.max(0, attemptDelta), + now + ); + if (updated <= 0) { + continue; + } + CounterDelta delta = CounterDelta.between(current, status, retryable); + if (!delta.isZero()) { + batchMapper.adjustCounters( + current.getBatchId(), + delta.completed, + delta.processing, + delta.failed, + delta.pending, + delta.uploaded, + delta.skipped, + delta.cancelled, + delta.retryableFailed, + now + ); + } + refreshBatch(current.getBatchId()); + return true; + } + return false; + } + + /** + * 校验文件项状态机，拒绝迟到任务覆盖终态。 + * + * @param current 当前状态 + * @param next 目标状态 + * @return 是否允许迁移 + */ + private boolean isAllowedTransition(DocumentImportBatchItemStatus current, + DocumentImportBatchItemStatus next) { + if (current == next) { + return true; + } + return switch (next) { + case UPLOADING -> current == DocumentImportBatchItemStatus.PENDING; + case UPLOADED -> current == DocumentImportBatchItemStatus.UPLOADING; + case RUNNING -> current == DocumentImportBatchItemStatus.PENDING + || current == DocumentImportBatchItemStatus.FAILED; + case PENDING -> current == DocumentImportBatchItemStatus.RUNNING + || current == DocumentImportBatchItemStatus.FAILED; + case FAILED, COMPLETED -> current == DocumentImportBatchItemStatus.RUNNING + || current == DocumentImportBatchItemStatus.PENDING; + case SKIPPED -> current == DocumentImportBatchItemStatus.UPLOADED; + case CANCELLED -> current == DocumentImportBatchItemStatus.PENDING + || current == DocumentImportBatchItemStatus.UPLOADING + || current == DocumentImportBatchItemStatus.UPLOADED; + }; + } + + /** + * 将批次文件项绑定到创建后的文档。 + * + * @param itemId 文件项 ID + * @param documentId 文档 ID + */ + @org.springframework.transaction.annotation.Transactional + public void bindDocument(BigInteger itemId, BigInteger documentId) { + DocumentImportBatchItem item = requireItem(itemId); + if (documentId.equals(item.getDocumentId()) + && DocumentImportBatchItemStatus.PENDING.name().equals(item.getStatus())) { + return; + } + Date now = new Date(); + int updated = itemMapper.bindDocument(itemId, documentId, now); + if (updated <= 0) { + throw new BusinessException("导入文件状态已变化，请刷新后重试"); + } + batchMapper.adjustCounters(item.getBatchId(), + 0, 0, 0, 0, -1, 0, 0, 0, now); + refreshBatch(item.getBatchId()); + } + + /** + * 原子完成文件上传并增量更新批次上传数。 + * + * @param itemId 文件项 ID + * @param filePath 存储路径 + * @param storageLocator 可恢复存储定位符 + * @return 文件项仍处于上传中且完成成功时返回 {@code true} + */ + @org.springframework.transaction.annotation.Transactional + public boolean completeUpload(BigInteger itemId, + String filePath, + String storageLocator) { + DocumentImportBatchItem item = requireItem(itemId); + if (DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus())) { + return filePath.equals(item.getFilePath()) + && storageLocator.equals(item.getStorageLocator()); + } + if (!DocumentImportBatchItemStatus.UPLOADING.name().equals(item.getStatus())) { + return false; + } + Date now = new Date(); + int updated = itemMapper.completeUpload( + itemId, + filePath, + storageLocator, + now + ); + if (updated <= 0) { + return false; + } + batchMapper.adjustCounters(item.getBatchId(), + 0, 0, 0, 0, 1, 0, 0, 0, now); + refreshBatch(item.getBatchId()); + return true; + } + + /** + * 记录文件项成功后需要清理的历史文档。 + * + * @param itemId 文件项 ID + * @param replacedDocumentId 历史文档 ID + */ + @org.springframework.transaction.annotation.Transactional + public void markReplacement(BigInteger itemId, BigInteger replacedDocumentId) { + Date now = new Date(); + int updated = itemMapper.markReplacement(itemId, replacedDocumentId, now); + if (updated <= 0) { + throw new BusinessException("重复文件状态已变化，请刷新后重试"); + } + } + + /** + * 清除已完成的历史文档覆盖标记。 + * + * @param itemId 文件项 ID + * @param replacedDocumentId 历史文档 ID + */ + @org.springframework.transaction.annotation.Transactional + public void clearReplacement(BigInteger itemId, BigInteger replacedDocumentId) { + itemMapper.clearReplacement(itemId, replacedDocumentId, new Date()); + } + + /** + * 汇总并持久化批次状态。 + * + * @param batchId 批次 ID + * @return 最新状态 + */ + public DocumentImportBatchDtos.StatusResponse refreshBatch(BigInteger batchId) { + DocumentImportBatch batch = requireBatch(batchId); + Date now = new Date(); + String nextStatus = batch.getStatus(); + Date nextFinishedAt = batch.getFinishedAt(); + if (batch.getImportMode() == null) { + if (!DocumentImportBatchStatus.CANCELLED.name().equals(batch.getStatus())) { + nextStatus = valueOrZero(batch.getUploadedCount()) == valueOrZero(batch.getTotalCount()) + ? DocumentImportBatchStatus.READY.name() + : DocumentImportBatchStatus.UPLOADING.name(); + } + } else if (!DocumentImportBatchStatus.CANCELLED.name().equals(batch.getStatus()) + && !DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())) { + int terminalCount = valueOrZero(batch.getCompletedCount()) + + valueOrZero(batch.getFailedCount()) + + valueOrZero(batch.getSkippedCount()) + + valueOrZero(batch.getCancelledCount()); + if (terminalCount >= valueOrZero(batch.getTotalCount()) + && valueOrZero(batch.getProcessingCount()) == 0 + && valueOrZero(batch.getPendingCount()) == 0) { + nextStatus = valueOrZero(batch.getFailedCount()) > 0 + ? DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name() + : DocumentImportBatchStatus.COMPLETED.name(); + nextFinishedAt = now; + } else { + nextStatus = DocumentImportBatchStatus.RUNNING.name(); + } + } + if (!java.util.Objects.equals(nextStatus, batch.getStatus()) + || !java.util.Objects.equals(nextFinishedAt, batch.getFinishedAt())) { + batch.setStatus(nextStatus); + batch.setFinishedAt(nextFinishedAt); + batch.setModified(now); + batchService.updateById(batch, false); + } + return toStatusResponse(batch); + } + + /** + * 将批次标记为已中断，保留已完成文件并允许批量继续。 + * + * @param batchId 批次 ID + */ + public void markInterrupted(BigInteger batchId) { + if (batchId == null) { + return; + } + Date now = new Date(); + DocumentImportBatch update = new DocumentImportBatch(); + update.setStatus(DocumentImportBatchStatus.INTERRUPTED.name()); + update.setModified(now); + batchMapper.updateByQuery(update, + QueryWrapper.create() + .eq(DocumentImportBatch::getId, batchId) + .eq(DocumentImportBatch::getStatus, DocumentImportBatchStatus.RUNNING.name())); + } + + /** + * 将批次转换为状态响应。 + * + * @param batch 批次实体 + * @return 状态响应 + */ + public DocumentImportBatchDtos.StatusResponse toStatusResponse(DocumentImportBatch batch) { + DocumentImportBatchDtos.StatusResponse response = new DocumentImportBatchDtos.StatusResponse(); + response.setBatchId(batch.getId()); + response.setImportMode(batch.getImportMode()); + response.setStatus(batch.getStatus()); + response.setTotalCount(valueOrZero(batch.getTotalCount())); + response.setTotalBytes(batch.getTotalBytes() == null ? 0L : batch.getTotalBytes()); + response.setCompletedCount(valueOrZero(batch.getCompletedCount())); + response.setProcessingCount(valueOrZero(batch.getProcessingCount())); + response.setFailedCount(valueOrZero(batch.getFailedCount())); + response.setPendingCount(valueOrZero(batch.getPendingCount())); + response.setSkippedCount(valueOrZero(batch.getSkippedCount())); + response.setCancelledCount(valueOrZero(batch.getCancelledCount())); + response.setRetryableFailedCount(valueOrZero(batch.getRetryableFailedCount())); + int total = Math.max(1, valueOrZero(batch.getTotalCount())); + int terminalCount = valueOrZero(batch.getCompletedCount()) + + valueOrZero(batch.getFailedCount()) + + valueOrZero(batch.getSkippedCount()) + + valueOrZero(batch.getCancelledCount()); + response.setProgressPercent(Math.min(100, terminalCount * 100 / total)); + response.setStartedAt(batch.getStartedAt()); + response.setFinishedAt(batch.getFinishedAt()); + return response; + } + + private int valueOrZero(Integer value) { + return value == null ? 0 : value; + } + + /** + * 文件项状态变化对应的批次计数增量。 + */ + private static final class CounterDelta { + private int completed; + private int processing; + private int failed; + private int pending; + private int uploaded; + private int skipped; + private int cancelled; + private int retryableFailed; + + /** + * 计算文件项状态迁移前后的计数差值。 + * + * @param current 当前文件项 + * @param nextStatus 新状态 + * @param nextRetryable 新状态是否允许重试 + * @return 计数差值 + */ + private static CounterDelta between(DocumentImportBatchItem current, + DocumentImportBatchItemStatus nextStatus, + boolean nextRetryable) { + CounterDelta delta = new CounterDelta(); + apply(delta, DocumentImportBatchItemStatus.valueOf(current.getStatus()), + Boolean.TRUE.equals(current.getRetryable()), -1); + apply(delta, nextStatus, nextRetryable, 1); + return delta; + } + + /** + * 将一个状态映射到对应计数桶。 + * + * @param delta 待修改的差值 + * @param status 文件项状态 + * @param retryable 是否允许重试 + * @param direction 增加或减少方向 + */ + private static void apply(CounterDelta delta, + DocumentImportBatchItemStatus status, + boolean retryable, + int direction) { + switch (status) { + case COMPLETED -> delta.completed += direction; + case RUNNING -> delta.processing += direction; + case FAILED -> { + delta.failed += direction; + if (retryable) { + delta.retryableFailed += direction; + } + } + case SKIPPED -> delta.skipped += direction; + case CANCELLED -> delta.cancelled += direction; + case PENDING, UPLOADING, UPLOADED -> delta.pending += direction; + default -> throw new IllegalStateException("未知批次文件状态: " + status); + } + if (status == DocumentImportBatchItemStatus.UPLOADED) { + delta.uploaded += direction; + } + } + + /** + * 判断所有增量是否均为零。 + * + * @return 是否没有计数变化 + */ + private boolean isZero() { + return completed == 0 + && processing == 0 + && failed == 0 + && pending == 0 + && uploaded == 0 + && skipped == 0 + && cancelled == 0 + && retryableFailed == 0; + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBulkProperties.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBulkProperties.java new file mode 100644 index 00000000..ff78fa61 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportBulkProperties.java @@ -0,0 +1,192 @@ +package tech.easyflow.ai.documentimport.task; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.unit.DataSize; + +import java.time.Duration; + +/** + * 文档批量导入容量与并发缺省配置。 + * + * @author Codex + * @since 2026-07-31 + */ +@ConfigurationProperties(prefix = "easyflow.ai.document-import.bulk") +public class DocumentImportBulkProperties { + + private int maxFileCount = 2000; + private DataSize maxTotalSize = DataSize.ofGigabytes(1); + private DataSize maxFileSize = DataSize.ofMegabytes(100); + private int uploadConcurrency = 3; + private int parseMaxRunning = 2; + private int splitMaxRunning = 2; + private int indexMaxRunning = 2; + private int perBatchParseMaxRunning = 2; + private int pendingDispatchBatchSize = 100; + private Duration pendingDispatchInterval = Duration.ofSeconds(2); + private Duration pendingRedispatchDelay = Duration.ofSeconds(5); + private Duration pendingTimeout = Duration.ofHours(24); + private Duration parseSubmitTimeout = Duration.ofSeconds(120); + private Duration interruptionTimeout = Duration.ofMinutes(10); + private int maxTaskAttempts = 3; + + public int getMaxFileCount() { + return maxFileCount; + } + + public void setMaxFileCount(int maxFileCount) { + this.maxFileCount = maxFileCount; + } + + public DataSize getMaxTotalSize() { + return maxTotalSize; + } + + public void setMaxTotalSize(DataSize maxTotalSize) { + this.maxTotalSize = maxTotalSize; + } + + public DataSize getMaxFileSize() { + return maxFileSize; + } + + public void setMaxFileSize(DataSize maxFileSize) { + this.maxFileSize = maxFileSize; + } + + public int getUploadConcurrency() { + return uploadConcurrency; + } + + public void setUploadConcurrency(int uploadConcurrency) { + this.uploadConcurrency = uploadConcurrency; + } + + public int getParseMaxRunning() { + return parseMaxRunning; + } + + public void setParseMaxRunning(int parseMaxRunning) { + this.parseMaxRunning = parseMaxRunning; + } + + /** + * 获取全局分块任务并发上限。 + * + * @return 分块任务并发上限 + */ + public int getSplitMaxRunning() { + return splitMaxRunning; + } + + /** + * 设置全局分块任务并发上限。 + * + * @param splitMaxRunning 分块任务并发上限 + */ + public void setSplitMaxRunning(int splitMaxRunning) { + this.splitMaxRunning = splitMaxRunning; + } + + public int getIndexMaxRunning() { + return indexMaxRunning; + } + + public void setIndexMaxRunning(int indexMaxRunning) { + this.indexMaxRunning = indexMaxRunning; + } + + public int getPerBatchParseMaxRunning() { + return perBatchParseMaxRunning; + } + + public void setPerBatchParseMaxRunning(int perBatchParseMaxRunning) { + this.perBatchParseMaxRunning = perBatchParseMaxRunning; + } + + public int getPendingDispatchBatchSize() { + return pendingDispatchBatchSize; + } + + public void setPendingDispatchBatchSize(int pendingDispatchBatchSize) { + this.pendingDispatchBatchSize = pendingDispatchBatchSize; + } + + public Duration getPendingDispatchInterval() { + return pendingDispatchInterval; + } + + public void setPendingDispatchInterval(Duration pendingDispatchInterval) { + this.pendingDispatchInterval = pendingDispatchInterval; + } + + public Duration getPendingRedispatchDelay() { + return pendingRedispatchDelay; + } + + public void setPendingRedispatchDelay(Duration pendingRedispatchDelay) { + this.pendingRedispatchDelay = pendingRedispatchDelay; + } + + /** + * 获取待处理任务最长排队时间。 + * + * @return 最长排队时间 + */ + public Duration getPendingTimeout() { + return pendingTimeout; + } + + /** + * 设置待处理任务最长排队时间。 + * + * @param pendingTimeout 最长排队时间 + */ + public void setPendingTimeout(Duration pendingTimeout) { + this.pendingTimeout = pendingTimeout; + } + + /** + * 获取解析服务任务提交超时时间。 + * + * @return 任务提交超时时间 + */ + public Duration getParseSubmitTimeout() { + return parseSubmitTimeout; + } + + /** + * 设置解析服务任务提交超时时间。 + * + * @param parseSubmitTimeout 任务提交超时时间 + */ + public void setParseSubmitTimeout(Duration parseSubmitTimeout) { + this.parseSubmitTimeout = parseSubmitTimeout; + } + + public Duration getInterruptionTimeout() { + return interruptionTimeout; + } + + public void setInterruptionTimeout(Duration interruptionTimeout) { + this.interruptionTimeout = interruptionTimeout; + } + + /** + * 获取单文件每阶段最大执行次数。 + * + * @return 最大执行次数 + */ + public int getMaxTaskAttempts() { + return maxTaskAttempts; + } + + /** + * 设置单文件每阶段最大执行次数。 + * + * @param maxTaskAttempts 最大执行次数 + */ + public void setMaxTaskAttempts(int maxTaskAttempts) { + this.maxTaskAttempts = maxTaskAttempts; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportChunkSnapshotService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportChunkSnapshotService.java new file mode 100644 index 00000000..c5166d7b --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportChunkSnapshotService.java @@ -0,0 +1,94 @@ +package tech.easyflow.ai.documentimport.task; + +import com.alibaba.fastjson2.JSON; +import org.springframework.stereotype.Service; +import tech.easyflow.ai.document.support.DocumentInputStreamSupport; +import tech.easyflow.ai.documentimport.DocumentImportDtos; +import tech.easyflow.ai.easyagents.CustomMultipartFile; +import tech.easyflow.common.filestorage.FileStorageService; +import tech.easyflow.common.util.StringUtil; +import tech.easyflow.common.web.exceptions.BusinessException; + +import javax.annotation.Resource; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +/** + * 自动导入分块快照持久化服务。 + * + *

快照写入对象存储，向量化任务只保存稳定路径，避免依赖短期预览缓存。

+ * + * @author Codex + * @since 2026-07-31 + */ +@Service +public class DocumentImportChunkSnapshotService { + + private static final long MAX_SNAPSHOT_BYTES = 256L * 1024L * 1024L; + + @Resource(name = "default") + private FileStorageService storageService; + + /** + * 持久化预览会话及其最终分块。 + * + * @param session 预览会话 + * @return 快照存储路径 + */ + public String save(DocumentImportDtos.PreviewSession session) { + if (session == null || session.getKnowledgeId() == null || session.getDocumentId() == null + || session.getDocumentChunks() == null || session.getDocumentChunks().isEmpty()) { + throw new BusinessException("分块快照内容不完整"); + } + String fileName = session.getDocumentId() + "-chunks.json"; + byte[] payload = JSON.toJSONBytes(session); + if (payload.length > MAX_SNAPSHOT_BYTES) { + throw new BusinessException("分块快照过大，请调整文档后重试"); + } + CustomMultipartFile file = new CustomMultipartFile( + payload, fileName, fileName, "application/json"); + String path = storageService.save(file, + "knowledge-import-snapshots/" + session.getKnowledgeId() + "/" + session.getDocumentId()); + if (!StringUtil.hasText(path)) { + throw new BusinessException("分块快照保存失败"); + } + return path; + } + + /** + * 从稳定存储恢复预览会话。 + * + * @param path 快照路径 + * @return 预览会话 + */ + public DocumentImportDtos.PreviewSession load(String path) { + if (!StringUtil.hasText(path)) { + throw new BusinessException("分块快照不存在，请重试"); + } + try (InputStream inputStream = storageService.readStream(path)) { + byte[] payload = DocumentInputStreamSupport.readBytes(inputStream, MAX_SNAPSHOT_BYTES); + String json = new String(payload, StandardCharsets.UTF_8); + DocumentImportDtos.PreviewSession session = + JSON.parseObject(json, DocumentImportDtos.PreviewSession.class); + if (session == null || session.getDocumentChunks() == null + || session.getDocumentChunks().isEmpty()) { + throw new BusinessException("分块快照无有效内容，请重试"); + } + return session; + } catch (IOException error) { + throw new BusinessException("分块快照读取失败，请重试"); + } + } + + /** + * 删除已完成向量化的分块快照。 + * + * @param path 快照路径 + */ + public void delete(String path) { + if (StringUtil.hasText(path)) { + storageService.delete(path); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitor.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitor.java new file mode 100644 index 00000000..26306ef0 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportPendingTaskMonitor.java @@ -0,0 +1,59 @@ +package tech.easyflow.ai.documentimport.task; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import tech.easyflow.common.cache.DistributedScheduledLock; + +/** + * 文档导入待处理任务恢复调度器。 + * + * @author Codex + * @since 2026-07-31 + */ +@Component +public class DocumentImportPendingTaskMonitor { + + private final KnowledgeDocumentImportTaskAppService appService; + + /** + * 创建文档导入恢复调度器。 + * + * @param appService 文档导入任务服务 + */ + public DocumentImportPendingTaskMonitor(KnowledgeDocumentImportTaskAppService appService) { + this.appService = appService; + } + + /** + * 周期性投递仍处于等待状态的解析与向量化任务。 + */ + @Scheduled( + fixedDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}", + initialDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}" + ) + @DistributedScheduledLock( + key = "easyflow:schedule:document-import:pending-dispatch", + leaseSeconds = 30L + ) + public void dispatchPendingTasks() { + appService.dispatchPendingTasks(); + } + + /** + * 检测排队超时与长时间无心跳的任务。 + */ + @Scheduled( + fixedDelayString = "${easyflow.ai.document-import.bulk.interruption-scan-interval:60s}", + initialDelayString = "${easyflow.ai.document-import.bulk.interruption-scan-interval:60s}" + ) + @DistributedScheduledLock( + key = "easyflow:schedule:document-import:interruption-scan", + leaseSeconds = 30L + ) + public void recoverTimedOutTasks() { + appService.expireTimedOutParseSubmissions(); + appService.recoverInterruptedTasks(); + appService.expireTimedOutPendingTasks(); + appService.cleanupCompletedReplacements(); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskConsumer.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskConsumer.java new file mode 100644 index 00000000..9cb88345 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskConsumer.java @@ -0,0 +1,94 @@ +package tech.easyflow.ai.documentimport.task; + +import com.alibaba.fastjson2.JSON; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import tech.easyflow.common.mq.config.MQProperties; +import tech.easyflow.common.mq.core.MQConsumerHandler; +import tech.easyflow.common.mq.core.MQMessage; +import tech.easyflow.common.mq.core.MQSubscription; + +import java.util.List; + +/** + * 文档分块任务消费者。 + * + * @author Codex + * @since 2026-08-02 + */ +@Component +public class DocumentImportSplitTaskConsumer implements MQConsumerHandler { + + private static final Logger LOG = + LoggerFactory.getLogger(DocumentImportSplitTaskConsumer.class); + + private final KnowledgeDocumentImportTaskAppService appService; + private final MQProperties mqProperties; + + /** + * 创建分块任务消费者。 + * + * @param appService 文档导入应用服务 + * @param mqProperties MQ 配置 + */ + public DocumentImportSplitTaskConsumer( + KnowledgeDocumentImportTaskAppService appService, + MQProperties mqProperties) { + this.appService = appService; + this.mqProperties = mqProperties; + } + + /** + * 获取分块任务订阅。 + * + * @return MQ 订阅信息 + */ + @Override + public MQSubscription subscription() { + MQSubscription subscription = new MQSubscription(); + subscription.setTopic(DocumentImportTaskMqConstants.SPLIT_TOPIC); + subscription.setConsumerGroup(DocumentImportTaskMqConstants.SPLIT_GROUP); + subscription.setShardCount(resolveShardCount()); + return subscription; + } + + /** + * 处理一批分块任务消息。 + * + * @param messages MQ 消息 + */ + @Override + public void handle(List messages) { + LOG.info("文档分块消费者收到消息批次: count={}", + messages == null ? 0 : messages.size()); + for (MQMessage message : messages) { + DocumentImportTaskMessage event = + JSON.parseObject(message.getBody(), DocumentImportTaskMessage.class); + if (event == null || event.getTaskId() == null) { + LOG.warn("文档分块消费者跳过非法消息: streamMessageId={}, messageId={}", + message == null ? null : message.getStreamMessageId(), + message == null ? null : message.getMessageId()); + continue; + } + try { + appService.handleSplitTask(event.getTaskId()); + } catch (Exception error) { + LOG.error("文档分块消费者处理失败: taskId={}, messageId={}, streamMessageId={}", + event.getTaskId(), message.getMessageId(), + message.getStreamMessageId(), error); + throw error; + } + } + } + + /** + * 获取当前 Redis Stream 分片数。 + * + * @return 分片数 + */ + private int resolveShardCount() { + return Math.max( + mqProperties.getRedis().getChatPersistShardCount(), 1); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskProducer.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskProducer.java new file mode 100644 index 00000000..0742a15b --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportSplitTaskProducer.java @@ -0,0 +1,58 @@ +package tech.easyflow.ai.documentimport.task; + +import com.alibaba.fastjson2.JSON; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import tech.easyflow.common.mq.core.MQMessage; +import tech.easyflow.common.mq.core.MQProducer; + +import java.math.BigInteger; +import java.util.Date; + +/** + * 文档分块任务消息生产者。 + * + * @author Codex + * @since 2026-08-02 + */ +@Service +public class DocumentImportSplitTaskProducer { + + private static final Logger LOG = + LoggerFactory.getLogger(DocumentImportSplitTaskProducer.class); + + private final MQProducer mqProducer; + + /** + * 创建分块任务消息生产者。 + * + * @param mqProducer MQ 生产者 + */ + public DocumentImportSplitTaskProducer(MQProducer mqProducer) { + this.mqProducer = mqProducer; + } + + /** + * 发送分块任务消息。 + * + * @param taskId 任务 ID + */ + public void send(BigInteger taskId) { + DocumentImportTaskMessage event = new DocumentImportTaskMessage(); + event.setTaskId(taskId); + event.setOccurredAt(new Date()); + + MQMessage message = new MQMessage(); + message.setMessageId("split-" + taskId); + message.setTopic(DocumentImportTaskMqConstants.SPLIT_TOPIC); + message.setKey(String.valueOf(taskId)); + message.setCreatedAt(event.getOccurredAt()); + message.setBody(JSON.toJSONString(event)); + LOG.info("准备投递文档分块 MQ 消息: topic={}, taskId={}, messageId={}", + message.getTopic(), taskId, message.getMessageId()); + String recordId = mqProducer.send(message); + LOG.info("文档分块 MQ 消息投递完成: topic={}, taskId={}, messageId={}, recordId={}", + message.getTopic(), taskId, message.getMessageId(), recordId); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportStaleBatchMonitor.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportStaleBatchMonitor.java new file mode 100644 index 00000000..10c06d61 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportStaleBatchMonitor.java @@ -0,0 +1,120 @@ +package tech.easyflow.ai.documentimport.task; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import tech.easyflow.ai.documentimport.ImportCallerContext; +import tech.easyflow.ai.documentimport.ImportCallerType; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; +import tech.easyflow.common.cache.DistributedScheduledLock; +import tech.easyflow.common.util.StringUtil; + +import java.time.Duration; +import java.util.Date; +import java.util.List; + +/** + * 文档导入未完成批次与取消对象清理调度器。 + * + * @author Codex + * @since 2026-08-02 + */ +@Component +public class DocumentImportStaleBatchMonitor { + + private static final Logger LOG = + LoggerFactory.getLogger(DocumentImportStaleBatchMonitor.class); + private static final Duration INCOMPLETE_TIMEOUT = Duration.ofMinutes(30); + private static final int BATCH_SIZE = 100; + + private final DocumentImportBatchMapper batchMapper; + private final DocumentImportBatchAppService batchAppService; + + /** + * 创建未完成批次清理调度器。 + * + * @param batchMapper 批次 Mapper + * @param batchAppService 批次应用服务 + */ + public DocumentImportStaleBatchMonitor( + DocumentImportBatchMapper batchMapper, + DocumentImportBatchAppService batchAppService) { + this.batchMapper = batchMapper; + this.batchAppService = batchAppService; + } + + /** + * 分批回收长时间无进展的上传批次，并重试对象清理。 + */ + @Scheduled( + fixedDelayString = + "${easyflow.ai.document-import.bulk.stale-batch-scan-interval:60s}", + initialDelayString = + "${easyflow.ai.document-import.bulk.stale-batch-scan-interval:60s}" + ) + @DistributedScheduledLock( + key = "easyflow:schedule:document-import:stale-batch-cleanup", + leaseSeconds = 50L + ) + public void cleanupStaleBatches() { + Date cutoff = new Date( + System.currentTimeMillis() - INCOMPLETE_TIMEOUT.toMillis() + ); + List candidates = + batchMapper.selectStaleIncompleteBatches(cutoff, BATCH_SIZE); + for (DocumentImportBatch batch : candidates) { + cancelCandidate(batch, cutoff); + } + batchAppService.cleanupCancelledStoredObjects(BATCH_SIZE); + } + + /** + * 通过数据库条件更新尝试取消单个候选批次。 + * + * @param batch 候选批次 + * @param cutoff 最后进展截止时间 + */ + private void cancelCandidate(DocumentImportBatch batch, Date cutoff) { + if (batch.getCallerType() == null || batch.getCallerId() == null) { + LOG.warn("跳过调用者信息不完整的超时导入批次: batchId={}", batch.getId()); + return; + } + try { + ImportCallerContext caller = new ImportCallerContext( + ImportCallerType.valueOf(batch.getCallerType()), + batch.getCallerId() + ); + boolean cancelled = batchAppService.cancelStaleBatch( + batch.getKnowledgeId(), + batch.getId(), + caller, + cutoff + ); + if (cancelled && StringUtil.hasText(batch.getIdempotencyKeyHash())) { + int released = batchMapper.releaseSubmissionFingerprint( + batch.getId(), + batch.getIdempotencyKeyHash(), + cutoff, + new Date() + ); + if (released <= 0) { + LOG.warn( + "超时导入批次已取消但提交指纹未释放: batchId={}", + batch.getId() + ); + } + } + } catch (IllegalArgumentException error) { + LOG.error( + "超时导入批次调用者类型无效: batchId={}, callerType={}", + batch.getId(), + batch.getCallerType(), + error + ); + } catch (RuntimeException error) { + LOG.error("回收超时导入批次失败: batchId={}", batch.getId(), error); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskMqConstants.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskMqConstants.java index 0ae88649..8700a0fd 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskMqConstants.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskMqConstants.java @@ -13,6 +13,8 @@ public final class DocumentImportTaskMqConstants { public static final String PARSE_TOPIC = "knowledge-document-parse"; public static final String PARSE_GROUP = "knowledge-document-parse-group"; + public static final String SPLIT_TOPIC = "knowledge-document-split"; + public static final String SPLIT_GROUP = "knowledge-document-split-group"; public static final String INDEX_TOPIC = "knowledge-document-index"; public static final String INDEX_GROUP = "knowledge-document-index-group"; } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskStatusStreamService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskStatusStreamService.java index aab1d578..4c571e0a 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskStatusStreamService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskStatusStreamService.java @@ -147,6 +147,7 @@ public class DocumentImportTaskStatusStreamService { payload.put("parseCurrentStage", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_PARSE_CURRENT_STAGE)); payload.put("parseStatusMessage", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_PARSE_STATUS_MESSAGE)); payload.put("lastTaskError", document.getLastTaskError()); + payload.put("lastTaskErrorCode", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE)); payload.put("taskModifiedAt", document.getTaskModifiedAt()); return payload; } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java index e963a438..7f8b0612 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppService.java @@ -30,6 +30,7 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionSynchronization; import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import tech.easyflow.ai.config.SearcherFactory; import tech.easyflow.ai.document.model.DocumentParseArtifacts; @@ -37,14 +38,21 @@ import tech.easyflow.ai.document.model.DocumentParseScenario; import tech.easyflow.ai.document.model.DocumentParseTaskInfo; import tech.easyflow.ai.document.model.DocumentParsedResult; import tech.easyflow.ai.document.model.DocumentSourceRef; +import tech.easyflow.ai.document.exception.DocumentParseBridgeException; import tech.easyflow.ai.document.service.DocumentParseBridgeService; +import tech.easyflow.ai.document.support.DocumentInputStreamSupport; import tech.easyflow.ai.documentimport.DocumentImportDtos; import tech.easyflow.ai.documentimport.DocumentImportKeys; import tech.easyflow.ai.documentimport.DocumentImportPreviewService; import tech.easyflow.ai.easyagents.CustomMultipartFile; import tech.easyflow.ai.entity.DocumentChunk; import tech.easyflow.ai.entity.DocumentCollection; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.entity.DocumentImportBatchItem; import tech.easyflow.ai.entity.DocumentImportTask; +import tech.easyflow.ai.enums.DocumentImportBatchItemStage; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; +import tech.easyflow.ai.enums.DocumentImportMode; import tech.easyflow.ai.enums.DocumentImportTaskPhase; import tech.easyflow.ai.enums.DocumentImportTaskStatus; import tech.easyflow.ai.enums.DocumentProcessStatus; @@ -55,9 +63,13 @@ import tech.easyflow.ai.mapper.DocumentMapper; import tech.easyflow.ai.service.DocumentChunkService; import tech.easyflow.ai.service.DocumentCollectionService; import tech.easyflow.ai.service.DocumentImportTaskService; +import tech.easyflow.ai.service.DocumentImportBatchItemService; import tech.easyflow.ai.service.ModelService; +import tech.easyflow.ai.service.DocumentService; import tech.easyflow.ai.support.DocumentStoreLifecycleSupport; import tech.easyflow.common.domain.Result; +import tech.easyflow.common.cache.RedisLockExecutor; +import tech.easyflow.common.cache.RedisLockExecutor.LockHandle; import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.util.FileUtil; import tech.easyflow.common.util.StringUtil; @@ -67,19 +79,27 @@ import javax.annotation.Resource; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; import java.net.URLConnection; +import java.time.Duration; import java.util.Base64; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -99,7 +119,19 @@ public class KnowledgeDocumentImportTaskAppService { private static final String OFFICE_PPTX_PAGE_STRATEGY = "OFFICE_PPTX_PAGE"; private static final String OFFICE_XLSX_ROW_WINDOW_STRATEGY = "OFFICE_XLSX_ROW_WINDOW"; private static final String SEARCH_RENDER_MARKDOWN_METADATA_KEY = "renderMarkdown"; + private static final String TASK_ERROR_PARSE_SERVICE_UNAVAILABLE = "parse_service_unavailable"; + private static final String TASK_ERROR_PARSE_SERVICE_TIMEOUT = "parse_service_timeout"; + private static final String TASK_ERROR_DOCUMENT_SOURCE_UNAVAILABLE = "document_source_unavailable"; + private static final String TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE = "unsupported_document_source"; + private static final String TASK_ERROR_INVALID_PARSE_REQUEST = "invalid_parse_request"; + private static final String TASK_ERROR_PARSE_FAILED = "parse_failed"; + private static final String TASK_ERROR_PENDING_TIMEOUT = "pending_timeout"; + private static final String TASK_ERROR_EXECUTION_INTERRUPTED = "execution_interrupted"; + private static final String TASK_ERROR_SPLIT_FAILED = "split_failed"; + private static final String TASK_ERROR_INDEX_FAILED = "index_failed"; + private static final Pattern HTTP_SERVER_ERROR_PATTERN = Pattern.compile("\\bstatus=5\\d{2}\\b"); private static final Pattern MARKDOWN_IMAGE_PATTERN = Pattern.compile("!\\[(?:[^\\]]*)\\]\\(([^)]+)\\)"); + private final FlexIDKeyGenerator flexIdKeyGenerator = new FlexIDKeyGenerator(); @Resource private DocumentMapper documentMapper; @@ -122,12 +154,31 @@ public class KnowledgeDocumentImportTaskAppService { @Resource private DocumentImportTaskService documentImportTaskService; + @Resource + private DocumentImportBatchItemService documentImportBatchItemService; + + @Autowired + @Lazy + private DocumentService documentService; + + @Resource + private DocumentImportBatchTracker documentImportBatchTracker; + + @Resource + private DocumentImportBulkProperties bulkProperties; + + @Resource + private RedisLockExecutor redisLockExecutor; + @Resource private DocumentImportParseMonitorProperties parseMonitorProperties; @Resource private DocumentImportPreviewService documentImportPreviewService; + @Resource + private DocumentImportChunkSnapshotService documentImportChunkSnapshotService; + @Resource private RagIngestionService ragIngestionService; @@ -137,6 +188,9 @@ public class KnowledgeDocumentImportTaskAppService { @Resource private DocumentImportParseTaskProducer parseTaskProducer; + @Resource + private DocumentImportSplitTaskProducer splitTaskProducer; + @Resource private DocumentImportIndexTaskProducer indexTaskProducer; @@ -200,17 +254,10 @@ public class KnowledgeDocumentImportTaskAppService { fileExt, shouldUseDocumentParseBridge(fileExt) ? "async-bridge" : "sync-default"); documentImportTaskStatusStreamService.publishAfterCommit(document.getId()); - if (shouldUseDocumentParseBridge(fileExt)) { - LOG.info("文档解析任务准备通过 MQ 异步投递: knowledgeId={}, documentId={}, taskId={}, fileExt={}", - knowledge.getId(), document.getId(), task.getId(), fileExt); - dispatchParseTaskAfterCommit(task.getId()); - scheduleParseTaskFallback(task.getId()); - } else { - LOG.info("文档解析任务准备同步执行: knowledgeId={}, documentId={}, taskId={}, fileExt={}", - knowledge.getId(), document.getId(), task.getId(), fileExt); - selfProxy.handleParseTask(task.getId()); - document = requireDocument(document.getId()); - } + LOG.info("文档解析任务准备异步投递: knowledgeId={}, documentId={}, taskId={}, fileExt={}", + knowledge.getId(), document.getId(), task.getId(), fileExt); + dispatchParseTaskAfterCommit(task.getId()); + scheduleParseTaskFallback(task.getId()); DocumentImportDtos.TaskCreateResponse response = new DocumentImportDtos.TaskCreateResponse(); response.setDocumentId(document.getId()); @@ -219,6 +266,409 @@ public class KnowledgeDocumentImportTaskAppService { return Result.ok(response); } + /** + * 为已上传完成的批次文件创建文档与解析任务。 + * + * @param batch 导入批次 + * @param items 批次文件项 + */ + @Transactional + public void createBatchImportTasks(DocumentImportBatch batch, + List items) { + DocumentCollection knowledge = assertDocumentCollection(batch.getKnowledgeId()); + List taskIds = new ArrayList(); + for (DocumentImportBatchItem item : items) { + if (DocumentImportBatchItemStatus.SKIPPED.name().equals(item.getStatus())) { + continue; + } + if (!DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus()) + || !StringUtil.hasText(item.getFilePath())) { + throw new BusinessException("批次包含未上传完成的文件"); + } + String fileExt = normalizeFileExtension(item.getFileName(), item.getFilePath()); + assertSupportedImportFile(fileExt); + Date now = new Date(); + tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document(); + document.setId(generateId(document)); + document.setCollectionId(knowledge.getId()); + document.setDocumentPath(item.getFilePath()); + document.setTitle(item.getFileName()); + document.setDocumentType(fileExt); + document.setCreated(now); + document.setModified(now); + document.setCreatedBy(resolveOperatorId()); + document.setModifiedBy(resolveOperatorId()); + document.setProcessStatus(DocumentProcessStatus.PARSING.name()); + document.setTotalChunks(0); + document.setCompletedChunks(0); + document.setFailedChunks(0); + document.setProgressPercent(0); + document.setTaskModifiedAt(now); + Map options = buildInitialOptions(fileExt); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_MODE, batch.getImportMode()); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID, batch.getId().toString()); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID, item.getId().toString()); + options.put(DocumentImportKeys.KEY_DOCUMENT_IMPORT_RELATIVE_PATH, item.getRelativePath()); + document.setOptions(options); + documentMapper.insert(document); + + DocumentImportTask task = createTask(document, DocumentImportTaskPhase.PARSE, + buildDocumentPayload(document)); + documentImportBatchTracker.bindDocument(item.getId(), document.getId()); + taskIds.add(task.getId()); + documentImportTaskStatusStreamService.publishAfterCommit(document.getId()); + } + int initialDispatchCount = Math.min( + taskIds.size(), + Math.max(1, bulkProperties.getPerBatchParseMaxRunning()) + ); + runAfterCommit(() -> taskIds.subList(0, initialDispatchCount) + .forEach(parseTaskProducer::send)); + } + + /** + * 重试批次中所有失败文件，单个文件重试失败不会中止其他文件。 + * + * @param batchId 批次 ID + */ + public void retryBatchFailures(BigInteger batchId) { + retryBatchFailures(batchId, Set.of()); + } + + /** + * 重试批次中选定的失败文件，单个文件重试失败不会中止其他文件。 + * + * @param batchId 批次 ID + * @param fileKeys 指定文件键；为空时重试全部可恢复失败项 + */ + public void retryBatchFailures(BigInteger batchId, Set fileKeys) { + QueryWrapper query = QueryWrapper.create() + .eq(DocumentImportBatchItem::getBatchId, batchId) + .eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.FAILED.name()) + .eq(DocumentImportBatchItem::getRetryable, true); + if (fileKeys != null && !fileKeys.isEmpty()) { + query.in(DocumentImportBatchItem::getClientFileKey, fileKeys); + } + List failedItems = documentImportBatchItemService.list( + query + ); + for (DocumentImportBatchItem item : failedItems) { + try { + selfProxy.retryBatchItemInNewTransaction(item.getId()); + } catch (Exception error) { + LOG.error("批次失败项重试启动失败: batchId={}, itemId={}, documentId={}", + batchId, item.getId(), item.getDocumentId(), error); + updateBatchItem(item.getId(), + DocumentImportBatchItemStage.valueOf(item.getStage()), + DocumentImportBatchItemStatus.FAILED, + truncateError(error.getMessage())); + } + } + } + + /** + * 在独立事务中重试一个批次失败项，避免单文件异常影响同批次其他文件。 + * + * @param itemId 批次文件项 ID + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void retryBatchItemInNewTransaction(BigInteger itemId) { + retryBatchItem(documentImportBatchTracker.requireItem(itemId)); + } + + /** + * 定时重新投递待处理任务，承接容量不足和消息投递异常后的恢复。 + */ + public void dispatchPendingTasks() { + int limit = Math.max(1, bulkProperties.getPendingDispatchBatchSize()); + Date redispatchBefore = new Date(System.currentTimeMillis() + - Math.max(1L, bulkProperties.getPendingRedispatchDelay().toMillis())); + List tasks = + documentImportTaskMapper.selectPendingFairly(redispatchBefore, limit); + for (DocumentImportTask task : tasks) { + int touched = documentImportTaskMapper.touchPendingForDispatch( + task.getId(), redispatchBefore, new Date(), resolveOperatorId()); + if (touched != 1) { + continue; + } + if (DocumentImportTaskPhase.PARSE.name().equals(task.getPhase())) { + parseTaskProducer.send(task.getId()); + } else if (DocumentImportTaskPhase.SPLIT.name().equals(task.getPhase())) { + splitTaskProducer.send(task.getId()); + } else if (DocumentImportTaskPhase.INDEX.name().equals(task.getPhase())) { + indexTaskProducer.send(task.getId()); + } + } + } + + /** + * 检测超过最长排队时间的待处理任务。 + */ + public void expireTimedOutPendingTasks() { + Date cutoff = new Date(System.currentTimeMillis() + - Math.max(1L, bulkProperties.getPendingTimeout().toMillis())); + List staleTasks = documentImportTaskService.list( + QueryWrapper.create() + .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.PENDING.name()) + .le(DocumentImportTask::getCreated, cutoff) + .orderBy(DocumentImportTask::getCreated, true) + .limit(Math.max(1, bulkProperties.getPendingDispatchBatchSize())) + ); + for (DocumentImportTask task : staleTasks) { + selfProxy.expireTimedOutPendingTask(task.getId(), cutoff); + } + } + + /** + * 检测已进入解析阶段但未取得服务任务 ID 的超时提交。 + */ + public void expireTimedOutParseSubmissions() { + Date cutoff = new Date(System.currentTimeMillis() + - Math.max(1L, bulkProperties.getParseSubmitTimeout().toMillis())); + List staleTasks = documentImportTaskService.list( + QueryWrapper.create() + .eq(DocumentImportTask::getPhase, DocumentImportTaskPhase.PARSE.name()) + .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.RUNNING.name()) + .isNull(DocumentImportTask::getProviderTaskId) + .le(DocumentImportTask::getStartedAt, cutoff) + .orderBy(DocumentImportTask::getStartedAt, true) + .limit(Math.max(1, bulkProperties.getPendingDispatchBatchSize())) + ); + for (DocumentImportTask task : staleTasks) { + selfProxy.expireTimedOutParseSubmission(task.getId(), cutoff); + } + } + + /** + * 将一个未取得服务任务 ID 的超时提交标记为失败并释放并发名额。 + * + * @param taskId 任务 ID + * @param cutoff 提交超时时间边界 + */ + @Transactional + public void expireTimedOutParseSubmission(BigInteger taskId, Date cutoff) { + DocumentImportTask current = requireTask(taskId); + if (!DocumentImportTaskPhase.PARSE.name().equals(current.getPhase()) + || !DocumentImportTaskStatus.RUNNING.name().equals(current.getStatus()) + || StringUtil.hasText(current.getProviderTaskId()) + || current.getStartedAt() == null + || current.getStartedAt().after(cutoff)) { + return; + } + Date now = new Date(); + String errorMessage = "文档解析服务响应超时，请重试"; + DocumentImportTask update = new DocumentImportTask(); + update.setStatus(DocumentImportTaskStatus.FAILED.name()); + update.setErrorSummary(errorMessage); + update.setFailureCode(TASK_ERROR_PARSE_SERVICE_TIMEOUT); + update.setLeaseUntil(null); + update.setFinishedAt(now); + update.setModified(now); + update.setModifiedBy(resolveOperatorId()); + int updated = documentImportTaskMapper.updateByQuery(update, + QueryWrapper.create() + .eq(DocumentImportTask::getId, taskId) + .eq(DocumentImportTask::getPhase, DocumentImportTaskPhase.PARSE.name()) + .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.RUNNING.name()) + .eq(DocumentImportTask::getExecutionToken, current.getExecutionToken()) + .isNull(DocumentImportTask::getProviderTaskId) + .le(DocumentImportTask::getStartedAt, cutoff)); + if (updated <= 0) { + return; + } + finishRecoveredTask(current, now, errorMessage, TASK_ERROR_PARSE_SERVICE_TIMEOUT); + } + + /** + * 将超过最长排队时间的任务标记为失败。 + * + * @param taskId 任务 ID + * @param cutoff 排队超时时间边界 + */ + @Transactional + public void expireTimedOutPendingTask(BigInteger taskId, Date cutoff) { + DocumentImportTask current = requireTask(taskId); + if (!DocumentImportTaskStatus.PENDING.name().equals(current.getStatus()) + || current.getCreated() == null + || current.getCreated().after(cutoff)) { + return; + } + Date now = new Date(); + String errorMessage = "任务排队超时，请重试"; + DocumentImportTask update = new DocumentImportTask(); + update.setStatus(DocumentImportTaskStatus.FAILED.name()); + update.setErrorSummary(errorMessage); + update.setFailureCode(TASK_ERROR_PENDING_TIMEOUT); + update.setFinishedAt(now); + update.setModified(now); + update.setModifiedBy(resolveOperatorId()); + int updated = documentImportTaskMapper.updateByQuery(update, + QueryWrapper.create() + .eq(DocumentImportTask::getId, taskId) + .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.PENDING.name()) + .le(DocumentImportTask::getCreated, cutoff)); + if (updated <= 0) { + return; + } + finishRecoveredTask(current, now, errorMessage, TASK_ERROR_PENDING_TIMEOUT); + } + + /** + * 检测所有长时间无心跳的运行任务，避免孤儿任务持续占用并发名额。 + */ + public void recoverInterruptedTasks() { + Date cutoff = new Date(System.currentTimeMillis() + - Math.max(1L, bulkProperties.getInterruptionTimeout().toMillis())); + Date now = new Date(); + List staleTasks = + documentImportTaskMapper.selectExpiredRunningTasks( + now, + cutoff, + Math.max(1, bulkProperties.getPendingDispatchBatchSize()) + ); + for (DocumentImportTask task : staleTasks) { + selfProxy.recoverInterruptedTask(task.getId(), cutoff); + } + } + + /** + * 重试已完成导入项遗留的历史文档覆盖清理。 + */ + public void cleanupCompletedReplacements() { + List items = documentImportBatchItemService.list( + QueryWrapper.create() + .eq(DocumentImportBatchItem::getStatus, DocumentImportBatchItemStatus.COMPLETED.name()) + .isNotNull(DocumentImportBatchItem::getReplacedDocumentId) + .orderBy(DocumentImportBatchItem::getModified, true) + .limit(Math.max(1, bulkProperties.getPendingDispatchBatchSize())) + ); + for (DocumentImportBatchItem item : items) { + cleanupCompletedReplacement(item.getId()); + } + } + + /** + * 清理一个已被成功导入文档替代的历史文档。 + * + * @param itemId 新导入批次项 ID + */ + public void cleanupCompletedReplacement(BigInteger itemId) { + DocumentImportBatchItem item = documentImportBatchTracker.requireItem(itemId); + BigInteger replacedDocumentId = item.getReplacedDocumentId(); + if (replacedDocumentId == null) { + return; + } + LockHandle lockHandle = redisLockExecutor.tryAcquire( + "easyflow:lock:document-import:replacement:" + replacedDocumentId, + Duration.ZERO, + Duration.ofSeconds(30) + ); + if (lockHandle == null) { + return; + } + try { + item = documentImportBatchTracker.requireItem(itemId); + replacedDocumentId = item.getReplacedDocumentId(); + if (replacedDocumentId == null) { + return; + } + tech.easyflow.ai.entity.Document replacement = + documentMapper.selectOneById(item.getDocumentId()); + tech.easyflow.ai.entity.Document historical = + documentMapper.selectOneById(replacedDocumentId); + if (replacement == null || historical == null + || replacedDocumentId.equals(item.getDocumentId())) { + documentImportBatchTracker.clearReplacement(itemId, replacedDocumentId); + return; + } + if (!DocumentProcessStatus.COMPLETED.name().equals(replacement.getProcessStatus())) { + return; + } + if (!Objects.equals(replacement.getCollectionId(), historical.getCollectionId())) { + LOG.error("拒绝跨知识库清理被覆盖文档: itemId={}, documentId={}, replacedDocumentId={}", + itemId, item.getDocumentId(), replacedDocumentId); + documentImportBatchTracker.clearReplacement(itemId, replacedDocumentId); + return; + } + if (!documentService.removeDoc(replacedDocumentId.toString())) { + LOG.warn("被覆盖文档暂未清理，等待下次重试: itemId={}, replacedDocumentId={}", + itemId, replacedDocumentId); + return; + } + documentMapper.deleteById(replacedDocumentId); + documentImportBatchTracker.clearReplacement(itemId, replacedDocumentId); + LOG.info("被覆盖的历史文档已清理: itemId={}, documentId={}, replacedDocumentId={}", + itemId, item.getDocumentId(), replacedDocumentId); + } catch (Exception error) { + LOG.error("清理被覆盖的历史文档失败，等待下次重试: itemId={}, replacedDocumentId={}", + itemId, replacedDocumentId, error); + } finally { + lockHandle.release(); + } + } + + /** + * 将一个失去心跳的运行任务标记为失败并释放并发名额。 + * + * @param taskId 任务 ID + * @param cutoff 失联时间边界 + */ + @Transactional + public void recoverInterruptedTask(BigInteger taskId, Date cutoff) { + DocumentImportTask current = requireTask(taskId); + Date now = new Date(); + boolean leaseExpired = current.getLeaseUntil() != null + ? !current.getLeaseUntil().after(now) + : current.getModified() != null + && !current.getModified().after(cutoff); + if (!DocumentImportTaskStatus.RUNNING.name().equals(current.getStatus()) + || !leaseExpired) { + return; + } + String errorMessage = current.getBatchId() == null + ? "任务执行中断，请重试" + : "任务执行中断，请继续批次"; + int updated = documentImportTaskMapper.failExpiredOwned( + taskId, + current.getExecutionToken(), + errorMessage, + TASK_ERROR_EXECUTION_INTERRUPTED, + now, + cutoff, + resolveOperatorId() + ); + if (updated <= 0) { + return; + } + finishRecoveredTask(current, now, errorMessage, TASK_ERROR_EXECUTION_INTERRUPTED); + } + + /** + * 收口调度器已经原子置为失败的任务，并同步文档与批次状态。 + * + * @param task 任务实体 + * @param now 收口时间 + * @param errorMessage 用户可见错误信息 + * @param errorCode 稳定错误码 + */ + private void finishRecoveredTask(DocumentImportTask task, + Date now, + String errorMessage, + String errorCode) { + DocumentImportBatchItemStage stage = toBatchItemStage(task.getPhase()); + tech.easyflow.ai.entity.Document document = documentMapper.selectOneById(task.getDocumentId()); + if (document == null) { + LOG.warn("自动清理缺少关联文档的导入任务: taskId={}, documentId={}, phase={}, batchId={}", + task.getId(), task.getDocumentId(), task.getPhase(), task.getBatchId()); + } else { + document.setProcessStatus(toFailureProcessStatus(task.getPhase()).name()); + document.setProgressPercent(0); + setDocumentTaskError(document, errorMessage, errorCode); + persistDocumentTaskState(document, now); + } + finishRecoveredBatchState(task, stage, errorMessage, errorCode); + } + /** * 查询任务详情。 * @@ -295,18 +745,42 @@ public class KnowledgeDocumentImportTaskAppService { throw new BusinessException("当前文档状态不允许开始向量化"); } - DocumentImportDtos.PreviewSession session = resolveIndexPreviewSession(knowledge, document, request.getPreviewSessionId()); + String existingSnapshotPath = optionAsString( + document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH); + DocumentImportDtos.PreviewSession session = resolveIndexPreviewSession( + knowledge, + document, + request.getPreviewSessionId(), + StringUtil.hasText(request.getPreviewSessionId()) ? null : existingSnapshotPath + ); int totalChunks = session.getDocumentChunks().size(); if (totalChunks <= 0) { throw new BusinessException("未生成有效分块，无法开始向量化"); } - mergeDocumentPreviewOptions(document, session); - updateDocumentIndexing(document, totalChunks); - DocumentImportTask task = createTask(document, DocumentImportTaskPhase.INDEX, Map.of( - "previewSessionId", session.getSessionId(), - "totalChunks", totalChunks - )); + boolean createdSnapshot = StringUtil.hasText(request.getPreviewSessionId()) + || !StringUtil.hasText(existingSnapshotPath); + String snapshotPath = createdSnapshot + ? documentImportChunkSnapshotService.save(session) + : existingSnapshotPath; + mergeDocumentPreviewOptions(document, session, snapshotPath); + if (!claimDocumentIndexing(document, totalChunks)) { + if (createdSnapshot) { + deleteChunkSnapshotAfterCompletion(snapshotPath); + } + throw new BusinessException("文档状态已变化，请刷新后重试"); + } + Map payload = new LinkedHashMap(); + if (StringUtil.hasText(session.getSessionId())) { + payload.put("previewSessionId", session.getSessionId()); + } + payload.put("chunkSnapshotPath", snapshotPath); + payload.put("totalChunks", totalChunks); + DocumentImportTask task = createTask(document, DocumentImportTaskPhase.INDEX, payload); + updateBatchItem(task.getBatchItemId(), + DocumentImportBatchItemStage.INDEX, + DocumentImportBatchItemStatus.PENDING, + null); LOG.info("文档向量化任务已创建: knowledgeId={}, documentId={}, taskId={}, previewSessionId={}, totalChunks={}", knowledge.getId(), document.getId(), task.getId(), session.getSessionId(), totalChunks); dispatchIndexTaskAfterCommit(task.getId()); @@ -331,17 +805,18 @@ public class KnowledgeDocumentImportTaskAppService { if (!DocumentProcessStatus.PARSE_FAILED.name().equals(document.getProcessStatus())) { throw new BusinessException("当前文档不支持重试解析"); } - String fileExt = normalizeFileExtension(document.getTitle(), document.getDocumentPath()); + if (!claimDocumentParseRetry(document)) { + throw new BusinessException("文档状态已变化，请刷新后重试"); + } resetDocumentForParseRetry(document); DocumentImportTask task = createTask(document, DocumentImportTaskPhase.PARSE, buildDocumentPayload(document)); - if (shouldUseDocumentParseBridge(fileExt)) { - dispatchParseTaskAfterCommit(task.getId()); - scheduleParseTaskFallback(task.getId()); - return Result.ok(buildTaskStartResponse(task, DocumentProcessStatus.PARSING)); - } - selfProxy.handleParseTask(task.getId()); - tech.easyflow.ai.entity.Document current = requireDocument(document.getId()); - return Result.ok(buildTaskStartResponse(task, DocumentProcessStatus.valueOf(current.getProcessStatus()))); + updateBatchItem(task.getBatchItemId(), + DocumentImportBatchItemStage.PARSE, + DocumentImportBatchItemStatus.PENDING, + null); + dispatchParseTaskAfterCommit(task.getId()); + scheduleParseTaskFallback(task.getId()); + return Result.ok(buildTaskStartResponse(task, DocumentProcessStatus.PARSING)); } /** @@ -358,12 +833,200 @@ public class KnowledgeDocumentImportTaskAppService { return startIndexTask(startRequest); } + /** + * 根据文档失败阶段执行统一重试。 + * + * @param request 重试请求 + * @return 重试后的任务状态 + */ + @Transactional + public Result retryFailedTask( + DocumentImportDtos.TaskRetryRequest request) { + DocumentCollection knowledge = assertDocumentCollection(request.getKnowledgeId()); + tech.easyflow.ai.entity.Document document = + requireDocumentForKnowledge(request.getDocumentId(), knowledge.getId()); + String status = document.getProcessStatus(); + if (DocumentProcessStatus.PARSE_FAILED.name().equals(status)) { + return retryParseTask(request); + } + if (DocumentProcessStatus.INDEX_FAILED.name().equals(status)) { + return retryIndexTask(request); + } + if (!DocumentProcessStatus.SPLIT_FAILED.name().equals(status)) { + throw new BusinessException("当前文档无需重试"); + } + if (!claimDocumentSplitRetry(document)) { + throw new BusinessException("文档状态已变化，请刷新后重试"); + } + BigInteger batchId = optionAsBigInteger( + document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID); + BigInteger batchItemId = optionAsBigInteger( + document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID); + DocumentImportTask splitTask = + enqueueAutomaticSplit(batchId, batchItemId, document); + DocumentImportDtos.TaskStartIndexResponse response = new DocumentImportDtos.TaskStartIndexResponse(); + response.setTaskId(splitTask.getId()); + response.setProcessStatus(DocumentProcessStatus.SPLITTING.name()); + return Result.ok(response); + } + + private void retryBatchItem(DocumentImportBatchItem item) { + if (item.getDocumentId() == null) { + throw new BusinessException("失败文件尚未生成文档"); + } + int completedRetries = item.getAttemptCount() == null ? 0 : item.getAttemptCount(); + if (completedRetries >= Math.max(1, bulkProperties.getMaxTaskAttempts()) - 1) { + documentImportBatchTracker.transitionItem( + item.getId(), + DocumentImportBatchItemStage.valueOf(item.getStage()), + DocumentImportBatchItemStatus.FAILED, + item.getErrorSummary(), + false, + 0, + item.getFailureCode() + ); + throw new BusinessException("该文件已达到最大重试次数"); + } + tech.easyflow.ai.entity.Document document = requireDocument(item.getDocumentId()); + DocumentImportDtos.TaskRetryRequest request = new DocumentImportDtos.TaskRetryRequest(); + request.setKnowledgeId(item.getKnowledgeId()); + request.setDocumentId(item.getDocumentId()); + String status = document.getProcessStatus(); + if (!DocumentProcessStatus.PARSE_FAILED.name().equals(status) + && !DocumentProcessStatus.SPLIT_FAILED.name().equals(status) + && !DocumentProcessStatus.INDEX_FAILED.name().equals(status)) { + throw new BusinessException("当前失败项状态不支持重试"); + } + retryFailedTask(request); + } + + /** + * 持久化一个自动分块任务，并在当前事务提交后投递。 + * + * @param batchId 批次 ID + * @param batchItemId 文件项 ID + * @param document 文档实体 + * @return 已创建的分块任务 + */ + private DocumentImportTask enqueueAutomaticSplit( + BigInteger batchId, + BigInteger batchItemId, + tech.easyflow.ai.entity.Document document) { + if (batchId == null || batchItemId == null) { + throw new BusinessException("自动分块任务缺少批次归属"); + } + DocumentImportBatch batch = documentImportBatchTracker.requireBatch(batchId); + Map payload = new LinkedHashMap(); + payload.put("strategyConfigJson", batch.getRequestedStrategyJson()); + Date now = new Date(); + document.setProcessStatus(DocumentProcessStatus.SPLITTING.name()); + document.setProgressPercent(0); + clearDocumentTaskError(document); + persistDocumentTaskState(document, now); + updateBatchItem(batchItemId, + DocumentImportBatchItemStage.SPLIT, + DocumentImportBatchItemStatus.PENDING, + null); + DocumentImportTask task = + createTask(document, DocumentImportTaskPhase.SPLIT, payload); + dispatchSplitTaskAfterCommit(task.getId()); + scheduleSplitTaskFallback(task.getId()); + return task; + } + + /** + * 处理持久化分块任务消息。 + * + * @param taskId 任务 ID + */ + public void handleSplitTask(BigInteger taskId) { + DocumentImportTask task = requireTask(taskId); + if (!DocumentImportTaskPhase.SPLIT.name().equals(task.getPhase())) { + LOG.warn("忽略非分块阶段任务: taskId={}, phase={}", taskId, task.getPhase()); + return; + } + if (DocumentImportTaskStatus.COMPLETED.name().equals(task.getStatus()) + || DocumentImportTaskStatus.FAILED.name().equals(task.getStatus())) { + LOG.info("分块任务已结束，跳过重复处理: taskId={}, status={}", + taskId, task.getStatus()); + return; + } + LockHandle executionLock = redisLockExecutor.tryAcquire( + "easyflow:lock:document-import:document:" + task.getDocumentId(), + Duration.ZERO, + Duration.ofMinutes(30) + ); + if (executionLock == null) { + LOG.info("文档已有导入任务执行中，延后分块任务: taskId={}, documentId={}", + taskId, task.getDocumentId()); + return; + } + String snapshotPath = null; + String previewSessionId = null; + try { + if (!selfProxy.tryMarkTaskRunning(taskId)) { + LOG.info("分块任务未抢占成功，跳过本次执行: taskId={}", taskId); + return; + } + task = requireTask(taskId); + tech.easyflow.ai.entity.Document document = + documentMapper.selectOneById(task.getDocumentId()); + if (document == null) { + finishMissingDocumentTask(task); + return; + } + updateBatchItem(task.getBatchItemId(), + DocumentImportBatchItemStage.SPLIT, + DocumentImportBatchItemStatus.RUNNING, + null); + if (!touchRunningTask(task) || !executionLock.renew()) { + throw new TaskOwnershipLostException(task.getId()); + } + + StrategyConfig requestedStrategy = resolveSplitStrategy(task); + DocumentCollection knowledge = + assertDocumentCollection(task.getKnowledgeId()); + DocumentImportDtos.PreviewSession session = + buildPreviewSessionForDocument( + knowledge, document, requestedStrategy); + previewSessionId = documentImportPreviewService.put(session); + session.setSessionId(previewSessionId); + int totalChunks = session.getDocumentChunks().size(); + if (totalChunks <= 0) { + throw new BusinessException("未生成有效分块，无法开始向量化"); + } + snapshotPath = documentImportChunkSnapshotService.save(session); + if (!touchRunningTask(task) || !executionLock.renew()) { + throw new TaskOwnershipLostException(task.getId()); + } + if (!selfProxy.completeSplitTask( + task, document, session, snapshotPath, totalChunks)) { + deleteOwnedSplitArtifacts(previewSessionId, snapshotPath); + LOG.warn("分块任务所有权已失效，忽略迟到结果: taskId={}, documentId={}", + task.getId(), document.getId()); + } + } catch (TaskOwnershipLostException ownershipLost) { + deleteOwnedSplitArtifacts(previewSessionId, snapshotPath); + LOG.warn("分块任务执行令牌已失效: taskId={}", taskId); + } catch (Exception error) { + deleteOwnedSplitArtifacts(previewSessionId, snapshotPath); + String errorMessage = truncateError(error.getMessage()); + LOG.error("文档分块任务失败: taskId={}, documentId={}", + taskId, task.getDocumentId(), error); + if (!selfProxy.failSplitTask( + task, task.getDocumentId(), errorMessage, TASK_ERROR_SPLIT_FAILED)) { + LOG.warn("分块任务所有权已失效，忽略迟到失败: taskId={}", taskId); + } + } finally { + executionLock.release(); + } + } + /** * 处理解析任务消息。 * * @param taskId 任务 ID */ - @Transactional public void handleParseTask(BigInteger taskId) { DocumentImportTask task = requireTask(taskId); if (!DocumentImportTaskPhase.PARSE.name().equals(task.getPhase())) { @@ -375,13 +1038,21 @@ public class KnowledgeDocumentImportTaskAppService { LOG.info("解析任务已结束，跳过重复处理: taskId={}, status={}", taskId, task.getStatus()); return; } - if (!tryMarkTaskRunning(taskId)) { + if (!selfProxy.tryMarkTaskRunning(taskId)) { LOG.info("解析任务未抢占成功，跳过本次执行: taskId={}", taskId); return; } task = requireTask(taskId); + tech.easyflow.ai.entity.Document document = documentMapper.selectOneById(task.getDocumentId()); + if (document == null) { + finishMissingDocumentTask(task); + return; + } + updateBatchItem(task.getBatchItemId(), + DocumentImportBatchItemStage.PARSE, + DocumentImportBatchItemStatus.RUNNING, + null); - tech.easyflow.ai.entity.Document document = requireDocument(task.getDocumentId()); LOG.info("开始执行文档解析任务: taskId={}, documentId={}, knowledgeId={}, currentStatus={}", taskId, document.getId(), task.getKnowledgeId(), document.getProcessStatus()); @@ -394,7 +1065,8 @@ public class KnowledgeDocumentImportTaskAppService { } } catch (Exception e) { LOG.error("文档解析任务失败: taskId={}, documentId={}", taskId, document.getId(), e); - markParseFailed(task, document, truncateError(e.getMessage())); + String errorCode = resolveParseFailureCode(e); + markParseFailed(task, document, resolveParseFailureMessage(e, errorCode), errorCode); } } @@ -405,6 +1077,7 @@ public class KnowledgeDocumentImportTaskAppService { QueryWrapper queryWrapper = QueryWrapper.create() .eq(DocumentImportTask::getPhase, DocumentImportTaskPhase.PARSE.name()) .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.RUNNING.name()) + .isNotNull(DocumentImportTask::getProviderTaskId) .orderBy(DocumentImportTask::getModified, true) .limit(parseMonitorProperties.getBatchSize()); List runningTasks = documentImportTaskService.list(queryWrapper); @@ -438,7 +1111,11 @@ public class KnowledgeDocumentImportTaskAppService { || !StringUtil.hasText(task.getProviderTaskId())) { return; } - tech.easyflow.ai.entity.Document document = requireDocument(task.getDocumentId()); + tech.easyflow.ai.entity.Document document = documentMapper.selectOneById(task.getDocumentId()); + if (document == null) { + finishMissingDocumentTask(task); + return; + } if (!DocumentProcessStatus.PARSING.name().equals(document.getProcessStatus())) { return; } @@ -450,7 +1127,8 @@ public class KnowledgeDocumentImportTaskAppService { syncBridgeParseTask(task, document, fileExt); } catch (Exception e) { LOG.error("文档解析任务收敛失败: taskId={}, documentId={}", taskId, document.getId(), e); - markParseFailed(task, document, truncateError(e.getMessage())); + String errorCode = resolveParseFailureCode(e); + markParseFailed(task, document, resolveParseFailureMessage(e, errorCode), errorCode); } } @@ -459,7 +1137,6 @@ public class KnowledgeDocumentImportTaskAppService { * * @param taskId 任务 ID */ - @Transactional public void handleIndexTask(BigInteger taskId) { DocumentImportTask task = requireTask(taskId); if (!DocumentImportTaskPhase.INDEX.name().equals(task.getPhase())) { @@ -471,56 +1148,99 @@ public class KnowledgeDocumentImportTaskAppService { LOG.info("向量化任务已结束，跳过重复处理: taskId={}, status={}", taskId, task.getStatus()); return; } - if (!tryMarkTaskRunning(taskId)) { - LOG.info("向量化任务未抢占成功，跳过本次执行: taskId={}", taskId); + LockHandle executionLock = redisLockExecutor.tryAcquire( + "easyflow:lock:document-import:document:" + task.getDocumentId(), + Duration.ZERO, + Duration.ofMinutes(30) + ); + if (executionLock == null) { + LOG.info("文档已有导入任务执行中，延后向量化任务: taskId={}, documentId={}", + taskId, task.getDocumentId()); return; } - task = requireTask(taskId); - - tech.easyflow.ai.entity.Document document = requireDocument(task.getDocumentId()); - LOG.info("开始执行文档向量化任务: taskId={}, documentId={}, knowledgeId={}, currentStatus={}", - taskId, document.getId(), task.getKnowledgeId(), document.getProcessStatus()); - StoreExecutionContext storeContext = null; - List storedChunks = new ArrayList(); try { - DocumentCollection knowledge = assertDocumentCollection(task.getKnowledgeId()); - DocumentImportDtos.PreviewSession session = resolveIndexPreviewSession( - knowledge, - document, - asString(task.getPayloadJson().get("previewSessionId")) - ); - List chunks = session.getDocumentChunks(); - if (chunks == null || chunks.isEmpty()) { - throw new BusinessException("预览会话无有效分块"); + if (!selfProxy.tryMarkTaskRunning(taskId)) { + LOG.info("向量化任务未抢占成功，跳过本次执行: taskId={}", taskId); + return; } + task = requireTask(taskId); + tech.easyflow.ai.entity.Document document = documentMapper.selectOneById(task.getDocumentId()); + if (document == null) { + finishMissingDocumentTask(task); + return; + } + updateBatchItem(task.getBatchItemId(), + DocumentImportBatchItemStage.INDEX, + DocumentImportBatchItemStatus.RUNNING, + null); - clearPersistedChunks(document.getId()); - storeContext = prepareStoreContext(document); - int totalChunks = chunks.size(); - int completedChunks = 0; - for (int start = 0; start < chunks.size(); start += INDEX_BATCH_SIZE) { - int end = Math.min(start + INDEX_BATCH_SIZE, chunks.size()); - List batch = new ArrayList(chunks.subList(start, end)); - LOG.info("文档向量化任务开始处理批次: taskId={}, documentId={}, batchStart={}, batchEnd={}, batchSize={}, totalChunks={}", - taskId, document.getId(), start, end, batch.size(), totalChunks); - storeDocumentChunks(storeContext, batch); - storedChunks.addAll(batch); - persistChunkBatch(document, batch); - completedChunks += batch.size(); - updateDocumentIndexProgress(document.getId(), totalChunks, completedChunks); + LOG.info("开始执行文档向量化任务: taskId={}, documentId={}, knowledgeId={}, currentStatus={}", + taskId, document.getId(), task.getKnowledgeId(), document.getProcessStatus()); + StoreExecutionContext storeContext = null; + List storedChunks = new ArrayList(); + try { + DocumentCollection knowledge = assertDocumentCollection(task.getKnowledgeId()); + DocumentImportDtos.PreviewSession session = resolveIndexPreviewSession( + knowledge, + document, + asString(task.getPayloadJson().get("previewSessionId")), + asString(task.getPayloadJson().get("chunkSnapshotPath")) + ); + List chunks = session.getDocumentChunks(); + if (chunks == null || chunks.isEmpty()) { + throw new BusinessException("预览会话无有效分块"); + } + assertUniqueChunkIds(chunks); + + if (!touchRunningTask(task) || !executionLock.renew()) { + throw new TaskOwnershipLostException(task.getId()); + } + clearPersistedChunks(document.getId()); + storeContext = prepareStoreContext(document); + int totalChunks = chunks.size(); + int completedChunks = 0; + for (int start = 0; start < chunks.size(); start += INDEX_BATCH_SIZE) { + int end = Math.min(start + INDEX_BATCH_SIZE, chunks.size()); + List batch = new ArrayList(chunks.subList(start, end)); + LOG.info("文档向量化任务开始处理批次: taskId={}, documentId={}, batchStart={}, batchEnd={}, batchSize={}, totalChunks={}", + taskId, document.getId(), start, end, batch.size(), totalChunks); + storeDocumentChunks(storeContext, batch); + storedChunks.addAll(batch); + persistChunkBatch(document, batch); + completedChunks += batch.size(); + updateDocumentIndexProgress(document.getId(), totalChunks, completedChunks); + if (!touchRunningTask(task) || !executionLock.renew()) { + throw new TaskOwnershipLostException(task.getId()); + } + } + updateKnowledgeAfterStore(storeContext); + selfProxy.completeIndexTask(task, document, totalChunks); + if (StringUtil.hasText(session.getSessionId())) { + documentImportPreviewService.remove(session.getSessionId()); + } + deleteChunkSnapshotAfterCompletion( + asString(task.getPayloadJson().get("chunkSnapshotPath"))); + } catch (TaskOwnershipLostException ownershipLost) { + // 旧执行失去 token 后不能清理可能已由新执行写入的索引或分块。 + LOG.warn("向量化任务执行令牌已失效，停止迟到回滚: taskId={}, documentId={}", + taskId, document.getId()); + } catch (Exception e) { + LOG.error("文档向量化任务失败: taskId={}, documentId={}", taskId, document.getId(), e); + if (!ownsRunningTask(task)) { + LOG.warn("向量化任务已失去执行权，跳过迟到失败和清理: taskId={}, documentId={}", + taskId, document.getId()); + return; + } + clearPersistedChunks(document.getId()); + if (storeContext != null && !storedChunks.isEmpty()) { + rollbackStoredChunks(taskId, document.getId(), storeContext, storedChunks); + } + markIndexFailed(task, document, "分块或向量化失败，请重试"); + } finally { + closeStoreContext(storeContext); } - updateKnowledgeAfterStore(storeContext); - markIndexCompleted(task, document, totalChunks); - documentImportPreviewService.remove(session.getSessionId()); - } catch (Exception e) { - LOG.error("文档向量化任务失败: taskId={}, documentId={}", taskId, document.getId(), e); - clearPersistedChunks(document.getId()); - if (storeContext != null && !storedChunks.isEmpty()) { - rollbackStoredChunks(taskId, document.getId(), storeContext, storedChunks); - } - markIndexFailed(task, document, truncateError(e.getMessage())); } finally { - closeStoreContext(storeContext); + executionLock.release(); } } @@ -595,9 +1315,10 @@ public class KnowledgeDocumentImportTaskAppService { } options.put(DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN, parsedKnowledgeContent.documentRenderMarkdown); options.put(DocumentImportKeys.KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY, parsedKnowledgeContent.parseArtifactSummary); + options.remove(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE); clearDocumentParseProgress(options); - Date now = new Date(); + boolean automaticBatch = isAutomaticBatch(task.getBatchId()); document.setContent(parsedKnowledgeContent.documentLlmContent); document.setDocumentType(sourceFormat); document.setOptions(options); @@ -607,12 +1328,123 @@ public class KnowledgeDocumentImportTaskAppService { document.setCompletedChunks(0); document.setFailedChunks(0); document.setLastTaskError(null); - persistDocumentTaskState(document, now); + if (!selfProxy.completeParseTask(task, document, automaticBatch)) { + LOG.warn("解析任务所有权已失效，忽略迟到结果: taskId={}, documentId={}", + task.getId(), document.getId()); + return; + } LOG.info("文档解析任务完成: taskId={}, documentId={}, processStatus={}, providerTaskId={}, contentLength={}", - task.getId(), document.getId(), DocumentProcessStatus.READY_FOR_SEGMENT.name(), providerTaskId, + task.getId(), document.getId(), document.getProcessStatus(), providerTaskId, parsedKnowledgeContent.documentLlmContent.length()); + } - finishTask(task, now, DocumentImportTaskStatus.COMPLETED, null); + /** + * 原子完成解析任务、文档状态与批次项状态。 + * + * @param task 解析任务 + * @param document 已填充解析结果的文档 + * @param automaticBatch 是否为自动导入批次 + * @return 是否仍持有任务终态写入权 + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean completeParseTask(DocumentImportTask task, + tech.easyflow.ai.entity.Document document, + boolean automaticBatch) { + Date now = new Date(); + if (!finishTask(task, now, DocumentImportTaskStatus.COMPLETED, null)) { + return false; + } + persistDocumentTaskState(document, now); + if (automaticBatch) { + // 先持久化 SPLIT/PENDING，再在提交后投递，确保服务退出后可由调度器恢复。 + enqueueAutomaticSplit( + task.getBatchId(), task.getBatchItemId(), document); + } else { + updateBatchItem(task.getBatchItemId(), + DocumentImportBatchItemStage.DONE, + DocumentImportBatchItemStatus.COMPLETED, + null); + } + return true; + } + + /** + * 原子完成分块任务、策略快照和 INDEX 待处理任务。 + * + * @param task 分块任务 + * @param document 文档实体 + * @param session 分块结果 + * @param snapshotPath 分块快照路径 + * @param totalChunks 分块总数 + * @return 是否仍持有本轮分块任务执行权 + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean completeSplitTask( + DocumentImportTask task, + tech.easyflow.ai.entity.Document document, + DocumentImportDtos.PreviewSession session, + String snapshotPath, + int totalChunks) { + Date now = new Date(); + if (!finishTask( + task, now, DocumentImportTaskStatus.COMPLETED, null, null)) { + return false; + } + mergeDocumentPreviewOptions(document, session, snapshotPath); + updateDocumentIndexing(document, totalChunks); + persistAppliedStrategy(task.getBatchItemId(), session.getStrategyConfig()); + + Map payload = new LinkedHashMap(); + if (StringUtil.hasText(session.getSessionId())) { + payload.put("previewSessionId", session.getSessionId()); + } + payload.put("chunkSnapshotPath", snapshotPath); + payload.put("totalChunks", totalChunks); + DocumentImportTask indexTask = + createTask(document, DocumentImportTaskPhase.INDEX, payload); + updateBatchItem(task.getBatchItemId(), + DocumentImportBatchItemStage.INDEX, + DocumentImportBatchItemStatus.PENDING, + null); + dispatchIndexTaskAfterCommit(indexTask.getId()); + scheduleIndexTaskFallback(indexTask.getId()); + return true; + } + + /** + * 原子写入分块任务、文档和批次项失败状态。 + * + * @param task 分块任务 + * @param documentId 文档 ID + * @param errorMessage 用户可见错误信息 + * @param failureCode 稳定失败码 + * @return 是否仍持有本轮分块任务执行权 + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean failSplitTask( + DocumentImportTask task, + BigInteger documentId, + String errorMessage, + String failureCode) { + Date now = new Date(); + if (!finishTask( + task, now, DocumentImportTaskStatus.FAILED, + errorMessage, failureCode)) { + return false; + } + tech.easyflow.ai.entity.Document current = requireDocument(documentId); + current.setProcessStatus(DocumentProcessStatus.SPLIT_FAILED.name()); + current.setProgressPercent(0); + setDocumentTaskError(current, errorMessage, failureCode); + persistDocumentTaskState(current, now); + updateBatchItemFailure( + task.getBatchItemId(), + DocumentImportBatchItemStage.SPLIT, + errorMessage, + failureCode, + canRetryBatchItem(task.getBatchItemId()) + ); + return true; } /** @@ -711,17 +1543,181 @@ public class KnowledgeDocumentImportTaskAppService { rewriteArtifactImageReferences(parsedResult.getArtifacts(), storedImageUrls); } + /** + * 收口关联文档已经不存在的孤儿任务。 + * + * @param task 孤儿任务 + */ + private void finishMissingDocumentTask(DocumentImportTask task) { + Date now = new Date(); + String errorMessage = "关联文档已删除，任务已自动清理"; + if (!finishTask(task, now, DocumentImportTaskStatus.FAILED, errorMessage)) { + return; + } + DocumentImportBatchItemStage stage = toBatchItemStage(task.getPhase()); + finishRecoveredBatchState( + task, stage, errorMessage, TASK_ERROR_EXECUTION_INTERRUPTED); + LOG.warn("自动清理缺少关联文档的导入任务: taskId={}, documentId={}, phase={}, batchId={}", + task.getId(), task.getDocumentId(), task.getPhase(), task.getBatchId()); + } + + /** + * 尽力同步孤儿或超时任务的批次状态，批次数据缺失时不回滚任务清理结果。 + * + * @param task 任务实体 + * @param stage 失败阶段 + * @param errorMessage 用户可见错误信息 + * @param failureCode 稳定失败码 + */ + private void finishRecoveredBatchState(DocumentImportTask task, + DocumentImportBatchItemStage stage, + String errorMessage, + String failureCode) { + try { + updateBatchItemFailure( + task.getBatchItemId(), + stage, + errorMessage, + failureCode, + canRetryBatchItem(task.getBatchItemId())); + if (task.getBatchId() != null) { + documentImportBatchTracker.markInterrupted(task.getBatchId()); + } + } catch (Exception error) { + LOG.error("同步已回收任务的批次状态失败: taskId={}, batchId={}, batchItemId={}", + task.getId(), task.getBatchId(), task.getBatchItemId(), error); + } + } + + /** + * 将底层解析异常转换为稳定错误码。 + * + * @param error 底层异常 + * @return 稳定错误码；无法归类时返回 {@code null} + */ + private String resolveParseFailureCode(Throwable error) { + Throwable current = error; + String fallbackCode = null; + while (current != null) { + if (current instanceof DocumentParseBridgeException bridgeError) { + String bridgeCode = bridgeError.getCode(); + if ("unsupported_source".equals(bridgeCode)) { + return TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE; + } + if ("request_build_failed".equals(bridgeCode)) { + return TASK_ERROR_INVALID_PARSE_REQUEST; + } + if ("source_load_failed".equals(bridgeCode)) { + return TASK_ERROR_DOCUMENT_SOURCE_UNAVAILABLE; + } + if ("service_not_enabled".equals(bridgeCode)) { + return TASK_ERROR_PARSE_SERVICE_UNAVAILABLE; + } + fallbackCode = TASK_ERROR_PARSE_FAILED; + } + if (current instanceof SocketTimeoutException) { + return TASK_ERROR_PARSE_SERVICE_TIMEOUT; + } + if (current instanceof ConnectException + || current instanceof NoRouteToHostException + || current instanceof UnknownHostException) { + return TASK_ERROR_PARSE_SERVICE_UNAVAILABLE; + } + String message = current.getMessage(); + if (StringUtil.hasText(message)) { + String normalized = message.toLowerCase(Locale.ROOT); + if (normalized.contains("timed out") + || normalized.contains("timeout") + || normalized.contains("超时")) { + return TASK_ERROR_PARSE_SERVICE_TIMEOUT; + } + if (HTTP_SERVER_ERROR_PATTERN.matcher(normalized).find() + || normalized.contains("service unavailable") + || normalized.contains("connection refused") + || normalized.contains("failed to connect") + || normalized.contains("no route to host") + || normalized.contains("unknown host")) { + return TASK_ERROR_PARSE_SERVICE_UNAVAILABLE; + } + } + current = current.getCause(); + } + return fallbackCode == null ? TASK_ERROR_PARSE_FAILED : fallbackCode; + } + + /** + * 将底层解析异常转换为可安全展示的错误信息。 + * + * @param error 底层异常 + * @param errorCode 稳定错误码 + * @return 用户可见错误信息 + */ + private String resolveParseFailureMessage(Throwable error, String errorCode) { + if (TASK_ERROR_PARSE_SERVICE_UNAVAILABLE.equals(errorCode)) { + return "文档解析服务暂不可用，请稍后重试"; + } + if (TASK_ERROR_PARSE_SERVICE_TIMEOUT.equals(errorCode)) { + return "文档解析服务响应超时，请重试"; + } + if (TASK_ERROR_DOCUMENT_SOURCE_UNAVAILABLE.equals(errorCode)) { + return "文档文件读取失败，请联系管理员"; + } + return truncateError(error == null ? null : error.getMessage()); + } + + /** + * 将解析任务标记为失败。 + * + * @param task 解析任务 + * @param document 文档实体 + * @param errorMessage 用户可见错误信息 + * @param errorCode 稳定错误码 + */ private void markParseFailed(DocumentImportTask task, tech.easyflow.ai.entity.Document document, - String errorMessage) { + String errorMessage, + String errorCode) { + KnowledgeDocumentImportTaskAppService executor = selfProxy == null ? this : selfProxy; + if (!executor.failParseTask(task, document, errorMessage, errorCode)) { + LOG.warn("解析任务所有权已失效，忽略迟到失败: taskId={}, documentId={}", + task.getId(), document.getId()); + } + } + + /** + * 原子写入解析任务、文档与批次项失败状态。 + * + * @param task 解析任务 + * @param document 文档实体 + * @param errorMessage 用户可见错误信息 + * @param errorCode 稳定错误码 + * @return 是否仍持有任务终态写入权 + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean failParseTask(DocumentImportTask task, + tech.easyflow.ai.entity.Document document, + String errorMessage, + String errorCode) { Date now = new Date(); + if (!finishTask( + task, now, DocumentImportTaskStatus.FAILED, + errorMessage, errorCode)) { + return false; + } document.setProcessStatus(DocumentProcessStatus.PARSE_FAILED.name()); - document.setLastTaskError(errorMessage); + setDocumentTaskError(document, errorMessage, errorCode); persistDocumentTaskState(document, now); LOG.warn("文档解析任务失败: taskId={}, documentId={}, processStatus={}, error={}", task.getId(), document.getId(), DocumentProcessStatus.PARSE_FAILED.name(), errorMessage); - finishTask(task, now, DocumentImportTaskStatus.FAILED, errorMessage); + updateBatchItemFailure( + task.getBatchItemId(), + DocumentImportBatchItemStage.PARSE, + errorMessage, + errorCode, + isRetryableParseFailure(errorCode) + && canRetryBatchItem(task.getBatchItemId())); + return true; } /** @@ -1150,34 +2146,72 @@ public class KnowledgeDocumentImportTaskAppService { return path.replaceFirst("^\\./+", ""); } - private void markIndexCompleted(DocumentImportTask task, - tech.easyflow.ai.entity.Document document, - int totalChunks) { + /** + * 原子完成向量化任务、文档状态与批次项状态。 + * + * @param task 向量化任务 + * @param document 文档实体 + * @param totalChunks 分块总数 + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void completeIndexTask(DocumentImportTask task, + tech.easyflow.ai.entity.Document document, + int totalChunks) { Date now = new Date(); + if (!finishTask(task, now, DocumentImportTaskStatus.COMPLETED, null)) { + throw new TaskOwnershipLostException(task.getId()); + } document.setProcessStatus(DocumentProcessStatus.COMPLETED.name()); document.setTotalChunks(totalChunks); document.setCompletedChunks(totalChunks); document.setFailedChunks(0); document.setProgressPercent(100); - document.setLastTaskError(null); + clearDocumentTaskError(document); persistDocumentTaskState(document, now); LOG.info("文档向量化任务完成: taskId={}, documentId={}, processStatus={}, totalChunks={}", task.getId(), document.getId(), DocumentProcessStatus.COMPLETED.name(), totalChunks); - finishTask(task, now, DocumentImportTaskStatus.COMPLETED, null); + updateBatchItem(task.getBatchItemId(), + DocumentImportBatchItemStage.DONE, + DocumentImportBatchItemStatus.COMPLETED, + null); } private void markIndexFailed(DocumentImportTask task, tech.easyflow.ai.entity.Document document, String errorMessage) { - tech.easyflow.ai.entity.Document current = requireDocument(document.getId()); + KnowledgeDocumentImportTaskAppService executor = selfProxy == null ? this : selfProxy; + if (!executor.failIndexTask(task, document, errorMessage)) { + LOG.warn("向量化任务所有权已失效，忽略迟到失败: taskId={}, documentId={}", + task.getId(), document.getId()); + } + } + + /** + * 原子写入向量化任务、文档与批次项失败状态。 + * + * @param task 向量化任务 + * @param document 文档实体 + * @param errorMessage 用户可见错误信息 + * @return 是否仍持有任务终态写入权 + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean failIndexTask(DocumentImportTask task, + tech.easyflow.ai.entity.Document document, + String errorMessage) { Date now = new Date(); + if (!finishTask( + task, now, DocumentImportTaskStatus.FAILED, + errorMessage, TASK_ERROR_INDEX_FAILED)) { + return false; + } + tech.easyflow.ai.entity.Document current = requireDocument(document.getId()); current.setProcessStatus(DocumentProcessStatus.INDEX_FAILED.name()); current.setTotalChunks(defaultInt(current.getTotalChunks())); current.setCompletedChunks(0); current.setFailedChunks(defaultInt(current.getTotalChunks())); current.setProgressPercent(0); - current.setLastTaskError(errorMessage); + setDocumentTaskError(current, errorMessage, TASK_ERROR_INDEX_FAILED); persistDocumentTaskState(current, now); LOG.warn("文档向量化任务失败: taskId={}, documentId={}, processStatus={}, completedChunks={}, totalChunks={}, error={}", task.getId(), @@ -1187,7 +2221,13 @@ public class KnowledgeDocumentImportTaskAppService { defaultInt(current.getTotalChunks()), errorMessage); - finishTask(task, now, DocumentImportTaskStatus.FAILED, errorMessage); + updateBatchItemFailure( + task.getBatchItemId(), + DocumentImportBatchItemStage.INDEX, + errorMessage, + TASK_ERROR_INDEX_FAILED, + canRetryBatchItem(task.getBatchItemId())); + return true; } /** @@ -1196,27 +2236,87 @@ public class KnowledgeDocumentImportTaskAppService { * @param taskId 任务 ID * @return 是否抢占成功 */ - private boolean tryMarkTaskRunning(BigInteger taskId) { - Date now = new Date(); - DocumentImportTask update = new DocumentImportTask(); - update.setStatus(DocumentImportTaskStatus.RUNNING.name()); - update.setStartedAt(now); - update.setModified(now); - update.setModifiedBy(resolveOperatorId()); + @Transactional(propagation = Propagation.REQUIRES_NEW) + public boolean tryMarkTaskRunning(BigInteger taskId) { + DocumentImportTask task = requireTask(taskId); + String phase = task.getPhase(); + String lockKey = "easyflow:lock:document-import:capacity:" + phase; + LockHandle lockHandle = redisLockExecutor.tryAcquire( + lockKey, Duration.ZERO, Duration.ofSeconds(5)); + if (lockHandle == null) { + return false; + } + try { + int globalLimit = resolvePhaseRunningLimit(phase); + long globalRunning = documentImportTaskService.count( + QueryWrapper.create() + .eq(DocumentImportTask::getPhase, phase) + .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.RUNNING.name()) + ); + if (globalRunning >= Math.max(1, globalLimit)) { + return false; + } + if (task.getBatchId() != null && DocumentImportTaskPhase.PARSE.name().equals(phase)) { + long batchRunning = documentImportTaskService.count( + QueryWrapper.create() + .eq(DocumentImportTask::getBatchId, task.getBatchId()) + .eq(DocumentImportTask::getPhase, phase) + .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.RUNNING.name()) + ); + if (batchRunning >= Math.max(1, bulkProperties.getPerBatchParseMaxRunning())) { + return false; + } + } - QueryWrapper queryWrapper = QueryWrapper.create() - .eq(DocumentImportTask::getId, taskId) - .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.PENDING.name()); - return documentImportTaskMapper.updateByQuery(update, queryWrapper) > 0; + Date now = new Date(); + String executionToken = UUID.randomUUID().toString(); + Date leaseUntil = new Date( + now.getTime() + + Math.max( + 1L, + bulkProperties.getInterruptionTimeout().toMillis())); + return documentImportTaskMapper.claimPending( + taskId, + executionToken, + leaseUntil, + now, + resolveOperatorId() + ) > 0; + } finally { + lockHandle.release(); + } } - private void updateTaskProvider(DocumentImportTask task, String providerTaskId) { - task.setProviderTaskId(providerTaskId); - task.setModified(new Date()); + /** + * 原子写入解析服务任务 ID，避免超时回收后迟到响应重新激活本地任务。 + * + * @param task 本地解析任务 + * @param providerTaskId 解析服务任务 ID + * @return 本地任务仍在运行且写入成功时返回 {@code true} + */ + private boolean updateTaskProvider(DocumentImportTask task, String providerTaskId) { + Date now = new Date(); Map payload = new LinkedHashMap(task.getPayloadJson()); payload.put("providerTaskId", providerTaskId); + DocumentImportTask update = new DocumentImportTask(); + update.setProviderTaskId(providerTaskId); + update.setPayloadJson(payload); + update.setModified(now); + update.setModifiedBy(resolveOperatorId()); + int updated = documentImportTaskMapper.updateByQuery(update, + QueryWrapper.create() + .eq(DocumentImportTask::getId, task.getId()) + .eq(DocumentImportTask::getPhase, DocumentImportTaskPhase.PARSE.name()) + .eq(DocumentImportTask::getStatus, DocumentImportTaskStatus.RUNNING.name()) + .eq(DocumentImportTask::getExecutionToken, task.getExecutionToken()) + .isNull(DocumentImportTask::getProviderTaskId)); + if (updated <= 0) { + return false; + } + task.setProviderTaskId(providerTaskId); task.setPayloadJson(payload); - documentImportTaskService.updateById(task); + task.setModified(now); + return true; } private void updateDocumentIndexing(tech.easyflow.ai.entity.Document document, int totalChunks) { @@ -1226,10 +2326,99 @@ public class KnowledgeDocumentImportTaskAppService { document.setCompletedChunks(0); document.setFailedChunks(0); document.setProgressPercent(0); - document.setLastTaskError(null); + clearDocumentTaskError(document); persistDocumentTaskState(document, now); } + /** + * 按当前失败或待处理状态原子领取向量化启动权。 + * + * @param document 文档实体 + * @param totalChunks 分块总数 + * @return 是否领取成功 + */ + private boolean claimDocumentIndexing(tech.easyflow.ai.entity.Document document, int totalChunks) { + Date now = new Date(); + String expectedStatus = document.getProcessStatus(); + tech.easyflow.ai.entity.Document update = new tech.easyflow.ai.entity.Document(); + update.setProcessStatus(DocumentProcessStatus.INDEXING.name()); + update.setTotalChunks(totalChunks); + update.setCompletedChunks(0); + update.setFailedChunks(0); + update.setProgressPercent(0); + update.setOptions(document.getOptions()); + update.setTaskModifiedAt(now); + update.setModified(now); + update.setModifiedBy(resolveOperatorId()); + int updated = documentMapper.updateByQuery(update, + QueryWrapper.create() + .eq(tech.easyflow.ai.entity.Document::getId, document.getId()) + .eq(tech.easyflow.ai.entity.Document::getProcessStatus, expectedStatus)); + if (updated <= 0) { + return false; + } + document.setProcessStatus(DocumentProcessStatus.INDEXING.name()); + document.setTotalChunks(totalChunks); + document.setCompletedChunks(0); + document.setFailedChunks(0); + document.setProgressPercent(0); + document.setTaskModifiedAt(now); + documentImportTaskStatusStreamService.publishAfterCommit(document.getId()); + return true; + } + + /** + * 按解析失败状态原子领取重试权。 + * + * @param document 文档实体 + * @return 是否领取成功 + */ + private boolean claimDocumentParseRetry(tech.easyflow.ai.entity.Document document) { + Date now = new Date(); + tech.easyflow.ai.entity.Document update = new tech.easyflow.ai.entity.Document(); + update.setProcessStatus(DocumentProcessStatus.PARSING.name()); + update.setProgressPercent(0); + update.setTaskModifiedAt(now); + update.setModified(now); + update.setModifiedBy(resolveOperatorId()); + int updated = documentMapper.updateByQuery(update, + QueryWrapper.create() + .eq(tech.easyflow.ai.entity.Document::getId, document.getId()) + .eq(tech.easyflow.ai.entity.Document::getProcessStatus, + DocumentProcessStatus.PARSE_FAILED.name())); + if (updated > 0) { + document.setProcessStatus(DocumentProcessStatus.PARSING.name()); + return true; + } + return false; + } + + /** + * 按分块失败状态原子领取重试权。 + * + * @param document 文档实体 + * @return 是否领取成功 + */ + private boolean claimDocumentSplitRetry(tech.easyflow.ai.entity.Document document) { + Date now = new Date(); + tech.easyflow.ai.entity.Document update = new tech.easyflow.ai.entity.Document(); + update.setProcessStatus(DocumentProcessStatus.SPLITTING.name()); + update.setProgressPercent(0); + update.setTaskModifiedAt(now); + update.setModified(now); + update.setModifiedBy(resolveOperatorId()); + int updated = documentMapper.updateByQuery(update, + QueryWrapper.create() + .eq(tech.easyflow.ai.entity.Document::getId, document.getId()) + .eq(tech.easyflow.ai.entity.Document::getProcessStatus, + DocumentProcessStatus.SPLIT_FAILED.name())); + if (updated > 0) { + document.setProcessStatus(DocumentProcessStatus.SPLITTING.name()); + return true; + } + return false; + } + private void updateDocumentIndexProgress(BigInteger documentId, int totalChunks, int completedChunks) { int progressPercent = Math.min(100, totalChunks <= 0 ? 0 : (completedChunks * 100) / totalChunks); tech.easyflow.ai.entity.Document document = requireDocument(documentId); @@ -1245,7 +2434,7 @@ public class KnowledgeDocumentImportTaskAppService { private void resetDocumentForParseRetry(tech.easyflow.ai.entity.Document document) { Date now = new Date(); document.setProcessStatus(DocumentProcessStatus.PARSING.name()); - document.setLastTaskError(null); + clearDocumentTaskError(document); document.setProgressPercent(0); Map options = copyOptions(document.getOptions()); clearDocumentParseProgress(options); @@ -1303,15 +2492,47 @@ public class KnowledgeDocumentImportTaskAppService { * @param status 终态 * @param errorSummary 错误摘要 */ - private void finishTask(DocumentImportTask task, - Date now, - DocumentImportTaskStatus status, - String errorSummary) { - task.setStatus(status.name()); - task.setErrorSummary(errorSummary); - task.setFinishedAt(now); - task.setModified(now); - documentImportTaskService.updateById(task, false); + private boolean finishTask(DocumentImportTask task, + Date now, + DocumentImportTaskStatus status, + String errorSummary) { + return finishTask(task, now, status, errorSummary, null); + } + + /** + * 使用执行令牌收口任务终态。 + * + * @param task 任务实体 + * @param now 当前时间 + * @param status 终态 + * @param errorSummary 错误摘要 + * @param failureCode 稳定失败码 + * @return 是否仍持有执行权 + */ + private boolean finishTask(DocumentImportTask task, + Date now, + DocumentImportTaskStatus status, + String errorSummary, + String failureCode) { + int updated = documentImportTaskMapper.finishOwned( + task.getId(), + task.getExecutionToken(), + status.name(), + errorSummary, + failureCode, + now, + resolveOperatorId() + ); + if (updated > 0) { + task.setStatus(status.name()); + task.setErrorSummary(errorSummary); + task.setFailureCode(failureCode); + task.setLeaseUntil(null); + task.setFinishedAt(now); + task.setModified(now); + return true; + } + return false; } private DocumentImportDtos.PreviewSession buildPreviewSessionForDocument(DocumentCollection knowledge, @@ -2000,7 +3221,8 @@ public class KnowledgeDocumentImportTaskAppService { } private void mergeDocumentPreviewOptions(tech.easyflow.ai.entity.Document document, - DocumentImportDtos.PreviewSession session) { + DocumentImportDtos.PreviewSession session, + String chunkSnapshotPath) { Map options = copyOptions(document.getOptions()); options.put(DocumentImportKeys.KEY_DOCUMENT_STRATEGY_CODE, session.getStrategyConfig().getStrategyCode()); options.put(DocumentImportKeys.KEY_DOCUMENT_STRATEGY_LABEL, resolveStrategyLabel(session.getSourceFormat(), session.getStrategyConfig())); @@ -2009,18 +3231,29 @@ public class KnowledgeDocumentImportTaskAppService { session.getAnalysis() == null ? new LinkedHashMap() : session.getAnalysis().getFeatures()); options.put(DocumentImportKeys.KEY_DOCUMENT_SOURCE_FILE_EXT, session.getSourceFormat()); options.put(DocumentImportKeys.KEY_DOCUMENT_PREVIEW_VERSION, "v2"); + if (StringUtil.hasText(chunkSnapshotPath)) { + options.put(DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH, chunkSnapshotPath); + } document.setOptions(options); } private DocumentImportDtos.PreviewSession resolveIndexPreviewSession(DocumentCollection knowledge, tech.easyflow.ai.entity.Document document, - String previewSessionId) { + String previewSessionId, + String chunkSnapshotPath) { + String persistedSnapshotPath = StringUtil.hasText(chunkSnapshotPath) + ? chunkSnapshotPath + : optionAsString(document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH); + if (StringUtil.hasText(persistedSnapshotPath)) { + DocumentImportDtos.PreviewSession session = + documentImportChunkSnapshotService.load(persistedSnapshotPath); + assertPreviewSessionOwner(knowledge, document, session); + return session; + } if (StringUtil.hasText(previewSessionId)) { DocumentImportDtos.PreviewSession session = documentImportPreviewService.getRequired(previewSessionId); - if (!knowledge.getId().equals(session.getKnowledgeId()) || !document.getId().equals(session.getDocumentId())) { - throw new BusinessException("预览会话与当前文档不匹配"); - } + assertPreviewSessionOwner(knowledge, document, session); return session; } StrategyConfig storedStrategy = readStoredStrategy(document); @@ -2030,6 +3263,22 @@ public class KnowledgeDocumentImportTaskAppService { return rebuilt; } + /** + * 校验分块预览或快照归属，防止跨知识库复用分块。 + * + * @param knowledge 当前知识库 + * @param document 当前文档 + * @param session 待校验会话 + */ + private void assertPreviewSessionOwner(DocumentCollection knowledge, + tech.easyflow.ai.entity.Document document, + DocumentImportDtos.PreviewSession session) { + if (session == null || !knowledge.getId().equals(session.getKnowledgeId()) + || !document.getId().equals(session.getDocumentId())) { + throw new BusinessException("分块快照与当前文档不匹配"); + } + } + @SuppressWarnings("unchecked") private StrategyConfig readStoredStrategy(tech.easyflow.ai.entity.Document document) { Object snapshot = document.getOptions() == null ? null : document.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_STRATEGY_SNAPSHOT); @@ -2049,12 +3298,11 @@ public class KnowledgeDocumentImportTaskAppService { private List buildDocumentChunks(tech.easyflow.ai.entity.Document document, List previewChunks) { - FlexIDKeyGenerator flexIDKeyGenerator = new FlexIDKeyGenerator(); List chunks = new ArrayList(); for (int i = 0; i < previewChunks.size(); i++) { RagChunk previewChunk = previewChunks.get(i); DocumentChunk chunk = new DocumentChunk(); - chunk.setId(new BigInteger(String.valueOf(flexIDKeyGenerator.generate(chunk, null)))); + chunk.setId(generateId(chunk)); chunk.setDocumentId(document.getId()); chunk.setDocumentCollectionId(document.getCollectionId()); chunk.setContent(previewChunk.getContent()); @@ -2088,9 +3336,15 @@ public class KnowledgeDocumentImportTaskAppService { DocumentImportTask task = new DocumentImportTask(); task.setDocumentId(document.getId()); task.setKnowledgeId(document.getCollectionId()); + task.setBatchId(optionAsBigInteger( + document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ID)); + task.setBatchItemId(optionAsBigInteger( + document.getOptions(), DocumentImportKeys.KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID)); task.setPhase(phase.name()); task.setStatus(DocumentImportTaskStatus.PENDING.name()); task.setPayloadJson(payload); + task.setAttemptNo(0); + task.setVersion(0); task.setCreated(now); task.setModified(now); task.setCreatedBy(resolveOperatorId()); @@ -2099,6 +3353,251 @@ public class KnowledgeDocumentImportTaskAppService { return task; } + private BigInteger optionAsBigInteger(Map options, String key) { + if (options == null || !StringUtil.hasText(key)) { + return null; + } + Object value = options.get(key); + if (value instanceof BigInteger) { + return (BigInteger) value; + } + String text = asString(value); + if (!StringUtil.hasText(text)) { + return null; + } + try { + return new BigInteger(text); + } catch (NumberFormatException error) { + return null; + } + } + + /** + * 从文档选项读取字符串。 + * + * @param options 文档选项 + * @param key 选项键 + * @return 字符串值 + */ + private String optionAsString(Map options, String key) { + return options == null ? null : asString(options.get(key)); + } + + private boolean isAutomaticBatch(BigInteger batchId) { + return batchId != null && documentImportBatchTracker.isAutoBatch(batchId); + } + + /** + * 获取阶段对应的全局运行上限。 + * + * @param phase 任务阶段 + * @return 全局运行上限 + */ + private int resolvePhaseRunningLimit(String phase) { + if (DocumentImportTaskPhase.PARSE.name().equals(phase)) { + return bulkProperties.getParseMaxRunning(); + } + if (DocumentImportTaskPhase.SPLIT.name().equals(phase)) { + return bulkProperties.getSplitMaxRunning(); + } + return bulkProperties.getIndexMaxRunning(); + } + + /** + * 将任务阶段映射为批次文件项阶段。 + * + * @param phase 任务阶段 + * @return 批次文件项阶段 + */ + private DocumentImportBatchItemStage toBatchItemStage(String phase) { + if (DocumentImportTaskPhase.PARSE.name().equals(phase)) { + return DocumentImportBatchItemStage.PARSE; + } + if (DocumentImportTaskPhase.SPLIT.name().equals(phase)) { + return DocumentImportBatchItemStage.SPLIT; + } + return DocumentImportBatchItemStage.INDEX; + } + + /** + * 将任务阶段映射为文档失败状态。 + * + * @param phase 任务阶段 + * @return 文档失败状态 + */ + private DocumentProcessStatus toFailureProcessStatus(String phase) { + if (DocumentImportTaskPhase.PARSE.name().equals(phase)) { + return DocumentProcessStatus.PARSE_FAILED; + } + if (DocumentImportTaskPhase.SPLIT.name().equals(phase)) { + return DocumentProcessStatus.SPLIT_FAILED; + } + return DocumentProcessStatus.INDEX_FAILED; + } + + /** + * 从分块任务载荷恢复提交时的策略快照。 + * + * @param task 分块任务 + * @return 请求策略；未传时返回默认自适应策略 + */ + private StrategyConfig resolveSplitStrategy(DocumentImportTask task) { + String strategyJson = + asString(task.getPayloadJson().get("strategyConfigJson")); + if (!StringUtil.hasText(strategyJson) + || "null".equalsIgnoreCase(strategyJson.trim())) { + return StrategyConfig.defaults(); + } + try { + StrategyConfig strategy = + JSON.parseObject(strategyJson, StrategyConfig.class); + return strategy == null ? StrategyConfig.defaults() : strategy; + } catch (RuntimeException error) { + throw new BusinessException("分块策略格式错误"); + } + } + + /** + * 持久化批次文件实际采用的策略。 + * + * @param itemId 批次文件项 ID + * @param strategyConfig 实际策略 + */ + private void persistAppliedStrategy( + BigInteger itemId, + StrategyConfig strategyConfig) { + if (itemId == null || strategyConfig == null) { + return; + } + DocumentImportBatchItem update = new DocumentImportBatchItem(); + update.setId(itemId); + update.setAppliedStrategyCode(strategyConfig.getStrategyCode()); + update.setStrategySnapshotJson( + JSON.toJSONString(strategyConfigToMap(strategyConfig))); + update.setModified(new Date()); + update.setModifiedBy(resolveOperatorId()); + // 策略回写使用部分实体，仅更新已设置字段，保留批次项其余非空列。 + documentImportBatchItemService.updateById(update); + } + + /** + * 清理本轮分块执行独占创建的临时工件。 + * + * @param previewSessionId 预览会话 ID + * @param snapshotPath 分块快照路径 + */ + private void deleteOwnedSplitArtifacts( + String previewSessionId, + String snapshotPath) { + if (StringUtil.hasText(previewSessionId)) { + documentImportPreviewService.remove(previewSessionId); + } + if (StringUtil.hasText(snapshotPath)) { + deleteChunkSnapshotAfterCompletion(snapshotPath); + } + } + + /** + * 判断批次项是否仍未达到最大尝试次数。 + * + * @param itemId 批次文件项 ID + * @return 是否允许继续重试 + */ + private boolean canRetryBatchItem(BigInteger itemId) { + if (itemId == null) { + return false; + } + DocumentImportBatchItem item = + documentImportBatchTracker.requireItem(itemId); + int completedRetries = + item.getAttemptCount() == null ? 0 : item.getAttemptCount(); + return completedRetries + < Math.max(1, bulkProperties.getMaxTaskAttempts()) - 1; + } + + /** + * 判断历史批次失败项能否按当前失败分类恢复重试资格。 + * + *

用于兼容修复前已经写入 {@code retryable=false} 的失败项。 + * 新失败项仍在写入终态时持久化重试资格，避免正常路径重复计算。

+ * + * @param item 批次失败项 + * @return 是否可以恢复重试资格 + */ + public boolean isRecoverableBatchFailure(DocumentImportBatchItem item) { + if (item == null + || !DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus()) + || (item.getAttemptCount() != null + && item.getAttemptCount() + >= Math.max(1, bulkProperties.getMaxTaskAttempts()) - 1)) { + return false; + } + DocumentImportBatchItemStage stage; + try { + stage = DocumentImportBatchItemStage.valueOf(item.getStage()); + } catch (IllegalArgumentException | NullPointerException error) { + return false; + } + return switch (stage) { + case PARSE -> isRetryableParseFailure(item.getFailureCode()); + case SPLIT, INDEX -> true; + case UPLOAD, DONE -> false; + }; + } + + /** + * 判断解析失败是否属于可恢复故障。 + * + *

只有已明确识别的输入或请求问题禁止重试。文档源暂时不可用、 + * 解析服务异常和未识别的系统错误允许人工重试，并继续受最大尝试 + * 次数约束，便于服务恢复或代码修复后沿用原任务继续处理。

+ * + * @param failureCode 稳定失败码 + * @return 是否可恢复 + */ + private boolean isRetryableParseFailure(String failureCode) { + return !TASK_ERROR_UNSUPPORTED_DOCUMENT_SOURCE.equals(failureCode) + && !TASK_ERROR_INVALID_PARSE_REQUEST.equals(failureCode); + } + + private void updateBatchItem(BigInteger itemId, + DocumentImportBatchItemStage stage, + DocumentImportBatchItemStatus status, + String errorSummary) { + if (itemId != null) { + documentImportBatchTracker.updateItem(itemId, stage, status, errorSummary); + } + } + + /** + * 更新批次文件项失败状态和稳定失败码。 + * + * @param itemId 批次文件项 ID + * @param stage 失败阶段 + * @param errorSummary 错误摘要 + * @param failureCode 稳定失败码 + * @param retryable 是否允许重试 + */ + private void updateBatchItemFailure( + BigInteger itemId, + DocumentImportBatchItemStage stage, + String errorSummary, + String failureCode, + boolean retryable) { + if (itemId == null) { + return; + } + documentImportBatchTracker.transitionItem( + itemId, + stage, + DocumentImportBatchItemStatus.FAILED, + errorSummary, + retryable, + 0, + failureCode + ); + } + private Map buildFilePayload(DocumentImportDtos.TaskCreateRequest request) { Map payload = new LinkedHashMap(); payload.put("filePath", request.getFilePath()); @@ -2189,16 +3688,23 @@ public class KnowledgeDocumentImportTaskAppService { StoreExecutionContext storeContext, List documentChunks) { try { - List ids = new ArrayList(); + Set uniqueIds = new LinkedHashSet(); for (DocumentChunk chunk : documentChunks) { - ids.add(chunk.getId()); + if (chunk != null && chunk.getId() != null) { + uniqueIds.add(chunk.getId()); + } } + List ids = new ArrayList(uniqueIds); LOG.warn("开始回滚文档向量化外部索引: taskId={}, documentId={}, knowledgeId={}, chunkCount={}", taskId, documentId, storeContext == null || storeContext.knowledge == null ? null : storeContext.knowledge.getId(), ids.size()); - storeContext.documentStore.delete(ids, storeContext.options); + StoreResult deleteResult = storeContext.documentStore.delete(ids, storeContext.options); + if (deleteResult == null || !deleteResult.isSuccess()) { + String failReason = deleteResult == null ? "未返回结果" : deleteResult.getFailReason(); + throw new IllegalStateException("向量存储回滚失败: " + failReason); + } if (storeContext.searcher != null) { for (BigInteger id : ids) { storeContext.searcher.deleteDocument(id); @@ -2348,27 +3854,74 @@ public class KnowledgeDocumentImportTaskAppService { private void submitBridgeParseTask(DocumentImportTask task, tech.easyflow.ai.entity.Document document, String fileExt) { - DocumentSourceRef sourceRef = new DocumentSourceRef(); - sourceRef.setFileName(document.getTitle()); - sourceRef.setFilePath(document.getDocumentPath()); - sourceRef.setContentType(resolveBridgeContentType(fileExt)); try { + DocumentSourceRef sourceRef = buildBridgeSourceRef(task, document, fileExt); LOG.info("文档解析桥接任务开始提交: taskId={}, documentId={}, fileName={}, fileExt={}", task.getId(), document.getId(), document.getTitle(), fileExt); String providerTaskId = documentParseBridgeService.submit(sourceRef, DocumentParseScenario.KNOWLEDGE_IMPORT).getTaskId(); if (!StringUtil.hasText(providerTaskId)) { throw new BusinessException("文档解析服务未返回任务ID"); } - updateTaskProvider(task, providerTaskId); + if (!updateTaskProvider(task, providerTaskId)) { + LOG.warn("解析任务提交完成时本地任务已结束，忽略迟到结果: taskId={}, documentId={}, providerTaskId={}", + task.getId(), document.getId(), providerTaskId); + return; + } LOG.info("文档解析桥接任务提交完成: taskId={}, documentId={}, providerTaskId={}", task.getId(), document.getId(), providerTaskId); } catch (BusinessException e) { throw e; } catch (Exception e) { - throw new BusinessException("文档解析失败：" + e.getMessage()); + throw new BusinessException(500, 1, "文档解析失败", e); } } + /** + * 构建解析桥接文档源。批次上传文件由服务端生成存储地址，直接通过受信任存储服务读取； + * 其他 URL 继续交由解析桥接层执行公网地址校验。 + * + * @param task 导入任务 + * @param document 文档实体 + * @param fileExt 文件后缀 + * @return 解析桥接文档源 + * @throws DocumentParseBridgeException 批次文件读取失败或超过大小限制时抛出 + */ + private DocumentSourceRef buildBridgeSourceRef(DocumentImportTask task, + tech.easyflow.ai.entity.Document document, + String fileExt) { + DocumentSourceRef sourceRef = buildBridgeSourceMetadata(document, fileExt); + if (task.getBatchId() == null) { + sourceRef.setFilePath(document.getDocumentPath()); + return sourceRef; + } + try (InputStream inputStream = storageService.readStream(document.getDocumentPath())) { + byte[] contentBytes = DocumentInputStreamSupport.readBytes( + inputStream, + bulkProperties.getMaxFileSize().toBytes() + ); + sourceRef.setContentBytes(contentBytes); + sourceRef.setSize((long) contentBytes.length); + return sourceRef; + } catch (Exception error) { + throw DocumentParseBridgeException.sourceLoadFailed("读取批量导入文件失败", error); + } + } + + /** + * 构建用于解析服务精确路由的文档源元数据。 + * + * @param document 文档实体 + * @param fileExt 文件后缀 + * @return 不包含文档内容的源元数据 + */ + private DocumentSourceRef buildBridgeSourceMetadata(tech.easyflow.ai.entity.Document document, + String fileExt) { + DocumentSourceRef sourceRef = new DocumentSourceRef(); + sourceRef.setFileName(document.getTitle()); + sourceRef.setContentType(resolveBridgeContentType(fileExt)); + return sourceRef; + } + /** * 单次查询桥接解析任务状态并收敛结果。 * @@ -2380,7 +3933,11 @@ public class KnowledgeDocumentImportTaskAppService { tech.easyflow.ai.entity.Document document, String fileExt) { try { - DocumentParseTaskInfo taskInfo = documentParseBridgeService.queryTaskInfo(task.getProviderTaskId()); + DocumentSourceRef sourceMetadata = buildBridgeSourceMetadata(document, fileExt); + DocumentParseTaskInfo taskInfo = documentParseBridgeService.queryTaskInfo( + task.getProviderTaskId(), + sourceMetadata + ); String providerStatus = taskInfo == null ? null : taskInfo.getStatus(); LOG.info("文档解析桥接任务单次收敛: taskId={}, documentId={}, providerTaskId={}, providerStatus={}, hasResult={}, error={}", task.getId(), @@ -2391,7 +3948,7 @@ public class KnowledgeDocumentImportTaskAppService { taskInfo == null ? null : taskInfo.getError()); if (isTaskSuccess(providerStatus)) { DocumentParsedResult result = taskInfo.getResult() == null - ? documentParseBridgeService.queryResult(task.getProviderTaskId()) + ? documentParseBridgeService.queryResult(task.getProviderTaskId(), sourceMetadata) : taskInfo.getResult(); markParseSuccess(task, document, result, fileExt, task.getProviderTaskId()); return; @@ -2399,12 +3956,14 @@ public class KnowledgeDocumentImportTaskAppService { if (isTaskFailed(providerStatus)) { throw new BusinessException(taskInfo == null ? "文档解析失败" : taskInfo.getError()); } + if (!touchRunningTask(task)) { + throw new TaskOwnershipLostException(task.getId()); + } updateDocumentParseProgress(document, taskInfo); - touchRunningTask(task); } catch (BusinessException e) { throw e; } catch (Exception e) { - throw new BusinessException("文档解析失败：" + e.getMessage()); + throw new BusinessException(500, 1, "文档解析失败", e); } } @@ -2417,6 +3976,15 @@ public class KnowledgeDocumentImportTaskAppService { runAfterCommit(() -> parseTaskProducer.send(taskId)); } + /** + * 在事务提交后投递分块任务消息。 + * + * @param taskId 任务 ID + */ + private void dispatchSplitTaskAfterCommit(BigInteger taskId) { + runAfterCommit(() -> splitTaskProducer.send(taskId)); + } + /** * 在事务提交后投递向量化任务消息。 * @@ -2457,6 +4025,15 @@ public class KnowledgeDocumentImportTaskAppService { scheduleTaskFallback(taskId, "index", () -> selfProxy.handleIndexTask(taskId)); } + /** + * 在事务提交后调度本地分块兜底执行。 + * + * @param taskId 任务 ID + */ + private void scheduleSplitTaskFallback(BigInteger taskId) { + scheduleTaskFallback(taskId, "split", () -> selfProxy.handleSplitTask(taskId)); + } + /** * 在事务提交后调度本地解析兜底执行。 * @@ -2531,10 +4108,62 @@ public class KnowledgeDocumentImportTaskAppService { * * @param task 任务实体 */ - private void touchRunningTask(DocumentImportTask task) { - task.setModified(new Date()); - task.setModifiedBy(resolveOperatorId()); - documentImportTaskService.updateById(task, false); + private boolean touchRunningTask(DocumentImportTask task) { + Date now = new Date(); + Date leaseUntil = new Date( + now.getTime() + + Math.max( + 1L, + bulkProperties.getInterruptionTimeout().toMillis())); + int updated = documentImportTaskMapper.renewLease( + task.getId(), + task.getExecutionToken(), + leaseUntil, + now, + resolveOperatorId() + ); + if (updated > 0) { + task.setModified(now); + task.setLeaseUntil(leaseUntil); + return true; + } + return false; + } + + /** + * 校验任务是否仍由当前执行令牌持有。 + * + * @param task 任务实体 + * @return 是否仍持有执行权 + */ + private boolean ownsRunningTask(DocumentImportTask task) { + if (task == null || !StringUtil.hasText(task.getExecutionToken())) { + return false; + } + return documentImportTaskService.count( + QueryWrapper.create() + .eq(DocumentImportTask::getId, task.getId()) + .eq(DocumentImportTask::getStatus, + DocumentImportTaskStatus.RUNNING.name()) + .eq(DocumentImportTask::getExecutionToken, + task.getExecutionToken()) + ) > 0; + } + + /** + * 向量化完成后尽力删除持久化分块快照。 + * + * @param snapshotPath 快照路径 + */ + private void deleteChunkSnapshotAfterCompletion(String snapshotPath) { + if (!StringUtil.hasText(snapshotPath)) { + return; + } + try { + documentImportChunkSnapshotService.delete(snapshotPath); + } catch (RuntimeException error) { + LOG.error("删除已完成文档的分块快照失败: path={}", snapshotPath, error); + } } private StrategyConfig resolveStrategyConfig(DocumentCollection knowledge, @@ -2584,6 +4213,21 @@ public class KnowledgeDocumentImportTaskAppService { return config; } + /** + * 任务已被看门狗或其他执行者收口时抛出的内部异常。 + */ + private static final class TaskOwnershipLostException extends RuntimeException { + + /** + * 创建任务所有权丢失异常。 + * + * @param taskId 任务 ID + */ + private TaskOwnershipLostException(BigInteger taskId) { + super("文档导入任务所有权已失效: " + taskId); + } + } + @SuppressWarnings("unchecked") private StrategyConfig readProfileConfig(Map options, String strategyCode) { if (!StringUtil.hasText(strategyCode)) { @@ -2793,8 +4437,24 @@ public class KnowledgeDocumentImportTaskAppService { } private BigInteger generateId(Object entity) { - FlexIDKeyGenerator generator = new FlexIDKeyGenerator(); - return new BigInteger(String.valueOf(generator.generate(entity, null))); + return new BigInteger(String.valueOf(flexIdKeyGenerator.generate(entity, null))); + } + + /** + * 校验单次向量化分块的 ID 完整性，避免外部存储已写入后才触发数据库主键冲突。 + * + * @param chunks 待向量化分块 + */ + private void assertUniqueChunkIds(List chunks) { + Set uniqueIds = new HashSet(chunks.size()); + for (DocumentChunk chunk : chunks) { + if (chunk == null || chunk.getId() == null) { + throw new IllegalStateException("文档分块缺少 ID"); + } + if (!uniqueIds.add(chunk.getId())) { + throw new IllegalStateException("检测到重复文档分块 ID: " + chunk.getId()); + } + } } private BigInteger resolveOperatorId() { @@ -2823,6 +4483,35 @@ public class KnowledgeDocumentImportTaskAppService { return normalized.contains("FAIL") || normalized.contains("ERROR") || normalized.contains("CANCEL"); } + /** + * 写入文档任务错误及稳定错误码。 + * + * @param document 文档实体 + * @param errorMessage 用户可见错误信息 + * @param errorCode 稳定错误码 + */ + private void setDocumentTaskError(tech.easyflow.ai.entity.Document document, + String errorMessage, + String errorCode) { + document.setLastTaskError(errorMessage); + Map options = copyOptions(document.getOptions()); + if (StringUtil.hasText(errorCode)) { + options.put(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE, errorCode); + } else { + options.remove(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE); + } + document.setOptions(options); + } + + /** + * 清除文档任务错误及稳定错误码。 + * + * @param document 文档实体 + */ + private void clearDocumentTaskError(tech.easyflow.ai.entity.Document document) { + setDocumentTaskError(document, null, null); + } + private Map copyOptions(Map options) { return options == null ? new LinkedHashMap() : new LinkedHashMap(options); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeImportBatchFacade.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeImportBatchFacade.java new file mode 100644 index 00000000..19cc49c4 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/documentimport/task/KnowledgeImportBatchFacade.java @@ -0,0 +1,1000 @@ +package tech.easyflow.ai.documentimport.task; + +import com.alibaba.fastjson2.JSON; +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.query.QueryWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext; +import tech.easyflow.ai.documentimport.DocumentImportBatchDtos; +import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult; +import tech.easyflow.ai.documentimport.ImportCallerContext; +import tech.easyflow.ai.documentimport.PublicDocumentImportDtos; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.entity.DocumentImportBatchItem; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; +import tech.easyflow.ai.enums.DocumentImportBatchStatus; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; +import tech.easyflow.ai.service.DocumentImportBatchItemService; +import tech.easyflow.ai.service.DocumentImportBatchService; +import tech.easyflow.common.util.StringUtil; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * 知识库 Public API 批量导入门面。 + * + *

门面仅补充 Public API 容量、调用者归属、幂等和响应映射， + * 解析、分块、快照、向量化及恢复继续复用现有批次链路。

+ * + * @author Codex + * @since 2026-08-02 + */ +@Service +public class KnowledgeImportBatchFacade { + + private static final Logger LOG = + LoggerFactory.getLogger(KnowledgeImportBatchFacade.class); + private static final int MAX_FILE_COUNT = 200; + private static final long MAX_FILE_BYTES = 100L * 1024L * 1024L; + private static final long MAX_TOTAL_BYTES = 200L * 1024L * 1024L; + private static final Duration SUBMISSION_DEDUPLICATION_WINDOW = + Duration.ofMinutes(10); + private static final Duration INCOMPLETE_SUBMISSION_TIMEOUT = + Duration.ofMinutes(30); + private static final Set SUPPORTED_EXTENSIONS = + Set.of("txt", "md", "pdf", "docx", "pptx", "xlsx"); + + private final DocumentImportBatchAppService batchAppService; + private final DocumentImportBatchTracker batchTracker; + private final DocumentImportBatchService batchService; + private final DocumentImportBatchItemService itemService; + private final DocumentImportBatchMapper batchMapper; + + /** + * 创建 Public API 批量导入门面。 + * + * @param batchAppService 批次应用服务 + * @param batchTracker 批次跟踪器 + * @param batchService 批次服务 + * @param itemService 批次项服务 + * @param batchMapper 批次 Mapper + */ + public KnowledgeImportBatchFacade(DocumentImportBatchAppService batchAppService, + DocumentImportBatchTracker batchTracker, + DocumentImportBatchService batchService, + DocumentImportBatchItemService itemService, + DocumentImportBatchMapper batchMapper) { + this.batchAppService = batchAppService; + this.batchTracker = batchTracker; + this.batchService = batchService; + this.itemService = itemService; + this.batchMapper = batchMapper; + } + + /** + * 提交一个 Multipart 批量导入任务。 + * + * @param caller 调用者上下文 + * @param metadata 批量元数据 + * @param files Multipart 文件 + * @return 异步任务响应 + */ + public PublicDocumentImportDtos.SubmitResponse submit( + ImportCallerContext caller, + PublicDocumentImportDtos.BatchMetadata metadata, + List files) { + if (caller == null || caller.getCallerType() == null + || caller.getCallerId() == null) { + throw new BusinessException("导入调用者信息不完整"); + } + PreparedSubmission prepared = prepareSubmission(metadata, files); + String submissionFingerprint = buildSubmissionFingerprint( + caller, + prepared.requestDigest + ); + Date deduplicationCutoff = new Date( + System.currentTimeMillis() + - SUBMISSION_DEDUPLICATION_WINDOW.toMillis() + ); + Date incompleteCutoff = new Date( + System.currentTimeMillis() + - INCOMPLETE_SUBMISSION_TIMEOUT.toMillis() + ); + DocumentImportBatch existing = + findBySubmissionFingerprint(caller, submissionFingerprint); + if (existing != null) { + if (shouldReuseSubmission( + existing, + deduplicationCutoff, + incompleteCutoff + )) { + return resolveExistingSubmission( + existing, + prepared.requestDigest + ); + } + if (!releaseExpiredSubmission( + existing, + caller, + submissionFingerprint, + deduplicationCutoff, + incompleteCutoff + )) { + return resolveCurrentSubmissionOrConflict( + caller, + submissionFingerprint, + prepared.requestDigest + ); + } + DocumentImportBatch concurrent = + findBySubmissionFingerprint(caller, submissionFingerprint); + if (concurrent != null) { + if (shouldReuseSubmission( + concurrent, + deduplicationCutoff, + incompleteCutoff + )) { + return resolveExistingSubmission( + concurrent, + prepared.requestDigest + ); + } + if (!releaseExpiredSubmission( + concurrent, + caller, + submissionFingerprint, + deduplicationCutoff, + incompleteCutoff + )) { + return resolveCurrentSubmissionOrConflict( + caller, + submissionFingerprint, + prepared.requestDigest + ); + } + } + } + + DocumentImportBatchDtos.CreateRequest createRequest = + new DocumentImportBatchDtos.CreateRequest(); + createRequest.setKnowledgeId(metadata.getKnowledgeId()); + createRequest.setFiles(prepared.files); + String strategyJson = JSON.toJSONString(metadata.getChunkStrategy()); + DocumentImportBatchCreateContext createContext = + new DocumentImportBatchCreateContext( + caller, + submissionFingerprint, + prepared.requestDigest, + normalizeDuplicatePolicy(metadata.getDuplicatePolicy()), + strategyJson + ); + + DocumentImportBatchDtos.CreateResponse created; + try { + created = batchAppService.createBatch(createRequest, createContext); + } catch (DuplicateKeyException duplicate) { + DocumentImportBatch concurrent = + findBySubmissionFingerprint(caller, submissionFingerprint); + if (concurrent == null) { + throw duplicate; + } + return resolveExistingSubmission( + concurrent, + prepared.requestDigest + ); + } + + try { + for (int index = 0; index < files.size(); index++) { + DocumentImportBatchDtos.ItemResponse item = created.getItems().get(index); + batchAppService.uploadItem( + metadata.getKnowledgeId(), + created.getBatchId(), + item.getItemId(), + files.get(index), + caller + ); + persistContentHash(item.getItemId(), prepared.contentHashes.get(index)); + } + DocumentImportBatchDtos.StartRequest startRequest = + new DocumentImportBatchDtos.StartRequest(); + startRequest.setKnowledgeId(metadata.getKnowledgeId()); + startRequest.setBatchId(created.getBatchId()); + startRequest.setImportMode("AUTO"); + startRequest.setDuplicatePolicy(metadata.getDuplicatePolicy()); + batchAppService.startBatch(startRequest, caller); + return toSubmitResponse( + batchAppService.requireOwnedBatch( + metadata.getKnowledgeId(), + created.getBatchId(), + caller + ) + ); + } catch (RuntimeException error) { + compensateFailedSubmission( + metadata.getKnowledgeId(), + created.getBatchId(), + caller, + submissionFingerprint + ); + throw error; + } + } + + /** + * 查询调用者拥有的批次状态。 + * + * @param caller 调用者上下文 + * @param taskId 批次任务 ID + * @param itemStatus 可选文件状态 + * @param pageNumber 页码 + * @param pageSize 每页数量 + * @return Public API 状态响应 + */ + public PublicDocumentImportDtos.StatusResponse getStatus( + ImportCallerContext caller, + BigInteger taskId, + String itemStatus, + long pageNumber, + long pageSize) { + if (pageNumber < 1 || pageSize < 1 || pageSize > 100) { + throw new BusinessException("分页参数无效，pageSize 最大为100"); + } + DocumentImportBatch batch = + batchAppService.requireBatchForCaller(taskId, caller); + QueryWrapper itemQuery = QueryWrapper.create() + .eq(DocumentImportBatchItem::getBatchId, batch.getId()) + .orderBy(DocumentImportBatchItem::getCreated, true) + .orderBy(DocumentImportBatchItem::getId, true); + if (StringUtil.hasText(itemStatus)) { + try { + itemQuery.eq( + DocumentImportBatchItem::getStatus, + DocumentImportBatchItemStatus.valueOf( + itemStatus.trim().toUpperCase(Locale.ROOT) + ).name() + ); + } catch (IllegalArgumentException error) { + throw new BusinessException("文件状态筛选值无效"); + } + } + Page page = + itemService.page(new Page<>(pageNumber, pageSize), itemQuery); + return toStatusResponse(batch, page); + } + + /** + * 查询调用者拥有任务的知识库 ID。 + * + * @param caller 调用者上下文 + * @param taskId 批次任务 ID + * @return 知识库 ID + */ + public BigInteger getOwnedKnowledgeId(ImportCallerContext caller, + BigInteger taskId) { + return batchAppService.requireBatchForCaller(taskId, caller).getKnowledgeId(); + } + + /** + * 重试当前调用者拥有的异常批次。 + * + * @param caller 调用者上下文 + * @param request 重试请求 + * @return 重试响应 + */ + public PublicDocumentImportDtos.RetryResponse retry( + ImportCallerContext caller, + PublicDocumentImportDtos.RetryRequest request) { + if (request == null || request.getTaskId() == null) { + throw new BusinessException("重试请求信息不完整"); + } + if (request.getFileKeys() != null && request.getFileKeys().isEmpty()) { + throw new BusinessException("fileKeys 不能为空数组；省略该参数可重试全部异常文件"); + } + Set fileKeys = new LinkedHashSet<>(); + if (request.getFileKeys() != null) { + if (request.getFileKeys().size() > MAX_FILE_COUNT) { + throw new BusinessException("fileKeys 数量不能超过200"); + } + for (String fileKey : request.getFileKeys()) { + if (!StringUtil.hasText(fileKey) + || fileKey.trim().length() > 64 + || !fileKeys.add(fileKey.trim())) { + throw new BusinessException("fileKeys 包含空值、重复值或超长值"); + } + } + } + DocumentImportBatchRetryResult result = + batchAppService.retryOwnedBatch( + request.getTaskId(), + caller, + fileKeys + ); + PublicDocumentImportDtos.RetryResponse response = + new PublicDocumentImportDtos.RetryResponse(); + response.setTaskId(result.getTaskId()); + response.setStatus(result.getStatus()); + response.setRetriedCount(result.getRetriedCount()); + return response; + } + + /** + * 校验提交内容并计算实际文件摘要。 + * + * @param metadata 批量元数据 + * @param files 上传文件 + * @return 准备完成的提交信息 + */ + private PreparedSubmission prepareSubmission( + PublicDocumentImportDtos.BatchMetadata metadata, + List files) { + if (metadata == null || metadata.getKnowledgeId() == null) { + throw new BusinessException("knowledgeId 不能为空"); + } + if (files == null || files.isEmpty() + || metadata.getFiles() == null || metadata.getFiles().isEmpty()) { + throw new BusinessException("请选择需要导入的文件"); + } + if (files.size() != metadata.getFiles().size()) { + throw new BusinessException("metadata.files 与文件 Part 数量不一致"); + } + if (files.size() > MAX_FILE_COUNT) { + throw new BusinessException(413, 41301, "单次最多上传200个文件"); + } + normalizeDuplicatePolicy(metadata.getDuplicatePolicy()); + + long totalBytes = 0L; + Set normalizedPaths = new HashSet<>(); + Set fileKeys = new HashSet<>(); + List contentHashes = new ArrayList<>(files.size()); + List measuredFiles = + new ArrayList<>(files.size()); + StringBuilder digestSource = new StringBuilder() + .append(metadata.getKnowledgeId()).append('\n') + .append(normalizeDuplicatePolicy(metadata.getDuplicatePolicy())).append('\n') + .append(JSON.toJSONString(metadata.getChunkStrategy())).append('\n'); + for (int index = 0; index < files.size(); index++) { + MultipartFile file = files.get(index); + DocumentImportBatchDtos.ManifestItem declared = metadata.getFiles().get(index); + validateManifestPair(file, declared, fileKeys, normalizedPaths); + FileFingerprint fingerprint = fingerprint(file); + if (fingerprint.actualBytes > MAX_FILE_BYTES) { + throw new BusinessException(413, 41302, "单个文件不能超过100MiB"); + } + totalBytes = Math.addExact(totalBytes, fingerprint.actualBytes); + if (totalBytes > MAX_TOTAL_BYTES) { + throw new BusinessException(413, 41303, "单次文件总大小不能超过200MiB"); + } + validateContentSignature(declared.getFileName(), fingerprint.prefix); + contentHashes.add(fingerprint.sha256); + measuredFiles.add(withMeasuredSize(declared, fingerprint.actualBytes)); + digestSource.append(declared.getClientFileKey()).append('\u0000') + .append(normalizeRelativePath( + declared.getRelativePath(), + declared.getFileName() + )).append('\u0000') + .append(fingerprint.actualBytes).append('\u0000') + .append(fingerprint.sha256).append('\n'); + } + return new PreparedSubmission( + sha256(digestSource.toString().getBytes(StandardCharsets.UTF_8)), + contentHashes, + measuredFiles + ); + } + + /** + * 校验单个声明与 Multipart 文件的一致性。 + * + * @param file 实际文件 + * @param declared 声明项 + * @param fileKeys 已使用文件键 + * @param normalizedPaths 已使用规范路径 + */ + private void validateManifestPair(MultipartFile file, + DocumentImportBatchDtos.ManifestItem declared, + Set fileKeys, + Set normalizedPaths) { + if (file == null || file.isEmpty() || declared == null + || !StringUtil.hasText(declared.getClientFileKey()) + || !StringUtil.hasText(declared.getFileName())) { + throw new BusinessException("文件 Part 或 metadata.files 不完整"); + } + if (!fileKeys.add(declared.getClientFileKey())) { + throw new BusinessException("clientFileKey 在批次内必须唯一"); + } + if (!declared.getFileName().equals(file.getOriginalFilename())) { + throw new BusinessException("文件 Part 顺序或文件名与 metadata 不一致"); + } + String relativePath = + normalizeRelativePath(declared.getRelativePath(), declared.getFileName()); + if (!normalizedPaths.add(relativePath)) { + throw new BusinessException("规范化后的文件相对路径不能重复"); + } + assertSupportedExtension(declared.getFileName()); + } + + /** + * 使用服务端实际读取到的文件大小构造内部清单项。 + * + * @param declared Public API 声明项 + * @param actualBytes 服务端读取到的实际字节数 + * @return 内部批量导入清单项 + */ + private DocumentImportBatchDtos.ManifestItem withMeasuredSize( + DocumentImportBatchDtos.ManifestItem declared, + long actualBytes) { + DocumentImportBatchDtos.ManifestItem measured = + new DocumentImportBatchDtos.ManifestItem(); + measured.setClientFileKey(declared.getClientFileKey()); + measured.setFileName(declared.getFileName()); + measured.setRelativePath(declared.getRelativePath()); + measured.setFileSize(actualBytes); + return measured; + } + + /** + * 流式计算实际大小、SHA-256 和小型签名前缀。 + * + * @param file 上传文件 + * @return 文件指纹 + */ + private FileFingerprint fingerprint(MultipartFile file) { + MessageDigest digest = newSha256Digest(); + byte[] buffer = new byte[8192]; + byte[] prefix = new byte[1024]; + int prefixLength = 0; + long actualBytes = 0L; + try (InputStream input = file.getInputStream()) { + int read; + while ((read = input.read(buffer)) >= 0) { + if (read == 0) { + continue; + } + if (prefixLength < prefix.length) { + int copyLength = Math.min(read, prefix.length - prefixLength); + System.arraycopy(buffer, 0, prefix, prefixLength, copyLength); + prefixLength += copyLength; + } + digest.update(buffer, 0, read); + actualBytes = Math.addExact(actualBytes, read); + if (actualBytes > MAX_FILE_BYTES) { + throw new BusinessException(413, 41302, "单个文件不能超过100MiB"); + } + } + } catch (IOException error) { + throw new BusinessException(500, 50021, "读取上传文件失败", error); + } + byte[] actualPrefix = new byte[prefixLength]; + System.arraycopy(prefix, 0, actualPrefix, 0, prefixLength); + return new FileFingerprint(actualBytes, toHex(digest.digest()), actualPrefix); + } + + /** + * 校验基础文件签名，阻止明显的扩展名伪装。 + * + * @param fileName 文件名 + * @param prefix 文件前缀 + */ + private void validateContentSignature(String fileName, byte[] prefix) { + String extension = extension(fileName); + boolean valid; + if ("pdf".equals(extension)) { + valid = new String(prefix, StandardCharsets.ISO_8859_1).contains("%PDF-"); + } else if ("docx".equals(extension) + || "pptx".equals(extension) + || "xlsx".equals(extension)) { + valid = prefix.length >= 4 + && prefix[0] == 'P' + && prefix[1] == 'K' + && (prefix[2] == 3 || prefix[2] == 5 || prefix[2] == 7); + } else { + valid = true; + for (byte value : prefix) { + if (value == 0) { + valid = false; + break; + } + } + } + if (!valid) { + throw new BusinessException(415, 41501, "文件内容与扩展名不匹配"); + } + } + + /** + * 处理已存在的服务端提交指纹记录。 + * + * @param existing 已存在批次 + * @param requestDigest 当前请求摘要 + * @return 原任务响应 + */ + private PublicDocumentImportDtos.SubmitResponse resolveExistingSubmission( + DocumentImportBatch existing, + String requestDigest) { + if (!requestDigest.equals(existing.getRequestDigest())) { + throw new BusinessException( + 409, + 40901, + "导入请求指纹冲突，请重新提交" + ); + } + return toSubmitResponse(existing); + } + + /** + * 生成调用者隔离的服务端提交指纹。 + * + * @param caller 调用者上下文 + * @param requestDigest 请求内容摘要 + * @return 服务端提交指纹 + */ + private String buildSubmissionFingerprint(ImportCallerContext caller, + String requestDigest) { + String source = caller.getCallerType().name() + '\n' + + caller.getCallerId() + '\n' + + requestDigest; + return sha256(source.getBytes(StandardCharsets.UTF_8)); + } + + /** + * 判断已有提交是否仍应复用。 + * + *

执行中的任务始终复用；上传中或待启动任务在超时前复用； + * 已结束任务仅在终态后的短去重窗口内复用。

+ * + * @param batch 已存在批次 + * @param deduplicationCutoff 终态去重窗口起点 + * @param incompleteCutoff 未完成提交超时起点 + * @return 是否复用原任务 + */ + private boolean shouldReuseSubmission(DocumentImportBatch batch, + Date deduplicationCutoff, + Date incompleteCutoff) { + if (DocumentImportBatchStatus.CANCELLED.name().equals(batch.getStatus())) { + return false; + } + if (DocumentImportBatchStatus.UPLOADING.name().equals(batch.getStatus()) + || DocumentImportBatchStatus.READY.name().equals(batch.getStatus())) { + Date lastProgressAt = batch.getModified() == null + ? batch.getCreated() + : batch.getModified(); + return lastProgressAt == null + || !lastProgressAt.before(incompleteCutoff); + } + if (DocumentImportBatchStatus.RUNNING.name().equals(batch.getStatus())) { + return true; + } + Date finishedAt = batch.getFinishedAt() != null + ? batch.getFinishedAt() + : batch.getModified(); + if (finishedAt == null) { + finishedAt = batch.getCreated(); + } + return finishedAt == null + || !finishedAt.before(deduplicationCutoff); + } + + /** + * 原子取消超时的未完成提交并释放已过期指纹。 + * + * @param batch 待释放批次 + * @param caller 调用者上下文 + * @param submissionFingerprint 服务端提交指纹 + * @param deduplicationCutoff 终态去重窗口起点 + * @param incompleteCutoff 未完成提交超时起点 + * @return 是否已安全释放指纹 + */ + private boolean releaseExpiredSubmission( + DocumentImportBatch batch, + ImportCallerContext caller, + String submissionFingerprint, + Date deduplicationCutoff, + Date incompleteCutoff) { + if (DocumentImportBatchStatus.UPLOADING.name().equals(batch.getStatus()) + || DocumentImportBatchStatus.READY.name().equals(batch.getStatus())) { + boolean cancelled = batchAppService.cancelStaleBatch( + batch.getKnowledgeId(), + batch.getId(), + caller, + incompleteCutoff + ); + if (!cancelled) { + return false; + } + } + return batchMapper.releaseSubmissionFingerprint( + batch.getId(), + submissionFingerprint, + deduplicationCutoff, + new Date() + ) > 0; + } + + /** + * 在并发状态变化后返回当前任务，或给出明确冲突。 + * + * @param caller 调用者上下文 + * @param submissionFingerprint 服务端提交指纹 + * @param requestDigest 请求摘要 + * @return 当前有效任务 + */ + private PublicDocumentImportDtos.SubmitResponse + resolveCurrentSubmissionOrConflict( + ImportCallerContext caller, + String submissionFingerprint, + String requestDigest) { + DocumentImportBatch current = + findBySubmissionFingerprint(caller, submissionFingerprint); + if (current != null + && !DocumentImportBatchStatus.CANCELLED.name().equals( + current.getStatus())) { + return resolveExistingSubmission(current, requestDigest); + } + throw new BusinessException( + 409, + 40902, + "相同导入请求状态正在变化，请稍后重试" + ); + } + + /** + * 按调用者和服务端提交指纹查询批次。 + * + * @param caller 调用者上下文 + * @param submissionFingerprint 服务端提交指纹 + * @return 已存在批次 + */ + private DocumentImportBatch findBySubmissionFingerprint( + ImportCallerContext caller, + String submissionFingerprint) { + return batchService.getOne( + QueryWrapper.create() + .eq(DocumentImportBatch::getCallerType, caller.getCallerType().name()) + .eq(DocumentImportBatch::getCallerId, caller.getCallerId()) + .eq( + DocumentImportBatch::getIdempotencyKeyHash, + submissionFingerprint + ) + .limit(1) + ); + } + + /** + * 提交失败时取消未启动批次、清理对象并释放提交指纹。 + * + * @param knowledgeId 知识库 ID + * @param batchId 批次 ID + * @param caller 调用者上下文 + * @param submissionFingerprint 服务端提交指纹 + */ + private void compensateFailedSubmission(BigInteger knowledgeId, + BigInteger batchId, + ImportCallerContext caller, + String submissionFingerprint) { + try { + batchAppService.cancelBatch(knowledgeId, batchId, caller); + } catch (RuntimeException compensationError) { + LOG.error("Public API 批量提交失败后取消批次异常: batchId={}", + batchId, compensationError); + } + batchMapper.releaseSubmissionFingerprint( + batchId, + submissionFingerprint, + new Date(), + new Date() + ); + } + + /** + * 持久化已校验的内容哈希。 + * + * @param itemId 批次项 ID + * @param contentHash 内容 SHA-256 + */ + private void persistContentHash(BigInteger itemId, String contentHash) { + DocumentImportBatchItem update = new DocumentImportBatchItem(); + update.setId(itemId); + update.setContentSha256(contentHash); + update.setModified(new Date()); + // 使用默认忽略 null 的更新语义，避免部分实体将其余非空列覆盖为 null。 + if (!itemService.updateById(update)) { + throw new BusinessException(500, 50022, "保存文件内容摘要失败"); + } + } + + /** + * 转换提交响应。 + * + * @param batch 批次 + * @return 提交响应 + */ + private PublicDocumentImportDtos.SubmitResponse toSubmitResponse( + DocumentImportBatch batch) { + PublicDocumentImportDtos.SubmitResponse response = + new PublicDocumentImportDtos.SubmitResponse(); + response.setTaskId(batch.getId()); + response.setStatus(mapBatchStatus(batch)); + response.setTotalCount(valueOrZero(batch.getTotalCount())); + response.setTotalBytes(batch.getTotalBytes() == null ? 0L : batch.getTotalBytes()); + response.setCreatedAt(batch.getCreated()); + return response; + } + + /** + * 转换状态响应。 + * + * @param batch 批次 + * @param itemPage 文件分页 + * @return 状态响应 + */ + private PublicDocumentImportDtos.StatusResponse toStatusResponse( + DocumentImportBatch batch, + Page itemPage) { + DocumentImportBatchDtos.StatusResponse internal = + batchTracker.toStatusResponse(batch); + PublicDocumentImportDtos.StatusResponse response = + new PublicDocumentImportDtos.StatusResponse(); + response.setTaskId(batch.getId()); + response.setKnowledgeId(batch.getKnowledgeId()); + response.setStatus(mapBatchStatus(batch)); + response.setProgressPercent(internal.getProgressPercent()); + PublicDocumentImportDtos.Counts counts = new PublicDocumentImportDtos.Counts(); + counts.setTotal(internal.getTotalCount()); + counts.setCompleted(internal.getCompletedCount()); + counts.setProcessing(internal.getProcessingCount()); + counts.setPending(internal.getPendingCount()); + counts.setFailed(internal.getFailedCount()); + counts.setSkipped(internal.getSkippedCount()); + counts.setRetryableFailed(internal.getRetryableFailedCount()); + response.setCounts(counts); + response.setCanRetry(internal.getRetryableFailedCount() > 0 + && (DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name().equals(batch.getStatus()) + || DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus()))); + PublicDocumentImportDtos.ItemPage items = + new PublicDocumentImportDtos.ItemPage(); + items.setPageNumber(itemPage.getPageNumber()); + items.setPageSize(itemPage.getPageSize()); + items.setTotal(itemPage.getTotalRow()); + items.setRecords(itemPage.getRecords().stream().map(this::toItemRecord).toList()); + response.setItems(items); + return response; + } + + /** + * 转换文件状态记录。 + * + * @param item 批次项 + * @return Public 文件记录 + */ + private PublicDocumentImportDtos.ItemRecord toItemRecord( + DocumentImportBatchItem item) { + PublicDocumentImportDtos.ItemRecord record = + new PublicDocumentImportDtos.ItemRecord(); + record.setFileKey(item.getClientFileKey()); + record.setRelativePath(item.getRelativePath()); + record.setDocumentId(item.getDocumentId()); + record.setStage(item.getStage()); + record.setStatus(item.getStatus()); + record.setAttemptCount(valueOrZero(item.getAttemptCount())); + record.setRetryable(Boolean.TRUE.equals(item.getRetryable())); + if (StringUtil.hasText(item.getFailureCode()) + || StringUtil.hasText(item.getErrorSummary())) { + PublicDocumentImportDtos.ItemError error = + new PublicDocumentImportDtos.ItemError(); + error.setCode(StringUtil.hasText(item.getFailureCode()) + ? item.getFailureCode() + : String.valueOf(item.getStage()).toUpperCase(Locale.ROOT) + "_FAILED"); + error.setMessage(StringUtil.hasText(item.getErrorSummary()) + ? item.getErrorSummary() + : "文件处理失败"); + record.setError(error); + } + return record; + } + + /** + * 映射稳定 Public 批次状态。 + * + * @param batch 批次 + * @return Public 状态 + */ + private String mapBatchStatus(DocumentImportBatch batch) { + String status = batch.getStatus(); + if (DocumentImportBatchStatus.UPLOADING.name().equals(status) + || DocumentImportBatchStatus.READY.name().equals(status)) { + return "QUEUED"; + } + if (DocumentImportBatchStatus.COMPLETED.name().equals(status)) { + return "SUCCEEDED"; + } + if (DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name().equals(status) + && valueOrZero(batch.getFailedCount()) >= valueOrZero(batch.getTotalCount()) + && valueOrZero(batch.getCompletedCount()) == 0 + && valueOrZero(batch.getSkippedCount()) == 0) { + return "FAILED"; + } + return status; + } + + /** + * 规范化重复文件策略。 + * + * @param value 原始策略 + * @return 规范策略 + */ + private String normalizeDuplicatePolicy(String value) { + String normalized = StringUtil.hasText(value) + ? value.trim().toUpperCase(Locale.ROOT) + : "SKIP"; + if (!Set.of("SKIP", "OVERWRITE", "REIMPORT").contains(normalized)) { + throw new BusinessException("duplicatePolicy 仅支持 SKIP、OVERWRITE、REIMPORT"); + } + return normalized; + } + + /** + * 规范化文件夹相对路径。 + * + * @param relativePath 相对路径 + * @param fileName 文件名 + * @return 规范路径 + */ + private String normalizeRelativePath(String relativePath, String fileName) { + String normalized = StringUtil.hasText(relativePath) + ? relativePath.replace('\\', '/') + : fileName; + if (normalized.length() > 1024 || normalized.startsWith("/") + || normalized.endsWith("/") || normalized.contains("//")) { + throw new BusinessException("文件相对路径无效"); + } + String[] segments = normalized.split("/"); + if (segments.length > 64) { + throw new BusinessException("文件相对路径层级过深"); + } + for (String segment : segments) { + if (!StringUtil.hasText(segment) || ".".equals(segment) + || "..".equals(segment)) { + throw new BusinessException("文件相对路径无效"); + } + } + return String.join("/", segments); + } + + /** + * 校验文件扩展名。 + * + * @param fileName 文件名 + */ + private void assertSupportedExtension(String fileName) { + if (!SUPPORTED_EXTENSIONS.contains(extension(fileName))) { + throw new BusinessException(415, 41502, "暂不支持该文件格式"); + } + } + + /** + * 获取小写文件扩展名。 + * + * @param fileName 文件名 + * @return 扩展名 + */ + private String extension(String fileName) { + int dot = fileName == null ? -1 : fileName.lastIndexOf('.'); + return dot < 0 + ? "" + : fileName.substring(dot + 1).toLowerCase(Locale.ROOT); + } + + /** + * 计算 SHA-256。 + * + * @param bytes 输入字节 + * @return 十六进制摘要 + */ + private String sha256(byte[] bytes) { + return toHex(newSha256Digest().digest(bytes)); + } + + /** + * 创建 SHA-256 摘要器。 + * + * @return 摘要器 + */ + private MessageDigest newSha256Digest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException error) { + throw new IllegalStateException("JVM 不支持 SHA-256", error); + } + } + + /** + * 将字节转换为小写十六进制。 + * + * @param bytes 输入字节 + * @return 十六进制字符串 + */ + private String toHex(byte[] bytes) { + StringBuilder hex = new StringBuilder(bytes.length * 2); + for (byte value : bytes) { + hex.append(Character.forDigit((value >>> 4) & 0x0f, 16)); + hex.append(Character.forDigit(value & 0x0f, 16)); + } + return hex.toString(); + } + + /** + * 将可空整数转换为零。 + * + * @param value 可空整数 + * @return 非空整数 + */ + private int valueOrZero(Integer value) { + return value == null ? 0 : value; + } + + /** + * 已准备的提交信息。 + */ + private static final class PreparedSubmission { + private final String requestDigest; + private final List contentHashes; + private final List files; + + /** + * 创建准备结果。 + * + * @param requestDigest 请求摘要 + * @param contentHashes 文件内容摘要 + * @param files 已写入服务端实测大小的内部文件清单 + */ + private PreparedSubmission( + String requestDigest, + List contentHashes, + List files) { + this.requestDigest = requestDigest; + this.contentHashes = contentHashes; + this.files = files; + } + } + + /** + * 流式文件指纹。 + */ + private static final class FileFingerprint { + private final long actualBytes; + private final String sha256; + private final byte[] prefix; + + /** + * 创建文件指纹。 + * + * @param actualBytes 实际字节数 + * @param sha256 内容摘要 + * @param prefix 签名前缀 + */ + private FileFingerprint(long actualBytes, String sha256, byte[] prefix) { + this.actualBytes = actualBytes; + this.sha256 = sha256; + this.prefix = prefix; + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatch.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatch.java new file mode 100644 index 00000000..3735d85b --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatch.java @@ -0,0 +1,333 @@ +package tech.easyflow.ai.entity; + +import com.mybatisflex.annotation.Column; +import com.mybatisflex.annotation.Id; +import com.mybatisflex.annotation.KeyType; +import com.mybatisflex.annotation.Table; +import tech.easyflow.common.entity.DateEntity; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Date; + +/** + * 知识库文档批量导入批次。 + * + * @author Codex + * @since 2026-07-31 + */ +@Table(value = "tb_document_import_batch", comment = "知识库文档批量导入批次") +public class DocumentImportBatch extends DateEntity implements Serializable { + + @Id(keyType = KeyType.Generator, value = "snowFlakeId") + private BigInteger id; + + @Column(comment = "知识库ID") + private BigInteger knowledgeId; + + @Column(comment = "调用者类型") + private String callerType; + + @Column(comment = "调用者ID") + private BigInteger callerId; + + @Column(comment = "幂等键哈希") + private String idempotencyKeyHash; + + @Column(comment = "请求摘要") + private String requestDigest; + + @Column(comment = "重复文件策略") + private String duplicatePolicy; + + @Column(comment = "请求分块策略") + private String requestedStrategyJson; + + @Column(comment = "重试代次") + private Integer retryGeneration; + + @Column(comment = "乐观锁版本") + private Integer version; + + @Column(comment = "导入模式") + private String importMode; + + @Column(comment = "批次状态") + private String status; + + @Column(comment = "文件总数") + private Integer totalCount; + + @Column(comment = "文件总字节数") + private Long totalBytes; + + @Column(comment = "完成数") + private Integer completedCount; + + @Column(comment = "处理中数量") + private Integer processingCount; + + @Column(comment = "失败数") + private Integer failedCount; + + @Column(comment = "等待数") + private Integer pendingCount; + + @Column(comment = "已上传数") + private Integer uploadedCount; + + @Column(comment = "跳过数") + private Integer skippedCount; + + @Column(comment = "取消数") + private Integer cancelledCount; + + @Column(comment = "可重试失败数") + private Integer retryableFailedCount; + + @Column(comment = "开始时间") + private Date startedAt; + + @Column(comment = "结束时间") + private Date finishedAt; + + @Column(comment = "创建时间") + private Date created; + + @Column(comment = "创建人") + private BigInteger createdBy; + + @Column(comment = "修改时间") + private Date modified; + + @Column(comment = "修改人") + private BigInteger modifiedBy; + + public BigInteger getId() { + return id; + } + + public void setId(BigInteger id) { + this.id = id; + } + + public BigInteger getKnowledgeId() { + return knowledgeId; + } + + public void setKnowledgeId(BigInteger knowledgeId) { + this.knowledgeId = knowledgeId; + } + + public String getCallerType() { + return callerType; + } + + public void setCallerType(String callerType) { + this.callerType = callerType; + } + + public BigInteger getCallerId() { + return callerId; + } + + public void setCallerId(BigInteger callerId) { + this.callerId = callerId; + } + + public String getIdempotencyKeyHash() { + return idempotencyKeyHash; + } + + public void setIdempotencyKeyHash(String idempotencyKeyHash) { + this.idempotencyKeyHash = idempotencyKeyHash; + } + + public String getRequestDigest() { + return requestDigest; + } + + public void setRequestDigest(String requestDigest) { + this.requestDigest = requestDigest; + } + + public String getDuplicatePolicy() { + return duplicatePolicy; + } + + public void setDuplicatePolicy(String duplicatePolicy) { + this.duplicatePolicy = duplicatePolicy; + } + + public String getRequestedStrategyJson() { + return requestedStrategyJson; + } + + public void setRequestedStrategyJson(String requestedStrategyJson) { + this.requestedStrategyJson = requestedStrategyJson; + } + + public Integer getRetryGeneration() { + return retryGeneration; + } + + public void setRetryGeneration(Integer retryGeneration) { + this.retryGeneration = retryGeneration; + } + + public Integer getVersion() { + return version; + } + + public void setVersion(Integer version) { + this.version = version; + } + + public String getImportMode() { + return importMode; + } + + public void setImportMode(String importMode) { + this.importMode = importMode; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Integer getTotalCount() { + return totalCount; + } + + public void setTotalCount(Integer totalCount) { + this.totalCount = totalCount; + } + + public Long getTotalBytes() { + return totalBytes; + } + + public void setTotalBytes(Long totalBytes) { + this.totalBytes = totalBytes; + } + + public Integer getCompletedCount() { + return completedCount; + } + + public void setCompletedCount(Integer completedCount) { + this.completedCount = completedCount; + } + + public Integer getProcessingCount() { + return processingCount; + } + + public void setProcessingCount(Integer processingCount) { + this.processingCount = processingCount; + } + + public Integer getFailedCount() { + return failedCount; + } + + public void setFailedCount(Integer failedCount) { + this.failedCount = failedCount; + } + + public Integer getPendingCount() { + return pendingCount; + } + + public void setPendingCount(Integer pendingCount) { + this.pendingCount = pendingCount; + } + + public Integer getUploadedCount() { + return uploadedCount; + } + + public void setUploadedCount(Integer uploadedCount) { + this.uploadedCount = uploadedCount; + } + + public Integer getSkippedCount() { + return skippedCount; + } + + public void setSkippedCount(Integer skippedCount) { + this.skippedCount = skippedCount; + } + + public Integer getCancelledCount() { + return cancelledCount; + } + + public void setCancelledCount(Integer cancelledCount) { + this.cancelledCount = cancelledCount; + } + + public Integer getRetryableFailedCount() { + return retryableFailedCount; + } + + public void setRetryableFailedCount(Integer retryableFailedCount) { + this.retryableFailedCount = retryableFailedCount; + } + + public Date getStartedAt() { + return startedAt; + } + + public void setStartedAt(Date startedAt) { + this.startedAt = startedAt; + } + + public Date getFinishedAt() { + return finishedAt; + } + + public void setFinishedAt(Date finishedAt) { + this.finishedAt = finishedAt; + } + + @Override + public Date getCreated() { + return created; + } + + @Override + public void setCreated(Date created) { + this.created = created; + } + + public BigInteger getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(BigInteger createdBy) { + this.createdBy = createdBy; + } + + @Override + public Date getModified() { + return modified; + } + + @Override + public void setModified(Date modified) { + this.modified = modified; + } + + public BigInteger getModifiedBy() { + return modifiedBy; + } + + public void setModifiedBy(BigInteger modifiedBy) { + this.modifiedBy = modifiedBy; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatchItem.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatchItem.java new file mode 100644 index 00000000..0a7308a1 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportBatchItem.java @@ -0,0 +1,300 @@ +package tech.easyflow.ai.entity; + +import com.mybatisflex.annotation.Column; +import com.mybatisflex.annotation.Id; +import com.mybatisflex.annotation.KeyType; +import com.mybatisflex.annotation.Table; +import tech.easyflow.common.entity.DateEntity; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Date; + +/** + * 知识库文档批量导入文件项。 + * + * @author Codex + * @since 2026-07-31 + */ +@Table(value = "tb_document_import_batch_item", comment = "知识库文档批量导入文件项") +public class DocumentImportBatchItem extends DateEntity implements Serializable { + + @Id(keyType = KeyType.Generator, value = "snowFlakeId") + private BigInteger id; + + @Column(comment = "批次ID") + private BigInteger batchId; + + @Column(comment = "知识库ID") + private BigInteger knowledgeId; + + @Column(comment = "文档ID") + private BigInteger documentId; + + @Column(comment = "待覆盖的历史文档ID") + private BigInteger replacedDocumentId; + + @Column(comment = "客户端文件键") + private String clientFileKey; + + @Column(comment = "文件名") + private String fileName; + + @Column(comment = "文件夹相对路径") + private String relativePath; + + @Column(comment = "文件大小") + private Long fileSize; + + @Column(comment = "存储路径") + private String filePath; + + @Column(comment = "可恢复存储定位符") + private String storageLocator; + + @Column(comment = "是否等待清理存储对象") + private Boolean cleanupPending; + + @Column(comment = "文件内容SHA-256") + private String contentSha256; + + @Column(comment = "当前阶段") + private String stage; + + @Column(comment = "当前状态") + private String status; + + @Column(comment = "错误摘要") + private String errorSummary; + + @Column(comment = "稳定失败码") + private String failureCode; + + @Column(comment = "实际分块策略编码") + private String appliedStrategyCode; + + @Column(comment = "分块策略快照") + private String strategySnapshotJson; + + @Column(comment = "是否允许批量重试") + private Boolean retryable; + + @Column(comment = "重试次数") + private Integer attemptCount; + + @Column(comment = "创建时间") + private Date created; + + @Column(comment = "创建人") + private BigInteger createdBy; + + @Column(comment = "修改时间") + private Date modified; + + @Column(comment = "修改人") + private BigInteger modifiedBy; + + public BigInteger getId() { + return id; + } + + public void setId(BigInteger id) { + this.id = id; + } + + public BigInteger getBatchId() { + return batchId; + } + + public void setBatchId(BigInteger batchId) { + this.batchId = batchId; + } + + public BigInteger getKnowledgeId() { + return knowledgeId; + } + + public void setKnowledgeId(BigInteger knowledgeId) { + this.knowledgeId = knowledgeId; + } + + public BigInteger getDocumentId() { + return documentId; + } + + public void setDocumentId(BigInteger documentId) { + this.documentId = documentId; + } + + public BigInteger getReplacedDocumentId() { + return replacedDocumentId; + } + + public void setReplacedDocumentId(BigInteger replacedDocumentId) { + this.replacedDocumentId = replacedDocumentId; + } + + public String getClientFileKey() { + return clientFileKey; + } + + public void setClientFileKey(String clientFileKey) { + this.clientFileKey = clientFileKey; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getRelativePath() { + return relativePath; + } + + public void setRelativePath(String relativePath) { + this.relativePath = relativePath; + } + + public Long getFileSize() { + return fileSize; + } + + public void setFileSize(Long fileSize) { + this.fileSize = fileSize; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public String getStorageLocator() { + return storageLocator; + } + + public void setStorageLocator(String storageLocator) { + this.storageLocator = storageLocator; + } + + public Boolean getCleanupPending() { + return cleanupPending; + } + + public void setCleanupPending(Boolean cleanupPending) { + this.cleanupPending = cleanupPending; + } + + public String getContentSha256() { + return contentSha256; + } + + public void setContentSha256(String contentSha256) { + this.contentSha256 = contentSha256; + } + + public String getStage() { + return stage; + } + + public void setStage(String stage) { + this.stage = stage; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getErrorSummary() { + return errorSummary; + } + + public void setErrorSummary(String errorSummary) { + this.errorSummary = errorSummary; + } + + public String getFailureCode() { + return failureCode; + } + + public void setFailureCode(String failureCode) { + this.failureCode = failureCode; + } + + public String getAppliedStrategyCode() { + return appliedStrategyCode; + } + + public void setAppliedStrategyCode(String appliedStrategyCode) { + this.appliedStrategyCode = appliedStrategyCode; + } + + public String getStrategySnapshotJson() { + return strategySnapshotJson; + } + + public void setStrategySnapshotJson(String strategySnapshotJson) { + this.strategySnapshotJson = strategySnapshotJson; + } + + public Boolean getRetryable() { + return retryable; + } + + public void setRetryable(Boolean retryable) { + this.retryable = retryable; + } + + public Integer getAttemptCount() { + return attemptCount; + } + + public void setAttemptCount(Integer attemptCount) { + this.attemptCount = attemptCount; + } + + @Override + public Date getCreated() { + return created; + } + + @Override + public void setCreated(Date created) { + this.created = created; + } + + public BigInteger getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(BigInteger createdBy) { + this.createdBy = createdBy; + } + + @Override + public Date getModified() { + return modified; + } + + @Override + public void setModified(Date modified) { + this.modified = modified; + } + + public BigInteger getModifiedBy() { + return modifiedBy; + } + + public void setModifiedBy(BigInteger modifiedBy) { + this.modifiedBy = modifiedBy; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportTask.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportTask.java index 04935e11..066ef39e 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportTask.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/entity/DocumentImportTask.java @@ -31,6 +31,12 @@ public class DocumentImportTask extends DateEntity implements Serializable { @Column(comment = "知识库ID") private BigInteger knowledgeId; + @Column(comment = "批次ID") + private BigInteger batchId; + + @Column(comment = "批次文件项ID") + private BigInteger batchItemId; + @Column(comment = "任务阶段") private String phase; @@ -46,6 +52,21 @@ public class DocumentImportTask extends DateEntity implements Serializable { @Column(comment = "错误摘要") private String errorSummary; + @Column(comment = "稳定失败码") + private String failureCode; + + @Column(comment = "执行尝试次数") + private Integer attemptNo; + + @Column(comment = "执行令牌") + private String executionToken; + + @Column(comment = "租约到期时间") + private Date leaseUntil; + + @Column(comment = "乐观锁版本") + private Integer version; + @Column(comment = "开始时间") private Date startedAt; @@ -88,6 +109,22 @@ public class DocumentImportTask extends DateEntity implements Serializable { this.knowledgeId = knowledgeId; } + public BigInteger getBatchId() { + return batchId; + } + + public void setBatchId(BigInteger batchId) { + this.batchId = batchId; + } + + public BigInteger getBatchItemId() { + return batchItemId; + } + + public void setBatchItemId(BigInteger batchItemId) { + this.batchItemId = batchItemId; + } + public String getPhase() { return phase; } @@ -128,6 +165,46 @@ public class DocumentImportTask extends DateEntity implements Serializable { this.errorSummary = errorSummary; } + public String getFailureCode() { + return failureCode; + } + + public void setFailureCode(String failureCode) { + this.failureCode = failureCode; + } + + public Integer getAttemptNo() { + return attemptNo; + } + + public void setAttemptNo(Integer attemptNo) { + this.attemptNo = attemptNo; + } + + public String getExecutionToken() { + return executionToken; + } + + public void setExecutionToken(String executionToken) { + this.executionToken = executionToken; + } + + public Date getLeaseUntil() { + return leaseUntil; + } + + public void setLeaseUntil(Date leaseUntil) { + this.leaseUntil = leaseUntil; + } + + public Integer getVersion() { + return version; + } + + public void setVersion(Integer version) { + this.version = version; + } + public Date getStartedAt() { return startedAt; } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchItemStage.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchItemStage.java new file mode 100644 index 00000000..e2daa5a6 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchItemStage.java @@ -0,0 +1,25 @@ +package tech.easyflow.ai.enums; + +/** + * 文档批量导入项当前阶段。 + * + * @author Codex + * @since 2026-07-31 + */ +public enum DocumentImportBatchItemStage { + + /** 上传阶段。 */ + UPLOAD, + + /** 解析阶段。 */ + PARSE, + + /** 分块阶段。 */ + SPLIT, + + /** 向量化阶段。 */ + INDEX, + + /** 全流程结束。 */ + DONE +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchItemStatus.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchItemStatus.java new file mode 100644 index 00000000..ceb45f28 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchItemStatus.java @@ -0,0 +1,34 @@ +package tech.easyflow.ai.enums; + +/** + * 文档批量导入项状态。 + * + * @author Codex + * @since 2026-07-31 + */ +public enum DocumentImportBatchItemStatus { + + /** 等待处理。 */ + PENDING, + + /** 文件正在上传。 */ + UPLOADING, + + /** 正在处理。 */ + RUNNING, + + /** 文件已上传。 */ + UPLOADED, + + /** 处理失败。 */ + FAILED, + + /** 处理完成。 */ + COMPLETED, + + /** 因重复而跳过。 */ + SKIPPED, + + /** 文件随未启动批次一并取消。 */ + CANCELLED +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchStatus.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchStatus.java new file mode 100644 index 00000000..bcf5a155 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportBatchStatus.java @@ -0,0 +1,31 @@ +package tech.easyflow.ai.enums; + +/** + * 文档批量导入状态。 + * + * @author Codex + * @since 2026-07-31 + */ +public enum DocumentImportBatchStatus { + + /** 文件上传中。 */ + UPLOADING, + + /** 文件已上传，等待选择导入方式。 */ + READY, + + /** 批次处理中。 */ + RUNNING, + + /** 批次处理已中断，可继续。 */ + INTERRUPTED, + + /** 部分文件失败，可继续失败项。 */ + PARTIAL_SUCCEEDED, + + /** 未启动的上传批次已取消。 */ + CANCELLED, + + /** 批次全部完成。 */ + COMPLETED +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportMode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportMode.java new file mode 100644 index 00000000..67dce52b --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportMode.java @@ -0,0 +1,20 @@ +package tech.easyflow.ai.enums; + +/** + * 知识库文档导入模式。 + * + * @author Codex + * @since 2026-07-31 + */ +public enum DocumentImportMode { + + /** + * 解析完成后由用户确认分块策略。 + */ + MANUAL, + + /** + * 自动完成解析、分块、向量化和入库。 + */ + AUTO +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportTaskPhase.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportTaskPhase.java index cf2f3083..f8c7f95a 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportTaskPhase.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentImportTaskPhase.java @@ -13,6 +13,11 @@ public enum DocumentImportTaskPhase { */ PARSE, + /** + * 文档分块阶段。 + */ + SPLIT, + /** * 向量化阶段。 */ diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentProcessStatus.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentProcessStatus.java index 305210e3..fbbd1067 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentProcessStatus.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/DocumentProcessStatus.java @@ -28,6 +28,16 @@ public enum DocumentProcessStatus { */ READY_FOR_SEGMENT, + /** + * 自动分块处理中。 + */ + SPLITTING, + + /** + * 自动分块失败。 + */ + SPLIT_FAILED, + /** * 已确认分块，可开始向量化。 */ @@ -54,6 +64,18 @@ public enum DocumentProcessStatus { * @return 是否运行中 */ public boolean isProcessing() { - return this == PARSING || this == INDEXING; + return this == PARSING || this == SPLITTING || this == INDEXING; + } + + /** + * 判断状态名称是否属于运行中状态。 + * + * @param status 状态名称 + * @return 是否运行中 + */ + public static boolean isProcessing(String status) { + return PARSING.name().equals(status) + || SPLITTING.name().equals(status) + || INDEXING.name().equals(status); } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/KnowledgeApiPermissionScope.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/KnowledgeApiPermissionScope.java new file mode 100644 index 00000000..057da078 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/enums/KnowledgeApiPermissionScope.java @@ -0,0 +1,52 @@ +package tech.easyflow.ai.enums; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * 知识库 Public API 产品权限范围。 + * + * @author Codex + * @since 2026-08-02 + */ +public enum KnowledgeApiPermissionScope { + + /** + * 知识库读取权限。 + */ + KNOWLEDGE_READ, + + /** + * 知识库导入权限。 + */ + KNOWLEDGE_IMPORT, + + /** + * 知识库维护权限。 + */ + KNOWLEDGE_MAINTENANCE; + + /** + * 根据三个权限开关构造稳定 Scope 集合。 + * + * @param readEnabled 是否开启读取 + * @param importEnabled 是否开启导入 + * @param maintenanceEnabled 是否开启维护 + * @return 已开启的权限 Scope + */ + public static Set enabledScopes(boolean readEnabled, + boolean importEnabled, + boolean maintenanceEnabled) { + Set scopes = new LinkedHashSet<>(); + if (readEnabled) { + scopes.add(KNOWLEDGE_READ.name()); + } + if (importEnabled) { + scopes.add(KNOWLEDGE_IMPORT.name()); + } + if (maintenanceEnabled) { + scopes.add(KNOWLEDGE_MAINTENANCE.name()); + } + return scopes; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java new file mode 100644 index 00000000..41433c4a --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchItemMapper.java @@ -0,0 +1,292 @@ +package tech.easyflow.ai.mapper; + +import com.mybatisflex.core.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Update; +import tech.easyflow.ai.entity.DocumentImportBatchItem; + +import java.math.BigInteger; +import java.util.Date; + +/** + * 文档批量导入文件项映射层。 + * + * @author Codex + * @since 2026-07-31 + */ +public interface DocumentImportBatchItemMapper extends BaseMapper { + + /** + * 原子领取文件上传权并刷新批次进度时间。 + * + *

文件项与批次在同一条 MySQL 多表更新中加锁，确保上传领取和 + * 超时取消之间不存在旧快照窗口。

+ * + * @param batchId 批次 ID + * @param itemId 文件项 ID + * @param knowledgeId 知识库 ID + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch batch " + + "INNER JOIN tb_document_import_batch_item item " + + "ON item.batch_id=batch.id SET " + + "item.stage='UPLOAD', item.status='UPLOADING', " + + "item.error_summary=NULL, item.failure_code=NULL, " + + "item.retryable=0, item.modified=#{modified}, " + + "batch.modified=#{modified}, batch.version=batch.version + 1 " + + "WHERE batch.id=#{batchId} AND batch.knowledge_id=#{knowledgeId} " + + "AND batch.status IN ('UPLOADING','READY') " + + "AND item.id=#{itemId} AND item.status='PENDING' " + + "AND item.cleanup_pending=0 AND item.storage_locator IS NULL") + int claimUpload( + @Param("batchId") BigInteger batchId, + @Param("itemId") BigInteger itemId, + @Param("knowledgeId") BigInteger knowledgeId, + @Param("modified") Date modified + ); + + /** + * 按预期状态原子迁移文件项。 + * + * @param id 文件项 ID + * @param expectedStatus 预期状态 + * @param stage 新阶段 + * @param status 新状态 + * @param errorSummary 错误摘要 + * @param failureCode 稳定失败码 + * @param retryable 是否允许批量重试 + * @param attemptDelta 重试次数增量 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch_item SET " + + "stage=#{stage}, status=#{status}, error_summary=#{errorSummary}, " + + "failure_code=#{failureCode}, " + + "retryable=#{retryable}, attempt_count=attempt_count + #{attemptDelta}, " + + "modified=#{modified} WHERE id=#{id} AND status=#{expectedStatus}") + int transitionStatus( + @Param("id") BigInteger id, + @Param("expectedStatus") String expectedStatus, + @Param("stage") String stage, + @Param("status") String status, + @Param("errorSummary") String errorSummary, + @Param("failureCode") String failureCode, + @Param("retryable") boolean retryable, + @Param("attemptDelta") int attemptDelta, + @Param("modified") Date modified + ); + + /** + * 将上传项原子绑定到创建后的文档。 + * + * @param id 文件项 ID + * @param documentId 文档 ID + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch_item SET " + + "document_id=#{documentId}, stage='PARSE', status='PENDING', " + + "error_summary=NULL, retryable=0, modified=#{modified} " + + "WHERE id=#{id} AND status='UPLOADED' AND document_id IS NULL") + int bindDocument( + @Param("id") BigInteger id, + @Param("documentId") BigInteger documentId, + @Param("modified") Date modified + ); + + /** + * 在物理写入前持久化可恢复存储定位符。 + * + * @param id 文件项 ID + * @param storageLocator 可恢复存储定位符 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch_item SET " + + "storage_locator=#{storageLocator}, cleanup_pending=0, " + + "modified=#{modified} " + + "WHERE id=#{id} AND status='UPLOADING' " + + "AND storage_locator IS NULL AND cleanup_pending=0") + int registerUploadWriteIntent( + @Param("id") BigInteger id, + @Param("storageLocator") String storageLocator, + @Param("modified") Date modified + ); + + /** + * 在物理写入前原子撤销上传领取与写意图登记。 + * + *

同时兼容登记未提交和提交结果未知两种情况；仅清除空定位符或 + * 本次预期定位符，避免覆盖其他请求的新写入意图。

+ * + * @param id 文件项 ID + * @param storageLocator 本次预期存储定位符，可为空 + * @param errorSummary 上传准备失败摘要 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch batch " + + "INNER JOIN tb_document_import_batch_item item " + + "ON item.batch_id=batch.id SET " + + "item.status='PENDING', item.storage_locator=NULL, " + + "item.file_path=NULL, item.cleanup_pending=0, " + + "item.error_summary=#{errorSummary}, item.failure_code=NULL, " + + "item.retryable=0, item.modified=#{modified}, " + + "batch.modified=#{modified}, batch.version=batch.version + 1 " + + "WHERE item.id=#{id} AND item.status='UPLOADING' " + + "AND item.cleanup_pending=0 " + + "AND (item.storage_locator IS NULL " + + "OR item.storage_locator=#{storageLocator})") + int abortUploadBeforeWrite( + @Param("id") BigInteger id, + @Param("storageLocator") String storageLocator, + @Param("errorSummary") String errorSummary, + @Param("modified") Date modified + ); + + /** + * 在文件仍处于上传中时原子绑定存储路径并完成上传。 + * + * @param id 文件项 ID + * @param filePath 存储路径 + * @param storageLocator 可恢复存储定位符 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch_item SET " + + "file_path=#{filePath}, status='UPLOADED', error_summary=NULL, " + + "cleanup_pending=0, retryable=0, modified=#{modified} " + + "WHERE id=#{id} AND status='UPLOADING' " + + "AND storage_locator=#{storageLocator}") + int completeUpload( + @Param("id") BigInteger id, + @Param("filePath") String filePath, + @Param("storageLocator") String storageLocator, + @Param("modified") Date modified + ); + + /** + * 将指定可恢复写意图标记为等待清理。 + * + * @param id 文件项 ID + * @param storageLocator 可恢复存储定位符 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch_item SET " + + "cleanup_pending=1, modified=#{modified} " + + "WHERE id=#{id} AND status IN ('UPLOADING','CANCELLED') " + + "AND storage_locator=#{storageLocator}") + int markUploadCleanupPending( + @Param("id") BigInteger id, + @Param("storageLocator") String storageLocator, + @Param("modified") Date modified + ); + + /** + * 将已取消文件项的现有存储引用标记为等待清理。 + * + * @param id 文件项 ID + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch_item SET " + + "cleanup_pending=1, modified=#{modified} " + + "WHERE id=#{id} AND status='CANCELLED' " + + "AND (storage_locator IS NOT NULL OR file_path IS NOT NULL)") + int markCancelledCleanupPending( + @Param("id") BigInteger id, + @Param("modified") Date modified + ); + + /** + * 为新导入项记录待覆盖的历史文档。 + * + * @param id 文件项 ID + * @param replacedDocumentId 历史文档 ID + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch_item SET " + + "replaced_document_id=#{replacedDocumentId}, modified=#{modified} " + + "WHERE id=#{id} AND status='UPLOADED' AND replaced_document_id IS NULL") + int markReplacement( + @Param("id") BigInteger id, + @Param("replacedDocumentId") BigInteger replacedDocumentId, + @Param("modified") Date modified + ); + + /** + * 完成历史文档清理后原子清除覆盖标记。 + * + * @param id 文件项 ID + * @param replacedDocumentId 历史文档 ID + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch_item SET " + + "replaced_document_id=NULL, modified=#{modified} " + + "WHERE id=#{id} AND replaced_document_id=#{replacedDocumentId}") + int clearReplacement( + @Param("id") BigInteger id, + @Param("replacedDocumentId") BigInteger replacedDocumentId, + @Param("modified") Date modified + ); + + /** + * 对象删除成功后原子清理上传中项并恢复待上传状态。 + * + * @param id 文件项 ID + * @param storageLocator 已删除对象的恢复定位符 + * @param filePath 已删除对象的兼容存储路径 + * @param recoveryMessage 恢复为待上传时的提示 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch batch " + + "INNER JOIN tb_document_import_batch_item item " + + "ON item.batch_id=batch.id SET " + + "item.stage='UPLOAD', item.error_summary=#{recoveryMessage}, " + + "item.failure_code=NULL, item.retryable=0, item.status='PENDING', " + + "item.cleanup_pending=0, item.storage_locator=NULL, " + + "item.file_path=NULL, item.modified=#{modified}, " + + "batch.modified=#{modified}, batch.version=batch.version + 1 " + + "WHERE item.id=#{id} AND item.cleanup_pending=1 " + + "AND item.status='UPLOADING' " + + "AND item.storage_locator <=> #{storageLocator} " + + "AND item.file_path <=> #{filePath}") + int completeUploadingStorageCleanup( + @Param("id") BigInteger id, + @Param("storageLocator") String storageLocator, + @Param("filePath") String filePath, + @Param("recoveryMessage") String recoveryMessage, + @Param("modified") Date modified + ); + + /** + * 对象删除成功后原子清理已取消项并保持取消状态。 + * + * @param id 文件项 ID + * @param storageLocator 已删除对象的恢复定位符 + * @param filePath 已删除对象的兼容存储路径 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch batch " + + "INNER JOIN tb_document_import_batch_item item " + + "ON item.batch_id=batch.id SET " + + "item.cleanup_pending=0, item.storage_locator=NULL, " + + "item.file_path=NULL, item.modified=#{modified}, " + + "batch.modified=#{modified}, batch.version=batch.version + 1 " + + "WHERE item.id=#{id} AND item.cleanup_pending=1 " + + "AND item.status='CANCELLED' " + + "AND item.storage_locator <=> #{storageLocator} " + + "AND item.file_path <=> #{filePath}") + int completeCancelledStorageCleanup( + @Param("id") BigInteger id, + @Param("storageLocator") String storageLocator, + @Param("filePath") String filePath, + @Param("modified") Date modified + ); +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchMapper.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchMapper.java new file mode 100644 index 00000000..041d7b23 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportBatchMapper.java @@ -0,0 +1,167 @@ +package tech.easyflow.ai.mapper; + +import com.mybatisflex.core.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; +import tech.easyflow.ai.entity.DocumentImportBatch; + +import java.math.BigInteger; +import java.util.Date; +import java.util.List; + +/** + * 文档批量导入批次映射层。 + * + * @author Codex + * @since 2026-07-31 + */ +public interface DocumentImportBatchMapper extends BaseMapper { + + /** + * 锁定并读取调用者拥有的批次，保证后续重试基于最新状态。 + * + * @param batchId 批次 ID + * @param callerType 调用者类型 + * @param callerId 调用者 ID + * @return 批次；不存在时返回 null + */ + @Select("SELECT * FROM tb_document_import_batch " + + "WHERE id=#{batchId} AND caller_type=#{callerType} " + + "AND caller_id=#{callerId} FOR UPDATE") + DocumentImportBatch selectOwnedForUpdate( + @Param("batchId") BigInteger batchId, + @Param("callerType") String callerType, + @Param("callerId") BigInteger callerId + ); + + /** + * 按文件项状态迁移增量更新批次计数。 + * + * @param batchId 批次 ID + * @param completedDelta 完成数增量 + * @param processingDelta 处理中数量增量 + * @param failedDelta 失败数增量 + * @param pendingDelta 等待数增量 + * @param uploadedDelta 已上传数增量 + * @param skippedDelta 跳过数增量 + * @param cancelledDelta 取消数增量 + * @param retryableFailedDelta 可重试失败数增量 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch SET " + + "completed_count=GREATEST(0, completed_count + #{completedDelta}), " + + "processing_count=GREATEST(0, processing_count + #{processingDelta}), " + + "failed_count=GREATEST(0, failed_count + #{failedDelta}), " + + "pending_count=GREATEST(0, pending_count + #{pendingDelta}), " + + "uploaded_count=GREATEST(0, uploaded_count + #{uploadedDelta}), " + + "skipped_count=GREATEST(0, skipped_count + #{skippedDelta}), " + + "cancelled_count=GREATEST(0, cancelled_count + #{cancelledDelta}), " + + "retryable_failed_count=GREATEST(0, retryable_failed_count + #{retryableFailedDelta}), " + + "modified=#{modified} WHERE id=#{batchId}") + int adjustCounters( + @Param("batchId") BigInteger batchId, + @Param("completedDelta") int completedDelta, + @Param("processingDelta") int processingDelta, + @Param("failedDelta") int failedDelta, + @Param("pendingDelta") int pendingDelta, + @Param("uploadedDelta") int uploadedDelta, + @Param("skippedDelta") int skippedDelta, + @Param("cancelledDelta") int cancelledDelta, + @Param("retryableFailedDelta") int retryableFailedDelta, + @Param("modified") Date modified + ); + + /** + * 原子领取 Public API 重试代次。 + * + * @param batchId 批次 ID + * @param callerType 调用者类型 + * @param callerId 调用者 ID + * @param expectedGeneration 预期重试代次 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch SET " + + "status='RUNNING', finished_at=NULL, " + + "retry_generation=retry_generation + 1, " + + "version=version + 1, modified=#{modified} " + + "WHERE id=#{batchId} AND caller_type=#{callerType} AND caller_id=#{callerId} " + + "AND retry_generation=#{expectedGeneration} " + + "AND status IN ('PARTIAL_SUCCEEDED','INTERRUPTED')") + int claimRetry( + @Param("batchId") BigInteger batchId, + @Param("callerType") String callerType, + @Param("callerId") BigInteger callerId, + @Param("expectedGeneration") int expectedGeneration, + @Param("modified") Date modified + ); + + /** + * 原子领取无进展的未完成批次取消权。 + * + * @param batchId 批次 ID + * @param knowledgeId 知识库 ID + * @param callerType 调用者类型 + * @param callerId 调用者 ID + * @param incompleteCutoff 最后进展截止时间 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch SET " + + "status='CANCELLED', finished_at=#{modified}, " + + "version=version + 1, modified=#{modified} " + + "WHERE id=#{batchId} AND knowledge_id=#{knowledgeId} " + + "AND caller_type=#{callerType} AND caller_id=#{callerId} " + + "AND status IN ('UPLOADING','READY') " + + "AND ((modified IS NOT NULL AND modified < #{incompleteCutoff}) " + + "OR (modified IS NULL AND created < #{incompleteCutoff}))") + int claimStaleCancellation( + @Param("batchId") BigInteger batchId, + @Param("knowledgeId") BigInteger knowledgeId, + @Param("callerType") String callerType, + @Param("callerId") BigInteger callerId, + @Param("incompleteCutoff") Date incompleteCutoff, + @Param("modified") Date modified + ); + + /** + * 分批查询无进展的未完成批次。 + * + * @param incompleteCutoff 最后进展截止时间 + * @param limit 最大返回数量 + * @return 待回收批次 + */ + @Select("SELECT * FROM tb_document_import_batch " + + "WHERE status IN ('UPLOADING','READY') " + + "AND ((modified IS NOT NULL AND modified < #{incompleteCutoff}) " + + "OR (modified IS NULL AND created < #{incompleteCutoff})) " + + "ORDER BY COALESCE(modified, created), id LIMIT #{limit}") + List selectStaleIncompleteBatches( + @Param("incompleteCutoff") Date incompleteCutoff, + @Param("limit") int limit + ); + + /** + * 释放已取消或超过去重窗口的服务端提交指纹。 + * + * @param batchId 批次 ID + * @param submissionFingerprint 当前服务端提交指纹 + * @param deduplicationCutoff 去重窗口起点 + * @param modified 修改时间 + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_batch SET " + + "idempotency_key_hash=NULL, modified=#{modified}, version=version + 1 " + + "WHERE id=#{batchId} AND idempotency_key_hash=#{submissionFingerprint} " + + "AND (status='CANCELLED' OR (" + + "COALESCE(finished_at, modified, created) < #{deduplicationCutoff} " + + "AND status IN ('INTERRUPTED','PARTIAL_SUCCEEDED','COMPLETED')))") + int releaseSubmissionFingerprint( + @Param("batchId") BigInteger batchId, + @Param("submissionFingerprint") String submissionFingerprint, + @Param("deduplicationCutoff") Date deduplicationCutoff, + @Param("modified") Date modified + ); +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportTaskMapper.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportTaskMapper.java index 90e508ca..0ef03589 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportTaskMapper.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/mapper/DocumentImportTaskMapper.java @@ -1,8 +1,15 @@ package tech.easyflow.ai.mapper; import com.mybatisflex.core.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; import tech.easyflow.ai.entity.DocumentImportTask; +import java.math.BigInteger; +import java.util.Date; +import java.util.List; + /** * 文档导入任务映射层。 * @@ -10,4 +17,169 @@ import tech.easyflow.ai.entity.DocumentImportTask; * @since 2026-04-14 */ public interface DocumentImportTaskMapper extends BaseMapper { + + /** + * 按任务阶段和批次轮转查询待投递任务，避免大批次独占投递窗口。 + * + * @param redispatchBefore 允许重新投递的修改时间边界 + * @param limit 最大任务数 + * @return 公平排序后的待投递任务 + */ + @Select("SELECT task.* FROM tb_document_import_task task JOIN (" + + "SELECT task.id, ROW_NUMBER() OVER (" + + "PARTITION BY task.phase, COALESCE(task.batch_id, task.id) " + + "ORDER BY task.created, task.id) AS lane_row " + + "FROM tb_document_import_task task " + + "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore}" + + ") ranked ON ranked.id=task.id " + + "ORDER BY ranked.lane_row, task.created, task.id LIMIT #{limit}") + List selectPendingFairly( + @Param("redispatchBefore") Date redispatchBefore, + @Param("limit") int limit + ); + + /** + * 仅在任务仍待处理时原子更新时间戳，取得本轮重新投递资格。 + * + * @param id 任务 ID + * @param redispatchBefore 允许重新投递的修改时间边界 + * @param now 当前时间 + * @param operatorId 操作人 ID + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_task SET modified=#{now}, " + + "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 " + + "WHERE id=#{id} AND status='PENDING' " + + "AND modified <= #{redispatchBefore}") + int touchPendingForDispatch( + @Param("id") BigInteger id, + @Param("redispatchBefore") Date redispatchBefore, + @Param("now") Date now, + @Param("operatorId") BigInteger operatorId + ); + + /** + * 查询租约已过期的运行任务，并兼容迁移前没有租约的历史任务。 + * + * @param now 当前时间 + * @param legacyCutoff 历史任务失联时间边界 + * @param limit 最大任务数 + * @return 已失去执行租约的运行任务 + */ + @Select("SELECT * FROM tb_document_import_task " + + "WHERE status='RUNNING' AND (" + + "(lease_until IS NOT NULL AND lease_until <= #{now}) OR " + + "(lease_until IS NULL AND modified <= #{legacyCutoff})) " + + "ORDER BY COALESCE(lease_until, modified), id LIMIT #{limit}") + List selectExpiredRunningTasks( + @Param("now") Date now, + @Param("legacyCutoff") Date legacyCutoff, + @Param("limit") int limit + ); + + /** + * 使用执行令牌原子领取一个待处理任务。 + * + * @param id 任务 ID + * @param executionToken 本轮执行令牌 + * @param leaseUntil 租约到期时间 + * @param now 当前时间 + * @param operatorId 操作人 ID + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_task SET status='RUNNING', " + + "attempt_no=COALESCE(attempt_no, 0) + 1, " + + "execution_token=#{executionToken}, lease_until=#{leaseUntil}, " + + "started_at=COALESCE(started_at, #{now}), finished_at=NULL, " + + "error_summary=NULL, failure_code=NULL, modified=#{now}, " + + "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 " + + "WHERE id=#{id} AND status='PENDING'") + int claimPending( + @Param("id") BigInteger id, + @Param("executionToken") String executionToken, + @Param("leaseUntil") Date leaseUntil, + @Param("now") Date now, + @Param("operatorId") BigInteger operatorId + ); + + /** + * 在仍持有执行令牌时续租任务。 + * + * @param id 任务 ID + * @param executionToken 本轮执行令牌 + * @param leaseUntil 新租约到期时间 + * @param now 当前时间 + * @param operatorId 操作人 ID + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_task SET lease_until=#{leaseUntil}, " + + "modified=#{now}, modified_by=#{operatorId}, " + + "version=COALESCE(version, 0) + 1 " + + "WHERE id=#{id} AND status='RUNNING' " + + "AND execution_token=#{executionToken}") + int renewLease( + @Param("id") BigInteger id, + @Param("executionToken") String executionToken, + @Param("leaseUntil") Date leaseUntil, + @Param("now") Date now, + @Param("operatorId") BigInteger operatorId + ); + + /** + * 在仍持有执行令牌时写入任务终态。 + * + * @param id 任务 ID + * @param executionToken 本轮执行令牌 + * @param status 任务终态 + * @param errorSummary 错误摘要 + * @param failureCode 稳定失败码 + * @param now 当前时间 + * @param operatorId 操作人 ID + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_task SET status=#{status}, " + + "error_summary=#{errorSummary}, failure_code=#{failureCode}, " + + "lease_until=NULL, finished_at=#{now}, modified=#{now}, " + + "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 " + + "WHERE id=#{id} AND status='RUNNING' " + + "AND execution_token=#{executionToken}") + int finishOwned( + @Param("id") BigInteger id, + @Param("executionToken") String executionToken, + @Param("status") String status, + @Param("errorSummary") String errorSummary, + @Param("failureCode") String failureCode, + @Param("now") Date now, + @Param("operatorId") BigInteger operatorId + ); + + /** + * 仅在当前执行令牌仍持有已过期租约时将任务标记为失败。 + * + * @param id 任务 ID + * @param executionToken 本轮执行令牌 + * @param errorSummary 错误摘要 + * @param failureCode 稳定失败码 + * @param now 当前时间 + * @param legacyCutoff 历史任务失联时间边界 + * @param operatorId 操作人 ID + * @return 更新行数 + */ + @Update("UPDATE tb_document_import_task SET status='FAILED', " + + "error_summary=#{errorSummary}, failure_code=#{failureCode}, " + + "lease_until=NULL, finished_at=#{now}, modified=#{now}, " + + "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 " + + "WHERE id=#{id} AND status='RUNNING' " + + "AND execution_token <=> #{executionToken} AND (" + + "(lease_until IS NOT NULL AND lease_until <= #{now}) OR " + + "(lease_until IS NULL AND modified <= #{legacyCutoff}))") + int failExpiredOwned( + @Param("id") BigInteger id, + @Param("executionToken") String executionToken, + @Param("errorSummary") String errorSummary, + @Param("failureCode") String failureCode, + @Param("now") Date now, + @Param("legacyCutoff") Date legacyCutoff, + @Param("operatorId") BigInteger operatorId + ); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentImportBatchItemService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentImportBatchItemService.java new file mode 100644 index 00000000..9ac87fcf --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentImportBatchItemService.java @@ -0,0 +1,13 @@ +package tech.easyflow.ai.service; + +import com.mybatisflex.core.service.IService; +import tech.easyflow.ai.entity.DocumentImportBatchItem; + +/** + * 文档批量导入文件项服务。 + * + * @author Codex + * @since 2026-07-31 + */ +public interface DocumentImportBatchItemService extends IService { +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentImportBatchService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentImportBatchService.java new file mode 100644 index 00000000..259b664d --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentImportBatchService.java @@ -0,0 +1,13 @@ +package tech.easyflow.ai.service; + +import com.mybatisflex.core.service.IService; +import tech.easyflow.ai.entity.DocumentImportBatch; + +/** + * 文档批量导入批次服务。 + * + * @author Codex + * @since 2026-07-31 + */ +public interface DocumentImportBatchService extends IService { +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentService.java index 3da7c32d..facb6853 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/DocumentService.java @@ -19,7 +19,37 @@ import java.util.List; */ public interface DocumentService extends IService { - Page getDocumentList(String knowledgeId , int pageSize, int pageNum, String fileName); + /** + * 按知识库和文件标题查询文档分页。 + * + * @param knowledgeId 知识库 ID + * @param pageSize 每页条数 + * @param pageNum 页码 + * @param fileName 可选的文件标题筛选 + * @return 文档分页 + */ + Page getDocumentList( + String knowledgeId, + int pageSize, + int pageNum, + String fileName + ); + + /** + * 按知识库和文档 ID 查询文档分页。 + * + * @param knowledgeId 知识库 ID + * @param pageSize 每页条数 + * @param pageNum 页码 + * @param documentId 可选的文档 ID + * @return 文档分页 + */ + Page getDocumentListById( + String knowledgeId, + int pageSize, + int pageNum, + BigInteger documentId + ); boolean removeDoc(String id); @@ -44,4 +74,6 @@ public interface DocumentService extends IService { Result retryParseTask(DocumentImportDtos.TaskRetryRequest request); Result retryIndexTask(DocumentImportDtos.TaskRetryRequest request); + + Result retryFailedTask(DocumentImportDtos.TaskRetryRequest request); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/KnowledgeSharePermissionService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/KnowledgeSharePermissionService.java index ceccdcf6..cc583423 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/KnowledgeSharePermissionService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/KnowledgeSharePermissionService.java @@ -27,6 +27,27 @@ public interface KnowledgeSharePermissionService { */ void replaceApiShareEnabled(BigInteger apiKeyId, boolean enabled); + /** + * 替换访问令牌的三类知识库 Public API 权限。 + * + * @param apiKeyId 系统访问令牌 ID + * @param readEnabled 是否开启读取 + * @param importEnabled 是否开启导入 + * @param maintenanceEnabled 是否开启维护 + */ + void replaceApiPermissions(BigInteger apiKeyId, + boolean readEnabled, + boolean importEnabled, + boolean maintenanceEnabled); + + /** + * 查询访问令牌已开启的知识库 Public API 权限。 + * + * @param apiKeyId 系统访问令牌 ID + * @return 权限 Scope 集合 + */ + Set getApiPermissionScopes(BigInteger apiKeyId); + /** * 断言当前令牌具备知识库分享权限。 * diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java index 1bc673b8..a1f91908 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImpl.java @@ -486,6 +486,10 @@ public class DocumentCollectionServiceImpl extends ServiceImpl + implements DocumentImportBatchItemService { +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentImportBatchServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentImportBatchServiceImpl.java new file mode 100644 index 00000000..28acdc83 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentImportBatchServiceImpl.java @@ -0,0 +1,19 @@ +package tech.easyflow.ai.service.impl; + +import com.mybatisflex.spring.service.impl.ServiceImpl; +import org.springframework.stereotype.Service; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; +import tech.easyflow.ai.service.DocumentImportBatchService; + +/** + * 文档批量导入批次服务实现。 + * + * @author Codex + * @since 2026-07-31 + */ +@Service +public class DocumentImportBatchServiceImpl + extends ServiceImpl + implements DocumentImportBatchService { +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java index 14ea261d..d74da2cd 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/DocumentServiceImpl.java @@ -106,6 +106,45 @@ public class DocumentServiceImpl extends ServiceImpl i @Override public Page getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) { + return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null); + } + + /** + * 按知识库和文档 ID 查询文档分页。 + * + * @param knowledgeId 知识库 ID + * @param pageSize 每页条数 + * @param pageNum 页码 + * @param documentId 可选的文档 ID + * @return 文档分页 + */ + @Override + public Page getDocumentListById( + String knowledgeId, + int pageSize, + int pageNum, + BigInteger documentId + ) { + return queryDocumentList(knowledgeId, pageSize, pageNum, null, documentId); + } + + /** + * 执行文档分页查询。 + * + * @param knowledgeId 知识库 ID + * @param pageSize 每页条数 + * @param pageNum 页码 + * @param fileName 可选的文件标题筛选 + * @param documentId 可选的文档 ID + * @return 文档分页 + */ + private Page queryDocumentList( + String knowledgeId, + int pageSize, + int pageNum, + String fileName, + BigInteger documentId + ) { QueryWrapper queryWrapper=QueryWrapper.create() .select( DOCUMENT.ALL_COLUMNS, @@ -120,19 +159,23 @@ public class DocumentServiceImpl extends ServiceImpl i if (fileName != null && !fileName.trim().isEmpty()) { queryWrapper.and(DOCUMENT.TITLE.like(fileName)); } + if (documentId != null) { + queryWrapper.and(DOCUMENT.ID.eq(documentId)); + } // 分组 queryWrapper.groupBy(DOCUMENT.ID); - Page documentVoPage = documentMapper.paginateAs(pageNum, pageSize, queryWrapper, Document.class); - return documentVoPage; + return documentMapper.paginateAs(pageNum, pageSize, queryWrapper, Document.class); } /** - * 根据文档id删除文件 + * 删除文档的向量、搜索索引、分块、存储文件和主记录。 * - * @param id 文档id - * @return + * @param id 文档 ID + * @return 全部数据库清理成功时返回 true + * @throws BusinessException 文档仍在处理中时抛出 */ @Override + @Transactional public boolean removeDoc(String id) { // 查询该文档对应哪些分割的字段，先删除 QueryWrapper queryWrapperDocument = QueryWrapper.create().eq(Document::getId, id); @@ -140,8 +183,7 @@ public class DocumentServiceImpl extends ServiceImpl i if (oneByQuery == null) { return false; } - if (DocumentProcessStatus.PARSING.name().equals(oneByQuery.getProcessStatus()) - || DocumentProcessStatus.INDEXING.name().equals(oneByQuery.getProcessStatus())) { + if (DocumentProcessStatus.isProcessing(oneByQuery.getProcessStatus())) { throw new BusinessException("文档处理中，暂不允许删除"); } DocumentCollection knowledge = knowledgeService.getById(oneByQuery.getCollectionId()); @@ -149,28 +191,40 @@ public class DocumentServiceImpl extends ServiceImpl i return false; } - // 存储到知识库 - DocumentStore documentStore = knowledge.toDocumentStore(); - if (documentStore == null) { - return false; - } - + QueryWrapper queryWrapper = QueryWrapper.create() + .select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id); + List chunkIds = documentChunkMapper.selectListByQueryAs( + queryWrapper, + BigInteger.class + ); + DocumentStore documentStore = null; try { - Model model = modelService.getById(knowledge.getVectorEmbedModelId()); - if (model == null) { - return false; + if (!chunkIds.isEmpty()) { + documentStore = knowledge.toDocumentStore(); + if (documentStore == null) { + return false; + } + Model model = modelService.getById( + knowledge.getVectorEmbedModelId() + ); + if (model == null) { + return false; + } + StoreOptions options = StoreOptions.ofCollectionName( + knowledge.getVectorStoreCollection() + ); + EmbeddingOptions embeddingOptions = new EmbeddingOptions(); + embeddingOptions.setModel(model.getModelName()); + options.setEmbeddingOptions(embeddingOptions); + StoreResult deleteResult = documentStore.delete(chunkIds, options); + if (deleteResult == null || !deleteResult.isSuccess()) { + String failReason = deleteResult == null + ? "未返回结果" + : deleteResult.getFailReason(); + Log.error("删除文档向量失败: documentId={}, reason={}", id, failReason); + throw new BusinessException("文档向量删除失败"); + } } - // 设置向量模型 - StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection()); - EmbeddingOptions embeddingOptions = new EmbeddingOptions(); - embeddingOptions.setModel(model.getModelName()); - options.setEmbeddingOptions(embeddingOptions); - options.setCollectionName(knowledge.getVectorStoreCollection()); - // 查询文本分割表tb_document_chunk中对应的有哪些数据，找出来删除 - QueryWrapper queryWrapper = QueryWrapper.create() - .select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id); - List chunkIds = documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class); - documentStore.delete(chunkIds, options); // 删除搜索引擎中的数据 DocumentSearcher searcher = searcherFactory.getSearcher(); if (searcher != null) { @@ -181,9 +235,16 @@ public class DocumentServiceImpl extends ServiceImpl i return false; } // 再删除指定路径下的文件 - Document document = documentMapper.selectOneByQuery(queryWrapperDocument); - storageService.delete(document.getDocumentPath()); - return true; + String chunkSnapshotPath = oneByQuery.getOptions() == null + ? null + : asString(oneByQuery.getOptions().get( + DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH)); + if (StringUtil.hasText(chunkSnapshotPath)) { + storageService.delete(chunkSnapshotPath); + } + storageService.delete(oneByQuery.getDocumentPath()); + // 主记录必须最后删除；否则接口返回成功后文档仍会出现在列表中。 + return documentMapper.deleteById(oneByQuery.getId()) > 0; } finally { DocumentStoreLifecycleSupport.closeQuietly(documentStore); } @@ -1012,4 +1073,9 @@ public class DocumentServiceImpl extends ServiceImpl i public Result retryIndexTask(DocumentImportDtos.TaskRetryRequest request) { return importTaskAppService.retryIndexTask(request); } + + @Override + public Result retryFailedTask(DocumentImportDtos.TaskRetryRequest request) { + return importTaskAppService.retryFailedTask(request); + } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/KnowledgeSharePermissionServiceImpl.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/KnowledgeSharePermissionServiceImpl.java index 503a78c7..85e70fb1 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/KnowledgeSharePermissionServiceImpl.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/service/impl/KnowledgeSharePermissionServiceImpl.java @@ -3,8 +3,13 @@ package tech.easyflow.ai.service.impl; import com.mybatisflex.core.query.QueryWrapper; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import tech.easyflow.ai.enums.KnowledgeApiPermissionScope; import tech.easyflow.ai.enums.KnowledgeShareActionScope; import tech.easyflow.ai.service.KnowledgeSharePermissionService; +import tech.easyflow.common.cache.RedisLockExecutor; +import tech.easyflow.common.cache.RedisLockExecutor.LockHandle; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.system.entity.SysApiKey; import tech.easyflow.system.entity.SysApiKeyResource; @@ -15,9 +20,9 @@ import tech.easyflow.system.service.SysApiKeyService; import javax.annotation.Resource; import java.math.BigInteger; +import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -30,10 +35,51 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis public static final String RESOURCE_TYPE_KNOWLEDGE = "KNOWLEDGE"; - private static final Map> URI_SCOPE_MAPPING = new LinkedHashMap<>(); + private static final String PERMISSION_LOCK_PREFIX = + "easyflow:lock:knowledge-api-permission:"; + private static final Duration PERMISSION_LOCK_WAIT = Duration.ofSeconds(2); + private static final Duration PERMISSION_LOCK_LEASE = Duration.ofSeconds(15); + + private static final Map> API_SCOPE_URI_MAPPING = new LinkedHashMap<>(); + private static final Map> LEGACY_ACTION_URI_MAPPING = new LinkedHashMap<>(); static { - URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.VIEW.name(), List.of( + API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_READ.name(), List.of( + "/public-api/knowledge-share/detail", + "/public-api/knowledge-share/search", + "/public-api/knowledge-share/document/page", + "/public-api/knowledge-share/document/download", + "/public-api/knowledge-share/documentChunk/page", + "/public-api/knowledge-share/faq/page", + "/public-api/knowledge-share/faq/detail", + "/public-api/knowledge-share/faq/exportExcel" + )); + API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name(), List.of( + "/public-api/knowledge-share/document/import/batch", + "/public-api/knowledge-share/document/import/batch/status", + "/public-api/knowledge-share/document/import/batch/retry", + "/public-api/knowledge-share/document/import/analyze", + "/public-api/knowledge-share/document/import/preview", + "/public-api/knowledge-share/document/import/commit", + "/public-api/knowledge-share/document/import/task/create", + "/public-api/knowledge-share/document/import/task/detail", + "/public-api/knowledge-share/document/import/task/preview", + "/public-api/knowledge-share/document/import/task/startIndex", + "/public-api/knowledge-share/document/import/task/retryParse", + "/public-api/knowledge-share/document/import/task/retryIndex", + "/public-api/knowledge-share/faq/importExcel", + "/public-api/knowledge-share/faq/downloadImportTemplate" + )); + API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name(), List.of( + "/public-api/knowledge-share/document/remove", + "/public-api/knowledge-share/documentChunk/update", + "/public-api/knowledge-share/documentChunk/remove", + "/public-api/knowledge-share/faq/save", + "/public-api/knowledge-share/faq/update", + "/public-api/knowledge-share/faq/remove" + )); + + LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.VIEW.name(), List.of( "/public-api/knowledge-share/detail", "/public-api/knowledge-share/document/page", "/public-api/knowledge-share/document/download", @@ -42,10 +88,10 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis "/public-api/knowledge-share/faq/page", "/public-api/knowledge-share/faq/detail" )); - URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.SEARCH.name(), List.of( + LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.SEARCH.name(), List.of( "/public-api/knowledge-share/search" )); - URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_CREATE.name(), List.of( + LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_CREATE.name(), List.of( "/public-api/knowledge-share/document/import/analyze", "/public-api/knowledge-share/document/import/preview", "/public-api/knowledge-share/document/import/commit", @@ -54,18 +100,21 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis "/public-api/knowledge-share/document/import/task/startIndex", "/public-api/knowledge-share/document/import/task/retryParse", "/public-api/knowledge-share/document/import/task/retryIndex", + "/public-api/knowledge-share/document/import/batch", + "/public-api/knowledge-share/document/import/batch/status", + "/public-api/knowledge-share/document/import/batch/retry", "/public-api/knowledge-share/faq/save" )); - URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_UPDATE.name(), List.of( + LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_UPDATE.name(), List.of( "/public-api/knowledge-share/documentChunk/update", "/public-api/knowledge-share/faq/update" )); - URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_DELETE.name(), List.of( + LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_DELETE.name(), List.of( "/public-api/knowledge-share/document/remove", "/public-api/knowledge-share/documentChunk/remove", "/public-api/knowledge-share/faq/remove" )); - URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.IMPORT_EXPORT.name(), List.of( + LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.IMPORT_EXPORT.name(), List.of( "/public-api/knowledge-share/faq/importExcel", "/public-api/knowledge-share/faq/exportExcel", "/public-api/knowledge-share/faq/downloadImportTemplate" @@ -78,6 +127,8 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis private SysApiKeyResourceService resourceService; @Resource private SysApiKeyResourceMappingService mappingService; + @Resource + private RedisLockExecutor redisLockExecutor; @Override @Transactional(rollbackFor = Exception.class) @@ -97,32 +148,48 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis throw new BusinessException("动作范围不能为空"); } - mappingService.removeScopedMappings(apiKeyId, RESOURCE_TYPE_KNOWLEDGE, knowledgeId); - List rows = new ArrayList<>(); - for (String scope : normalizedScopes) { - List uris = URI_SCOPE_MAPPING.get(scope); - if (uris == null || uris.isEmpty()) { - continue; + Runnable releaseLock = acquirePermissionMutationLock(apiKeyId); + try { + mappingService.removeScopedMappings( + apiKeyId, + RESOURCE_TYPE_KNOWLEDGE, + knowledgeId + ); + List rows = new ArrayList<>(); + for (String legacyScope : normalizedScopes) { + List uris = LEGACY_ACTION_URI_MAPPING.get(legacyScope); + if (uris == null || uris.isEmpty()) { + continue; + } + for (String uri : uris) { + rows.add(buildMapping( + apiKeyId, + knowledgeId, + uri, + requireApiScope(uri) + )); + } } - for (String uri : uris) { - SysApiKeyResource resource = ensureResource(uri); - SysApiKeyResourceMapping row = new SysApiKeyResourceMapping(); - row.setApiKeyId(apiKeyId); - row.setApiKeyResourceId(resource.getId()); - row.setResourceType(RESOURCE_TYPE_KNOWLEDGE); - row.setResourceTargetId(knowledgeId); - row.setActionScope(scope); - rows.add(row); + if (!rows.isEmpty()) { + mappingService.saveBatch(rows); } - } - if (!rows.isEmpty()) { - mappingService.saveBatch(rows); + } finally { + releaseLock.run(); } } @Override @Transactional(rollbackFor = Exception.class) public void replaceApiShareEnabled(BigInteger apiKeyId, boolean enabled) { + replaceApiPermissions(apiKeyId, enabled, enabled, false); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void replaceApiPermissions(BigInteger apiKeyId, + boolean readEnabled, + boolean importEnabled, + boolean maintenanceEnabled) { if (apiKeyId == null) { throw new BusinessException("系统访问令牌不能为空"); } @@ -130,30 +197,73 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis if (apiKey == null) { throw new BusinessException("系统访问令牌不存在"); } - mappingService.removeScopedMappings(apiKeyId, RESOURCE_TYPE_KNOWLEDGE); - if (!enabled) { - return; - } + Runnable releaseLock = acquirePermissionMutationLock(apiKeyId); + try { + // 全局开关只替换全局授权，保留分享页配置的指定知识库权限。 + mappingService.remove( + QueryWrapper.create() + .eq(SysApiKeyResourceMapping::getApiKeyId, apiKeyId) + .eq( + SysApiKeyResourceMapping::getResourceType, + RESOURCE_TYPE_KNOWLEDGE + ) + .isNull( + SysApiKeyResourceMapping::getResourceTargetId + ) + ); + Set enabledScopes = KnowledgeApiPermissionScope.enabledScopes( + readEnabled, + importEnabled, + maintenanceEnabled + ); + if (enabledScopes.isEmpty()) { + return; + } - List rows = new ArrayList<>(); - for (String scope : KnowledgeShareActionScope.defaultApiScopes()) { - List uris = URI_SCOPE_MAPPING.get(scope); - if (uris == null || uris.isEmpty()) { - continue; + List rows = new ArrayList<>(); + for (String scope : enabledScopes) { + List uris = API_SCOPE_URI_MAPPING.get(scope); + if (uris == null || uris.isEmpty()) { + continue; + } + for (String uri : uris) { + rows.add(buildMapping(apiKeyId, null, uri, scope)); + } } - for (String uri : uris) { - SysApiKeyResource resource = ensureResource(uri); - SysApiKeyResourceMapping row = new SysApiKeyResourceMapping(); - row.setApiKeyId(apiKeyId); - row.setApiKeyResourceId(resource.getId()); - row.setResourceType(RESOURCE_TYPE_KNOWLEDGE); - row.setActionScope(scope); - rows.add(row); + if (!rows.isEmpty()) { + mappingService.saveBatch(rows); + } + } finally { + releaseLock.run(); + } + } + + @Override + public Set getApiPermissionScopes(BigInteger apiKeyId) { + if (apiKeyId == null) { + return Set.of(); + } + List mappings = mappingService.list( + QueryWrapper.create() + .select(SysApiKeyResourceMapping::getActionScope) + .eq(SysApiKeyResourceMapping::getApiKeyId, apiKeyId) + .eq(SysApiKeyResourceMapping::getResourceType, RESOURCE_TYPE_KNOWLEDGE) + .isNull(SysApiKeyResourceMapping::getResourceTargetId) + .in( + SysApiKeyResourceMapping::getActionScope, + API_SCOPE_URI_MAPPING.keySet() + ) + ); + if (mappings == null || mappings.isEmpty()) { + return Set.of(); + } + Set scopes = new java.util.LinkedHashSet<>(); + for (SysApiKeyResourceMapping mapping : mappings) { + if (mapping.getActionScope() != null) { + scopes.add(mapping.getActionScope()); } } - if (!rows.isEmpty()) { - mappingService.saveBatch(rows); - } + return scopes; } @Override @@ -161,9 +271,56 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis if (apiKeyId == null || knowledgeId == null) { throw new BusinessException("API 分享鉴权参数不完整"); } + if (!API_SCOPE_URI_MAPPING.containsKey(actionScope)) { + throw new IllegalArgumentException("未知的知识库 API 权限范围"); + } sysApiKeyService.checkResourceScope(apiKeyId, requestUri, RESOURCE_TYPE_KNOWLEDGE, knowledgeId, actionScope); } + /** + * 构造单条知识库 API 权限映射。 + * + * @param apiKeyId 访问令牌 ID + * @param knowledgeId 知识库 ID；为空表示全局授权 + * @param uri 请求 URI + * @param scope 产品权限 Scope + * @return 权限映射 + */ + private SysApiKeyResourceMapping buildMapping(BigInteger apiKeyId, + BigInteger knowledgeId, + String uri, + String scope) { + SysApiKeyResource resource = ensureResource(uri); + SysApiKeyResourceMapping row = new SysApiKeyResourceMapping(); + row.setApiKeyId(apiKeyId); + row.setApiKeyResourceId(resource.getId()); + row.setResourceType(RESOURCE_TYPE_KNOWLEDGE); + row.setResourceTargetId(knowledgeId); + row.setActionScope(scope); + return row; + } + + /** + * 按 URI 获取唯一的产品权限 Scope。 + * + * @param uri 请求 URI + * @return 产品权限 Scope + */ + private String requireApiScope(String uri) { + for (Map.Entry> entry : API_SCOPE_URI_MAPPING.entrySet()) { + if (entry.getValue().contains(uri)) { + return entry.getKey(); + } + } + throw new IllegalArgumentException("知识库接口未归类: " + uri); + } + + /** + * 获取或创建固定 URI 对应的 API 资源。 + * + * @param requestInterface 请求 URI + * @return API 资源 + */ private SysApiKeyResource ensureResource(String requestInterface) { QueryWrapper wrapper = QueryWrapper.create() .eq(SysApiKeyResource::getRequestInterface, requestInterface); @@ -177,4 +334,35 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis resourceService.save(resource); return resource; } + + /** + * 获取访问令牌知识库权限变更锁，并在事务完成后释放。 + * + * @param apiKeyId 访问令牌 ID + * @return 非事务直接调用时使用的释放动作 + */ + private Runnable acquirePermissionMutationLock(BigInteger apiKeyId) { + LockHandle handle = redisLockExecutor.tryAcquire( + PERMISSION_LOCK_PREFIX + apiKeyId, + PERMISSION_LOCK_WAIT, + PERMISSION_LOCK_LEASE + ); + if (handle == null) { + throw new BusinessException("访问令牌权限正在更新，请稍后重试"); + } + if (TransactionSynchronizationManager.isSynchronizationActive() + && TransactionSynchronizationManager.isActualTransactionActive()) { + TransactionSynchronizationManager.registerSynchronization( + new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + handle.release(); + } + } + ); + return () -> { + }; + } + return handle::release; + } } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java index 5ad3fe4b..db2014e1 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/document/service/impl/DocumentParseBridgeServiceImplTest.java @@ -108,6 +108,26 @@ public class DocumentParseBridgeServiceImplTest { Assert.assertEquals("# demo", taskInfo.getResult().getPreferredText()); } + /** + * 验证带 PDF 源信息的任务查询会直达 PDF 服务，不试探无关的 PPTX 服务。 + */ + @Test + public void shouldQueryPdfTaskAgainstResolvedService() { + FakePdfDocumentParseService pdfService = new FakePdfDocumentParseService(); + pdfService.taskStatusValue = "completed"; + FakePptxDocumentParseService pptxService = new FakePptxDocumentParseService(); + DocumentParseBridgeServiceImpl bridgeService = + buildBridgeService(pdfService, pptxService, null, pdfService); + + DocumentParseTaskInfo taskInfo = bridgeService.queryTaskInfo("task-1", buildSource()); + DocumentParsedResult result = bridgeService.queryResult("task-1", buildSource()); + + Assert.assertEquals("completed", taskInfo.getStatus()); + Assert.assertEquals("# demo", result.getPreferredText()); + Assert.assertEquals(0, pptxService.queryTaskInfoCallCount); + Assert.assertEquals(0, pptxService.queryResultCallCount); + } + /** * 验证缺少底层服务时抛出稳定错误码。 */ @@ -390,6 +410,8 @@ public class DocumentParseBridgeServiceImplTest { private static class FakePptxDocumentParseService implements PptxDocumentParseService { private int parseCallCount; + private int queryTaskInfoCallCount; + private int queryResultCallCount; @Override public ParseResponse parse(ParseRequest request) { @@ -415,6 +437,13 @@ public class DocumentParseBridgeServiceImplTest { @Override public ParseResponse queryResult(String taskId) { + queryResultCallCount++; + throw new UnsupportedOperationException(); + } + + @Override + public ParseTaskInfo queryTaskInfo(String taskId) { + queryTaskInfoCallCount++; throw new UnsupportedOperationException(); } } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java new file mode 100644 index 00000000..90efbed7 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchAppServiceTest.java @@ -0,0 +1,913 @@ +package tech.easyflow.ai.documentimport.task; + +import com.mybatisflex.core.query.QueryWrapper; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.ai.documentimport.DocumentImportBatchDtos; +import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult; +import tech.easyflow.ai.documentimport.ImportCallerContext; +import tech.easyflow.ai.documentimport.ImportCallerType; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.entity.DocumentImportBatchItem; +import tech.easyflow.ai.enums.DocumentImportBatchItemStage; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; +import tech.easyflow.ai.enums.DocumentImportBatchStatus; +import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; +import tech.easyflow.ai.mapper.DocumentMapper; +import tech.easyflow.ai.service.DocumentImportBatchItemService; +import tech.easyflow.ai.service.DocumentImportBatchService; +import tech.easyflow.common.cache.RedisLockExecutor; +import tech.easyflow.common.filestorage.FileStorageService; +import tech.easyflow.common.filestorage.FileStorageWriteHandle; +import tech.easyflow.common.filestorage.FileStorageWriteResult; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.Date; +import java.util.List; +import java.util.Set; + +/** + * {@link DocumentImportBatchAppService} 批次启动与重复策略回归测试。 + * + * @author Codex + * @since 2026-07-31 + */ +public class DocumentImportBatchAppServiceTest { + + /** + * 验证上传领取通过单条多表更新同步刷新批次进度时间。 + */ + @Test + public void uploadShouldAtomicallyClaimItemAndTouchBatch() { + TestContext context = createContext(); + DocumentImportBatch batch = context.batchService.getOne( + QueryWrapper.create() + ); + batch.setStatus(DocumentImportBatchStatus.UPLOADING.name()); + DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(41)); + DocumentImportBatchItem uploaded = pendingUploadItem(BigInteger.valueOf(41)); + uploaded.setStatus(DocumentImportBatchItemStatus.UPLOADED.name()); + uploaded.setFilePath("/stored/demo.docx"); + FileStorageWriteHandle handle = writeHandle(pending); + String locator = handle.encodeLocator(); + uploaded.setStorageLocator(locator); + uploaded.setCleanupPending(false); + Mockito.when(context.batchTracker.requireItem(pending.getId())) + .thenReturn(pending, uploaded); + Mockito.when(context.itemMapper.claimUpload( + Mockito.eq(batch.getId()), + Mockito.eq(pending.getId()), + Mockito.eq(batch.getKnowledgeId()), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.itemMapper.registerUploadWriteIntent( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.batchTracker.completeUpload( + pending.getId(), + "/stored/demo.docx", + locator + )).thenReturn(true); + FileStorageService storage = Mockito.mock(FileStorageService.class); + Mockito.when(storage.prepareRecoverableWrite( + Mockito.anyString(), + Mockito.anyString() + )).thenReturn(handle); + Mockito.when(storage.saveRecoverable( + Mockito.any(MultipartFile.class), + Mockito.eq(handle) + )).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator)); + setStorageService(context.service, storage); + MultipartFile file = uploadFile("demo.docx", pending.getFileSize()); + + DocumentImportBatchDtos.ItemResponse response = context.service.uploadItem( + batch.getKnowledgeId(), + batch.getId(), + pending.getId(), + file, + publicCaller() + ); + + Assert.assertEquals( + DocumentImportBatchItemStatus.UPLOADED.name(), + response.getStatus() + ); + org.mockito.InOrder order = Mockito.inOrder(context.itemMapper, storage); + order.verify(context.itemMapper).claimUpload( + Mockito.eq(batch.getId()), + Mockito.eq(pending.getId()), + Mockito.eq(batch.getKnowledgeId()), + Mockito.any(Date.class) + ); + order.verify(context.itemMapper).registerUploadWriteIntent( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + ); + order.verify(storage).saveRecoverable(file, handle); + } + + /** + * 验证对象绑定失败后先清理，再恢复为可重新上传状态。 + */ + @Test + public void uploadBindFailureShouldCleanupAndAllowSameItemRetry() { + TestContext context = createContext(); + DocumentImportBatch batch = context.batchService.getOne( + QueryWrapper.create() + ); + batch.setStatus(DocumentImportBatchStatus.UPLOADING.name()); + DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(42)); + FileStorageWriteHandle handle = writeHandle(pending); + String locator = handle.encodeLocator(); + DocumentImportBatchItem cleanup = pendingUploadItem(pending.getId()); + cleanup.setStatus(DocumentImportBatchItemStatus.UPLOADING.name()); + cleanup.setStorageLocator(locator); + cleanup.setFilePath("/stored/demo.docx"); + cleanup.setCleanupPending(true); + DocumentImportBatchItem retryPending = pendingUploadItem(pending.getId()); + DocumentImportBatchItem uploaded = pendingUploadItem(pending.getId()); + uploaded.setStatus(DocumentImportBatchItemStatus.UPLOADED.name()); + uploaded.setStorageLocator(locator); + uploaded.setFilePath("/stored/demo.docx"); + Mockito.when(context.batchTracker.requireItem(pending.getId())) + .thenReturn(pending, pending, cleanup, retryPending, uploaded); + Mockito.when(context.itemMapper.claimUpload( + Mockito.eq(batch.getId()), + Mockito.eq(pending.getId()), + Mockito.eq(batch.getKnowledgeId()), + Mockito.any(Date.class) + )).thenReturn(1, 1); + Mockito.when(context.itemMapper.registerUploadWriteIntent( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + )).thenReturn(1, 1); + Mockito.when(context.batchTracker.completeUpload( + pending.getId(), + "/stored/demo.docx", + locator + )).thenReturn(false, true); + Mockito.when(context.itemMapper.markUploadCleanupPending( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.itemMapper.completeUploadingStorageCleanup( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.eq("/stored/demo.docx"), + Mockito.anyString(), + Mockito.any(Date.class) + )).thenReturn(1); + FileStorageService storage = Mockito.mock(FileStorageService.class); + Mockito.when(storage.prepareRecoverableWrite( + Mockito.anyString(), + Mockito.anyString() + )).thenReturn(handle); + Mockito.when(storage.saveRecoverable( + Mockito.any(MultipartFile.class), + Mockito.eq(handle) + )).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator)); + setStorageService(context.service, storage); + MultipartFile file = uploadFile("demo.docx", pending.getFileSize()); + + try { + context.service.uploadItem( + batch.getKnowledgeId(), + batch.getId(), + pending.getId(), + file, + publicCaller() + ); + Assert.fail("Expected cancelled upload rejection"); + } catch (BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("上传批次已取消")); + } + + DocumentImportBatchDtos.ItemResponse retried = context.service.uploadItem( + batch.getKnowledgeId(), + batch.getId(), + pending.getId(), + file, + publicCaller() + ); + + Assert.assertEquals( + DocumentImportBatchItemStatus.UPLOADED.name(), + retried.getStatus() + ); + org.mockito.InOrder order = Mockito.inOrder(context.itemMapper, storage); + order.verify(context.itemMapper).registerUploadWriteIntent( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + ); + order.verify(storage).saveRecoverable(file, handle); + order.verify(context.itemMapper).markUploadCleanupPending( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + ); + order.verify(storage).deleteRecoverable(handle); + order.verify(context.itemMapper).completeUploadingStorageCleanup( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.eq("/stored/demo.docx"), + Mockito.anyString(), + Mockito.any(Date.class) + ); + Mockito.verify(context.itemMapper, Mockito.times(2)).claimUpload( + Mockito.eq(batch.getId()), + Mockito.eq(pending.getId()), + Mockito.eq(batch.getKnowledgeId()), + Mockito.any(Date.class) + ); + Mockito.verify(context.batchTracker, Mockito.never()).transitionItem( + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.anyBoolean(), Mockito.anyInt() + ); + } + + /** + * 验证批次并发取消后，对象清理不会把文件项恢复为待上传。 + */ + @Test + public void uploadCleanupShouldPreserveConcurrentCancellation() { + TestContext context = createContext(); + DocumentImportBatch batch = context.batchService.getOne( + QueryWrapper.create() + ); + batch.setStatus(DocumentImportBatchStatus.UPLOADING.name()); + DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(44)); + FileStorageWriteHandle handle = writeHandle(pending); + String locator = handle.encodeLocator(); + DocumentImportBatchItem cancelled = pendingUploadItem(pending.getId()); + cancelled.setStatus(DocumentImportBatchItemStatus.CANCELLED.name()); + cancelled.setStorageLocator(locator); + cancelled.setFilePath("/stored/demo.docx"); + cancelled.setCleanupPending(true); + Mockito.when(context.batchTracker.requireItem(pending.getId())) + .thenReturn(pending, pending, cancelled); + Mockito.when(context.itemMapper.claimUpload( + Mockito.eq(batch.getId()), + Mockito.eq(pending.getId()), + Mockito.eq(batch.getKnowledgeId()), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.itemMapper.registerUploadWriteIntent( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.batchTracker.completeUpload( + pending.getId(), + "/stored/demo.docx", + locator + )).thenReturn(false); + Mockito.when(context.itemMapper.markUploadCleanupPending( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.itemMapper.completeCancelledStorageCleanup( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.eq("/stored/demo.docx"), + Mockito.any(Date.class) + )).thenReturn(1); + FileStorageService storage = Mockito.mock(FileStorageService.class); + Mockito.when(storage.prepareRecoverableWrite( + Mockito.anyString(), + Mockito.anyString() + )).thenReturn(handle); + Mockito.when(storage.saveRecoverable( + Mockito.any(MultipartFile.class), + Mockito.eq(handle) + )).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator)); + setStorageService(context.service, storage); + + try { + context.service.uploadItem( + batch.getKnowledgeId(), + batch.getId(), + pending.getId(), + uploadFile("demo.docx", pending.getFileSize()), + publicCaller() + ); + Assert.fail("Expected cancelled upload rejection"); + } catch (BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("上传批次已取消")); + } + + Mockito.verify(storage).deleteRecoverable(handle); + Mockito.verify(context.itemMapper).completeCancelledStorageCleanup( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.eq("/stored/demo.docx"), + Mockito.any(Date.class) + ); + } + + /** + * 验证对象删除后的数据库恢复失败会保留待办，并由下一轮幂等完成。 + */ + @Test + public void cleanupRecoveryFailureShouldRemainRetryable() { + TestContext context = createContext(); + DocumentImportBatchItem cleanup = pendingUploadItem(BigInteger.valueOf(45)); + FileStorageWriteHandle handle = writeHandle(cleanup); + String locator = handle.encodeLocator(); + cleanup.setStatus(DocumentImportBatchItemStatus.UPLOADING.name()); + cleanup.setStorageLocator(locator); + cleanup.setFilePath("/stored/demo.docx"); + cleanup.setCleanupPending(true); + Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class))) + .thenReturn(List.of(cleanup)); + Mockito.when(context.itemMapper.completeUploadingStorageCleanup( + Mockito.eq(cleanup.getId()), + Mockito.eq(locator), + Mockito.eq("/stored/demo.docx"), + Mockito.anyString(), + Mockito.any(Date.class) + )).thenThrow(new RuntimeException("database unavailable")) + .thenReturn(1); + FileStorageService storage = Mockito.mock(FileStorageService.class); + setStorageService(context.service, storage); + + Assert.assertEquals(0, context.service.cleanupCancelledStoredObjects(10)); + Assert.assertEquals(1, context.service.cleanupCancelledStoredObjects(10)); + + Mockito.verify(storage, Mockito.times(2)).deleteRecoverable(handle); + Mockito.verify(context.itemMapper, Mockito.times(2)) + .completeUploadingStorageCleanup( + Mockito.eq(cleanup.getId()), + Mockito.eq(locator), + Mockito.eq("/stored/demo.docx"), + Mockito.anyString(), + Mockito.any(Date.class) + ); + } + + /** + * 验证写意图登记已提交但调用抛异常时，写入前撤销会清理定位符。 + */ + @Test + public void uploadIntentUnknownCommitShouldAbortBeforePhysicalWrite() { + TestContext context = createContext(); + DocumentImportBatch batch = context.batchService.getOne( + QueryWrapper.create() + ); + batch.setStatus(DocumentImportBatchStatus.UPLOADING.name()); + DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(46)); + FileStorageWriteHandle handle = writeHandle(pending); + String locator = handle.encodeLocator(); + Mockito.when(context.batchTracker.requireItem(pending.getId())) + .thenReturn(pending); + Mockito.when(context.itemMapper.claimUpload( + Mockito.eq(batch.getId()), + Mockito.eq(pending.getId()), + Mockito.eq(batch.getKnowledgeId()), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.itemMapper.registerUploadWriteIntent( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + )).thenThrow(new RuntimeException("commit result unknown")); + Mockito.when(context.itemMapper.abortUploadBeforeWrite( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.anyString(), + Mockito.any(Date.class) + )).thenReturn(1); + FileStorageService storage = Mockito.mock(FileStorageService.class); + Mockito.when(storage.prepareRecoverableWrite( + Mockito.anyString(), + Mockito.anyString() + )).thenReturn(handle); + setStorageService(context.service, storage); + + try { + context.service.uploadItem( + batch.getKnowledgeId(), + batch.getId(), + pending.getId(), + uploadFile("demo.docx", pending.getFileSize()), + publicCaller() + ); + Assert.fail("Expected unknown commit exception"); + } catch (RuntimeException expected) { + Assert.assertEquals("commit result unknown", expected.getMessage()); + } + + Mockito.verify(context.itemMapper).abortUploadBeforeWrite( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.anyString(), + Mockito.any(Date.class) + ); + Mockito.verify(storage, Mockito.never()).saveRecoverable( + Mockito.any(MultipartFile.class), + Mockito.any(FileStorageWriteHandle.class) + ); + } + + /** + * 验证提交响应异常但数据库已完成上传时不会误删有效对象。 + */ + @Test + public void uploadCommitUnknownShouldKeepConfirmedUploadedObject() { + TestContext context = createContext(); + DocumentImportBatch batch = context.batchService.getOne( + QueryWrapper.create() + ); + batch.setStatus(DocumentImportBatchStatus.UPLOADING.name()); + DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(43)); + FileStorageWriteHandle handle = writeHandle(pending); + String locator = handle.encodeLocator(); + DocumentImportBatchItem committed = pendingUploadItem(pending.getId()); + committed.setStatus(DocumentImportBatchItemStatus.UPLOADED.name()); + committed.setFilePath("/stored/demo.docx"); + committed.setStorageLocator(locator); + committed.setCleanupPending(false); + Mockito.when(context.batchTracker.requireItem(pending.getId())) + .thenReturn(pending, committed); + Mockito.when(context.itemMapper.claimUpload( + Mockito.eq(batch.getId()), + Mockito.eq(pending.getId()), + Mockito.eq(batch.getKnowledgeId()), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.itemMapper.registerUploadWriteIntent( + Mockito.eq(pending.getId()), + Mockito.eq(locator), + Mockito.any(Date.class) + )).thenReturn(1); + Mockito.when(context.batchTracker.completeUpload( + pending.getId(), + "/stored/demo.docx", + locator + )).thenThrow(new RuntimeException("commit result unknown")); + FileStorageService storage = Mockito.mock(FileStorageService.class); + Mockito.when(storage.prepareRecoverableWrite( + Mockito.anyString(), + Mockito.anyString() + )).thenReturn(handle); + Mockito.when(storage.saveRecoverable( + Mockito.any(MultipartFile.class), + Mockito.eq(handle) + )).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator)); + setStorageService(context.service, storage); + + DocumentImportBatchDtos.ItemResponse response = context.service.uploadItem( + batch.getKnowledgeId(), + batch.getId(), + pending.getId(), + uploadFile("demo.docx", pending.getFileSize()), + publicCaller() + ); + + Assert.assertEquals( + DocumentImportBatchItemStatus.UPLOADED.name(), + response.getStatus() + ); + Mockito.verify(context.itemMapper, Mockito.never()) + .markUploadCleanupPending( + Mockito.any(), + Mockito.anyString(), + Mockito.any(Date.class) + ); + Mockito.verify(storage, Mockito.never()) + .deleteRecoverable(Mockito.any(FileStorageWriteHandle.class)); + } + + /** + * 验证同一知识库已有运行中的自动批次时拒绝再次启动。 + */ + @Test + public void startAutoShouldRejectOtherRunningBatch() { + TestContext context = createContext(); + Mockito.when(context.batchService.count(Mockito.any(QueryWrapper.class))).thenReturn(1L); + beginTransactionSynchronization(); + try { + context.service.startBatch(startRequest("AUTO", "SKIP")); + Assert.fail("Expected active automatic batch rejection"); + } catch (BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("已有自动导入批次")); + } finally { + completeTransactionSynchronization(TransactionSynchronization.STATUS_ROLLED_BACK); + } + Mockito.verify(context.batchMapper, Mockito.never()) + .updateByQuery(Mockito.any(DocumentImportBatch.class), Mockito.any()); + Mockito.verify(context.lockHandle).release(); + } + + /** + * 验证覆盖策略会持久化历史文档 ID，并继续创建新文档任务。 + */ + @Test + public void startOverwriteShouldRecordHistoricalDocument() { + TestContext context = createContext(); + Mockito.when(context.batchService.count(Mockito.any(QueryWrapper.class))).thenReturn(0L); + DocumentImportBatchItem uploaded = uploadedItem(BigInteger.valueOf(11), BigInteger.ONE); + DocumentImportBatchItem historical = uploadedItem(BigInteger.valueOf(21), BigInteger.valueOf(22)); + historical.setBatchId(BigInteger.valueOf(99)); + historical.setDocumentId(BigInteger.valueOf(23)); + historical.setStatus(DocumentImportBatchItemStatus.COMPLETED.name()); + historical.setCreated(new Date()); + Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class))) + .thenReturn(List.of(uploaded), List.of(historical)); + tech.easyflow.ai.entity.Document historicalDocument = new tech.easyflow.ai.entity.Document(); + historicalDocument.setId(historical.getDocumentId()); + historicalDocument.setCollectionId(BigInteger.TWO); + Mockito.when(context.documentMapper.selectListByQuery(Mockito.any())) + .thenReturn(List.of(historicalDocument)); + Mockito.when(context.batchMapper.updateByQuery( + Mockito.any(DocumentImportBatch.class), Mockito.any() + )).thenReturn(1); + Mockito.when(context.batchTracker.refreshBatch(BigInteger.ONE)) + .thenReturn(new DocumentImportBatchDtos.StatusResponse()); + + beginTransactionSynchronization(); + try { + context.service.startBatch(startRequest("AUTO", "OVERWRITE")); + } finally { + completeTransactionSynchronization(TransactionSynchronization.STATUS_COMMITTED); + } + + Mockito.verify(context.batchTracker) + .markReplacement(uploaded.getId(), historical.getDocumentId()); + Mockito.verify(context.taskAppService) + .createBatchImportTasks(Mockito.any(DocumentImportBatch.class), Mockito.anyList()); + Mockito.verify(context.lockHandle).release(); + } + + /** + * 验证重试代次由服务端读取并原子领取。 + */ + @Test + public void retryShouldClaimCurrentGeneration() { + TestContext context = createContext(); + DocumentImportBatch batch = context.batchService.getOne( + QueryWrapper.create() + ); + batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name()); + batch.setRetryGeneration(0); + DocumentImportBatchItem failed = uploadedItem( + BigInteger.valueOf(31), + batch.getId() + ); + failed.setStatus(DocumentImportBatchItemStatus.FAILED.name()); + failed.setRetryable(true); + Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class))) + .thenReturn(List.of(failed)); + Mockito.when(context.batchMapper.claimRetry( + Mockito.any(), + Mockito.anyString(), + Mockito.any(), + Mockito.anyInt(), + Mockito.any(Date.class) + )).thenReturn(1); + + beginTransactionSynchronization(); + DocumentImportBatchRetryResult first; + try { + first = context.service.retryOwnedBatch( + batch.getId(), + publicCaller(), + Set.of("file-key") + ); + } finally { + completeTransactionSynchronization( + TransactionSynchronization.STATUS_ROLLED_BACK + ); + } + + Assert.assertEquals(Integer.valueOf(1), first.getRetryGeneration()); + Assert.assertEquals(Integer.valueOf(1), first.getRetriedCount()); + Mockito.verify(context.batchMapper).claimRetry( + Mockito.any(), + Mockito.anyString(), + Mockito.any(), + Mockito.eq(0), + Mockito.any(Date.class) + ); + Mockito.verify(context.lockHandle).release(); + } + + /** + * 验证修复前的可恢复失败项会在重试领取事务中恢复重试资格和批次计数。 + */ + @Test + public void retryShouldRestoreLegacyRecoverableItem() { + TestContext context = createContext(); + DocumentImportBatch batch = context.batchService.getOne( + QueryWrapper.create() + ); + batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name()); + batch.setRetryGeneration(0); + DocumentImportBatchItem failed = uploadedItem( + BigInteger.valueOf(32), + batch.getId() + ); + failed.setStage(DocumentImportBatchItemStage.PARSE.name()); + failed.setStatus(DocumentImportBatchItemStatus.FAILED.name()); + failed.setRetryable(false); + failed.setFailureCode("parse_failed"); + failed.setErrorSummary("历史代码异常"); + Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class))) + .thenReturn(List.of(failed)); + Mockito.when(context.taskAppService.isRecoverableBatchFailure(failed)) + .thenReturn(true); + Mockito.when(context.batchTracker.transitionItem( + Mockito.eq(failed.getId()), + Mockito.eq(DocumentImportBatchItemStage.PARSE), + Mockito.eq(DocumentImportBatchItemStatus.FAILED), + Mockito.eq("历史代码异常"), + Mockito.eq(true), + Mockito.eq(0), + Mockito.eq("parse_failed") + )).thenReturn(true); + Mockito.when(context.batchMapper.claimRetry( + Mockito.any(), + Mockito.anyString(), + Mockito.any(), + Mockito.anyInt(), + Mockito.any(Date.class) + )).thenReturn(1); + + beginTransactionSynchronization(); + DocumentImportBatchRetryResult result; + try { + result = context.service.retryOwnedBatch( + batch.getId(), + publicCaller(), + Set.of() + ); + } finally { + completeTransactionSynchronization( + TransactionSynchronization.STATUS_ROLLED_BACK + ); + } + + Assert.assertEquals(Integer.valueOf(1), result.getRetriedCount()); + Assert.assertTrue(failed.getRetryable()); + Mockito.verify(context.batchTracker).transitionItem( + failed.getId(), + DocumentImportBatchItemStage.PARSE, + DocumentImportBatchItemStatus.FAILED, + "历史代码异常", + true, + 0, + "parse_failed" + ); + Mockito.verify(context.batchMapper).claimRetry( + Mockito.eq(batch.getId()), + Mockito.anyString(), + Mockito.any(), + Mockito.eq(0), + Mockito.any(Date.class) + ); + Mockito.verify(context.lockHandle).release(); + } + + /** + * 验证运行中的任务拒绝再次领取，调用方可继续查询原 taskId。 + */ + @Test + public void retryShouldRejectTaskThatIsAlreadyRunning() { + TestContext context = createContext(); + DocumentImportBatch batch = context.batchService.getOne( + QueryWrapper.create() + ); + batch.setStatus(DocumentImportBatchStatus.RUNNING.name()); + + beginTransactionSynchronization(); + try { + context.service.retryOwnedBatch( + BigInteger.ONE, + publicCaller(), + Set.of() + ); + Assert.fail("Expected running task rejection"); + } catch (BusinessException expected) { + Assert.assertTrue( + expected.getMessage().contains("当前任务状态不允许重试") + ); + } finally { + completeTransactionSynchronization( + TransactionSynchronization.STATUS_ROLLED_BACK + ); + } + Mockito.verify(context.batchMapper, Mockito.never()).claimRetry( + Mockito.any(), + Mockito.anyString(), + Mockito.any(), + Mockito.anyInt(), + Mockito.any(Date.class) + ); + Mockito.verify(context.lockHandle).release(); + } + + /** + * 验证候选快照过期后若任务已恢复进展，不会继续取消或清理文件项。 + */ + @Test + public void staleCancellationShouldStopWhenConditionalClaimMisses() { + TestContext context = createContext(); + Mockito.when(context.batchMapper.claimStaleCancellation( + Mockito.any(), + Mockito.any(), + Mockito.anyString(), + Mockito.any(), + Mockito.any(Date.class), + Mockito.any(Date.class) + )).thenReturn(0); + + beginTransactionSynchronization(); + boolean cancelled; + try { + cancelled = context.service.cancelStaleBatch( + BigInteger.TWO, + BigInteger.ONE, + publicCaller(), + new Date(System.currentTimeMillis() - 30L * 60L * 1000L) + ); + } finally { + completeTransactionSynchronization( + TransactionSynchronization.STATUS_ROLLED_BACK + ); + } + + Assert.assertFalse(cancelled); + Mockito.verify(context.itemService, Mockito.never()) + .list(Mockito.any(QueryWrapper.class)); + Mockito.verifyNoInteractions(context.itemMapper); + Mockito.verify(context.lockHandle).release(); + } + + private TestContext createContext() { + DocumentImportBatchService batchService = Mockito.mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = Mockito.mock(DocumentImportBatchItemService.class); + DocumentImportBatchTracker batchTracker = Mockito.mock(DocumentImportBatchTracker.class); + KnowledgeDocumentImportTaskAppService taskAppService = + Mockito.mock(KnowledgeDocumentImportTaskAppService.class); + DocumentImportBatchMapper batchMapper = Mockito.mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = + Mockito.mock(DocumentImportBatchItemMapper.class); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class); + RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class); + Mockito.when(redisLockExecutor.tryAcquire( + Mockito.anyString(), Mockito.any(), Mockito.any() + )).thenReturn(lockHandle); + + DocumentImportBatch batch = new DocumentImportBatch(); + batch.setId(BigInteger.ONE); + batch.setKnowledgeId(BigInteger.TWO); + batch.setStatus(DocumentImportBatchStatus.READY.name()); + Mockito.when(batchTracker.requireBatch(BigInteger.ONE)).thenReturn(batch); + Mockito.when(batchService.getOne(Mockito.any(QueryWrapper.class))) + .thenReturn(batch); + Mockito.when(batchMapper.selectOwnedForUpdate( + Mockito.any(), Mockito.anyString(), Mockito.any() + )).thenReturn(batch); + + DocumentImportBatchAppService service = new DocumentImportBatchAppService( + batchService, + itemService, + batchTracker, + new DocumentImportBulkProperties(), + taskAppService, + batchMapper, + itemMapper, + documentMapper, + redisLockExecutor + ); + return new TestContext( + service, batchService, itemService, batchTracker, + taskAppService, batchMapper, itemMapper, documentMapper, lockHandle + ); + } + + /** + * 创建 Public API 测试调用者。 + * + * @return Public API 调用者 + */ + private ImportCallerContext publicCaller() { + return new ImportCallerContext( + ImportCallerType.PUBLIC_API, + BigInteger.valueOf(77) + ); + } + + private DocumentImportBatchDtos.StartRequest startRequest(String mode, String duplicatePolicy) { + DocumentImportBatchDtos.StartRequest request = new DocumentImportBatchDtos.StartRequest(); + request.setKnowledgeId(BigInteger.TWO); + request.setBatchId(BigInteger.ONE); + request.setImportMode(mode); + request.setDuplicatePolicy(duplicatePolicy); + return request; + } + + private DocumentImportBatchItem uploadedItem(BigInteger itemId, BigInteger batchId) { + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(itemId); + item.setBatchId(batchId); + item.setKnowledgeId(BigInteger.TWO); + item.setClientFileKey("file-key"); + item.setFileName("demo.docx"); + item.setFilePath("/demo.docx"); + item.setStatus(DocumentImportBatchItemStatus.UPLOADED.name()); + return item; + } + + private DocumentImportBatchItem pendingUploadItem(BigInteger itemId) { + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(itemId); + item.setBatchId(BigInteger.ONE); + item.setKnowledgeId(BigInteger.TWO); + item.setClientFileKey("file-key-" + itemId); + item.setFileName("demo.docx"); + item.setRelativePath("demo.docx"); + item.setFileSize(4L); + item.setStage("UPLOAD"); + item.setStatus(DocumentImportBatchItemStatus.PENDING.name()); + item.setCleanupPending(false); + return item; + } + + private FileStorageWriteHandle writeHandle(DocumentImportBatchItem item) { + return new FileStorageWriteHandle( + "mock-storage", + "", + "/tmp/easyflow-test", + "knowledge-import/" + item.getBatchId() + "/" + item.getId(), + item.getId() + ".docx" + ); + } + + private MultipartFile uploadFile(String fileName, long size) { + MultipartFile file = Mockito.mock(MultipartFile.class); + Mockito.when(file.getOriginalFilename()).thenReturn(fileName); + Mockito.when(file.getSize()).thenReturn(size); + Mockito.when(file.isEmpty()).thenReturn(false); + return file; + } + + private void setStorageService(DocumentImportBatchAppService service, + FileStorageService storageService) { + try { + Field field = DocumentImportBatchAppService.class.getDeclaredField( + "storageService" + ); + field.setAccessible(true); + field.set(service, storageService); + } catch (ReflectiveOperationException error) { + throw new AssertionError("Failed to inject storage service", error); + } + } + + private void beginTransactionSynchronization() { + TransactionSynchronizationManager.initSynchronization(); + } + + private void completeTransactionSynchronization(int status) { + List synchronizations = + TransactionSynchronizationManager.getSynchronizations(); + for (TransactionSynchronization synchronization : synchronizations) { + if (status == TransactionSynchronization.STATUS_COMMITTED) { + synchronization.afterCommit(); + } + synchronization.afterCompletion(status); + } + TransactionSynchronizationManager.clearSynchronization(); + } + + /** + * 测试依赖集合。 + */ + private record TestContext( + DocumentImportBatchAppService service, + DocumentImportBatchService batchService, + DocumentImportBatchItemService itemService, + DocumentImportBatchTracker batchTracker, + KnowledgeDocumentImportTaskAppService taskAppService, + DocumentImportBatchMapper batchMapper, + DocumentImportBatchItemMapper itemMapper, + DocumentMapper documentMapper, + RedisLockExecutor.LockHandle lockHandle + ) { + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java new file mode 100644 index 00000000..16fa78e1 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportBatchTrackerTest.java @@ -0,0 +1,186 @@ +package tech.easyflow.ai.documentimport.task; + +import org.junit.Test; +import tech.easyflow.ai.documentimport.DocumentImportBatchDtos; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.entity.DocumentImportBatchItem; +import tech.easyflow.ai.enums.DocumentImportBatchItemStage; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; +import tech.easyflow.ai.enums.DocumentImportBatchStatus; +import tech.easyflow.ai.enums.DocumentImportMode; +import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; +import tech.easyflow.ai.service.DocumentImportBatchItemService; +import tech.easyflow.ai.service.DocumentImportBatchService; + +import java.math.BigInteger; +import java.util.Date; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 文档批量导入状态汇总测试。 + */ +public class DocumentImportBatchTrackerTest { + + /** + * 验证运行批次的完成、处理、失败与等待数量。 + */ + @Test + public void shouldAggregateRunningBatchProgress() { + DocumentImportBatchService batchService = mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class); + DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class); + DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 4); + batch.setCompletedCount(1); + batch.setProcessingCount(1); + batch.setFailedCount(1); + batch.setPendingCount(1); + when(batchService.getById(batch.getId())).thenReturn(batch); + + DocumentImportBatchTracker tracker = + new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper); + DocumentImportBatchDtos.StatusResponse response = tracker.refreshBatch(batch.getId()); + + assertEquals(1, response.getCompletedCount().intValue()); + assertEquals(1, response.getProcessingCount().intValue()); + assertEquals(1, response.getFailedCount().intValue()); + assertEquals(1, response.getPendingCount().intValue()); + assertEquals(50, response.getProgressPercent().intValue()); + assertEquals(DocumentImportBatchStatus.RUNNING.name(), response.getStatus()); + verify(batchService, never()).updateById(batch, false); + } + + /** + * 验证中断批次在汇总失败项后仍保留可继续状态。 + */ + @Test + public void shouldPreserveInterruptedStatusUntilUserContinues() { + DocumentImportBatchService batchService = mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class); + DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class); + DocumentImportBatch batch = batch(DocumentImportBatchStatus.INTERRUPTED, 2); + batch.setCompletedCount(1); + batch.setFailedCount(1); + batch.setRetryableFailedCount(1); + when(batchService.getById(batch.getId())).thenReturn(batch); + + DocumentImportBatchTracker tracker = + new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper); + DocumentImportBatchDtos.StatusResponse response = tracker.refreshBatch(batch.getId()); + + assertEquals(DocumentImportBatchStatus.INTERRUPTED.name(), response.getStatus()); + assertEquals(100, response.getProgressPercent().intValue()); + assertEquals(1, response.getRetryableFailedCount().intValue()); + } + + /** + * 验证失败项重试通过状态 CAS 增量迁移计数。 + */ + @Test + public void shouldMoveRetryableFailureBackToPendingAtomically() { + DocumentImportBatchService batchService = mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class); + DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class); + DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1); + batch.setFailedCount(1); + batch.setRetryableFailedCount(1); + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(BigInteger.TEN); + item.setBatchId(batch.getId()); + item.setStage(DocumentImportBatchItemStage.INDEX.name()); + item.setStatus(DocumentImportBatchItemStatus.FAILED.name()); + item.setRetryable(true); + when(itemService.getById(item.getId())).thenReturn(item); + when(itemMapper.transitionStatus( + eq(item.getId()), + eq(DocumentImportBatchItemStatus.FAILED.name()), + eq(DocumentImportBatchItemStage.INDEX.name()), + eq(DocumentImportBatchItemStatus.PENDING.name()), + eq(null), + eq(null), + eq(false), + eq(1), + any(Date.class) + )).thenReturn(1); + when(batchService.getById(batch.getId())).thenReturn(batch); + + DocumentImportBatchTracker tracker = + new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper); + + assertTrue(tracker.updateItem(item.getId(), + DocumentImportBatchItemStage.INDEX, + DocumentImportBatchItemStatus.PENDING, + null)); + verify(batchMapper).adjustCounters( + eq(batch.getId()), + eq(0), + eq(0), + eq(-1), + eq(1), + eq(0), + eq(0), + eq(0), + eq(-1), + any(Date.class) + ); + } + + /** + * 验证迟到任务不能把已完成文件重新改为处理中。 + */ + @Test + public void shouldRejectLateTransitionFromCompletedToRunning() { + DocumentImportBatchService batchService = mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class); + DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class); + DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class); + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(BigInteger.TEN); + item.setBatchId(BigInteger.ONE); + item.setStatus(DocumentImportBatchItemStatus.COMPLETED.name()); + when(itemService.getById(item.getId())).thenReturn(item); + + DocumentImportBatchTracker tracker = + new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper); + + assertFalse(tracker.updateItem(item.getId(), + DocumentImportBatchItemStage.INDEX, + DocumentImportBatchItemStatus.RUNNING, + null)); + verify(itemMapper, never()).transitionStatus( + any(), any(), any(), any(), any(), any(), anyBoolean(), anyInt(), any(Date.class)); + } + + private DocumentImportBatch batch(DocumentImportBatchStatus status, int totalCount) { + DocumentImportBatch batch = new DocumentImportBatch(); + batch.setId(BigInteger.ONE); + batch.setKnowledgeId(BigInteger.TWO); + batch.setImportMode(DocumentImportMode.AUTO.name()); + batch.setStatus(status.name()); + batch.setTotalCount(totalCount); + batch.setTotalBytes(100L); + batch.setCompletedCount(0); + batch.setProcessingCount(0); + batch.setFailedCount(0); + batch.setPendingCount(0); + batch.setUploadedCount(0); + batch.setSkippedCount(0); + batch.setCancelledCount(0); + batch.setRetryableFailedCount(0); + return batch; + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportChunkSnapshotServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportChunkSnapshotServiceTest.java new file mode 100644 index 00000000..32d93f43 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportChunkSnapshotServiceTest.java @@ -0,0 +1,68 @@ +package tech.easyflow.ai.documentimport.task; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.ai.documentimport.DocumentImportDtos; +import tech.easyflow.ai.entity.DocumentChunk; +import tech.easyflow.common.filestorage.FileStorageService; + +import java.io.ByteArrayInputStream; +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +/** + * {@link DocumentImportChunkSnapshotService} 持久化恢复测试。 + */ +public class DocumentImportChunkSnapshotServiceTest { + + /** + * 验证分块快照可跨缓存写入并从文件存储完整恢复。 + * + * @throws Exception 反射注入或文件读取异常 + */ + @Test + public void shouldPersistAndRestoreChunkSnapshot() throws Exception { + String storedPath = "http://localhost/snapshots/9-chunks.json"; + AtomicReference storedBytes = new AtomicReference(); + FileStorageService storageService = Mockito.mock(FileStorageService.class); + Mockito.when(storageService.save( + Mockito.any(MultipartFile.class), + Mockito.anyString() + )).thenAnswer(invocation -> { + MultipartFile file = invocation.getArgument(0); + storedBytes.set(file.getBytes()); + return storedPath; + }); + Mockito.when(storageService.readStream(storedPath)) + .thenAnswer(invocation -> new ByteArrayInputStream(storedBytes.get())); + + DocumentImportChunkSnapshotService service = new DocumentImportChunkSnapshotService(); + Field storageField = DocumentImportChunkSnapshotService.class + .getDeclaredField("storageService"); + storageField.setAccessible(true); + storageField.set(service, storageService); + + DocumentChunk chunk = new DocumentChunk(); + chunk.setId(BigInteger.valueOf(11)); + chunk.setDocumentId(BigInteger.valueOf(9)); + chunk.setDocumentCollectionId(BigInteger.valueOf(7)); + chunk.setContent("稳定分块"); + DocumentImportDtos.PreviewSession session = new DocumentImportDtos.PreviewSession(); + session.setKnowledgeId(BigInteger.valueOf(7)); + session.setDocumentId(BigInteger.valueOf(9)); + session.setDocumentChunks(List.of(chunk)); + + String path = service.save(session); + DocumentImportDtos.PreviewSession restored = service.load(path); + + Assert.assertEquals(storedPath, path); + Assert.assertEquals(session.getKnowledgeId(), restored.getKnowledgeId()); + Assert.assertEquals(session.getDocumentId(), restored.getDocumentId()); + Assert.assertEquals(1, restored.getDocumentChunks().size()); + Assert.assertEquals("稳定分块", restored.getDocumentChunks().get(0).getContent()); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskStatusStreamServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskStatusStreamServiceTest.java index d77168b8..1d68b073 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskStatusStreamServiceTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/DocumentImportTaskStatusStreamServiceTest.java @@ -6,11 +6,15 @@ import org.mockito.ArgumentMatchers; import org.mockito.Mockito; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import tech.easyflow.ai.documentimport.DocumentImportKeys; import tech.easyflow.ai.entity.Document; import tech.easyflow.ai.mapper.DocumentMapper; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.math.BigInteger; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; /** @@ -18,6 +22,33 @@ import java.util.concurrent.atomic.AtomicReference; */ public class DocumentImportTaskStatusStreamServiceTest { + /** + * 验证状态流会携带稳定错误码，供前端进行本地化展示。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void buildDocumentPayloadShouldIncludeTaskErrorCode() throws Exception { + Document document = new Document(); + document.setId(BigInteger.valueOf(88)); + document.setCollectionId(BigInteger.valueOf(99)); + document.setOptions(new LinkedHashMap(Map.of( + DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE, + "parse_service_unavailable" + ))); + DocumentImportTaskStatusStreamService service = new DocumentImportTaskStatusStreamService(); + Method method = DocumentImportTaskStatusStreamService.class.getDeclaredMethod( + "buildDocumentPayload", + Document.class + ); + method.setAccessible(true); + + @SuppressWarnings("unchecked") + Map payload = (Map) method.invoke(service, document); + + Assert.assertEquals("parse_service_unavailable", payload.get("lastTaskErrorCode")); + } + /** * 验证文档状态变更会向 Redis 广播文档 ID。 * diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java index 8bd91add..1db7838f 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeDocumentImportTaskAppServiceTest.java @@ -3,28 +3,53 @@ package tech.easyflow.ai.documentimport.task; import com.easyagents.document.core.entity.DocumentBlock; import com.easyagents.document.core.entity.DocumentImage; import com.easyagents.document.core.entity.DocumentTable; +import com.easyagents.core.model.embedding.EmbeddingModel; +import com.easyagents.core.store.DocumentStore; +import com.easyagents.core.store.StoreOptions; +import com.easyagents.core.store.StoreResult; import com.easyagents.rag.ingestion.model.StrategyConfig; +import com.easyagents.search.engine.service.DocumentSearcher; +import org.apache.ibatis.annotations.Update; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.ai.document.exception.DocumentParseBridgeException; import tech.easyflow.ai.document.model.DocumentParseArtifacts; import tech.easyflow.ai.document.model.DocumentParsedResult; +import tech.easyflow.ai.document.model.DocumentSourceRef; import tech.easyflow.ai.documentimport.DocumentImportKeys; import tech.easyflow.ai.entity.DocumentChunk; +import tech.easyflow.ai.entity.DocumentCollection; +import tech.easyflow.ai.entity.DocumentImportBatchItem; import tech.easyflow.ai.entity.DocumentImportTask; +import tech.easyflow.ai.enums.DocumentImportBatchItemStage; +import tech.easyflow.ai.enums.DocumentImportBatchItemStatus; import tech.easyflow.ai.enums.DocumentImportTaskStatus; +import tech.easyflow.ai.enums.DocumentImportTaskPhase; import tech.easyflow.ai.enums.DocumentProcessStatus; +import tech.easyflow.ai.mapper.DocumentImportTaskMapper; import tech.easyflow.ai.mapper.DocumentMapper; +import tech.easyflow.ai.service.DocumentImportBatchItemService; import tech.easyflow.ai.service.DocumentImportTaskService; +import tech.easyflow.ai.service.DocumentService; +import tech.easyflow.common.cache.RedisLockExecutor; import tech.easyflow.common.filestorage.FileStorageService; +import java.io.ByteArrayInputStream; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.math.BigInteger; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; +import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -38,6 +63,666 @@ import java.util.concurrent.atomic.AtomicReference; */ public class KnowledgeDocumentImportTaskAppServiceTest { + /** + * 验证待处理任务重新投递只更新必要字段，避免自定义查询结果覆盖非空列。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void dispatchPendingTasksShouldTouchPendingWithCas() + throws Exception { + BigInteger taskId = BigInteger.valueOf(29); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setPhase(DocumentImportTaskPhase.PARSE.name()); + task.setStatus(DocumentImportTaskStatus.PENDING.name()); + + DocumentImportTaskMapper taskMapper = + Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.selectPendingFairly( + Mockito.any(Date.class), Mockito.anyInt())) + .thenReturn(List.of(task)); + Mockito.when(taskMapper.touchPendingForDispatch( + Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class), + Mockito.any(BigInteger.class))) + .thenReturn(1); + DocumentImportParseTaskProducer producer = + Mockito.mock(DocumentImportParseTaskProducer.class); + + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportTaskMapper", taskMapper); + setField(service, "parseTaskProducer", producer); + setField(service, "bulkProperties", new DocumentImportBulkProperties()); + + service.dispatchPendingTasks(); + + ArgumentCaptor selectCutoffCaptor = + ArgumentCaptor.forClass(Date.class); + Mockito.verify(taskMapper).selectPendingFairly( + selectCutoffCaptor.capture(), Mockito.anyInt()); + ArgumentCaptor touchCutoffCaptor = + ArgumentCaptor.forClass(Date.class); + Mockito.verify(taskMapper).touchPendingForDispatch( + Mockito.eq(taskId), touchCutoffCaptor.capture(), Mockito.any(Date.class), + Mockito.any(BigInteger.class)); + Assert.assertEquals(selectCutoffCaptor.getValue(), touchCutoffCaptor.getValue()); + Mockito.verify(producer).send(taskId); + } + + /** + * 验证任务已离开待处理状态时不再发送重复消息。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void dispatchPendingTasksShouldSkipChangedTask() + throws Exception { + BigInteger taskId = BigInteger.valueOf(30); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setPhase(DocumentImportTaskPhase.PARSE.name()); + + DocumentImportTaskMapper taskMapper = + Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.selectPendingFairly( + Mockito.any(Date.class), Mockito.anyInt())) + .thenReturn(List.of(task)); + Mockito.when(taskMapper.touchPendingForDispatch( + Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class), + Mockito.any(BigInteger.class))) + .thenReturn(0); + DocumentImportParseTaskProducer producer = + Mockito.mock(DocumentImportParseTaskProducer.class); + + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportTaskMapper", taskMapper); + setField(service, "parseTaskProducer", producer); + setField(service, "bulkProperties", new DocumentImportBulkProperties()); + + service.dispatchPendingTasks(); + + Mockito.verify(producer, Mockito.never()).send(Mockito.any()); + } + + /** + * 验证重新投递资格更新包含待处理状态和过期时间边界，确保竞争更新仅一个成功。 + * + * @throws Exception 映射方法不存在时抛出 + */ + @Test + public void touchPendingForDispatchSqlShouldFenceConcurrentDispatch() + throws Exception { + Method method = DocumentImportTaskMapper.class.getMethod( + "touchPendingForDispatch", + BigInteger.class, + Date.class, + Date.class, + BigInteger.class + ); + Update update = method.getAnnotation(Update.class); + String sql = String.join(" ", update.value()); + + Assert.assertTrue(sql.contains("status='PENDING'")); + Assert.assertTrue(sql.contains("modified <= #{redispatchBefore}")); + Assert.assertTrue(sql.contains("modified=#{now}")); + } + + /** + * 验证分块策略快照使用忽略 null 的部分实体更新。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void persistAppliedStrategyShouldKeepRequiredBatchItemFields() + throws Exception { + BigInteger itemId = BigInteger.valueOf(31); + DocumentImportBatchItemService itemService = + Mockito.mock(DocumentImportBatchItemService.class); + Mockito.when(itemService.updateById(Mockito.any())) + .thenReturn(true); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportBatchItemService", itemService); + + Method method = + KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod( + "persistAppliedStrategy", + BigInteger.class, + StrategyConfig.class + ); + method.setAccessible(true); + method.invoke(service, itemId, StrategyConfig.defaults()); + + ArgumentCaptor updateCaptor = + ArgumentCaptor.forClass(DocumentImportBatchItem.class); + Mockito.verify(itemService).updateById(updateCaptor.capture()); + Mockito.verify(itemService, Mockito.never()).updateById( + Mockito.any(), Mockito.anyBoolean()); + Assert.assertEquals(itemId, updateCaptor.getValue().getId()); + Assert.assertNotNull( + updateCaptor.getValue().getStrategySnapshotJson()); + } + + /** + * 验证 SPLIT 待处理任务使用 execution token 和租约原子领取。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void tryMarkSplitTaskRunningShouldCreateExecutionFence() + throws Exception { + BigInteger taskId = BigInteger.valueOf(35); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setPhase(DocumentImportTaskPhase.SPLIT.name()); + task.setStatus(DocumentImportTaskStatus.PENDING.name()); + + DocumentImportTaskService taskService = + Mockito.mock(DocumentImportTaskService.class); + Mockito.when(taskService.getById(taskId)).thenReturn(task); + Mockito.when(taskService.count( + Mockito.any(com.mybatisflex.core.query.QueryWrapper.class))) + .thenReturn(0L); + DocumentImportTaskMapper taskMapper = + Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.claimPending( + Mockito.eq(taskId), + Mockito.anyString(), + Mockito.any(Date.class), + Mockito.any(Date.class), + Mockito.nullable(BigInteger.class) + )).thenReturn(1); + RedisLockExecutor lockExecutor = + Mockito.mock(RedisLockExecutor.class); + RedisLockExecutor.LockHandle lockHandle = + Mockito.mock(RedisLockExecutor.LockHandle.class); + Mockito.when(lockExecutor.tryAcquire( + Mockito.anyString(), Mockito.any(), Mockito.any() + )).thenReturn(lockHandle); + + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportTaskService", taskService); + setField(service, "documentImportTaskMapper", taskMapper); + setField(service, "redisLockExecutor", lockExecutor); + setField(service, "bulkProperties", new DocumentImportBulkProperties()); + + Assert.assertTrue(service.tryMarkTaskRunning(taskId)); + + ArgumentCaptor token = ArgumentCaptor.forClass(String.class); + ArgumentCaptor lease = ArgumentCaptor.forClass(Date.class); + Mockito.verify(taskMapper).claimPending( + Mockito.eq(taskId), + token.capture(), + lease.capture(), + Mockito.any(Date.class), + Mockito.nullable(BigInteger.class) + ); + Assert.assertFalse(token.getValue().isBlank()); + Assert.assertTrue(lease.getValue().after(new Date())); + Mockito.verify(lockHandle).release(); + } + + /** + * 验证新文档完成后会清理待覆盖的历史文档并清除持久化待办。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void cleanupCompletedReplacementShouldDeleteHistoricalDocument() throws Exception { + BigInteger itemId = BigInteger.valueOf(901); + BigInteger documentId = BigInteger.valueOf(902); + BigInteger replacedDocumentId = BigInteger.valueOf(903); + BigInteger knowledgeId = BigInteger.valueOf(904); + + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(itemId); + item.setDocumentId(documentId); + item.setReplacedDocumentId(replacedDocumentId); + tech.easyflow.ai.entity.Document replacement = new tech.easyflow.ai.entity.Document(); + replacement.setId(documentId); + replacement.setCollectionId(knowledgeId); + replacement.setProcessStatus(DocumentProcessStatus.COMPLETED.name()); + tech.easyflow.ai.entity.Document historical = new tech.easyflow.ai.entity.Document(); + historical.setId(replacedDocumentId); + historical.setCollectionId(knowledgeId); + historical.setProcessStatus(DocumentProcessStatus.COMPLETED.name()); + + DocumentImportBatchTracker tracker = Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.requireItem(itemId)).thenReturn(item); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneById(documentId)).thenReturn(replacement); + Mockito.when(documentMapper.selectOneById(replacedDocumentId)).thenReturn(historical); + DocumentService documentService = Mockito.mock(DocumentService.class); + Mockito.when(documentService.removeDoc(replacedDocumentId.toString())).thenReturn(true); + RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class); + RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class); + Mockito.when(redisLockExecutor.tryAcquire( + Mockito.anyString(), Mockito.any(), Mockito.any() + )).thenReturn(lockHandle); + + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "documentMapper", documentMapper); + setField(service, "documentService", documentService); + setField(service, "redisLockExecutor", redisLockExecutor); + + service.cleanupCompletedReplacement(itemId); + + Mockito.verify(documentService).removeDoc(replacedDocumentId.toString()); + Mockito.verify(documentMapper).deleteById(replacedDocumentId); + Mockito.verify(tracker).clearReplacement(itemId, replacedDocumentId); + Mockito.verify(lockHandle).release(); + } + + /** + * 验证失联且关联文档已删除的运行任务会被收口，避免持续占用并发名额。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void recoverInterruptedTaskShouldFailOrphanWithoutDocument() throws Exception { + BigInteger taskId = BigInteger.valueOf(41); + Date cutoff = new Date(System.currentTimeMillis() - 60_000L); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setDocumentId(BigInteger.valueOf(42)); + task.setPhase(DocumentImportTaskPhase.PARSE.name()); + task.setStatus(DocumentImportTaskStatus.RUNNING.name()); + task.setModified(new Date(cutoff.getTime() - 1_000L)); + + DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class); + Mockito.when(taskService.getById(taskId)).thenReturn(task); + DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.failExpiredOwned( + Mockito.eq(taskId), + Mockito.isNull(), + Mockito.anyString(), + Mockito.anyString(), + Mockito.any(Date.class), + Mockito.eq(cutoff), + Mockito.any() + )).thenReturn(1); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneById(task.getDocumentId())).thenReturn(null); + + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportTaskService", taskService); + setField(service, "documentImportTaskMapper", taskMapper); + setField(service, "documentMapper", documentMapper); + + service.recoverInterruptedTask(taskId, cutoff); + + Mockito.verify(taskMapper).failExpiredOwned( + Mockito.eq(taskId), + Mockito.isNull(), + Mockito.eq("任务执行中断，请重试"), + Mockito.eq("execution_interrupted"), + Mockito.any(Date.class), + Mockito.eq(cutoff), + Mockito.any() + ); + } + + /** + * 验证运行中断收口后仍保留批次文件的人工重试资格。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void recoveredBatchStateShouldRemainRetryableBeforeAttemptLimit() + throws Exception { + BigInteger batchId = BigInteger.valueOf(61); + BigInteger itemId = BigInteger.valueOf(62); + DocumentImportTask task = new DocumentImportTask(); + task.setBatchId(batchId); + task.setBatchItemId(itemId); + DocumentImportBatchItem item = new DocumentImportBatchItem(); + item.setId(itemId); + item.setAttemptCount(0); + + DocumentImportBatchTracker tracker = + Mockito.mock(DocumentImportBatchTracker.class); + Mockito.when(tracker.requireItem(itemId)).thenReturn(item); + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportBatchTracker", tracker); + setField(service, "bulkProperties", new DocumentImportBulkProperties()); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod( + "finishRecoveredBatchState", + DocumentImportTask.class, + DocumentImportBatchItemStage.class, + String.class, + String.class + ); + method.setAccessible(true); + + method.invoke( + service, + task, + DocumentImportBatchItemStage.PARSE, + "任务执行中断，请继续批次", + "execution_interrupted" + ); + + Mockito.verify(tracker).transitionItem( + itemId, + DocumentImportBatchItemStage.PARSE, + DocumentImportBatchItemStatus.FAILED, + "任务执行中断，请继续批次", + true, + 0, + "execution_interrupted" + ); + Mockito.verify(tracker).markInterrupted(batchId); + } + + /** + * 验证待处理超时按创建时间判定，即使重新投递刷新了修改时间也会正常收口。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void expireTimedOutPendingTaskShouldUseCreatedTime() throws Exception { + BigInteger taskId = BigInteger.valueOf(51); + BigInteger documentId = BigInteger.valueOf(52); + Date cutoff = new Date(System.currentTimeMillis() - 60_000L); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setDocumentId(documentId); + task.setPhase(DocumentImportTaskPhase.PARSE.name()); + task.setStatus(DocumentImportTaskStatus.PENDING.name()); + task.setCreated(new Date(cutoff.getTime() - 1_000L)); + task.setModified(new Date()); + + tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document(); + document.setId(documentId); + document.setCollectionId(BigInteger.valueOf(53)); + AtomicReference updatedDocumentRef = + new AtomicReference(); + + DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class); + Mockito.when(taskService.getById(taskId)).thenReturn(task); + DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.updateByQuery( + Mockito.any(DocumentImportTask.class), + Mockito.any() + )).thenReturn(1); + + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportTaskService", taskService); + setField(service, "documentImportTaskMapper", taskMapper); + setField(service, "documentMapper", mockDocumentMapper(document, updatedDocumentRef)); + setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService()); + + service.expireTimedOutPendingTask(taskId, cutoff); + + tech.easyflow.ai.entity.Document updatedDocument = updatedDocumentRef.get(); + Assert.assertNotNull(updatedDocument); + Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(), updatedDocument.getProcessStatus()); + Assert.assertEquals("任务排队超时，请重试", updatedDocument.getLastTaskError()); + Assert.assertEquals("pending_timeout", + updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE)); + } + + /** + * 验证未取得 MinerU 任务 ID 的超时提交会失败并释放运行状态。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void expireTimedOutParseSubmissionShouldExposeFriendlyTimeout() throws Exception { + BigInteger taskId = BigInteger.valueOf(61); + BigInteger documentId = BigInteger.valueOf(62); + Date cutoff = new Date(System.currentTimeMillis() - 60_000L); + DocumentImportTask task = new DocumentImportTask(); + task.setId(taskId); + task.setDocumentId(documentId); + task.setPhase(DocumentImportTaskPhase.PARSE.name()); + task.setStatus(DocumentImportTaskStatus.RUNNING.name()); + task.setStartedAt(new Date(cutoff.getTime() - 1_000L)); + + tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document(); + document.setId(documentId); + document.setCollectionId(BigInteger.valueOf(63)); + AtomicReference updatedDocumentRef = + new AtomicReference(); + + DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class); + Mockito.when(taskService.getById(taskId)).thenReturn(task); + DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.updateByQuery( + Mockito.any(DocumentImportTask.class), + Mockito.any() + )).thenReturn(1); + + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentImportTaskService", taskService); + setField(service, "documentImportTaskMapper", taskMapper); + setField(service, "documentMapper", mockDocumentMapper(document, updatedDocumentRef)); + setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService()); + + service.expireTimedOutParseSubmission(taskId, cutoff); + + ArgumentCaptor updateCaptor = ArgumentCaptor.forClass(DocumentImportTask.class); + Mockito.verify(taskMapper).updateByQuery(updateCaptor.capture(), Mockito.any()); + Assert.assertEquals(DocumentImportTaskStatus.FAILED.name(), updateCaptor.getValue().getStatus()); + Assert.assertEquals("文档解析服务响应超时，请重试", updateCaptor.getValue().getErrorSummary()); + + tech.easyflow.ai.entity.Document updatedDocument = updatedDocumentRef.get(); + Assert.assertNotNull(updatedDocument); + Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(), updatedDocument.getProcessStatus()); + Assert.assertEquals("文档解析服务响应超时，请重试", updatedDocument.getLastTaskError()); + Assert.assertEquals("parse_service_timeout", + updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE)); + } + + /** + * 验证 MinerU 任意 5xx 会归一化为服务暂不可用错误。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void resolveParseFailureCodeShouldRecognizeMineruServerErrors() throws Exception { + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod( + "resolveParseFailureCode", + Throwable.class + ); + method.setAccessible(true); + + for (int statusCode : new int[] {500, 502, 503, 504, 599}) { + Object code = method.invoke(service, + new RuntimeException("MinerU request failed: path=/tasks, status=" + statusCode + ", body=")); + Assert.assertEquals("status=" + statusCode, "parse_service_unavailable", code); + } + } + + /** + * 验证底层读取超时会归一化为解析服务超时错误。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void resolveParseFailureCodeShouldRecognizeSocketTimeout() throws Exception { + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod( + "resolveParseFailureCode", + Throwable.class + ); + method.setAccessible(true); + + Object code = method.invoke(service, + new RuntimeException("wrapped", new SocketTimeoutException("timeout"))); + + Assert.assertEquals("parse_service_timeout", code); + } + + /** + * 验证文档源读取失败优先于底层 UnknownHostException 分类，避免误报 MinerU 不可用。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void resolveParseFailureCodeShouldPreferDocumentSourceFailure() throws Exception { + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod( + "resolveParseFailureCode", + Throwable.class + ); + method.setAccessible(true); + + DocumentParseBridgeException sourceError = DocumentParseBridgeException.sourceLoadFailed( + "下载文档 URL 失败", + new UnknownHostException("远端文档地址不允许访问非公网目标") + ); + Object code = method.invoke(service, new RuntimeException("wrapped", sourceError)); + + Assert.assertEquals("document_source_unavailable", code); + } + + /** + * 验证解析桥接层的确定性输入错误会转换为不可重试错误码。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void resolveParseFailureCodeShouldClassifyPermanentBridgeFailures() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod("resolveParseFailureCode", Throwable.class); + method.setAccessible(true); + + Object unsupported = method.invoke( + service, + DocumentParseBridgeException.unsupportedSource("不支持的文件类型") + ); + Object invalidRequest = method.invoke( + service, + DocumentParseBridgeException.requestBuildFailed("解析请求无效") + ); + + Assert.assertEquals("unsupported_document_source", unsupported); + Assert.assertEquals("invalid_parse_request", invalidRequest); + } + + /** + * 验证未识别的系统异常仍会获得稳定错误码并允许后续人工重试。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void resolveParseFailureCodeShouldStabilizeUnknownFailures() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod("resolveParseFailureCode", Throwable.class); + method.setAccessible(true); + + Object code = method.invoke(service, new IllegalStateException("代码异常")); + + Assert.assertEquals("parse_failed", code); + } + + /** + * 验证未知系统异常和临时源读取失败允许人工重试，确定性输入错误除外。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void parseRetryabilityShouldDefaultToRecoverable() + throws Exception { + KnowledgeDocumentImportTaskAppService service = + new KnowledgeDocumentImportTaskAppService(); + Method method = KnowledgeDocumentImportTaskAppService.class + .getDeclaredMethod("isRetryableParseFailure", String.class); + method.setAccessible(true); + + Assert.assertTrue((Boolean) method.invoke(service, new Object[] {null})); + Assert.assertTrue((Boolean) method.invoke( + service, "document_source_unavailable")); + Assert.assertTrue((Boolean) method.invoke(service, "parse_failed")); + Assert.assertFalse((Boolean) method.invoke( + service, "unsupported_document_source")); + Assert.assertFalse((Boolean) method.invoke( + service, "invalid_parse_request")); + } + + /** + * 验证批次上传文件通过受信任存储服务读取为字节，避免内部附件 URL 进入公网 URL 校验。 + * + * @throws Exception 反射调用或存储桩异常 + */ + @Test + public void buildBridgeSourceRefShouldReadBatchFileFromStorage() throws Exception { + byte[] content = "batch-document".getBytes(StandardCharsets.UTF_8); + String storedUrl = "http://127.0.0.1:39000/easyflow/attachment/test.docx"; + FileStorageService storageService = Mockito.mock(FileStorageService.class); + Mockito.when(storageService.readStream(storedUrl)) + .thenReturn(new ByteArrayInputStream(content)); + + DocumentImportBulkProperties properties = new DocumentImportBulkProperties(); + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + setField(service, "storageService", storageService); + setField(service, "bulkProperties", properties); + + DocumentImportTask task = new DocumentImportTask(); + task.setBatchId(BigInteger.valueOf(61)); + tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document(); + document.setTitle("test.docx"); + document.setDocumentPath(storedUrl); + + Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod( + "buildBridgeSourceRef", + DocumentImportTask.class, + tech.easyflow.ai.entity.Document.class, + String.class + ); + method.setAccessible(true); + DocumentSourceRef sourceRef = (DocumentSourceRef) method.invoke(service, task, document, "docx"); + + Assert.assertArrayEquals(content, sourceRef.getContentBytes()); + Assert.assertNull(sourceRef.getFilePath()); + Assert.assertEquals(Long.valueOf(content.length), sourceRef.getSize()); + Mockito.verify(storageService).readStream(storedUrl); + } + + /** + * 验证非批次远程文档仍保留 URL，由文档源加载器继续执行 SSRF 防护。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void buildBridgeSourceRefShouldKeepExternalUrlForSsrfValidation() throws Exception { + FileStorageService storageService = Mockito.mock(FileStorageService.class); + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + setField(service, "storageService", storageService); + + DocumentImportTask task = new DocumentImportTask(); + tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document(); + document.setTitle("external.pdf"); + document.setDocumentPath("https://example.com/external.pdf"); + + Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod( + "buildBridgeSourceRef", + DocumentImportTask.class, + tech.easyflow.ai.entity.Document.class, + String.class + ); + method.setAccessible(true); + DocumentSourceRef sourceRef = (DocumentSourceRef) method.invoke(service, task, document, "pdf"); + + Assert.assertEquals("https://example.com/external.pdf", sourceRef.getFilePath()); + Assert.assertNull(sourceRef.getContentBytes()); + Mockito.verifyNoInteractions(storageService); + } + /** * 验证向量化失败会按整文档失败语义重置进度，并刷新任务错误信息。 * @@ -59,11 +744,16 @@ public class KnowledgeDocumentImportTaskAppServiceTest { persistedDocument.setLastTaskError("旧错误"); AtomicReference updatedDocumentRef = new AtomicReference(); - AtomicReference updatedTaskRef = new AtomicReference(); + DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.finishOwned( + Mockito.any(), Mockito.anyString(), Mockito.anyString(), + Mockito.any(), Mockito.anyString(), Mockito.any(), + Mockito.any() + )).thenReturn(1); KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); setField(service, "documentMapper", mockDocumentMapper(persistedDocument, updatedDocumentRef)); - setField(service, "documentImportTaskService", mockDocumentImportTaskService(updatedTaskRef)); + setField(service, "documentImportTaskMapper", taskMapper); setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService()); DocumentImportTask task = new DocumentImportTask(); @@ -72,6 +762,7 @@ public class KnowledgeDocumentImportTaskAppServiceTest { task.setKnowledgeId(knowledgeId); task.setStatus(DocumentImportTaskStatus.RUNNING.name()); task.setErrorSummary("旧错误"); + task.setExecutionToken("attempt-token"); tech.easyflow.ai.entity.Document inputDocument = new tech.easyflow.ai.entity.Document(); inputDocument.setId(documentId); @@ -93,11 +784,62 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Assert.assertEquals(Integer.valueOf(8), updatedDocument.getFailedChunks()); Assert.assertEquals(Integer.valueOf(0), updatedDocument.getProgressPercent()); Assert.assertEquals("新错误", updatedDocument.getLastTaskError()); + Assert.assertEquals("index_failed", + updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE)); - DocumentImportTask updatedTask = updatedTaskRef.get(); - Assert.assertNotNull(updatedTask); - Assert.assertEquals(DocumentImportTaskStatus.FAILED.name(), updatedTask.getStatus()); - Assert.assertEquals("新错误", updatedTask.getErrorSummary()); + Mockito.verify(taskMapper).finishOwned( + Mockito.eq(task.getId()), + Mockito.eq("attempt-token"), + Mockito.eq(DocumentImportTaskStatus.FAILED.name()), + Mockito.eq("新错误"), + Mockito.eq("index_failed"), + Mockito.any(Date.class), + Mockito.any() + ); + } + + /** + * 验证看门狗已经收口任务后，迟到执行者不能覆盖文档终态。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void markIndexFailedShouldIgnoreLostTaskOwnership() throws Exception { + BigInteger documentId = BigInteger.valueOf(81); + tech.easyflow.ai.entity.Document persistedDocument = new tech.easyflow.ai.entity.Document(); + persistedDocument.setId(documentId); + persistedDocument.setProcessStatus(DocumentProcessStatus.PARSE_FAILED.name()); + AtomicReference updatedDocumentRef = + new AtomicReference(); + DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class); + Mockito.when(taskMapper.updateByQuery( + Mockito.any(DocumentImportTask.class), + Mockito.any() + )).thenReturn(0); + + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + setField(service, "documentMapper", mockDocumentMapper(persistedDocument, updatedDocumentRef)); + setField(service, "documentImportTaskMapper", taskMapper); + + DocumentImportTask task = new DocumentImportTask(); + task.setId(BigInteger.valueOf(82)); + task.setDocumentId(documentId); + task.setStatus(DocumentImportTaskStatus.RUNNING.name()); + tech.easyflow.ai.entity.Document inputDocument = new tech.easyflow.ai.entity.Document(); + inputDocument.setId(documentId); + + Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod( + "markIndexFailed", + DocumentImportTask.class, + tech.easyflow.ai.entity.Document.class, + String.class + ); + method.setAccessible(true); + method.invoke(service, task, inputDocument, "迟到错误"); + + Assert.assertNull(updatedDocumentRef.get()); + Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(), + persistedDocument.getProcessStatus()); } /** @@ -232,6 +974,8 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Assert.assertEquals(2, chunks.size()); DocumentChunk firstChunk = chunks.get(0); + Assert.assertNotNull(firstChunk.getId()); + Assert.assertNotEquals(firstChunk.getId(), chunks.get(1).getId()); Assert.assertTrue(firstChunk.getContent().contains("Slide 1")); Assert.assertTrue(firstChunk.getContent().contains("本页介绍季度目标")); Assert.assertEquals("https://example.com/slides/slide-001.png", @@ -352,6 +1096,61 @@ public class KnowledgeDocumentImportTaskAppServiceTest { Assert.assertTrue(chunks.isEmpty()); } + /** + * 验证向量存储回滚会去重分块 ID，并在 Milvus 删除失败后停止清理搜索索引。 + * + * @throws Exception 反射调用异常 + */ + @Test + public void rollbackStoredChunksShouldValidateVectorDeleteResult() throws Exception { + KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); + DocumentStore documentStore = Mockito.mock(DocumentStore.class); + DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class); + StoreOptions storeOptions = StoreOptions.ofCollectionName("knowledge-test"); + Mockito.when(documentStore.delete(Mockito.anyCollection(), Mockito.same(storeOptions))) + .thenReturn(StoreResult.fail("milvus unavailable")); + + DocumentCollection knowledge = new DocumentCollection(); + knowledge.setId(BigInteger.valueOf(501)); + Class contextClass = Class.forName( + KnowledgeDocumentImportTaskAppService.class.getName() + "$StoreExecutionContext" + ); + Constructor constructor = contextClass.getDeclaredConstructor( + DocumentCollection.class, + EmbeddingModel.class, + DocumentStore.class, + StoreOptions.class, + DocumentSearcher.class + ); + constructor.setAccessible(true); + Object context = constructor.newInstance(knowledge, null, documentStore, storeOptions, searcher); + + DocumentChunk first = new DocumentChunk(); + first.setId(BigInteger.valueOf(601)); + DocumentChunk duplicate = new DocumentChunk(); + duplicate.setId(first.getId()); + Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod( + "rollbackStoredChunks", + BigInteger.class, + BigInteger.class, + contextClass, + List.class + ); + method.setAccessible(true); + method.invoke(service, + BigInteger.valueOf(701), + BigInteger.valueOf(702), + context, + List.of(first, duplicate)); + + @SuppressWarnings("rawtypes") + ArgumentCaptor idsCaptor = ArgumentCaptor.forClass(Collection.class); + Mockito.verify(documentStore).delete(idsCaptor.capture(), Mockito.same(storeOptions)); + Assert.assertEquals(1, idsCaptor.getValue().size()); + Assert.assertTrue(idsCaptor.getValue().contains(first.getId())); + Mockito.verifyNoInteractions(searcher); + } + private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument, AtomicReference updatedDocumentRef) { return (DocumentMapper) Proxy.newProxyInstance( @@ -370,20 +1169,6 @@ public class KnowledgeDocumentImportTaskAppServiceTest { ); } - private static DocumentImportTaskService mockDocumentImportTaskService(AtomicReference updatedTaskRef) { - return (DocumentImportTaskService) Proxy.newProxyInstance( - DocumentImportTaskService.class.getClassLoader(), - new Class[]{DocumentImportTaskService.class}, - (proxy, method, args) -> { - if ("updateById".equals(method.getName())) { - updatedTaskRef.set((DocumentImportTask) args[0]); - return true; - } - return defaultValue(method.getReturnType()); - } - ); - } - private static FileStorageService mockFileStorageService(AtomicReference savedPrePathRef, AtomicReference savedFilenameRef) { return (FileStorageService) Proxy.newProxyInstance( diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeImportBatchFacadeTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeImportBatchFacadeTest.java new file mode 100644 index 00000000..73195095 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/documentimport/task/KnowledgeImportBatchFacadeTest.java @@ -0,0 +1,431 @@ +package tech.easyflow.ai.documentimport.task; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext; +import tech.easyflow.ai.documentimport.DocumentImportBatchDtos; +import tech.easyflow.ai.documentimport.ImportCallerContext; +import tech.easyflow.ai.documentimport.ImportCallerType; +import tech.easyflow.ai.documentimport.PublicDocumentImportDtos; +import tech.easyflow.ai.entity.DocumentImportBatch; +import tech.easyflow.ai.enums.DocumentImportBatchStatus; +import tech.easyflow.ai.mapper.DocumentImportBatchMapper; +import tech.easyflow.ai.service.DocumentImportBatchItemService; +import tech.easyflow.ai.service.DocumentImportBatchService; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.io.ByteArrayInputStream; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * Public API 批量导入门面边界与服务端去重测试。 + * + * @author Codex + * @since 2026-08-02 + */ +public class KnowledgeImportBatchFacadeTest { + + /** + * 验证单次文件数超过 200 时在创建批次前拒绝。 + */ + @Test + public void submitShouldRejectMoreThanTwoHundredFiles() { + TestContext context = createContext(); + PublicDocumentImportDtos.BatchMetadata metadata = + new PublicDocumentImportDtos.BatchMetadata(); + metadata.setKnowledgeId(BigInteger.ONE); + List manifest = new ArrayList<>(); + List files = new ArrayList<>(); + for (int index = 0; index < 201; index++) { + String name = "file-" + index + ".txt"; + manifest.add(manifest(name, name, 1L)); + files.add(file(name, new byte[]{'a'})); + } + metadata.setFiles(manifest); + + try { + context.facade.submit(context.caller, metadata, files); + Assert.fail("Expected file count rejection"); + } catch (BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("最多上传200个文件")); + } + Mockito.verifyNoInteractions(context.batchAppService); + } + + /** + * 验证文件夹清单拒绝父目录穿越。 + */ + @Test + public void submitShouldRejectRelativePathTraversal() { + TestContext context = createContext(); + PublicDocumentImportDtos.BatchMetadata metadata = + metadata("demo.txt", "../demo.txt", 4L); + + try { + context.facade.submit( + context.caller, + metadata, + List.of(file( + "demo.txt", + "demo".getBytes(StandardCharsets.UTF_8) + )) + ); + Assert.fail("Expected relative path rejection"); + } catch (BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("相对路径无效")); + } + Mockito.verifyNoInteractions(context.batchAppService); + } + + /** + * 验证 Public API 可省略 fileSize，内部批次使用服务端实测大小。 + */ + @Test + public void submitShouldUseMeasuredSizeWhenMetadataOmitsFileSize() { + TestContext context = createContext(); + PublicDocumentImportDtos.BatchMetadata metadata = + metadata("demo.txt", "folder/demo.txt", 4L); + metadata.getFiles().get(0).setFileSize(null); + byte[] content = "demo".getBytes(StandardCharsets.UTF_8); + + DocumentImportBatchDtos.CreateResponse created = + new DocumentImportBatchDtos.CreateResponse(); + created.setBatchId(BigInteger.valueOf(91)); + DocumentImportBatchDtos.ItemResponse createdItem = + new DocumentImportBatchDtos.ItemResponse(); + createdItem.setItemId(BigInteger.valueOf(92)); + created.setItems(List.of(createdItem)); + Mockito.when(context.batchAppService.createBatch( + Mockito.any(), Mockito.any() + )).thenReturn(created); + Mockito.when(context.itemService.updateById(Mockito.any())) + .thenReturn(true); + DocumentImportBatch batch = new DocumentImportBatch(); + batch.setId(created.getBatchId()); + batch.setKnowledgeId(BigInteger.ONE); + batch.setStatus(DocumentImportBatchStatus.RUNNING.name()); + batch.setTotalCount(1); + batch.setTotalBytes((long) content.length); + batch.setCreated(new Date()); + Mockito.when(context.batchAppService.requireOwnedBatch( + BigInteger.ONE, created.getBatchId(), context.caller + )).thenReturn(batch); + + context.facade.submit( + context.caller, + metadata, + List.of(file("demo.txt", content)) + ); + + ArgumentCaptor request = + ArgumentCaptor.forClass(DocumentImportBatchDtos.CreateRequest.class); + Mockito.verify(context.batchAppService).createBatch( + request.capture(), Mockito.any()); + Assert.assertEquals( + Long.valueOf(content.length), + request.getValue().getFiles().get(0).getFileSize() + ); + Assert.assertNull(metadata.getFiles().get(0).getFileSize()); + } + + /** + * 验证相同提交由服务端指纹复用，包含仍在上传的并发请求。 + */ + @Test + public void submitShouldReuseTaskByServerFingerprint() { + TestContext context = createContext(); + PublicDocumentImportDtos.BatchMetadata metadata = + metadata("demo.txt", "folder/demo.txt", 4L); + List files = List.of(file( + "demo.txt", + "demo".getBytes(StandardCharsets.UTF_8) + )); + + DocumentImportBatchDtos.CreateResponse created = + new DocumentImportBatchDtos.CreateResponse(); + created.setBatchId(BigInteger.valueOf(91)); + DocumentImportBatchDtos.ItemResponse createdItem = + new DocumentImportBatchDtos.ItemResponse(); + createdItem.setItemId(BigInteger.valueOf(92)); + created.setItems(List.of(createdItem)); + Mockito.when(context.batchAppService.createBatch( + Mockito.any(), Mockito.any() + )).thenReturn(created); + Mockito.when(context.itemService.updateById(Mockito.any())) + .thenReturn(true); + DocumentImportBatch batch = new DocumentImportBatch(); + batch.setId(created.getBatchId()); + batch.setKnowledgeId(BigInteger.ONE); + batch.setStatus(DocumentImportBatchStatus.RUNNING.name()); + batch.setTotalCount(1); + batch.setTotalBytes(4L); + batch.setCreated(new Date()); + Mockito.when(context.batchAppService.requireOwnedBatch( + BigInteger.ONE, created.getBatchId(), context.caller + )).thenReturn(batch); + + context.facade.submit(context.caller, metadata, files); + + ArgumentCaptor hashUpdate = + ArgumentCaptor.forClass(tech.easyflow.ai.entity.DocumentImportBatchItem.class); + Mockito.verify(context.itemService).updateById(hashUpdate.capture()); + Assert.assertNotNull(hashUpdate.getValue().getContentSha256()); + Assert.assertEquals(createdItem.getItemId(), hashUpdate.getValue().getId()); + ArgumentCaptor createContext = + ArgumentCaptor.forClass(DocumentImportBatchCreateContext.class); + Mockito.verify(context.batchAppService).createBatch( + Mockito.any(), createContext.capture()); + batch.setRequestDigest(createContext.getValue().getRequestDigest()); + Mockito.when(context.batchService.getOne( + Mockito.any(com.mybatisflex.core.query.QueryWrapper.class))) + .thenReturn(batch); + + PublicDocumentImportDtos.SubmitResponse repeated = + context.facade.submit( + context.caller, metadata, files); + + Assert.assertEquals(batch.getId(), repeated.getTaskId()); + batch.setStatus(DocumentImportBatchStatus.UPLOADING.name()); + PublicDocumentImportDtos.SubmitResponse uploading = + context.facade.submit(context.caller, metadata, files); + + Assert.assertEquals(batch.getId(), uploading.getTaskId()); + Mockito.verify(context.batchAppService, Mockito.times(1)) + .createBatch(Mockito.any(), Mockito.any()); + } + + /** + * 验证长时间无进展的上传任务会先取消并释放指纹，再创建新任务。 + */ + @Test + public void submitShouldReplaceStaleIncompleteTask() { + TestContext context = createContext(); + PublicDocumentImportDtos.BatchMetadata metadata = + metadata("demo.txt", "folder/demo.txt", 4L); + List files = List.of(file( + "demo.txt", + "demo".getBytes(StandardCharsets.UTF_8) + )); + DocumentImportBatch stale = new DocumentImportBatch(); + stale.setId(BigInteger.valueOf(81)); + stale.setKnowledgeId(BigInteger.ONE); + stale.setStatus(DocumentImportBatchStatus.UPLOADING.name()); + stale.setModified(new Date( + System.currentTimeMillis() - 31L * 60L * 1000L + )); + Mockito.when(context.batchService.getOne( + Mockito.any(com.mybatisflex.core.query.QueryWrapper.class))) + .thenReturn(stale) + .thenReturn(null); + Mockito.when(context.batchAppService.cancelStaleBatch( + Mockito.eq(BigInteger.ONE), + Mockito.eq(stale.getId()), + Mockito.eq(context.caller), + Mockito.any(Date.class) + )).thenReturn(true); + Mockito.when(context.batchMapper.releaseSubmissionFingerprint( + Mockito.eq(stale.getId()), + Mockito.anyString(), + Mockito.any(Date.class), + Mockito.any(Date.class) + )).thenReturn(1); + + DocumentImportBatchDtos.CreateResponse created = + new DocumentImportBatchDtos.CreateResponse(); + created.setBatchId(BigInteger.valueOf(91)); + DocumentImportBatchDtos.ItemResponse item = + new DocumentImportBatchDtos.ItemResponse(); + item.setItemId(BigInteger.valueOf(92)); + created.setItems(List.of(item)); + Mockito.when(context.batchAppService.createBatch( + Mockito.any(), Mockito.any() + )).thenReturn(created); + Mockito.when(context.itemService.updateById(Mockito.any())) + .thenReturn(true); + DocumentImportBatch replacement = new DocumentImportBatch(); + replacement.setId(created.getBatchId()); + replacement.setKnowledgeId(BigInteger.ONE); + replacement.setStatus(DocumentImportBatchStatus.RUNNING.name()); + replacement.setTotalCount(1); + replacement.setTotalBytes(4L); + replacement.setCreated(new Date()); + Mockito.when(context.batchAppService.requireOwnedBatch( + BigInteger.ONE, created.getBatchId(), context.caller + )).thenReturn(replacement); + + PublicDocumentImportDtos.SubmitResponse response = + context.facade.submit(context.caller, metadata, files); + + Assert.assertEquals(replacement.getId(), response.getTaskId()); + Mockito.verify(context.batchAppService).cancelStaleBatch( + Mockito.eq(BigInteger.ONE), + Mockito.eq(stale.getId()), + Mockito.eq(context.caller), + Mockito.any(Date.class) + ); + Mockito.verify(context.batchMapper).releaseSubmissionFingerprint( + Mockito.eq(stale.getId()), + Mockito.anyString(), + Mockito.any(Date.class), + Mockito.any(Date.class) + ); + } + + /** + * 验证显式空文件键数组会被拒绝，省略时由服务端重试全部异常文件。 + */ + @Test + public void retryShouldDistinguishOmittedAndEmptyFileKeys() { + TestContext context = createContext(); + PublicDocumentImportDtos.RetryRequest empty = + new PublicDocumentImportDtos.RetryRequest(); + empty.setTaskId(BigInteger.ONE); + empty.setFileKeys(List.of()); + + try { + context.facade.retry(context.caller, empty); + Assert.fail("Expected empty fileKeys rejection"); + } catch (BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("不能为空数组")); + } + + PublicDocumentImportDtos.RetryRequest omitted = + new PublicDocumentImportDtos.RetryRequest(); + omitted.setTaskId(BigInteger.ONE); + tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult result = + new tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult( + BigInteger.ONE, + DocumentImportBatchStatus.RUNNING.name(), + 1, + 2 + ); + Mockito.when(context.batchAppService.retryOwnedBatch( + BigInteger.ONE, + context.caller, + java.util.Set.of() + )).thenReturn(result); + + PublicDocumentImportDtos.RetryResponse response = + context.facade.retry(context.caller, omitted); + + Assert.assertEquals(Integer.valueOf(2), response.getRetriedCount()); + } + + /** + * 创建单文件元数据。 + * + * @param fileName 文件名 + * @param relativePath 相对路径 + * @param fileSize 文件大小 + * @return 元数据 + */ + private PublicDocumentImportDtos.BatchMetadata metadata( + String fileName, + String relativePath, + long fileSize) { + PublicDocumentImportDtos.BatchMetadata metadata = + new PublicDocumentImportDtos.BatchMetadata(); + metadata.setKnowledgeId(BigInteger.ONE); + metadata.setFiles(List.of( + manifest(fileName, relativePath, fileSize))); + return metadata; + } + + /** + * 创建清单项。 + * + * @param fileName 文件名 + * @param relativePath 相对路径 + * @param fileSize 文件大小 + * @return 清单项 + */ + private DocumentImportBatchDtos.ManifestItem manifest( + String fileName, + String relativePath, + long fileSize) { + DocumentImportBatchDtos.ManifestItem item = + new DocumentImportBatchDtos.ManifestItem(); + item.setClientFileKey(relativePath); + item.setFileName(fileName); + item.setRelativePath(relativePath); + item.setFileSize(fileSize); + return item; + } + + /** + * 创建可重复读取的 MultipartFile。 + * + * @param fileName 文件名 + * @param bytes 文件内容 + * @return Multipart 文件 + */ + private MultipartFile file(String fileName, byte[] bytes) { + MultipartFile file = Mockito.mock(MultipartFile.class); + try { + Mockito.when(file.getInputStream()) + .thenAnswer(invocation -> new ByteArrayInputStream(bytes)); + } catch (java.io.IOException error) { + throw new IllegalStateException(error); + } + Mockito.when(file.getOriginalFilename()).thenReturn(fileName); + Mockito.when(file.getSize()).thenReturn((long) bytes.length); + Mockito.when(file.isEmpty()).thenReturn(bytes.length == 0); + return file; + } + + /** + * 创建测试依赖。 + * + * @return 测试上下文 + */ + private TestContext createContext() { + DocumentImportBatchAppService batchAppService = + Mockito.mock(DocumentImportBatchAppService.class); + DocumentImportBatchTracker batchTracker = + Mockito.mock(DocumentImportBatchTracker.class); + DocumentImportBatchService batchService = + Mockito.mock(DocumentImportBatchService.class); + DocumentImportBatchItemService itemService = + Mockito.mock(DocumentImportBatchItemService.class); + DocumentImportBatchMapper batchMapper = + Mockito.mock(DocumentImportBatchMapper.class); + KnowledgeImportBatchFacade facade = new KnowledgeImportBatchFacade( + batchAppService, + batchTracker, + batchService, + itemService, + batchMapper + ); + return new TestContext( + facade, + batchAppService, + batchService, + itemService, + batchMapper, + new ImportCallerContext( + ImportCallerType.PUBLIC_API, + BigInteger.valueOf(77) + ) + ); + } + + /** + * 测试依赖集合。 + */ + private record TestContext( + KnowledgeImportBatchFacade facade, + DocumentImportBatchAppService batchAppService, + DocumentImportBatchService batchService, + DocumentImportBatchItemService itemService, + DocumentImportBatchMapper batchMapper, + ImportCallerContext caller + ) { + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImplTest.java index 2e570ffe..ca1403c8 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImplTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentCollectionServiceImplTest.java @@ -10,6 +10,7 @@ import tech.easyflow.ai.config.SearcherFactory; import tech.easyflow.ai.enums.DocumentProcessStatus; import tech.easyflow.ai.mapper.DocumentChunkMapper; import tech.easyflow.ai.mapper.DocumentMapper; +import tech.easyflow.ai.mapper.FaqItemMapper; import java.io.Serializable; import java.lang.reflect.Field; @@ -92,6 +93,89 @@ public class DocumentCollectionServiceImplTest { Assert.assertEquals(completedChunkId, result.get(0).getId()); Assert.assertEquals("completed chunk", result.get(0).getContent()); Assert.assertEquals(String.valueOf(knowledgeId), searcher.lastKnowledgeId); + Assert.assertEquals( + tech.easyflow.ai.entity.DocumentCollection.TYPE_DOCUMENT, + result.get(0).getMetadata("resultType") + ); + Assert.assertEquals( + completedDocumentId, + result.get(0).getMetadata("documentId") + ); + Assert.assertEquals( + completedDocument.getTitle(), + result.get(0).getMetadata("sourceFileName") + ); + } + + /** + * 验证 FAQ 检索使用当前数据库记录回填稳定的 FAQ 来源信息。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void searchShouldFillFaqSourceMetadataFromDatabase() throws Exception { + BigInteger knowledgeId = BigInteger.ONE; + BigInteger faqId = BigInteger.valueOf(2001); + BigInteger categoryId = BigInteger.valueOf(3001); + tech.easyflow.ai.entity.DocumentCollection collection = + new tech.easyflow.ai.entity.DocumentCollection(); + collection.setId(knowledgeId); + collection.setCollectionType( + tech.easyflow.ai.entity.DocumentCollection.TYPE_FAQ + ); + collection.setOptions(new HashMap() {{ + put(KEY_DOC_RECALL_MAX_NUM, 5); + put(KEY_SIMILARITY_THRESHOLD, BigDecimal.ZERO); + }}); + + tech.easyflow.ai.entity.FaqItem faqItem = + new tech.easyflow.ai.entity.FaqItem(); + faqItem.setId(faqId); + faqItem.setCollectionId(knowledgeId); + faqItem.setCategoryId(categoryId); + faqItem.setQuestion("如何申请账号？"); + faqItem.setAnswerText("请联系管理员。"); + TestKeywordSearcher searcher = new TestKeywordSearcher( + List.of(buildHit(faqId, 0.85D)) + ); + + DocumentCollectionServiceImpl service = + new TestDocumentCollectionService(collection); + setField( + service, + "searcherFactory", + new SearcherFactory( + new StaticObjectProvider(searcher) + ) + ); + setField(service, "faqItemMapper", mockFaqItemMapper(faqItem)); + + tech.easyflow.ai.rag.KnowledgeRetrievalRequest request = + new tech.easyflow.ai.rag.KnowledgeRetrievalRequest(); + request.setKnowledgeId(knowledgeId); + request.setQuery("账号"); + request.setRetrievalMode( + com.easyagents.rag.retrieval.RetrievalMode.KEYWORD + ); + + List result = service.search(request); + + Assert.assertEquals(1, result.size()); + Document item = result.get(0); + Assert.assertEquals( + tech.easyflow.ai.entity.DocumentCollection.TYPE_FAQ, + item.getMetadata("resultType") + ); + Assert.assertEquals(faqId, item.getMetadata("faqId")); + Assert.assertEquals( + faqItem.getQuestion(), + item.getMetadata("question") + ); + Assert.assertEquals( + faqItem.getAnswerText(), + item.getMetadata("answerText") + ); + Assert.assertEquals(categoryId, item.getMetadata("categoryId")); } private static Document buildHit(BigInteger id, double score) { @@ -132,6 +216,27 @@ public class DocumentCollectionServiceImplTest { ); } + /** + * 创建返回固定 FAQ 的 Mapper 桩。 + * + * @param faqItem FAQ 数据 + * @return Mapper 桩 + */ + private static FaqItemMapper mockFaqItemMapper( + tech.easyflow.ai.entity.FaqItem faqItem + ) { + return (FaqItemMapper) Proxy.newProxyInstance( + FaqItemMapper.class.getClassLoader(), + new Class[]{FaqItemMapper.class}, + (proxy, method, args) -> { + if ("selectListByQuery".equals(method.getName())) { + return List.of(faqItem); + } + return defaultValue(method.getReturnType()); + } + ); + } + private static void setField(Object target, String fieldName, Object value) throws Exception { Field field = DocumentCollectionServiceImpl.class.getDeclaredField(fieldName); field.setAccessible(true); diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentServiceImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentServiceImplTest.java new file mode 100644 index 00000000..71eb1344 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/DocumentServiceImplTest.java @@ -0,0 +1,255 @@ +package tech.easyflow.ai.service.impl; + +import com.easyagents.core.store.DocumentStore; +import com.easyagents.core.store.StoreResult; +import com.mybatisflex.core.query.QueryWrapper; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import tech.easyflow.ai.config.SearcherFactory; +import tech.easyflow.ai.entity.Document; +import tech.easyflow.ai.entity.DocumentCollection; +import tech.easyflow.ai.entity.Model; +import tech.easyflow.ai.enums.DocumentProcessStatus; +import tech.easyflow.ai.mapper.DocumentChunkMapper; +import tech.easyflow.ai.mapper.DocumentMapper; +import tech.easyflow.ai.service.DocumentCollectionService; +import tech.easyflow.ai.service.ModelService; +import tech.easyflow.common.filestorage.FileStorageService; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.List; + +/** + * {@link DocumentServiceImpl} 文档维护回归测试。 + * + * @author Codex + * @since 2026-08-02 + */ +public class DocumentServiceImplTest { + + /** + * 验证删除链路在外部索引和分块清理后删除文档主记录。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void removeDocShouldDeleteDocumentRecord() throws Exception { + BigInteger documentId = BigInteger.valueOf(101); + BigInteger knowledgeId = BigInteger.valueOf(102); + BigInteger modelId = BigInteger.valueOf(103); + Document document = new Document(); + document.setId(documentId); + document.setCollectionId(knowledgeId); + document.setDocumentPath("storage://document.txt"); + document.setProcessStatus(DocumentProcessStatus.COMPLETED.name()); + + DocumentStore documentStore = Mockito.mock(DocumentStore.class); + Mockito.when(documentStore.delete( + Mockito.anyList(), Mockito.any())).thenReturn(StoreResult.success()); + DocumentCollection knowledge = Mockito.mock(DocumentCollection.class); + Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId); + Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test"); + Mockito.when(knowledge.toDocumentStore()).thenReturn(documentStore); + Model model = new Model(); + model.setModelName("embedding-test"); + + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneByQuery( + Mockito.any(QueryWrapper.class))).thenReturn(document); + Mockito.when(documentMapper.deleteById(documentId)).thenReturn(1); + DocumentChunkMapper chunkMapper = + Mockito.mock(DocumentChunkMapper.class); + Mockito.when(chunkMapper.selectListByQueryAs( + Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class))) + .thenReturn(List.of(BigInteger.valueOf(201))); + Mockito.when(chunkMapper.deleteByQuery( + Mockito.any(QueryWrapper.class))).thenReturn(1); + DocumentCollectionService knowledgeService = + Mockito.mock(DocumentCollectionService.class); + Mockito.when(knowledgeService.getById(knowledgeId)) + .thenReturn(knowledge); + ModelService modelService = Mockito.mock(ModelService.class); + Mockito.when(modelService.getById(modelId)).thenReturn(model); + FileStorageService storageService = + Mockito.mock(FileStorageService.class); + SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class); + + DocumentServiceImpl service = new DocumentServiceImpl(); + setField(service, "documentMapper", documentMapper); + setField(service, "documentChunkMapper", chunkMapper); + setField(service, "knowledgeService", knowledgeService); + setField(service, "modelService", modelService); + setField(service, "storageService", storageService); + setField(service, "searcherFactory", searcherFactory); + + Assert.assertTrue(service.removeDoc(documentId.toString())); + + Mockito.verify(documentMapper).deleteById(documentId); + Mockito.verify(storageService).delete(document.getDocumentPath()); + } + + /** + * 验证最后一个分块已单独删除时，文档删除跳过空向量请求并完成清理。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void removeDocShouldHandleDocumentWithoutChunks() throws Exception { + BigInteger documentId = BigInteger.valueOf(202); + BigInteger knowledgeId = BigInteger.valueOf(203); + BigInteger modelId = BigInteger.valueOf(204); + Document document = new Document(); + document.setId(documentId); + document.setCollectionId(knowledgeId); + document.setDocumentPath("storage://empty-document.txt"); + document.setProcessStatus(DocumentProcessStatus.COMPLETED.name()); + + DocumentCollection knowledge = Mockito.mock(DocumentCollection.class); + Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId); + Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test"); + Mockito.when(knowledge.toDocumentStore()).thenReturn(null); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneByQuery( + Mockito.any(QueryWrapper.class))).thenReturn(document); + Mockito.when(documentMapper.deleteById(documentId)).thenReturn(1); + DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class); + Mockito.when(chunkMapper.selectListByQueryAs( + Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class))) + .thenReturn(List.of()); + Mockito.when(chunkMapper.deleteByQuery( + Mockito.any(QueryWrapper.class))).thenReturn(0); + DocumentCollectionService knowledgeService = + Mockito.mock(DocumentCollectionService.class); + Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge); + ModelService modelService = Mockito.mock(ModelService.class); + FileStorageService storageService = Mockito.mock(FileStorageService.class); + SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class); + + DocumentServiceImpl service = new DocumentServiceImpl(); + setField(service, "documentMapper", documentMapper); + setField(service, "documentChunkMapper", chunkMapper); + setField(service, "knowledgeService", knowledgeService); + setField(service, "modelService", modelService); + setField(service, "storageService", storageService); + setField(service, "searcherFactory", searcherFactory); + + Assert.assertTrue(service.removeDoc(documentId.toString())); + + Mockito.verify(knowledge, Mockito.never()).toDocumentStore(); + Mockito.verifyNoInteractions(modelService); + Mockito.verify(documentMapper).deleteById(documentId); + Mockito.verify(storageService).delete(document.getDocumentPath()); + } + + /** + * 验证分块中的文档禁止删除，且不会触发任何外部或数据库清理。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void removeDocShouldRejectSplittingDocumentWithoutSideEffects() + throws Exception { + BigInteger documentId = BigInteger.valueOf(301); + Document document = new Document(); + document.setId(documentId); + document.setProcessStatus(DocumentProcessStatus.SPLITTING.name()); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneByQuery( + Mockito.any(QueryWrapper.class))).thenReturn(document); + DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class); + DocumentCollectionService knowledgeService = + Mockito.mock(DocumentCollectionService.class); + + DocumentServiceImpl service = new DocumentServiceImpl(); + setField(service, "documentMapper", documentMapper); + setField(service, "documentChunkMapper", chunkMapper); + setField(service, "knowledgeService", knowledgeService); + + try { + service.removeDoc(documentId.toString()); + Assert.fail("分块中的文档应拒绝删除"); + } catch (BusinessException expected) { + Assert.assertEquals("文档处理中，暂不允许删除", expected.getMessage()); + } + + Mockito.verifyNoInteractions(chunkMapper, knowledgeService); + Mockito.verify(documentMapper, Mockito.never()).deleteById(Mockito.any()); + } + + /** + * 验证向量删除失败时停止后续清理，避免接口返回虚假成功。 + * + * @throws Exception 反射注入异常 + */ + @Test + public void removeDocShouldStopWhenVectorDeleteFails() throws Exception { + BigInteger documentId = BigInteger.valueOf(401); + BigInteger knowledgeId = BigInteger.valueOf(402); + BigInteger modelId = BigInteger.valueOf(403); + Document document = new Document(); + document.setId(documentId); + document.setCollectionId(knowledgeId); + document.setProcessStatus(DocumentProcessStatus.COMPLETED.name()); + DocumentStore documentStore = Mockito.mock(DocumentStore.class); + Mockito.when(documentStore.delete( + Mockito.anyList(), Mockito.any())).thenReturn(StoreResult.fail("测试失败")); + DocumentCollection knowledge = Mockito.mock(DocumentCollection.class); + Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId); + Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test"); + Mockito.when(knowledge.toDocumentStore()).thenReturn(documentStore); + Model model = new Model(); + model.setModelName("embedding-test"); + DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class); + Mockito.when(documentMapper.selectOneByQuery( + Mockito.any(QueryWrapper.class))).thenReturn(document); + DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class); + Mockito.when(chunkMapper.selectListByQueryAs( + Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class))) + .thenReturn(List.of(BigInteger.valueOf(404))); + DocumentCollectionService knowledgeService = + Mockito.mock(DocumentCollectionService.class); + Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge); + ModelService modelService = Mockito.mock(ModelService.class); + Mockito.when(modelService.getById(modelId)).thenReturn(model); + FileStorageService storageService = Mockito.mock(FileStorageService.class); + SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class); + + DocumentServiceImpl service = new DocumentServiceImpl(); + setField(service, "documentMapper", documentMapper); + setField(service, "documentChunkMapper", chunkMapper); + setField(service, "knowledgeService", knowledgeService); + setField(service, "modelService", modelService); + setField(service, "storageService", storageService); + setField(service, "searcherFactory", searcherFactory); + + try { + service.removeDoc(documentId.toString()); + Assert.fail("向量删除失败时应停止文档删除"); + } catch (BusinessException expected) { + Assert.assertEquals("文档向量删除失败", expected.getMessage()); + } + + Mockito.verify(chunkMapper, Mockito.never()).deleteByQuery(Mockito.any()); + Mockito.verify(documentMapper, Mockito.never()).deleteById(Mockito.any()); + Mockito.verifyNoInteractions(storageService); + } + + /** + * 通过反射注入测试依赖。 + * + * @param target 目标对象 + * @param fieldName 字段名 + * @param value 字段值 + * @throws Exception 字段不存在或无法访问时抛出 + */ + private static void setField(Object target, + String fieldName, + Object value) throws Exception { + Field field = DocumentServiceImpl.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/KnowledgeSharePermissionServiceImplTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/KnowledgeSharePermissionServiceImplTest.java new file mode 100644 index 00000000..40f6cfdb --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/service/impl/KnowledgeSharePermissionServiceImplTest.java @@ -0,0 +1,137 @@ +package tech.easyflow.ai.service.impl; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import tech.easyflow.ai.enums.KnowledgeApiPermissionScope; +import tech.easyflow.common.cache.RedisLockExecutor; +import tech.easyflow.system.entity.SysApiKey; +import tech.easyflow.system.entity.SysApiKeyResource; +import tech.easyflow.system.entity.SysApiKeyResourceMapping; +import tech.easyflow.system.service.SysApiKeyResourceMappingService; +import tech.easyflow.system.service.SysApiKeyResourceService; +import tech.easyflow.system.service.SysApiKeyService; + +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 知识库 Public API 产品权限映射测试。 + * + * @author Codex + * @since 2026-08-02 + */ +public class KnowledgeSharePermissionServiceImplTest { + + /** + * 验证读取、导入和维护权限分别只生成各自接口映射。 + * + * @throws Exception 依赖注入失败 + */ + @Test + public void replacePermissionsShouldKeepReadImportMaintenanceSeparated() + throws Exception { + BigInteger apiKeyId = BigInteger.valueOf(701); + SysApiKeyService apiKeyService = Mockito.mock(SysApiKeyService.class); + SysApiKeyResourceService resourceService = + Mockito.mock(SysApiKeyResourceService.class); + SysApiKeyResourceMappingService mappingService = + Mockito.mock(SysApiKeyResourceMappingService.class); + RedisLockExecutor redisLockExecutor = + Mockito.mock(RedisLockExecutor.class); + RedisLockExecutor.LockHandle lockHandle = + Mockito.mock(RedisLockExecutor.LockHandle.class); + Mockito.when(redisLockExecutor.tryAcquire( + Mockito.anyString(), + Mockito.any(), + Mockito.any() + )).thenReturn(lockHandle); + Mockito.when(apiKeyService.getById(apiKeyId)).thenReturn(new SysApiKey()); + AtomicLong resourceId = new AtomicLong(800); + Mockito.when(resourceService.getOne( + Mockito.any(com.mybatisflex.core.query.QueryWrapper.class))) + .thenAnswer(invocation -> { + SysApiKeyResource resource = new SysApiKeyResource(); + resource.setId(BigInteger.valueOf(resourceId.incrementAndGet())); + return resource; + }); + + KnowledgeSharePermissionServiceImpl service = + new KnowledgeSharePermissionServiceImpl(); + setField(service, "sysApiKeyService", apiKeyService); + setField(service, "resourceService", resourceService); + setField(service, "mappingService", mappingService); + setField(service, "redisLockExecutor", redisLockExecutor); + + service.replaceApiPermissions(apiKeyId, true, false, false); + service.replaceApiPermissions(apiKeyId, false, true, false); + service.replaceApiPermissions(apiKeyId, false, false, true); + + @SuppressWarnings("unchecked") + ArgumentCaptor> mappings = + ArgumentCaptor.forClass(List.class); + Mockito.verify(mappingService, Mockito.times(3)).remove( + Mockito.any(com.mybatisflex.core.query.QueryWrapper.class) + ); + Mockito.verify(mappingService, Mockito.times(3)) + .saveBatch(mappings.capture()); + Mockito.verify(lockHandle, Mockito.times(3)).release(); + Assert.assertEquals(3, mappings.getAllValues().size()); + assertScopeMappings( + mappings.getAllValues().get(0), + KnowledgeApiPermissionScope.KNOWLEDGE_READ, + 8 + ); + assertScopeMappings( + mappings.getAllValues().get(1), + KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT, + 14 + ); + assertScopeMappings( + mappings.getAllValues().get(2), + KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE, + 6 + ); + } + + /** + * 断言一组全局知识库接口映射只包含指定权限。 + * + * @param mappings 接口映射 + * @param expectedScope 预期权限 + * @param expectedSize 预期接口数量 + */ + private void assertScopeMappings( + List mappings, + KnowledgeApiPermissionScope expectedScope, + int expectedSize) { + Assert.assertEquals(expectedSize, mappings.size()); + for (SysApiKeyResourceMapping mapping : mappings) { + Assert.assertEquals( + expectedScope.name(), + mapping.getActionScope() + ); + Assert.assertNull(mapping.getResourceTargetId()); + } + } + + /** + * 反射注入测试依赖。 + * + * @param target 目标对象 + * @param fieldName 字段名 + * @param value 字段值 + * @throws Exception 字段不存在或不可访问 + */ + private static void setField( + Object target, + String fieldName, + Object value) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/entity/SysApiKey.java b/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/entity/SysApiKey.java index 579f30d7..39cf36b3 100644 --- a/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/entity/SysApiKey.java +++ b/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/entity/SysApiKey.java @@ -26,6 +26,15 @@ public class SysApiKey extends SysApiKeyBase { @Column(ignore = true) private Boolean knowledgeShareEnabled; + @Column(ignore = true) + private Boolean knowledgeReadEnabled; + + @Column(ignore = true) + private Boolean knowledgeImportEnabled; + + @Column(ignore = true) + private Boolean knowledgeMaintenanceEnabled; + @Column(ignore = true) private Boolean workflowApiEnabled; @@ -56,6 +65,60 @@ public class SysApiKey extends SysApiKeyBase { this.knowledgeShareEnabled = knowledgeShareEnabled; } + /** + * 获取知识库读取权限开关。 + * + * @return 是否开启读取权限 + */ + public Boolean getKnowledgeReadEnabled() { + return knowledgeReadEnabled; + } + + /** + * 设置知识库读取权限开关。 + * + * @param knowledgeReadEnabled 是否开启读取权限 + */ + public void setKnowledgeReadEnabled(Boolean knowledgeReadEnabled) { + this.knowledgeReadEnabled = knowledgeReadEnabled; + } + + /** + * 获取知识库导入权限开关。 + * + * @return 是否开启导入权限 + */ + public Boolean getKnowledgeImportEnabled() { + return knowledgeImportEnabled; + } + + /** + * 设置知识库导入权限开关。 + * + * @param knowledgeImportEnabled 是否开启导入权限 + */ + public void setKnowledgeImportEnabled(Boolean knowledgeImportEnabled) { + this.knowledgeImportEnabled = knowledgeImportEnabled; + } + + /** + * 获取知识库维护权限开关。 + * + * @return 是否开启维护权限 + */ + public Boolean getKnowledgeMaintenanceEnabled() { + return knowledgeMaintenanceEnabled; + } + + /** + * 设置知识库维护权限开关。 + * + * @param knowledgeMaintenanceEnabled 是否开启维护权限 + */ + public void setKnowledgeMaintenanceEnabled(Boolean knowledgeMaintenanceEnabled) { + this.knowledgeMaintenanceEnabled = knowledgeMaintenanceEnabled; + } + public Boolean getWorkflowApiEnabled() { return workflowApiEnabled; } diff --git a/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/service/SysApiKeyService.java b/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/service/SysApiKeyService.java index 2f77e2d1..e5a18da7 100644 --- a/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/service/SysApiKeyService.java +++ b/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/service/SysApiKeyService.java @@ -13,7 +13,14 @@ import java.math.BigInteger; */ public interface SysApiKeyService extends IService { - void checkApikeyPermission(String apiKey, String requestURI); + /** + * 校验访问令牌是否具有接口权限。 + * + * @param apiKey 访问令牌明文 + * @param requestURI 请求 URI + * @return 已通过身份和接口权限校验的访问令牌 + */ + SysApiKey checkApikeyPermission(String apiKey, String requestURI); SysApiKey getSysApiKey(String apiKey); diff --git a/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/service/impl/SysApiKeyServiceImpl.java b/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/service/impl/SysApiKeyServiceImpl.java index ec3c323e..02b32e58 100644 --- a/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/service/impl/SysApiKeyServiceImpl.java +++ b/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/service/impl/SysApiKeyServiceImpl.java @@ -36,7 +36,7 @@ public class SysApiKeyServiceImpl extends ServiceImpl candidateRequestUris = getCandidateRequestUris(requestURI); QueryWrapper w = QueryWrapper.create(); @@ -55,6 +55,7 @@ public class SysApiKeyServiceImpl extends ServiceImpl getCandidateRequestUris(String requestURI) { diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/application-prod.yml b/easyflow-starter/easyflow-starter-all/src/main/resources/application-prod.yml index f07b76e2..2abbeca0 100644 --- a/easyflow-starter/easyflow-starter-all/src/main/resources/application-prod.yml +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/application-prod.yml @@ -47,7 +47,7 @@ easyflow: max-retry: 16 consumer-executor: core-size: 4 - max-size: 12 + max-size: 32 queue-capacity: 64 keep-alive-seconds: 60 pool: diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml b/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml index 5bbe3f0a..87a95c89 100644 --- a/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/application.yml @@ -10,6 +10,9 @@ server: enabled: true charset: UTF-8 # 必须设置 UTF-8，避免 WebFlux 流式返回（AI 场景）会乱码问题 force: true + tomcat: + # Public API 单次最多 200 个文件，额外为 metadata 与表单边界预留 Part。 + max-part-count: 205 spring: profiles: @@ -44,8 +47,8 @@ spring: servlet: multipart: max-file-size: 100MB - # 为 multipart 边界和请求头预留空间，文件本身仍由 M18 的 100 MiB 硬上限约束。 - max-request-size: 105MB + # Public API 业务上限为 200 MiB，为 metadata 与 multipart 边界预留空间。 + max-request-size: 220MB web: resources: # 示例：windows【file: C:\easyflow\attachment】 linux【file: /www/easyflow/attachment】 @@ -127,7 +130,7 @@ easyflow: max-retry: 16 consumer-executor: core-size: 16 - max-size: 24 + max-size: 32 queue-capacity: 64 keep-alive-seconds: 60 pool: @@ -198,6 +201,22 @@ easyflow: health: cache-ttl: 5s document-import: + bulk: + max-file-count: 2000 + max-total-size: 1GB + max-file-size: 100MB + upload-concurrency: 3 + parse-max-running: 2 + split-max-running: 2 + index-max-running: 2 + per-batch-parse-max-running: 2 + pending-dispatch-batch-size: 100 + pending-dispatch-interval: 2s + pending-redispatch-delay: 5s + pending-timeout: 24h + parse-submit-timeout: 120s + interruption-timeout: 10m + max-task-attempts: 3 status-broadcast-channel: easyflow:document-import:status parse-monitor: fixed-delay: 10000 @@ -249,7 +268,8 @@ easy-agents: provider: mineru mineru: # 统一文档解析桥接层直接复用 easy-agents 的 provider 配置，不在 easyflow 再复制一套配置体系 - base-url: https://hub.wust.edu.cn/modelServer/mineru-api + base-url: https://ontoweb.wust.edu.cn/mineru-api + submit-timeout-ms: 120000 default-lang-list: - ch diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V41__mysql_document_batch_import.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V41__mysql_document_batch_import.sql new file mode 100644 index 00000000..828cde12 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V41__mysql_document_batch_import.sql @@ -0,0 +1,52 @@ +CREATE TABLE `tb_document_import_batch` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `knowledge_id` bigint UNSIGNED NOT NULL COMMENT '知识库ID', + `import_mode` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '导入模式', + `status` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '批次状态', + `total_count` int NOT NULL DEFAULT 0 COMMENT '文件总数', + `total_bytes` bigint NOT NULL DEFAULT 0 COMMENT '文件总字节数', + `completed_count` int NOT NULL DEFAULT 0 COMMENT '完成数', + `processing_count` int NOT NULL DEFAULT 0 COMMENT '处理中数量', + `failed_count` int NOT NULL DEFAULT 0 COMMENT '失败数', + `pending_count` int NOT NULL DEFAULT 0 COMMENT '等待数', + `started_at` datetime NULL DEFAULT NULL COMMENT '开始时间', + `finished_at` datetime NULL DEFAULT NULL COMMENT '结束时间', + `created` datetime NULL DEFAULT NULL COMMENT '创建时间', + `created_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '创建人', + `modified` datetime NULL DEFAULT NULL COMMENT '修改时间', + `modified_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '修改人', + PRIMARY KEY (`id`) USING BTREE, + INDEX `idx_document_import_batch_knowledge_status` (`knowledge_id`, `status`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '知识库文档批量导入批次' ROW_FORMAT = DYNAMIC; + +CREATE TABLE `tb_document_import_batch_item` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `batch_id` bigint UNSIGNED NOT NULL COMMENT '批次ID', + `knowledge_id` bigint UNSIGNED NOT NULL COMMENT '知识库ID', + `document_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '文档ID', + `client_file_key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '客户端文件键', + `file_name` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '文件名', + `relative_path` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文件夹相对路径', + `file_size` bigint NOT NULL DEFAULT 0 COMMENT '文件大小', + `file_path` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '存储路径', + `stage` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '当前阶段', + `status` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '当前状态', + `error_summary` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '错误摘要', + `created` datetime NULL DEFAULT NULL COMMENT '创建时间', + `created_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '创建人', + `modified` datetime NULL DEFAULT NULL COMMENT '修改时间', + `modified_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '修改人', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_document_import_batch_item_key` (`batch_id`, `client_file_key`) USING BTREE, + INDEX `idx_document_import_batch_item_batch_status` (`batch_id`, `status`) USING BTREE, + INDEX `idx_document_import_batch_item_knowledge_key` (`knowledge_id`, `client_file_key`) USING BTREE, + INDEX `idx_document_import_batch_item_document` (`document_id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '知识库文档批量导入文件项' ROW_FORMAT = DYNAMIC; + +ALTER TABLE `tb_document_import_task` + ADD COLUMN `batch_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '批次ID' AFTER `knowledge_id`, + ADD COLUMN `batch_item_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '批次文件项ID' AFTER `batch_id`, + ADD INDEX `idx_document_import_task_batch_phase_status` (`batch_id`, `phase`, `status`) USING BTREE, + ADD INDEX `idx_document_import_task_batch_item` (`batch_item_id`) USING BTREE; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V42__mysql_document_batch_import_reliability.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V42__mysql_document_batch_import_reliability.sql new file mode 100644 index 00000000..26b5e074 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V42__mysql_document_batch_import_reliability.sql @@ -0,0 +1,42 @@ +ALTER TABLE `tb_document_import_batch` + ADD COLUMN `uploaded_count` int NOT NULL DEFAULT 0 COMMENT '已上传数' AFTER `pending_count`, + ADD COLUMN `skipped_count` int NOT NULL DEFAULT 0 COMMENT '跳过数' AFTER `uploaded_count`, + ADD COLUMN `cancelled_count` int NOT NULL DEFAULT 0 COMMENT '取消数' AFTER `skipped_count`, + ADD COLUMN `retryable_failed_count` int NOT NULL DEFAULT 0 COMMENT '可重试失败数' AFTER `cancelled_count`; + +ALTER TABLE `tb_document_import_batch_item` + ADD COLUMN `retryable` tinyint(1) NOT NULL DEFAULT 0 COMMENT '是否允许批量重试' AFTER `error_summary`, + ADD COLUMN `attempt_count` int NOT NULL DEFAULT 0 COMMENT '重试次数' AFTER `retryable`, + DROP INDEX `idx_document_import_batch_item_batch_status`, + ADD INDEX `idx_document_import_batch_item_batch_status` (`batch_id`, `status`, `modified`) USING BTREE; + +ALTER TABLE `tb_document_import_task` + ADD INDEX `idx_document_import_task_phase_status_modified` + (`phase`, `status`, `modified`, `batch_id`, `created`) USING BTREE; + +UPDATE `tb_document_import_batch` batch +SET batch.`uploaded_count` = ( + SELECT COUNT(*) + FROM `tb_document_import_batch_item` item + WHERE item.`batch_id` = batch.`id` + AND item.`status` = 'UPLOADED' + ), + batch.`skipped_count` = ( + SELECT COUNT(*) + FROM `tb_document_import_batch_item` item + WHERE item.`batch_id` = batch.`id` + AND item.`status` = 'SKIPPED' + ), + batch.`cancelled_count` = ( + SELECT COUNT(*) + FROM `tb_document_import_batch_item` item + WHERE item.`batch_id` = batch.`id` + AND item.`status` = 'CANCELLED' + ), + batch.`retryable_failed_count` = ( + SELECT COUNT(*) + FROM `tb_document_import_batch_item` item + WHERE item.`batch_id` = batch.`id` + AND item.`status` = 'FAILED' + AND item.`retryable` = 1 + ); diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V43__mysql_document_batch_import_overwrite.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V43__mysql_document_batch_import_overwrite.sql new file mode 100644 index 00000000..8b881079 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V43__mysql_document_batch_import_overwrite.sql @@ -0,0 +1,5 @@ +ALTER TABLE `tb_document_import_batch_item` + ADD COLUMN `replaced_document_id` bigint UNSIGNED NULL DEFAULT NULL + COMMENT '待覆盖的历史文档ID' AFTER `document_id`, + ADD INDEX `idx_document_import_batch_item_replacement` + (`replaced_document_id`, `status`, `modified`) USING BTREE; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V44__mysql_document_batch_import_retryable_backfill.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V44__mysql_document_batch_import_retryable_backfill.sql new file mode 100644 index 00000000..a387b59b --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V44__mysql_document_batch_import_retryable_backfill.sql @@ -0,0 +1,15 @@ +UPDATE `tb_document_import_batch_item` +SET `retryable` = 1 +WHERE `status` = 'FAILED' + AND `stage` IN ('PARSE', 'SPLIT', 'INDEX') + AND `document_id` IS NOT NULL + AND `retryable` = 0; + +UPDATE `tb_document_import_batch` batch +SET batch.`retryable_failed_count` = ( + SELECT COUNT(*) + FROM `tb_document_import_batch_item` item + WHERE item.`batch_id` = batch.`id` + AND item.`status` = 'FAILED' + AND item.`retryable` = 1 +); diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V45__mysql_knowledge_public_api_import.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V45__mysql_knowledge_public_api_import.sql new file mode 100644 index 00000000..9901527f --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V45__mysql_knowledge_public_api_import.sql @@ -0,0 +1,191 @@ +SET NAMES utf8mb4; + +INSERT INTO `tb_sys_api_key_resource` (`id`, `request_interface`, `title`) +VALUES + (366700000000000101, '/public-api/knowledge-share/detail', '知识库读取'), + (366700000000000102, '/public-api/knowledge-share/search', '知识库读取'), + (366700000000000103, '/public-api/knowledge-share/document/page', '知识库读取'), + (366700000000000104, '/public-api/knowledge-share/document/download', '知识库读取'), + (366700000000000105, '/public-api/knowledge-share/documentChunk/page', '知识库读取'), + (366700000000000106, '/public-api/knowledge-share/faq/page', '知识库读取'), + (366700000000000107, '/public-api/knowledge-share/faq/detail', '知识库读取'), + (366700000000000108, '/public-api/knowledge-share/faq/exportExcel', '知识库读取'), + (366700000000000109, '/public-api/knowledge-share/document/import/batch', '知识导入'), + (366700000000000110, '/public-api/knowledge-share/document/import/batch/status', '知识导入'), + (366700000000000111, '/public-api/knowledge-share/document/import/batch/retry', '知识导入'), + (366700000000000112, '/public-api/knowledge-share/document/import/analyze', '知识导入'), + (366700000000000113, '/public-api/knowledge-share/document/import/preview', '知识导入'), + (366700000000000114, '/public-api/knowledge-share/document/import/commit', '知识导入'), + (366700000000000115, '/public-api/knowledge-share/document/import/task/create', '知识导入'), + (366700000000000116, '/public-api/knowledge-share/document/import/task/detail', '知识导入'), + (366700000000000117, '/public-api/knowledge-share/document/import/task/preview', '知识导入'), + (366700000000000118, '/public-api/knowledge-share/document/import/task/startIndex', '知识导入'), + (366700000000000119, '/public-api/knowledge-share/document/import/task/retryParse', '知识导入'), + (366700000000000120, '/public-api/knowledge-share/document/import/task/retryIndex', '知识导入'), + (366700000000000121, '/public-api/knowledge-share/faq/importExcel', '知识导入'), + (366700000000000122, '/public-api/knowledge-share/faq/downloadImportTemplate', '知识导入'), + (366700000000000123, '/public-api/knowledge-share/document/remove', '知识库维护'), + (366700000000000124, '/public-api/knowledge-share/documentChunk/update', '知识库维护'), + (366700000000000125, '/public-api/knowledge-share/documentChunk/remove', '知识库维护'), + (366700000000000126, '/public-api/knowledge-share/faq/save', '知识库维护'), + (366700000000000127, '/public-api/knowledge-share/faq/update', '知识库维护'), + (366700000000000128, '/public-api/knowledge-share/faq/remove', '知识库维护') +ON DUPLICATE KEY UPDATE `title` = VALUES(`title`); + +SET @migrated_knowledge_api_key_count = ( + SELECT COUNT(DISTINCT mapping.`api_key_id`) + FROM `tb_sys_api_key_resource_mapping` mapping + WHERE mapping.`resource_type` = 'KNOWLEDGE' + AND mapping.`resource_target_id` IS NULL +); + +UPDATE `tb_sys_api_key_resource_mapping` mapping +JOIN `tb_sys_api_key_resource` resource + ON resource.`id` = mapping.`api_key_resource_id` +SET mapping.`action_scope` = CASE + WHEN resource.`request_interface` IN ( + '/public-api/knowledge-share/detail', + '/public-api/knowledge-share/search', + '/public-api/knowledge-share/document/page', + '/public-api/knowledge-share/document/download', + '/public-api/knowledge-share/documentChunk/page', + '/public-api/knowledge-share/faq/page', + '/public-api/knowledge-share/faq/detail', + '/public-api/knowledge-share/faq/exportExcel' + ) THEN 'KNOWLEDGE_READ' + WHEN resource.`request_interface` IN ( + '/public-api/knowledge-share/document/import/analyze', + '/public-api/knowledge-share/document/import/preview', + '/public-api/knowledge-share/document/import/commit', + '/public-api/knowledge-share/document/import/task/create', + '/public-api/knowledge-share/document/import/task/detail', + '/public-api/knowledge-share/document/import/task/preview', + '/public-api/knowledge-share/document/import/task/startIndex', + '/public-api/knowledge-share/document/import/task/retryParse', + '/public-api/knowledge-share/document/import/task/retryIndex', + '/public-api/knowledge-share/faq/importExcel', + '/public-api/knowledge-share/faq/downloadImportTemplate' + ) THEN 'KNOWLEDGE_IMPORT' + WHEN resource.`request_interface` IN ( + '/public-api/knowledge-share/document/remove', + '/public-api/knowledge-share/documentChunk/update', + '/public-api/knowledge-share/documentChunk/remove', + '/public-api/knowledge-share/faq/save', + '/public-api/knowledge-share/faq/update', + '/public-api/knowledge-share/faq/remove' + ) THEN 'KNOWLEDGE_MAINTENANCE' + ELSE mapping.`action_scope` +END +WHERE mapping.`resource_type` = 'KNOWLEDGE'; + +SET @revoked_maintenance_mapping_count = ( + SELECT COUNT(*) + FROM `tb_sys_api_key_resource_mapping` + WHERE `resource_type` = 'KNOWLEDGE' + AND `resource_target_id` IS NULL + AND `action_scope` = 'KNOWLEDGE_MAINTENANCE' +); + +DELETE FROM `tb_sys_api_key_resource_mapping` +WHERE `resource_type` = 'KNOWLEDGE' + AND `resource_target_id` IS NULL + AND `action_scope` = 'KNOWLEDGE_MAINTENANCE'; + +INSERT INTO `tb_sys_api_key_resource_mapping` + (`id`, `api_key_id`, `api_key_resource_id`, `resource_type`, + `resource_target_id`, `action_scope`) +SELECT + CAST( + 366900000000000000 + + ROW_NUMBER() OVER ( + ORDER BY token.`api_key_id`, + COALESCE(token.`resource_target_id`, 0), + resource.`id` + ) + AS UNSIGNED + ), + token.`api_key_id`, + resource.`id`, + 'KNOWLEDGE', + token.`resource_target_id`, + 'KNOWLEDGE_IMPORT' +FROM ( + SELECT DISTINCT mapping.`api_key_id`, mapping.`resource_target_id` + FROM `tb_sys_api_key_resource_mapping` mapping + WHERE mapping.`resource_type` = 'KNOWLEDGE' + AND mapping.`action_scope` = 'KNOWLEDGE_IMPORT' +) token +JOIN `tb_sys_api_key_resource` resource + ON resource.`request_interface` IN ( + '/public-api/knowledge-share/document/import/batch', + '/public-api/knowledge-share/document/import/batch/status', + '/public-api/knowledge-share/document/import/batch/retry' + ) +LEFT JOIN `tb_sys_api_key_resource_mapping` existing + ON existing.`api_key_id` = token.`api_key_id` + AND existing.`api_key_resource_id` = resource.`id` + AND existing.`resource_type` = 'KNOWLEDGE' + AND existing.`resource_target_id` <=> token.`resource_target_id` + AND existing.`action_scope` = 'KNOWLEDGE_IMPORT' +WHERE existing.`id` IS NULL; + +SELECT + @migrated_knowledge_api_key_count AS `migrated_knowledge_api_key_count`, + @revoked_maintenance_mapping_count AS `revoked_maintenance_mapping_count`; + +ALTER TABLE `tb_document_import_batch` + ADD COLUMN `caller_type` varchar(16) NULL DEFAULT NULL + COMMENT '调用者类型' AFTER `knowledge_id`, + ADD COLUMN `caller_id` bigint UNSIGNED NULL DEFAULT NULL + COMMENT '调用者ID' AFTER `caller_type`, + ADD COLUMN `idempotency_key_hash` char(64) NULL DEFAULT NULL + COMMENT '提交幂等键SHA-256' AFTER `caller_id`, + ADD COLUMN `request_digest` char(64) NULL DEFAULT NULL + COMMENT '请求摘要SHA-256' AFTER `idempotency_key_hash`, + ADD COLUMN `duplicate_policy` varchar(16) NULL DEFAULT NULL + COMMENT '重复文件策略' AFTER `request_digest`, + ADD COLUMN `requested_strategy_json` text NULL + COMMENT '请求分块策略' AFTER `duplicate_policy`, + ADD COLUMN `retry_generation` int NOT NULL DEFAULT 0 + COMMENT '重试代次' AFTER `requested_strategy_json`, + ADD COLUMN `last_retry_key_hash` char(64) NULL DEFAULT NULL + COMMENT '最近重试幂等键SHA-256' AFTER `retry_generation`, + ADD COLUMN `last_retry_generation` int NOT NULL DEFAULT 0 + COMMENT '最近重试响应代次' AFTER `last_retry_key_hash`, + ADD COLUMN `version` int NOT NULL DEFAULT 0 + COMMENT '乐观锁版本' AFTER `last_retry_generation`, + ADD UNIQUE INDEX `uk_document_import_batch_caller_idempotency` + (`caller_type`, `caller_id`, `idempotency_key_hash`) USING BTREE; + +UPDATE `tb_document_import_batch` +SET `caller_type` = 'ADMIN', + `caller_id` = COALESCE(`created_by`, 0), + `duplicate_policy` = COALESCE(`duplicate_policy`, 'SKIP') +WHERE `caller_type` IS NULL; + +ALTER TABLE `tb_document_import_batch_item` + ADD COLUMN `content_sha256` char(64) NULL DEFAULT NULL + COMMENT '文件内容SHA-256' AFTER `file_path`, + ADD COLUMN `failure_code` varchar(64) NULL DEFAULT NULL + COMMENT '稳定失败码' AFTER `error_summary`, + ADD COLUMN `applied_strategy_code` varchar(64) NULL DEFAULT NULL + COMMENT '实际分块策略编码' AFTER `failure_code`, + ADD COLUMN `strategy_snapshot_json` text NULL + COMMENT '分块策略快照' AFTER `applied_strategy_code`, + DROP INDEX `idx_document_import_batch_item_batch_status`, + ADD INDEX `idx_document_import_batch_item_batch_status` + (`batch_id`, `status`, `modified`, `id`) USING BTREE; + +ALTER TABLE `tb_document_import_task` + ADD COLUMN `failure_code` varchar(64) NULL DEFAULT NULL + COMMENT '稳定失败码' AFTER `error_summary`, + ADD COLUMN `attempt_no` int NOT NULL DEFAULT 0 + COMMENT '执行尝试次数' AFTER `failure_code`, + ADD COLUMN `execution_token` varchar(64) NULL DEFAULT NULL + COMMENT '执行令牌' AFTER `attempt_no`, + ADD COLUMN `lease_until` datetime NULL DEFAULT NULL + COMMENT '租约到期时间' AFTER `execution_token`, + ADD COLUMN `version` int NOT NULL DEFAULT 0 + COMMENT '乐观锁版本' AFTER `lease_until`, + ADD INDEX `idx_document_import_task_phase_status_lease` + (`phase`, `status`, `lease_until`, `created`, `id`) USING BTREE; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V46__mysql_document_import_retry_idempotency.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V46__mysql_document_import_retry_idempotency.sql new file mode 100644 index 00000000..178fbee3 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V46__mysql_document_import_retry_idempotency.sql @@ -0,0 +1,15 @@ +CREATE TABLE `tb_document_import_batch_retry_request` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `batch_id` bigint UNSIGNED NOT NULL COMMENT '批次ID', + `retry_key_hash` char(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '重试幂等键SHA-256', + `request_digest` char(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '重试请求摘要SHA-256', + `response_status` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '首次领取后的响应状态', + `retry_generation` int NOT NULL COMMENT '首次领取后的重试代次', + `created` datetime NULL DEFAULT NULL COMMENT '创建时间', + `modified` datetime NULL DEFAULT NULL COMMENT '修改时间', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_document_import_retry_batch_key` (`batch_id`, `retry_key_hash`) USING BTREE, + INDEX `idx_document_import_retry_batch_created` (`batch_id`, `created`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci + COMMENT = '知识库文档批量导入重试幂等请求' ROW_FORMAT = DYNAMIC; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V47__mysql_document_import_stale_cleanup.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V47__mysql_document_import_stale_cleanup.sql new file mode 100644 index 00000000..773bae2a --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V47__mysql_document_import_stale_cleanup.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS `tb_document_import_batch_retry_request`; + +ALTER TABLE `tb_document_import_batch` + DROP COLUMN `last_retry_key_hash`, + DROP COLUMN `last_retry_generation`, + ADD INDEX `idx_document_import_batch_status_modified` + (`status`, `modified`, `id`) USING BTREE; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V48__mysql_document_import_cleanup_index.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V48__mysql_document_import_cleanup_index.sql new file mode 100644 index 00000000..4ba0c290 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V48__mysql_document_import_cleanup_index.sql @@ -0,0 +1,3 @@ +ALTER TABLE `tb_document_import_batch_item` + ADD INDEX `idx_document_import_batch_item_cleanup` + (`stage`, `status`, `modified`, `id`) USING BTREE; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V49__mysql_document_import_recoverable_upload.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V49__mysql_document_import_recoverable_upload.sql new file mode 100644 index 00000000..7c19c806 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V49__mysql_document_import_recoverable_upload.sql @@ -0,0 +1,16 @@ +ALTER TABLE `tb_document_import_batch_item` + ADD COLUMN `storage_locator` varchar(2048) NULL DEFAULT NULL + COMMENT '可恢复存储定位符' AFTER `file_path`, + ADD COLUMN `cleanup_pending` tinyint(1) NOT NULL DEFAULT 0 + COMMENT '是否等待清理存储对象' AFTER `storage_locator`; + +UPDATE `tb_document_import_batch_item` +SET `cleanup_pending` = 1 +WHERE `stage` = 'UPLOAD' + AND `status` IN ('UPLOADING', 'CANCELLED') + AND `file_path` IS NOT NULL; + +ALTER TABLE `tb_document_import_batch_item` + DROP INDEX `idx_document_import_batch_item_cleanup`, + ADD INDEX `idx_document_import_batch_item_cleanup_pending` + (`cleanup_pending`, `modified`, `id`) USING BTREE; diff --git a/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json b/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json index f0653d48..411e12e8 100644 --- a/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json +++ b/easyflow-ui-admin/app/src/locales/langs/en-US/documentCollection.json @@ -78,6 +78,50 @@ "fileName": "File Name", "progressUpload": "Progress of file upload", "fileSize": "File size", + "batchUploadTitle": "Select files or drop them here", + "batchUploadDescription": "TXT, PDF, DOCX, MD, PPTX and XLSX. Up to 100MB per file and 1GB per folder.", + "batchUploadTip": "When upload completes, choose manual or automatic import.", + "selectFolder": "Select Folder", + "manualImport": "Manual Import", + "autoImport": "Auto Import", + "autoImportTip": "Automatically parse, chunk, vectorize and store every document.", + "manualImportStarted": "Documents are parsing and will be ready for manual chunking", + "autoImportStarted": "Automatic import started", + "uploadCompleteFirst": "Wait for every file to finish uploading", + "uploaded": "Uploaded", + "uploading": "Uploading", + "retryUpload": "Retry Upload", + "retry": "Retry", + "singleFileLimit": "Each file must be no larger than 100MB", + "folderSizeLimit": "The folder must be no larger than 1GB", + "fileCountLimit": "A batch can contain up to 2000 files", + "noSupportedFiles": "No supported documents found", + "unsupportedSkipped": "Unsupported files were skipped", + "createBatchFailed": "Failed to create upload batch", + "cancelBatchFailed": "Failed to cancel the upload batch. Please retry.", + "uploadFailed": "Upload failed. Please retry.", + "duplicatePolicy": "Duplicates", + "skipDuplicates": "Skip", + "overwriteDuplicates": "Replace", + "reimportDuplicates": "Reimport", + "batchStatus": "Automatic import status", + "batchRunning": "Running", + "batchInterrupted": "Interrupted", + "batchPartial": "Partially failed", + "batchFailed": "Import failed", + "batchCompleted": "Completed", + "completedCount": "Completed", + "processingCount": "Processing", + "failedCount": "Failed", + "pendingCount": "Pending", + "skippedCount": "Skipped", + "continueBatch": "Continue", + "splitOrIndexFailed": "Chunking or indexing failed. Please retry.", + "documentSourceUnavailable": "Unable to read the document file. Please contact the administrator.", + "parseServiceUnavailable": "The document parsing service is temporarily unavailable. Please retry later.", + "parseServiceTimeout": "The document parsing service timed out. Please retry later.", + "taskPendingTimeout": "The task waited too long in the queue. Please retry.", + "taskExecutionInterrupted": "The task was interrupted. Please retry.", "uploadCreateTip": "After upload, the document appears in the list first and is parsed asynchronously. Continue with chunking after parsing finishes.", "analysisTip": "The system analyzes multilingual structure first and recommends a splitting strategy. You can still adjust each file manually.", "manualStrategyTip": "The preview refreshes automatically when the chunking strategy changes. Start indexing after it looks right.", @@ -134,6 +178,8 @@ "PARSING": "Parsing", "PARSE_FAILED": "Parse Failed", "READY_FOR_SEGMENT": "Ready for Chunking", + "SPLITTING": "Chunking", + "SPLIT_FAILED": "Chunking Failed", "READY_FOR_INDEX": "Ready for Indexing", "INDEXING": "Indexing", "INDEX_FAILED": "Index Failed", diff --git a/easyflow-ui-admin/app/src/locales/langs/en-US/sysApiKey.json b/easyflow-ui-admin/app/src/locales/langs/en-US/sysApiKey.json index 9c132776..e6bcfbaa 100644 --- a/easyflow-ui-admin/app/src/locales/langs/en-US/sysApiKey.json +++ b/easyflow-ui-admin/app/src/locales/langs/en-US/sysApiKey.json @@ -14,7 +14,9 @@ "failure": "Failure" }, "permissions": "AuthInterface", - "knowledgeSharePermission": "Knowledge Share", + "knowledgeReadPermission": "Knowledge Read", + "knowledgeImportPermission": "Knowledge Import", + "knowledgeMaintenancePermission": "Knowledge Maintenance", "workflowApiPermission": "Workflow API", "addApiKeyNotice": "This operation will generate an API key. Please confirm whether to proceed" } diff --git a/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json b/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json index 959da088..f0fd760f 100644 --- a/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json +++ b/easyflow-ui-admin/app/src/locales/langs/zh-CN/documentCollection.json @@ -78,6 +78,50 @@ "fileName": "文件名称", "progressUpload": "文件上传进度", "fileSize": "文件大小", + "batchUploadTitle": "点击选择文件，或将文件拖到这里上传", + "batchUploadDescription": "支持 TXT、PDF、DOCX、MD、PPTX、XLSX；单个文件不超过 100MB，文件夹总大小不超过 1GB。", + "batchUploadTip": "上传完成后，可选择手动导入或自动导入。", + "selectFolder": "选择文件夹", + "manualImport": "手动导入", + "autoImport": "自动导入", + "autoImportTip": "自动完成解析、分块、向量化和入库，无需逐篇确认。", + "manualImportStarted": "文档已进入列表，解析完成后可继续选择分块策略", + "autoImportStarted": "自动导入已开始", + "uploadCompleteFirst": "请等待所有文件上传完成", + "uploaded": "已上传", + "uploading": "上传中", + "retryUpload": "重试上传", + "retry": "重试", + "singleFileLimit": "单个文件不能超过 100MB", + "folderSizeLimit": "文件夹总大小不能超过 1GB", + "fileCountLimit": "单批次文件数不能超过 2000", + "noSupportedFiles": "未找到支持的文档", + "unsupportedSkipped": "已跳过不支持的文件", + "createBatchFailed": "创建上传批次失败", + "cancelBatchFailed": "取消上传批次失败，请重试", + "uploadFailed": "文件上传失败，请重试", + "duplicatePolicy": "重复文件", + "skipDuplicates": "跳过", + "overwriteDuplicates": "覆盖", + "reimportDuplicates": "重新导入", + "batchStatus": "自动导入状态", + "batchRunning": "处理中", + "batchInterrupted": "已中断", + "batchPartial": "部分失败", + "batchFailed": "导入失败", + "batchCompleted": "已完成", + "completedCount": "完成", + "processingCount": "处理中", + "failedCount": "失败", + "pendingCount": "等待", + "skippedCount": "跳过", + "continueBatch": "继续", + "splitOrIndexFailed": "分块或向量化失败，请重试", + "documentSourceUnavailable": "文档文件读取失败，请联系管理员", + "parseServiceUnavailable": "文档解析服务暂不可用，请稍后重试", + "parseServiceTimeout": "文档解析服务响应超时，请稍后重试", + "taskPendingTimeout": "任务排队超时，请重试", + "taskExecutionInterrupted": "任务执行中断，请重试", "uploadCreateTip": "上传完成后，文档会先进入列表并异步解析，解析完成后再继续分块和向量化。", "analysisTip": "系统会先基于文档结构做中英文规则分析，再推荐拆分策略，你也可以逐个文件手动调整。", "manualStrategyTip": "调整分块策略后会自动刷新预览，确认效果后再启动向量化。", @@ -134,6 +178,8 @@ "PARSING": "解析中", "PARSE_FAILED": "解析失败", "READY_FOR_SEGMENT": "待分块", + "SPLITTING": "分块中", + "SPLIT_FAILED": "分块失败", "READY_FOR_INDEX": "待向量化", "INDEXING": "向量化中", "INDEX_FAILED": "向量化失败", diff --git a/easyflow-ui-admin/app/src/locales/langs/zh-CN/sysApiKey.json b/easyflow-ui-admin/app/src/locales/langs/zh-CN/sysApiKey.json index a638663f..f4772299 100644 --- a/easyflow-ui-admin/app/src/locales/langs/zh-CN/sysApiKey.json +++ b/easyflow-ui-admin/app/src/locales/langs/zh-CN/sysApiKey.json @@ -14,7 +14,9 @@ "failure": "已失效" }, "permissions": "授权接口", - "knowledgeSharePermission": "知识库分享授权", + "knowledgeReadPermission": "知识库读取", + "knowledgeImportPermission": "知识导入", + "knowledgeMaintenancePermission": "知识库维护", "workflowApiPermission": "工作流 API 调用授权", "addApiKeyNotice": "该操作会生成一个apiKey,请确认是否生成" } diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/Document.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/Document.vue index 1366906e..0feea3a6 100644 --- a/easyflow-ui-admin/app/src/views/ai/documentCollection/Document.vue +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/Document.vue @@ -13,6 +13,7 @@ import { api } from '#/api/request'; import bookIcon from '#/assets/ai/knowledge/book.svg'; import HeaderSearch from '#/components/headerSearch/HeaderSearch.vue'; import { createLazyComponentController } from '#/utils/lazy-component'; +import DocumentImportBatchStatus from '#/views/ai/documentCollection/DocumentImportBatchStatus.vue'; const ChunkDocumentTable = defineAsyncComponent( () => import('#/views/ai/documentCollection/ChunkDocumentTable.vue'), @@ -188,6 +189,7 @@ const headerButtons = [ ]; const panelMode = ref<'chunk' | 'list' | 'process'>('list'); const documentTableRef = ref(); +const batchStatusRefreshKey = ref(0); const documentTitle = ref(''); const handleSearch = (searchParams: string) => { documentTableRef.value?.search?.(searchParams); @@ -227,6 +229,7 @@ const backDoc = async () => { documentTitle.value = ''; await nextTick(); documentTableRef.value?.reload?.(); + batchStatusRefreshKey.value += 1; }; @@ -273,7 +276,16 @@ const backDoc = async () => { :buttons="canManageCurrentKnowledge ? headerButtons : []" @search="handleSearch" @button-click="handleButtonClick" - /> + > + +
@@ -343,6 +355,7 @@ const backDoc = async () => { :is="ImportKnowledgeDocFileComponent" v-if="ImportKnowledgeDocFileComponent" ref="importDocModalRef" + enable-bulk-auto :knowledge-id-prop="String(knowledgeId)" @imported="backDoc" /> @@ -386,6 +399,10 @@ const backDoc = async () => { margin: 0 auto; } +.doc-header :deep(.search-middle) { + flex: 1 1 360px; +} + .doc-content { display: flex; flex-direction: column; diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentCollection.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentCollection.vue index fa5a328d..bbfdd8b4 100644 --- a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentCollection.vue +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentCollection.vue @@ -284,7 +284,8 @@ const actions: ActionButton[] = [ text: $t('button.offline'), permission: '/api/v1/documentCollection/save', placement: 'menu', - visible: (row) => canAiResourceOffline(row.displayPublishStatus, row.publishStatus), + visible: (row) => + canAiResourceOffline(row.displayPublishStatus, row.publishStatus), onClick(row) { if (!ensureManageKnowledgeItem(row)) { return; @@ -298,7 +299,8 @@ const actions: ActionButton[] = [ tone: 'danger', permission: '/api/v1/documentCollection/remove', placement: 'menu', - visible: (row) => canAiResourceDelete(row.displayPublishStatus, row.publishStatus), + visible: (row) => + canAiResourceDelete(row.displayPublishStatus, row.publishStatus), onClick(row) { if (!ensureManageKnowledgeItem(row)) { return; @@ -325,8 +327,8 @@ const submitPublishAction = async (item: any) => { const confirmation = await confirmPublishSubmission({ api, confirmMessage: isRepublishAction(item) - ? $t('documentCollection.submitRepublishApprovalConfirm') - : $t('documentCollection.submitPublishApprovalConfirm'), + ? $t('documentCollection.submitRepublishApprovalConfirm') + : $t('documentCollection.submitPublishApprovalConfirm'), id: item.id, resourcePath: '/api/v1/documentCollection', title: $t('message.noticeTitle'), @@ -356,12 +358,9 @@ const submitOfflineAction = async (item: any) => { const impactRes = await api.get<{ data: OfflineImpactCheck; errorCode: number; - }>( - '/api/v1/documentCollection/offlineImpactCheck', - { - params: { id: item.id }, - }, - ); + }>('/api/v1/documentCollection/offlineImpactCheck', { + params: { id: item.id }, + }); if (impactRes.errorCode !== 0) { return; } @@ -399,9 +398,12 @@ const submitOfflineAction = async (item: any) => { } catch { return; } - const res = await api.post('/api/v1/documentCollection/submitOfflineApproval', { - id: item.id, - }); + const res = await api.post( + '/api/v1/documentCollection/submitOfflineApproval', + { + id: item.id, + }, + ); if (res.errorCode === 0) { ElMessage.success(res.message || $t('message.saveOkMessage')); reloadKnowledgeList(); diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.test.ts b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.test.ts new file mode 100644 index 00000000..d6c534ca --- /dev/null +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.test.ts @@ -0,0 +1,82 @@ +import { flushPromises, mount } from '@vue/test-utils'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import DocumentImportBatchStatus from './DocumentImportBatchStatus.vue'; + +const apiMocks = vi.hoisted(() => ({ + get: vi.fn(), + post: vi.fn(), +})); + +vi.mock('#/api/request', () => ({ api: apiMocks })); + +vi.mock('@easyflow/locales', () => ({ + $t: (key: string) => key, +})); + +function createBatch(status: 'COMPLETED' | 'RUNNING', completedCount: number) { + return { + batchId: 'batch-1', + completedCount, + failedCount: 0, + importMode: 'AUTO', + pendingCount: status === 'RUNNING' ? 1 : 0, + processingCount: 0, + progressPercent: status === 'COMPLETED' ? 100 : 50, + skippedCount: 0, + status, + totalCount: 2, + }; +} + +describe('documentImportBatchStatus', () => { + afterEach(() => { + vi.clearAllMocks(); + vi.useRealTimers(); + }); + + it('页面首次进入时不恢复已完成批次', async () => { + apiMocks.get.mockResolvedValue({ + data: createBatch('COMPLETED', 2), + errorCode: 0, + }); + + const wrapper = mount(DocumentImportBatchStatus, { + props: { knowledgeId: 'knowledge-1' }, + }); + await flushPromises(); + + expect(wrapper.find('.batch-status').exists()).toBe(false); + wrapper.unmount(); + }); + + it('当前页面跟踪的批次完成后继续展示结果', async () => { + vi.useFakeTimers(); + apiMocks.get + .mockResolvedValueOnce({ + data: createBatch('RUNNING', 1), + errorCode: 0, + }) + .mockResolvedValueOnce({ + data: createBatch('COMPLETED', 2), + errorCode: 0, + }); + + const wrapper = mount(DocumentImportBatchStatus, { + props: { knowledgeId: 'knowledge-1' }, + }); + await flushPromises(); + + expect(wrapper.find('.batch-status').exists()).toBe(true); + + await vi.advanceTimersByTimeAsync(3000); + await flushPromises(); + + expect(wrapper.find('.batch-status').exists()).toBe(true); + expect(wrapper.text()).toContain( + 'documentCollection.importDoc.batchCompleted', + ); + wrapper.unmount(); + }); +}); diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.vue new file mode 100644 index 00000000..c04e1f2f --- /dev/null +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentImportBatchStatus.vue @@ -0,0 +1,312 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentTable.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentTable.vue index 7f3e1ca5..e76c34c4 100644 --- a/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentTable.vue +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/DocumentTable.vue @@ -32,6 +32,7 @@ import { buildKnowledgeShareUrl } from '#/api/knowledge-share'; import { api, SseClient } from '#/api/request'; import documentIcon from '#/assets/ai/knowledge/document.svg'; import PageData from '#/components/page/PageData.vue'; +import { resolveDocumentTaskErrorText } from '#/views/ai/documentCollection/document-import-error'; import { buildKnowledgePath } from '#/views/ai/documentCollection/share-path'; interface DocumentStatusPayload { @@ -40,6 +41,7 @@ interface DocumentStatusPayload { failedChunks?: number; knowledgeId?: number | string; lastTaskError?: string; + lastTaskErrorCode?: string; parseCurrentStage?: string; parseStatusMessage?: string; processStatus?: string; @@ -86,6 +88,7 @@ const STREAM_RECONNECT_DELAY = 1500; const STREAM_RELOAD_DELAY = 250; const pageDataRef = ref(); +const retryingDocumentIds = ref>(new Set()); const taskStatusStreamClient = new SseClient(); let reconnectTimer: null | ReturnType = null; let reloadTimer: null | ReturnType = null; @@ -102,7 +105,7 @@ defineExpose({ }, }); -const processingStatuses = new Set(['INDEXING', 'PARSING']); +const processingStatuses = new Set(['INDEXING', 'PARSING', 'SPLITTING']); const isProcessingStatus = (status?: string) => processingStatuses.has(status || ''); @@ -113,7 +116,8 @@ const resolvedPermissions = computed(() => ({ canDownloadContent: props.permissions?.canDownloadContent ?? true, })); -const hasPermission = (key: PermissionKey) => Boolean(resolvedPermissions.value[key]); +const hasPermission = (key: PermissionKey) => + Boolean(resolvedPermissions.value[key]); const statusMetaMap: Record< string, @@ -142,6 +146,14 @@ const statusMetaMap: Record< icon: Loading, toneClass: 'status-pill--warning', }, + SPLIT_FAILED: { + icon: CloseBold, + toneClass: 'status-pill--danger', + }, + SPLITTING: { + icon: Loading, + toneClass: 'status-pill--warning', + }, READY_FOR_INDEX: { icon: Opportunity, toneClass: 'status-pill--primary', @@ -204,9 +216,21 @@ const parseStageLabels: Record = { }; const getProcessingHint = (row: any) => - row.parseStatusMessage || - parseStageLabels[row.parseCurrentStage || ''] || - ''; + row.parseStatusMessage || parseStageLabels[row.parseCurrentStage || ''] || ''; + +const getErrorText = (row: any) => { + const taskError = resolveDocumentTaskErrorText(row, $t); + if (taskError) { + return taskError; + } + if ( + row.processStatus === 'SPLIT_FAILED' || + row.processStatus === 'INDEX_FAILED' + ) { + return $t('documentCollection.importDoc.splitOrIndexFailed'); + } + return ''; +}; const clearReconnectTimer = () => { if (!reconnectTimer) { @@ -242,6 +266,7 @@ const patchDocumentRow = (payload: DocumentStatusPayload) => { completedChunks: payload.completedChunks, failedChunks: payload.failedChunks, lastTaskError: payload.lastTaskError, + lastTaskErrorCode: payload.lastTaskErrorCode, parseCurrentStage: payload.parseCurrentStage, parseStatusMessage: payload.parseStatusMessage, processStatus: payload.processStatus, @@ -363,17 +388,33 @@ const handleContinue = (row: any) => { emits('continueProcess', row); }; -const handleRetryParse = async (row: any) => { - await requestTaskAction( - '/api/v1/document/import/task/retryParse', - { - knowledgeId: props.knowledgeId, - documentId: row.id, - }, - getStatusLabel('PARSING'), - ); +const handleRetry = async (row: any) => { + const documentId = String(row.id); + if (retryingDocumentIds.value.has(documentId)) { + return; + } + retryingDocumentIds.value = new Set([ + documentId, + ...retryingDocumentIds.value, + ]); + try { + await requestTaskAction( + '/api/v1/document/import/task/retry', + { + knowledgeId: props.knowledgeId, + documentId: row.id, + }, + getStatusLabel('PARSING'), + ); + } finally { + const next = new Set(retryingDocumentIds.value); + next.delete(documentId); + retryingDocumentIds.value = next; + } }; +const isRetrying = (row: any) => retryingDocumentIds.value.has(String(row.id)); + const handleView = (row: any) => { emits('viewDoc', row.id); }; @@ -428,13 +469,18 @@ const primaryActionConfigs: Record< label: () => $t('button.viewSegmentation'), }, INDEX_FAILED: { - handler: handleContinue, - label: () => $t('button.continueProcess'), + handler: handleRetry, + label: () => $t('documentCollection.importDoc.retry'), permission: 'canCreateContent', }, PARSE_FAILED: { - handler: handleRetryParse, - label: () => $t('button.retryParse'), + handler: handleRetry, + label: () => $t('documentCollection.importDoc.retry'), + permission: 'canCreateContent', + }, + SPLIT_FAILED: { + handler: handleRetry, + label: () => $t('documentCollection.importDoc.retry'), permission: 'canCreateContent', }, READY_FOR_INDEX: { @@ -583,7 +629,8 @@ watch(
@@ -595,19 +642,24 @@ watch( {{ getProgressText(row) }} {{ getProcessingHint(row) }}
-
- {{ row.lastTaskError }} -
+
+ {{ getErrorText(row) }} +
+
@@ -624,6 +676,8 @@ watch( v-if="getPrimaryActionLabel(row)" link type="primary" + :disabled="isRetrying(row)" + :loading="isRetrying(row)" @click="handlePrimaryAction(row)" > {{ getPrimaryActionLabel(row) }} diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/ImportKnowledgeDocFile.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/ImportKnowledgeDocFile.vue index 65aa161c..42e7d2c4 100644 --- a/easyflow-ui-admin/app/src/views/ai/documentCollection/ImportKnowledgeDocFile.vue +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/ImportKnowledgeDocFile.vue @@ -5,7 +5,15 @@ import { useRoute } from 'vue-router'; import { EasyFlowFormModal } from '@easyflow/common-ui'; import { $t } from '@easyflow/locales'; -import { ElMessage } from 'element-plus'; +import { InfoFilled } from '@element-plus/icons-vue'; +import { + ElButton, + ElIcon, + ElMessage, + ElRadioButton, + ElRadioGroup, + ElTooltip, +} from 'element-plus'; import { api } from '#/api/request'; import ImportKnowledgeFileContainer from '#/views/ai/documentCollection/ImportKnowledgeFileContainer.vue'; @@ -24,6 +32,10 @@ const props = defineProps({ type: String, default: '', }, + enableBulkAuto: { + type: Boolean, + default: false, + }, }); const emits = defineEmits(['imported']); @@ -32,22 +44,43 @@ const route = useRoute(); const fileUploadRef = ref>(); const dialogVisible = ref(false); const submitting = ref(false); +const batchReady = ref(false); +const duplicatePolicy = ref<'OVERWRITE' | 'REIMPORT' | 'SKIP'>('SKIP'); const knowledgeId = computed( () => props.knowledgeIdProp || (route.query.id as string) || '', ); const resetDialogState = () => { + batchReady.value = false; + duplicatePolicy.value = 'SKIP'; fileUploadRef.value?.reset?.(); }; -const closeDialog = () => { +const handleBatchStateChange = (state: { ready: boolean }) => { + batchReady.value = state.ready; +}; + +const closeDialog = async () => { if (submitting.value) { return false; } - dialogVisible.value = false; - resetDialogState(); - return true; + try { + if (props.enableBulkAuto) { + await fileUploadRef.value?.cancelCurrentBatch?.(); + } else { + resetDialogState(); + } + dialogVisible.value = false; + batchReady.value = false; + duplicatePolicy.value = 'SKIP'; + return true; + } catch (error: any) { + ElMessage.error( + error?.message || $t('documentCollection.importDoc.cancelBatchFailed'), + ); + return false; + } }; const openDialog = () => { @@ -108,6 +141,39 @@ const createTasks = async () => { } }; +const startBatchImport = async (importMode: 'AUTO' | 'MANUAL') => { + const selectedBatchId = fileUploadRef.value?.getBatchId?.(); + if (!selectedBatchId || !fileUploadRef.value?.isBatchReady?.()) { + ElMessage.error($t('documentCollection.importDoc.uploadCompleteFirst')); + return; + } + submitting.value = true; + try { + const res = await props.requestClient.post( + '/api/v1/document/import/batch/start', + { + batchId: selectedBatchId, + duplicatePolicy: duplicatePolicy.value, + importMode, + knowledgeId: knowledgeId.value, + }, + ); + if (res.errorCode !== 0) { + return; + } + ElMessage.success( + importMode === 'AUTO' + ? $t('documentCollection.importDoc.autoImportStarted') + : $t('documentCollection.importDoc.manualImportStarted'), + ); + dialogVisible.value = false; + resetDialogState(); + emits('imported'); + } finally { + submitting.value = false; + } +}; + defineExpose({ closeDialog, openDialog, @@ -124,15 +190,69 @@ defineExpose({ :confirm-loading="submitting" :confirm-text="$t('button.importFile')" :submitting="submitting" + :show-footer="!enableBulkAuto" width="xl" @confirm="createTasks" >

- {{ $t('documentCollection.importDoc.uploadCreateTip') }} + {{ + $t( + enableBulkAuto + ? 'documentCollection.importDoc.batchUploadTip' + : 'documentCollection.importDoc.uploadCreateTip', + ) + }}

- + +
+ + {{ $t('documentCollection.importDoc.duplicatePolicy') }} + + + + {{ $t('documentCollection.importDoc.skipDuplicates') }} + + + {{ $t('documentCollection.importDoc.overwriteDuplicates') }} + + + {{ $t('documentCollection.importDoc.reimportDuplicates') }} + + +
+
+ + {{ $t('button.cancel') }} + + + {{ $t('documentCollection.importDoc.manualImport') }} + + + + {{ $t('documentCollection.importDoc.autoImport') }} + + + +
@@ -151,6 +271,36 @@ defineExpose({ color: var(--el-text-color-secondary); } +.import-dialog__footer { + display: flex; + flex-wrap: wrap; + gap: 8px; + justify-content: flex-end; + padding-top: 16px; + border-top: 1px solid var(--el-border-color-lighter); +} + +.duplicate-policy { + display: flex; + flex-wrap: wrap; + gap: 8px 12px; + align-items: center; + min-height: 32px; +} + +.duplicate-policy__label { + font-size: 13px; + color: var(--el-text-color-secondary); +} + +.import-dialog__footer :deep(.el-button + .el-button) { + margin-left: 0; +} + +.import-dialog__info { + margin-left: 6px; +} + :deep(.upload-demo) { width: 100%; } diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/ImportKnowledgeFileContainer.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/ImportKnowledgeFileContainer.vue index 4cf5fc13..a69cff27 100644 --- a/easyflow-ui-admin/app/src/views/ai/documentCollection/ImportKnowledgeFileContainer.vue +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/ImportKnowledgeFileContainer.vue @@ -1,14 +1,33 @@ @@ -116,10 +643,80 @@ function handleRemove(row: any) { .import-file-container { display: flex; flex-direction: column; - gap: 12px; + gap: 16px; } .import-file-container__table { margin-top: 2px; } + +.native-file-input { + display: none; +} + +.batch-drop-zone { + display: flex; + flex-direction: column; + gap: 8px; + align-items: center; + justify-content: center; + min-height: 208px; + padding: 24px; + cursor: pointer; + background: hsl(var(--surface-contrast-soft) / 54%); + border: 1px dashed var(--el-color-primary-light-5); + border-radius: 16px; + transition: + background-color 0.2s, + border-color 0.2s; +} + +.batch-drop-zone:hover, +.batch-drop-zone:focus-visible { + background: var(--el-color-primary-light-9); + border-color: var(--el-color-primary); + outline: none; +} + +.batch-drop-zone--disabled { + cursor: wait; + opacity: 0.72; +} + +.batch-drop-zone__icon { + font-size: 48px; + color: var(--el-color-primary); +} + +.batch-drop-zone__title { + font-size: 16px; + font-weight: 600; + color: var(--el-text-color-primary); +} + +.batch-drop-zone__description { + margin-bottom: 4px; + font-size: 13px; + color: var(--el-text-color-secondary); + text-align: center; +} + +.batch-table { + width: 100%; + height: 312px; + overflow: hidden; + border: 1px solid var(--el-border-color-lighter); + border-radius: 12px; +} + +:deep(.batch-file-name) { + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +:deep(.upload-state--success) { + color: var(--el-color-success); +} diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/KnowledgeShareManagement.vue b/easyflow-ui-admin/app/src/views/ai/documentCollection/KnowledgeShareManagement.vue index f55cfbf7..c188d490 100644 --- a/easyflow-ui-admin/app/src/views/ai/documentCollection/KnowledgeShareManagement.vue +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/KnowledgeShareManagement.vue @@ -11,18 +11,35 @@ import { api } from '#/api/request'; import { copyTextWithFeedback } from '#/utils/clipboard-feedback'; import { buildAbsoluteAppRouteUrl } from '#/utils/share-route-context'; +type EndpointEnumValue = { + description: string; + value: string; +}; + type EndpointParam = { - location: 'body' | 'query'; + enumValues?: readonly EndpointEnumValue[]; + location: 'body' | 'header' | 'metadata' | 'multipart' | 'query'; name: string; note?: string; required?: boolean; + type?: string; +}; + +type EndpointExample = { + content: string; + title: string; }; type EndpointDoc = { + contentType?: string; + examples?: EndpointExample[]; hint: string; method: 'GET' | 'POST'; + notes?: string[]; + openByDefault?: boolean; params: EndpointParam[]; path: string; + permission: '知识导入' | '知识库维护' | '知识库读取'; }; const props = defineProps({ @@ -53,9 +70,126 @@ const apiBaseUrl = computed(() => { return `${window.location.origin}${appBasePath}/public-api/knowledge-share`; }); +const formatJson = (value: unknown) => JSON.stringify(value, null, 2); + +const retrievalModeEnumValues: readonly EndpointEnumValue[] = [ + { value: 'VECTOR', description: '向量语义召回' }, + { value: 'KEYWORD', description: '关键词全文召回' }, + { value: 'HYBRID', description: '混合召回，默认值' }, +]; + +const chunkStrategyEnumValues: readonly EndpointEnumValue[] = [ + { value: 'AUTO', description: '自动选择分块策略，默认值' }, + { value: 'MARKDOWN_SECTION', description: '按 Markdown 标题层级分块' }, + { value: 'OUTLINE_SECTION', description: '按文档大纲或章节分块' }, + { value: 'QA_PAIR', description: '按问答对分块' }, + { value: 'PARAGRAPH_LENGTH', description: '按段落和长度分块' }, + { value: 'CUSTOM_REGEX', description: '按自定义正则表达式分块' }, +]; + +const duplicatePolicyEnumValues: readonly EndpointEnumValue[] = [ + { value: 'SKIP', description: '跳过历史重复文件，默认值' }, + { value: 'OVERWRITE', description: '导入成功后覆盖历史文档' }, + { value: 'REIMPORT', description: '保留历史文档并重新导入一份' }, +]; + +const importItemStatusEnumValues: readonly EndpointEnumValue[] = [ + { value: 'PENDING', description: '等待处理' }, + { value: 'UPLOADING', description: '文件正在上传' }, + { value: 'RUNNING', description: '正在处理' }, + { value: 'UPLOADED', description: '文件已上传' }, + { value: 'FAILED', description: '处理失败' }, + { value: 'COMPLETED', description: '处理完成' }, + { value: 'SKIPPED', description: '因重复而跳过' }, + { value: 'CANCELLED', description: '文件随未启动批次一并取消' }, +]; + +const parameterTypes: Record = { + answerHtml: 'string', + categoryId: 'string (ID)', + collectionId: 'string (ID)', + content: 'string', + documentId: 'string (ID)', + file: 'File', + fileKeys: 'string[]', + files: 'File[]', + id: 'string (ID)', + itemStatus: 'enum string', + keyword: 'string', + knowledgeId: 'string (ID)', + metadata: 'JSON', + 'metadata.chunkStrategy': 'object', + 'metadata.chunkStrategy.strategyCode': 'enum string', + 'metadata.duplicatePolicy': 'enum string', + 'metadata.files': 'object[]', + 'metadata.files[].clientFileKey': 'string', + 'metadata.files[].fileName': 'string', + 'metadata.files[].relativePath': 'string', + 'metadata.knowledgeId': 'string (ID)', + pageNumber: 'integer', + pageSize: 'integer', + question: 'string', + retrievalMode: 'enum string', + taskId: 'string (ID)', +}; + +const parameterDescriptions: Record = { + answerHtml: 'FAQ 答案 HTML', + categoryId: 'FAQ 分类 ID', + collectionId: '知识库 ID', + content: '分块正文', + documentId: '文档 ID', + file: '待上传文件', + fileKeys: '需要重试的文件键', + files: '待导入文件列表', + id: '数据 ID', + itemStatus: '文件处理状态筛选', + keyword: '检索关键词', + knowledgeId: '知识库 ID', + metadata: '导入元数据与文件清单', + 'metadata.chunkStrategy': '分块策略，省略时默认 AUTO', + 'metadata.chunkStrategy.strategyCode': '分块策略编码', + 'metadata.duplicatePolicy': '重复策略，默认 SKIP', + 'metadata.files': '与文件 Part 一一对应的清单', + 'metadata.files[].clientFileKey': '批次内唯一的文件键，最长 64 字符', + 'metadata.files[].fileName': '文件名，须与对应文件 Part 一致', + 'metadata.files[].relativePath': '文件夹内相对路径', + 'metadata.knowledgeId': '目标知识库 ID', + pageNumber: '页码，默认 1', + pageSize: '每页条数', + question: 'FAQ 问题', + retrievalMode: '召回方式', + taskId: '导入任务 ID', +}; + +const parameterLocationLabels: Record = { + body: 'JSON Body', + header: 'Header', + metadata: 'metadata JSON', + multipart: 'Multipart', + query: 'Query', +}; + +const buildJsonPostExample = ( + path: string, + body: unknown, + extraHeaders: string[] = [], +) => { + const headers = [ + " -H 'ApiKey: 你的访问令牌' \\", + ...extraHeaders.map((header) => ` -H '${header}' \\`), + " -H 'Content-Type: application/json' \\", + ]; + return [ + `curl -X POST '${apiBaseUrl.value}${path}' \\`, + ...headers, + ` --data '${formatJson(body)}'`, + ].join('\n'); +}; + const detailExample = computed(() => { return [ - `curl -X GET '${apiBaseUrl.value}/detail?knowledgeId=${props.knowledgeId}' \\`, + `curl -X GET '${apiBaseUrl.value}/detail?knowledgeId=${props.knowledgeId}&pageNumber=1&pageSize=50' \\`, " -H 'ApiKey: 你的访问令牌'", ].join('\n'); }); @@ -70,26 +204,252 @@ const searchExample = computed(() => { }); const endpointDocs = computed(() => { + const documentRemoveBody = { + knowledgeId: props.knowledgeId, + id: '文档ID', + }; + const chunkUpdateBody = { + knowledgeId: props.knowledgeId, + id: '分块ID', + content: '更新后的分块内容', + }; + const chunkRemoveBody = { + knowledgeId: props.knowledgeId, + id: '分块ID', + }; + const faqSaveBody = { + collectionId: props.knowledgeId, + question: '如何申请账号？', + answerHtml: '

请联系管理员开通账号。

', + }; + const faqUpdateBody = { + id: 'FAQ ID', + collectionId: props.knowledgeId, + question: '如何修改账号信息？', + answerHtml: '

请在个人中心修改。

', + }; + const faqRemoveBody = { + id: 'FAQ ID', + }; + const faqImportResponseExample = formatJson({ + errorCode: 0, + message: '成功', + data: { + totalCount: 100, + successCount: 98, + errorCount: 2, + errorRows: [ + { + rowNumber: 12, + categoryPath: '默认分类', + question: '示例问题', + reason: '问题不能为空或格式不正确', + }, + ], + }, + }); + const detailDocumentRecords = + props.collectionType === 'FAQ' + ? [] + : [ + { + id: '文档ID', + title: 'manual.pdf', + documentType: 'pdf', + contentType: 'application/pdf', + processStatus: 'INDEXED', + chunkCount: 12, + progressPercent: 100, + created: '2026-08-02 16:30:00', + modified: '2026-08-02 16:31:00', + }, + ]; + const detailResponseExample = formatJson({ + errorCode: 0, + message: '成功', + data: { + id: props.knowledgeId, + title: '示例知识库', + collectionType: props.collectionType, + documents: { + pageNumber: 1, + pageSize: 50, + totalPage: detailDocumentRecords.length > 0 ? 1 : 0, + totalRow: detailDocumentRecords.length, + records: detailDocumentRecords, + }, + }, + }); + const searchResponseExample = formatJson({ + errorCode: 0, + message: '成功', + data: + props.collectionType === 'FAQ' + ? [ + { + resultType: 'FAQ', + faqId: 'FAQ ID', + question: '如何申请账号？', + answerText: '请联系管理员开通账号。', + categoryId: 'FAQ 分类 ID', + content: '问题：如何申请账号？\n答案：请联系管理员开通账号。', + score: 0.8732, + hitSource: 'HYBRID', + }, + ] + : [ + { + resultType: 'DOCUMENT', + documentId: '文档ID', + documentName: 'manual.pdf', + sourceFileName: 'manual.pdf', + content: '命中的文档分块内容', + score: 0.8732, + hitSource: 'HYBRID', + }, + ], + }); + const retryBody = { + taskId: '导入接口返回的 taskId', + fileKeys: ['file-1'], + }; + const importMetadata = { + knowledgeId: props.knowledgeId, + chunkStrategy: { + strategyCode: 'AUTO', + }, + duplicatePolicy: 'SKIP', + files: [ + { + clientFileKey: 'file-1', + fileName: 'manual.pdf', + relativePath: 'docs/manual.pdf', + }, + ], + }; + const importCurlExample = [ + "FILE='./docs/manual.pdf'", + `KNOWLEDGE_ID='${props.knowledgeId}'`, + 'FILE_NAME=$(basename "$FILE")', + 'METADATA=$(printf \'{"knowledgeId":"%s","chunkStrategy":{"strategyCode":"AUTO"},"duplicatePolicy":"SKIP","files":[{"clientFileKey":"file-1","fileName":"%s","relativePath":"%s"}]}\' "$KNOWLEDGE_ID" "$FILE_NAME" "$FILE_NAME")', + '', + `curl -X POST '${apiBaseUrl.value}/document/import/batch' \\`, + " -H 'ApiKey: 你的访问令牌' \\", + ' -F "metadata=$METADATA;type=application/json" \\', + ' -F "files=@$FILE"', + ].join('\n'); + const importResponseExample = formatJson({ + errorCode: 0, + message: '成功', + data: { + taskId: '异步任务ID', + status: 'RUNNING', + totalCount: 1, + totalBytes: 123_456, + createdAt: '2026-08-02T16:30:00+08:00', + }, + }); + const importStatusExample = [ + `curl -G '${apiBaseUrl.value}/document/import/batch/status' \\`, + " -H 'ApiKey: 你的访问令牌' \\", + " --data-urlencode 'taskId=导入接口返回的 taskId' \\", + " --data-urlencode 'pageNumber=1' \\", + " --data-urlencode 'pageSize=20'", + ].join('\n'); + const importStatusResponseExample = formatJson({ + errorCode: 0, + message: '成功', + data: { + taskId: '异步任务ID', + knowledgeId: props.knowledgeId, + status: 'PARTIAL_SUCCEEDED', + progressPercent: 100, + counts: { + total: 2, + completed: 1, + processing: 0, + pending: 0, + failed: 1, + skipped: 0, + retryableFailed: 1, + }, + canRetry: true, + items: { + pageNumber: 1, + pageSize: 20, + total: 2, + records: [ + { + fileKey: 'file-1', + relativePath: 'docs/manual.pdf', + documentId: '文档ID', + stage: 'PARSE', + status: 'FAILED', + attemptCount: 1, + retryable: true, + error: { + code: 'parse_service_unavailable', + message: '文档解析服务暂时不可用', + }, + }, + ], + }, + }, + }); + const retryResponseExample = formatJson({ + errorCode: 0, + message: '成功', + data: { + taskId: '异步任务ID', + status: 'RUNNING', + retriedCount: 1, + }, + }); + const commonEndpoints: EndpointDoc[] = [ { method: 'GET', path: '/detail', hint: '获取知识库详情', - params: [{ name: 'knowledgeId', location: 'query', required: true }], + permission: '知识库读取', + params: [ + { name: 'knowledgeId', location: 'query', required: true }, + { name: 'pageNumber', location: 'query', note: '默认 1' }, + { name: 'pageSize', location: 'query', note: '默认 50，最大 100' }, + ], + notes: [ + '知识库原有字段保持在 data 顶层；documents 返回已上传文档的分页摘要。', + '文档摘要包含 ID、标题、类型、处理状态、分块数、处理进度和时间；FAQ 知识库返回空文档页。', + ], + examples: [ + { title: 'cURL 示例', content: detailExample.value }, + { title: '响应示例', content: detailResponseExample }, + ], }, { method: 'GET', path: '/search', hint: '知识检索', + permission: '知识库读取', params: [ { name: 'knowledgeId', location: 'query', required: true }, { name: 'keyword', location: 'query', required: true }, { name: 'retrievalMode', location: 'query', - note: '召回方式', + enumValues: retrievalModeEnumValues, + note: '召回方式，不传时默认 HYBRID', }, ], + notes: [ + 'resultType 用于区分 DOCUMENT 与 FAQ。', + '文档命中返回 documentId、documentName；FAQ 命中返回 faqId、question、answerText、categoryId。', + 'sourceFileName 作为文档名称兼容字段继续保留；FAQ 完整 HTML 可通过 /faq/detail 查询。', + ], + examples: [ + { title: 'cURL 示例', content: searchExample.value }, + { title: '响应示例', content: searchResponseExample }, + ], }, ]; const typeEndpoints: EndpointDoc[] = @@ -99,6 +459,7 @@ const endpointDocs = computed(() => { method: 'GET', path: '/faq/page', hint: 'FAQ 分页', + permission: '知识库读取', params: [ { name: 'knowledgeId', location: 'query', required: true }, { name: 'question', location: 'query', note: '问题关键字' }, @@ -107,16 +468,138 @@ const endpointDocs = computed(() => { { name: 'pageSize', location: 'query', note: '默认 10' }, ], }, + { + method: 'GET', + path: '/faq/detail', + hint: 'FAQ 详情', + permission: '知识库读取', + params: [ + { name: 'knowledgeId', location: 'query', required: true }, + { name: 'id', location: 'query', required: true, note: 'FAQ ID' }, + ], + }, { method: 'POST', path: '/faq/save', hint: '新增 FAQ', + permission: '知识库维护', + contentType: 'application/json', params: [ { name: 'collectionId', location: 'body', required: true }, { name: 'question', location: 'body', required: true }, { name: 'answerHtml', location: 'body', required: true }, { name: 'categoryId', location: 'body', note: '分类 ID' }, ], + examples: [ + { title: '请求体示例', content: formatJson(faqSaveBody) }, + { + title: 'cURL 示例', + content: buildJsonPostExample('/faq/save', faqSaveBody), + }, + ], + }, + { + method: 'POST', + path: '/faq/update', + hint: '修改 FAQ', + permission: '知识库维护', + contentType: 'application/json', + params: [ + { name: 'knowledgeId', location: 'query', required: true }, + { name: 'id', location: 'body', required: true, note: 'FAQ ID' }, + { + name: 'collectionId', + location: 'body', + note: '不传时保持原知识库 ID', + }, + { name: 'question', location: 'body', required: true }, + { name: 'answerHtml', location: 'body', required: true }, + { name: 'categoryId', location: 'body', note: '分类 ID' }, + ], + examples: [ + { title: '请求体示例', content: formatJson(faqUpdateBody) }, + { + title: 'cURL 示例', + content: buildJsonPostExample( + `/faq/update?knowledgeId=${props.knowledgeId}`, + faqUpdateBody, + ), + }, + ], + }, + { + method: 'POST', + path: '/faq/remove', + hint: '删除 FAQ', + permission: '知识库维护', + contentType: 'application/json', + params: [ + { name: 'knowledgeId', location: 'query', required: true }, + { name: 'id', location: 'body', required: true, note: 'FAQ ID' }, + ], + examples: [ + { title: '请求体示例', content: formatJson(faqRemoveBody) }, + { + title: 'cURL 示例', + content: buildJsonPostExample( + `/faq/remove?knowledgeId=${props.knowledgeId}`, + faqRemoveBody, + ), + }, + ], + }, + { + method: 'POST', + path: '/faq/importExcel', + hint: '导入 FAQ Excel', + permission: '知识导入', + contentType: 'multipart/form-data', + params: [ + { + name: 'collectionId', + location: 'multipart', + required: true, + }, + { name: 'file', location: 'multipart', required: true }, + ], + notes: [ + '仅支持 xlsx、xls 文件，单个文件不超过 10 MB，最多导入 5000 条。', + '请先下载导入模板并保持模板列结构。', + '由调用端自动生成 multipart boundary，不要手动设置 Content-Type 请求头。', + ], + examples: [ + { + title: 'cURL 示例', + content: [ + `curl -X POST '${apiBaseUrl.value}/faq/importExcel' \\`, + " -H 'ApiKey: 你的访问令牌' \\", + ` -F 'collectionId=${props.knowledgeId}' \\`, + " -F 'file=@./faq_import.xlsx'", + ].join('\n'), + }, + { + title: '响应示例', + content: faqImportResponseExample, + }, + ], + }, + { + method: 'GET', + path: '/faq/downloadImportTemplate', + hint: '下载 FAQ 导入模板', + permission: '知识导入', + params: [ + { name: 'knowledgeId', location: 'query', required: true }, + ], + }, + { + method: 'GET', + path: '/faq/exportExcel', + hint: '导出 FAQ Excel', + permission: '知识库读取', + params: [ + { name: 'knowledgeId', location: 'query', required: true }, + ], }, ] : [ @@ -124,46 +607,248 @@ const endpointDocs = computed(() => { method: 'GET', path: '/document/page', hint: '文档分页', + permission: '知识库读取', params: [ { name: 'knowledgeId', location: 'query', required: true }, - { name: 'title', location: 'query', note: '文档标题' }, + { name: 'documentId', location: 'query', note: '文档 ID' }, + { name: 'pageNumber', location: 'query', note: '默认 1' }, + { name: 'pageSize', location: 'query', note: '默认 10' }, + ], + }, + { + method: 'GET', + path: '/document/download', + hint: '下载文档', + permission: '知识库读取', + params: [ + { name: 'knowledgeId', location: 'query', required: true }, + { name: 'documentId', location: 'query', required: true }, + ], + }, + { + method: 'GET', + path: '/documentChunk/page', + hint: '文档分块分页', + permission: '知识库读取', + params: [ + { name: 'knowledgeId', location: 'query', required: true }, + { name: 'documentId', location: 'query', required: true }, { name: 'pageNumber', location: 'query', note: '默认 1' }, { name: 'pageSize', location: 'query', note: '默认 10' }, ], }, { method: 'POST', - path: '/document/import/task/create', - hint: '创建导入任务', + path: '/document/import/batch', + hint: '批量异步导入', + permission: '知识导入', + contentType: 'multipart/form-data', + openByDefault: true, params: [ - { name: 'knowledgeId', location: 'body', required: true }, { - name: 'fileName', - location: 'body', + name: 'metadata', + location: 'multipart', + type: 'JSON', required: true, - note: '文件名', + note: 'application/json，含知识库 ID、文件清单和可选分块策略', + }, + { + name: 'metadata.knowledgeId', + location: 'metadata', + required: true, + }, + { + name: 'metadata.chunkStrategy', + location: 'metadata', + }, + { + name: 'metadata.chunkStrategy.strategyCode', + location: 'metadata', + enumValues: chunkStrategyEnumValues, + note: '分块策略编码，不传时默认 AUTO', + }, + { + name: 'metadata.duplicatePolicy', + location: 'metadata', + enumValues: duplicatePolicyEnumValues, + note: '重复文件处理策略，不传时默认 SKIP', + }, + { + name: 'metadata.files', + location: 'metadata', + required: true, + }, + { + name: 'metadata.files[].clientFileKey', + location: 'metadata', + required: true, + }, + { + name: 'metadata.files[].fileName', + location: 'metadata', + required: true, + }, + { + name: 'metadata.files[].relativePath', + location: 'metadata', + note: '可省略，默认使用 fileName', + }, + { + name: 'files', + location: 'multipart', + type: 'File[]', + required: true, + note: '同名多文件 Part，总计不超过 200 MiB', + }, + ], + notes: [ + 'metadata 是 Content-Type 为 application/json 的 multipart Part，不是普通表单字符串。', + 'metadata.knowledgeId 填当前知识库 ID；chunkStrategy 可省略；duplicatePolicy 默认 SKIP。', + 'metadata.files 中每项填写 clientFileKey、fileName 和可选 relativePath；文件大小由服务端实际读取并统计。', + '支持 txt、md、pdf、docx、pptx、xlsx；单文件不超过 100 MiB，单批不超过 200 MiB，最多 200 个文件。', + 'metadata 最大 1 MiB；metadata.files 必须与 files Part 数量、顺序和文件名一致。', + 'chunkStrategy 可不传，默认 AUTO；duplicatePolicy 支持 SKIP、OVERWRITE、REIMPORT。', + '服务端会按调用者、元数据和文件内容生成请求指纹，任务完成后 10 分钟内的重复提交会返回原 taskId。', + '由调用端自动生成 multipart boundary，不要手动设置 Content-Type 请求头。', + ], + examples: [ + { + title: 'metadata Part 填写示例', + content: formatJson(importMetadata), + }, + { title: '可执行 cURL 示例', content: importCurlExample }, + { + title: 'HTTP 202 响应示例', + content: importResponseExample, + }, + ], + }, + { + method: 'GET', + path: '/document/import/batch/status', + hint: '查询导入状态', + permission: '知识导入', + params: [ + { name: 'taskId', location: 'query', required: true }, + { + name: 'itemStatus', + location: 'query', + enumValues: importItemStatusEnumValues, + note: '文件处理状态筛选，不传时查询全部状态', + }, + { name: 'pageNumber', location: 'query', note: '默认 1' }, + { + name: 'pageSize', + location: 'query', + note: '默认 20，最大 100', + }, + ], + notes: [ + '公开状态包括 QUEUED、RUNNING、SUCCEEDED、FAILED、PARTIAL_SUCCEEDED、INTERRUPTED、CANCELLED；进入终态后停止轮询。', + '只有 canRetry=true 的任务允许重试。', + ], + examples: [ + { title: 'cURL 示例', content: importStatusExample }, + { + title: '响应示例', + content: importStatusResponseExample, }, - { name: 'filePath', location: 'body', required: true, note: '上传后的文件路径' }, ], }, { method: 'POST', - path: '/document/import/task/preview', - hint: '生成分块预览', + path: '/document/import/batch/retry', + hint: '重试异常文件', + permission: '知识导入', + contentType: 'application/json', params: [ - { name: 'knowledgeId', location: 'body', required: true }, - { name: 'documentId', location: 'body', required: true }, - { name: 'files[0].strategyConfig', location: 'body', note: '拆分策略配置' }, + { name: 'taskId', location: 'body', required: true }, + { + name: 'fileKeys', + location: 'body', + type: 'string[]', + note: '省略时重试全部；最多 200 个，每项最长 64 字符', + }, + ], + notes: [ + '省略 fileKeys 时重试全部可恢复失败项；指定时只重试对应文件。', + 'fileKeys 不允许传空数组、空值、重复值或超过 64 字符的值，最多 200 个。', + '任务正在运行时不能重复领取重试，请继续使用 taskId 查询状态。', + ], + examples: [ + { title: '请求体示例', content: formatJson(retryBody) }, + { + title: 'cURL 示例', + content: buildJsonPostExample( + '/document/import/batch/retry', + retryBody, + ), + }, + { title: '响应示例', content: retryResponseExample }, ], }, { method: 'POST', - path: '/document/import/task/startIndex', - hint: '启动向量化', + path: '/document/remove', + hint: '删除文档', + permission: '知识库维护', + contentType: 'application/json', params: [ { name: 'knowledgeId', location: 'body', required: true }, - { name: 'documentId', location: 'body', required: true }, - { name: 'previewSessionId', location: 'body', note: '预览接口返回的会话 ID' }, + { name: 'id', location: 'body', required: true, note: '文档 ID' }, + ], + examples: [ + { title: '请求体示例', content: formatJson(documentRemoveBody) }, + { + title: 'cURL 示例', + content: buildJsonPostExample( + '/document/remove', + documentRemoveBody, + ), + }, + ], + }, + { + method: 'POST', + path: '/documentChunk/update', + hint: '更新文档分块', + permission: '知识库维护', + contentType: 'application/json', + params: [ + { name: 'knowledgeId', location: 'body', required: true }, + { name: 'id', location: 'body', required: true, note: '分块 ID' }, + { name: 'content', location: 'body', required: true }, + ], + examples: [ + { title: '请求体示例', content: formatJson(chunkUpdateBody) }, + { + title: 'cURL 示例', + content: buildJsonPostExample( + '/documentChunk/update', + chunkUpdateBody, + ), + }, + ], + }, + { + method: 'POST', + path: '/documentChunk/remove', + hint: '删除文档分块', + permission: '知识库维护', + contentType: 'application/json', + params: [ + { name: 'knowledgeId', location: 'body', required: true }, + { name: 'id', location: 'body', required: true, note: '分块 ID' }, + ], + examples: [ + { title: '请求体示例', content: formatJson(chunkRemoveBody) }, + { + title: 'cURL 示例', + content: buildJsonPostExample( + '/documentChunk/remove', + chunkRemoveBody, + ), + }, ], }, ]; @@ -173,13 +858,11 @@ const endpointDocs = computed(() => { })); }); -const formatEndpointParam = (param: EndpointParam) => { - const segments = [param.location, param.required ? '必填' : '可选']; - if (param.note) { - segments.push(param.note); - } - return `${param.name}（${segments.join('，')}）`; -}; +const getParameterType = (param: EndpointParam) => + param.type || parameterTypes[param.name] || 'string'; + +const getParameterDescription = (param: EndpointParam) => + param.note || parameterDescriptions[param.name] || '接口参数'; const createShare = async () => { if (!props.manageable) { @@ -271,7 +954,8 @@ const copyEndpointUrl = async (url: string) => {
@@ -287,14 +971,38 @@ const copyEndpointUrl = async (url: string) => { 请求头 ApiKey: 你的访问令牌 +
+ JSON 请求 + Content-Type: application/json +
+
+ 文件上传 + multipart/form-data + 由调用端自动生成 boundary +
当前知识库 ID {{ props.knowledgeId }}
+
+
批量导入调用流程
+
    +
  1. 提交 multipart 请求，保存 HTTP 202 响应中的 taskId。
    2. +
  2. 使用 taskId 查询任务状态，按需分页查看每个文件的结果。
    3. +
  3. + 当 canRetry=true 时，使用 taskId 重试全部异常文件，或通过 fileKeys + 指定部分文件。 +
    4. +
+
+
-
常用接口
+
接口列表
{ {{ endpoint.method }} {{ endpoint.path }} {{ endpoint.hint }} + {{ + endpoint.permission + }} { @click="copyEndpointUrl(endpoint.url)" />
-
- 参数 - + - {{ formatEndpointParam(param) }} - + + + + + + + + + + + + + + + + + + +
参数位置类型必填说明
+ {{ param.name }} + {{ parameterLocationLabels[param.location] }}{{ getParameterType(param) }}{{ param.required ? '是' : '否' }} +
{{ getParameterDescription(param) }}
+
    +
  • + {{ enumValue.value }} + {{ enumValue.description }} +
    • +
+
+
+ + {{ + endpoint.openByDefault + ? '必看：metadata、文件 Part 与完整示例' + : '查看请求与响应说明' + }} + +
+
+ Content-Type + {{ endpoint.contentType }} +
+
    +
  • + {{ note }} +
    • +
+
+
+ {{ example.title }} +
+
+ + 
{{ example.content }}
+
+
+
+
@@ -417,6 +1214,7 @@ const copyEndpointUrl = async (url: string) => { .api-doc { display: grid; + grid-template-columns: minmax(0, 1fr); gap: 18px; } @@ -446,9 +1244,32 @@ const copyEndpointUrl = async (url: string) => { border-radius: 10px; } +.api-doc__helper { + font-size: 12px; + color: var(--el-text-color-secondary); +} + .api-doc__section { display: grid; + grid-template-columns: minmax(0, 1fr); gap: 10px; + min-width: 0; +} + +.api-doc__import-flow { + padding: 16px; + background: var(--el-fill-color-extra-light); + border-radius: var(--el-border-radius-base); +} + +.api-doc__steps { + display: grid; + gap: 8px; + padding-left: 24px; + margin: 0; + font-size: 13px; + line-height: 1.7; + color: var(--el-text-color-regular); } .api-doc__section-title { @@ -459,12 +1280,16 @@ const copyEndpointUrl = async (url: string) => { .api-doc__endpoint-list { display: grid; + grid-template-columns: minmax(0, 1fr); gap: 8px; + min-width: 0; } .api-doc__endpoint { display: grid; + grid-template-columns: minmax(0, 1fr); gap: 6px; + min-width: 0; padding: 8px 0; } @@ -490,24 +1315,154 @@ const copyEndpointUrl = async (url: string) => { color: var(--el-text-color-secondary); } -.api-doc__endpoint-params { +.api-doc__permission { + padding: 2px 8px; + font-size: 12px; + color: var(--el-color-primary); + background: var(--el-color-primary-light-9); + border-radius: 999px; +} + +.api-doc__parameter-table-wrap { + width: 100%; + max-width: 100%; + min-width: 0; + overflow-x: auto; +} + +.api-doc__parameter-table { + width: 100%; + min-width: 640px; + font-size: 12px; + color: var(--el-text-color-regular); + table-layout: fixed; + border-spacing: 0; + border-collapse: separate; + border: 1px solid var(--el-border-color-lighter); + border-radius: var(--el-border-radius-base); +} + +.api-doc__parameter-table th, +.api-doc__parameter-table td { + padding: 8px 12px; + text-align: left; + vertical-align: top; + border-bottom: 1px solid var(--el-border-color-lighter); +} + +.api-doc__parameter-table th { + font-weight: 500; + color: var(--el-text-color-secondary); + white-space: nowrap; + background: var(--el-fill-color-extra-light); +} + +.api-doc__parameter-table th:first-child, +.api-doc__parameter-table td:first-child { + width: 28%; +} + +.api-doc__parameter-table th:nth-child(2), +.api-doc__parameter-table td:nth-child(2) { + width: 14%; + white-space: nowrap; +} + +.api-doc__parameter-table th:nth-child(3), +.api-doc__parameter-table td:nth-child(3) { + width: 12%; + white-space: nowrap; +} + +.api-doc__parameter-table th:nth-child(4), +.api-doc__parameter-table td:nth-child(4) { + width: 8%; + white-space: nowrap; +} + +.api-doc__parameter-table tbody tr:last-child td { + border-bottom: none; +} + +.api-doc__parameter-table td:last-child { + overflow-wrap: anywhere; +} + +.api-doc__parameter-table code { + white-space: normal; + overflow-wrap: anywhere; +} + +.api-doc__enum-list { + display: grid; + gap: 4px; + padding: 0; + margin: 6px 0 0; + list-style: none; +} + +.api-doc__enum-list li { display: flex; gap: 8px; align-items: flex-start; - flex-wrap: wrap; - font-size: 12px; +} + +.api-doc__enum-list code { + flex: 0 0 auto; + color: var(--el-text-color-primary); +} + +.api-doc__endpoint-details { + margin-top: 4px; + font-size: 13px; +} + +.api-doc__endpoint-details > summary { + width: fit-content; + color: var(--el-color-primary); + cursor: pointer; +} + +.api-doc__endpoint-details > summary:focus-visible { + border-radius: var(--el-border-radius-small); + outline: 2px solid var(--el-color-primary-light-5); + outline-offset: 2px; +} + +.api-doc__endpoint-details-content { + display: grid; + gap: 16px; + padding-top: 16px; +} + +.api-doc__content-type { + display: flex; + gap: 8px; + align-items: center; color: var(--el-text-color-secondary); } -.api-doc__params-label { - flex: 0 0 auto; - font-weight: 500; +.api-doc__content-type code { + color: var(--el-text-color-primary); } -.api-doc__param { - padding: 2px 8px; - background: var(--el-fill-color-light); - border-radius: 999px; +.api-doc__notes { + display: grid; + gap: 8px; + padding-left: 24px; + margin: 0; + line-height: 1.6; + color: var(--el-text-color-regular); +} + +.api-doc__example { + display: grid; + gap: 8px; +} + +.api-doc__example-title { + font-weight: 500; + color: var(--el-text-color-primary); } .api-doc__code { @@ -517,8 +1472,7 @@ const copyEndpointUrl = async (url: string) => { font-size: 12px; line-height: 1.7; color: var(--el-text-color-primary); - white-space: pre-wrap; - word-break: break-all; + white-space: pre; background: var(--el-fill-color-light); border-radius: 14px; } diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-error.test.ts b/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-error.test.ts new file mode 100644 index 00000000..e185b296 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-error.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveDocumentTaskErrorText } from './document-import-error'; + +const translate = (key: string) => `translated:${key}`; + +describe('resolveDocumentTaskErrorText', () => { + it('uses the stable backend error code', () => { + expect( + resolveDocumentTaskErrorText( + { lastTaskErrorCode: 'parse_service_unavailable' }, + translate, + ), + ).toBe('translated:documentCollection.importDoc.parseServiceUnavailable'); + }); + + it('supports the error code stored in document options', () => { + expect( + resolveDocumentTaskErrorText( + { options: { 'task.errorCode': 'pending_timeout' } }, + translate, + ), + ).toBe('translated:documentCollection.importDoc.taskPendingTimeout'); + }); + + it('normalizes legacy MinerU 503 errors', () => { + expect( + resolveDocumentTaskErrorText( + { + lastTaskError: + 'MinerU request failed: path=/tasks, status=503, body=', + }, + translate, + ), + ).toBe('translated:documentCollection.importDoc.parseServiceUnavailable'); + }); + + it('distinguishes legacy internal document access errors from MinerU errors', () => { + expect( + resolveDocumentTaskErrorText( + { + lastTaskError: + '下载文档 URL 失败: http://127.0.0.1/file.docx; 远端文档地址不允许访问非公网目标', + }, + translate, + ), + ).toBe('translated:documentCollection.importDoc.documentSourceUnavailable'); + }); + + it('uses the stable document source error code', () => { + expect( + resolveDocumentTaskErrorText( + { lastTaskErrorCode: 'document_source_unavailable' }, + translate, + ), + ).toBe('translated:documentCollection.importDoc.documentSourceUnavailable'); + }); + + it('keeps unknown business errors intact', () => { + expect( + resolveDocumentTaskErrorText( + { lastTaskError: '文档内容为空' }, + translate, + ), + ).toBe('文档内容为空'); + }); +}); diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-error.ts b/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-error.ts new file mode 100644 index 00000000..19e088ab --- /dev/null +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-error.ts @@ -0,0 +1,67 @@ +interface DocumentTaskErrorSource { + lastTaskError?: string; + lastTaskErrorCode?: string; + options?: Record; +} + +const ERROR_MESSAGE_KEYS: Record = { + document_source_unavailable: + 'documentCollection.importDoc.documentSourceUnavailable', + execution_interrupted: + 'documentCollection.importDoc.taskExecutionInterrupted', + parse_service_timeout: 'documentCollection.importDoc.parseServiceTimeout', + parse_service_unavailable: + 'documentCollection.importDoc.parseServiceUnavailable', + pending_timeout: 'documentCollection.importDoc.taskPendingTimeout', +}; + +const resolveLegacyErrorCode = (message: string) => { + const normalized = message.toLowerCase(); + if ( + normalized.includes('下载文档 url 失败') || + normalized.includes('读取批量导入文件失败') || + normalized.includes('远端文档地址不允许访问非公网目标') + ) { + return 'document_source_unavailable'; + } + if ( + normalized.includes('status=502') || + normalized.includes('status=503') || + normalized.includes('status=504') || + normalized.includes('connection refused') || + normalized.includes('failed to connect') || + normalized.includes('failed to call mineru endpoint') || + normalized.includes('no route to host') || + normalized.includes('unknown host') + ) { + return 'parse_service_unavailable'; + } + if ( + normalized.includes('timed out') || + normalized.includes('timeout') || + normalized.includes('超时') + ) { + return normalized.includes('排队') + ? 'pending_timeout' + : 'parse_service_timeout'; + } + if (normalized.includes('任务执行中断')) { + return 'execution_interrupted'; + } + return ''; +}; + +/** + * 将后端稳定错误码及历史原始错误转换为面向用户的本地化文案。 + */ +export const resolveDocumentTaskErrorText = ( + row: DocumentTaskErrorSource, + translate: (key: string) => string, +) => { + const storedCode = + row.lastTaskErrorCode || + String(row.options?.['task.errorCode'] || '') || + resolveLegacyErrorCode(row.lastTaskError || ''); + const messageKey = ERROR_MESSAGE_KEYS[storedCode]; + return messageKey ? translate(messageKey) : row.lastTaskError || ''; +}; diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-upload-response.test.ts b/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-upload-response.test.ts new file mode 100644 index 00000000..ba6f107d --- /dev/null +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-upload-response.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveDocumentUploadResponse } from './document-import-upload-response'; + +const fallbackMessage = '文件上传失败'; + +describe('resolveDocumentUploadResponse', () => { + it.each([ + ['数字错误码', '{"errorCode":0}'], + ['字符串错误码', '{"errorCode":"0"}'], + ['兼容 code 字段', '{"code":0}'], + ['带 BOM 的响应', '\uFEFF{"errorCode":0}'], + ])('接受%s', (_name, responseText) => { + expect( + resolveDocumentUploadResponse({ + fallbackMessage, + responseText, + status: 200, + }), + ).toEqual({ success: true }); + }); + + it('接受无响应体的 204 响应', () => { + expect( + resolveDocumentUploadResponse({ + fallbackMessage, + responseText: '', + status: 204, + }), + ).toEqual({ success: true }); + }); + + it('返回业务错误消息', () => { + expect( + resolveDocumentUploadResponse({ + fallbackMessage, + responseText: '{"errorCode":4091,"message":"文件已存在"}', + status: 409, + }), + ).toEqual({ + message: '文件已存在', + success: false, + }); + }); + + it('拒绝成功状态下的未知响应格式', () => { + expect( + resolveDocumentUploadResponse({ + fallbackMessage, + responseText: 'binary-response', + status: 200, + }), + ).toEqual({ + message: fallbackMessage, + success: false, + }); + }); + + it('拒绝缺少成功错误码的空响应', () => { + expect( + resolveDocumentUploadResponse({ + fallbackMessage, + responseText: '', + status: 200, + }), + ).toEqual({ + message: fallbackMessage, + success: false, + }); + }); +}); diff --git a/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-upload-response.ts b/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-upload-response.ts new file mode 100644 index 00000000..1f0bd9c1 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/ai/documentCollection/document-import-upload-response.ts @@ -0,0 +1,75 @@ +interface UploadResponsePayload { + code?: unknown; + error?: unknown; + errorCode?: unknown; + message?: unknown; + msg?: unknown; +} + +interface ResolveUploadResponseOptions { + fallbackMessage: string; + responseText: string; + status: number; + statusText?: string; +} + +interface UploadResponseResult { + message?: string; + success: boolean; +} + +function resolveMessage( + payload: undefined | UploadResponsePayload, + statusText: string | undefined, + fallbackMessage: string, +) { + const message = + payload?.message ?? payload?.error ?? payload?.msg ?? statusText; + return typeof message === 'string' && message.trim() + ? message.trim() + : fallbackMessage; +} + +export function resolveDocumentUploadResponse({ + fallbackMessage, + responseText, + status, + statusText, +}: ResolveUploadResponseOptions): UploadResponseResult { + const isHttpSuccess = status >= 200 && status < 300; + const normalizedBody = responseText.replace(/^\uFEFF/, '').trim(); + let payload: undefined | UploadResponsePayload; + + if (normalizedBody) { + try { + payload = JSON.parse(normalizedBody) as UploadResponsePayload; + } catch { + return { + message: fallbackMessage, + success: false, + }; + } + } + + if (!isHttpSuccess) { + return { + message: resolveMessage(payload, statusText, fallbackMessage), + success: false, + }; + } + + if (status === 204) { + return { success: true }; + } + + const rawErrorCode = payload?.errorCode ?? payload?.code; + const errorCode = Number(rawErrorCode); + if (!Number.isFinite(errorCode) || errorCode !== 0) { + return { + message: resolveMessage(payload, statusText, fallbackMessage), + success: false, + }; + } + + return { success: true }; +} diff --git a/easyflow-ui-admin/app/src/views/config/apikey/SysApiKeyModal.vue b/easyflow-ui-admin/app/src/views/config/apikey/SysApiKeyModal.vue index 346188ea..d18a5bb2 100644 --- a/easyflow-ui-admin/app/src/views/config/apikey/SysApiKeyModal.vue +++ b/easyflow-ui-admin/app/src/views/config/apikey/SysApiKeyModal.vue @@ -32,7 +32,10 @@ interface Entity { deptId: number | string; expiredAt: Date | null | string; permissionIds: (number | string)[]; // 绑定值：权限 ID 数组 - knowledgeShareEnabled: boolean; + knowledgeReadEnabled: boolean; + knowledgeImportEnabled: boolean; + knowledgeMaintenanceEnabled: boolean; + knowledgeShareEnabled?: boolean; workflowApiEnabled: boolean; id?: number; // 编辑时的主键 } @@ -51,7 +54,9 @@ const entity = ref({ deptId: '', expiredAt: null, permissionIds: [], - knowledgeShareEnabled: false, + knowledgeReadEnabled: false, + knowledgeImportEnabled: false, + knowledgeMaintenanceEnabled: false, workflowApiEnabled: false, }); // 加载状态 @@ -121,7 +126,14 @@ function getResourcePermissionList() { function createDefaultEntity(row: Partial = {}): Entity { const permissionIds = row.permissionIds || []; - const knowledgeShareEnabled = Boolean(row.knowledgeShareEnabled); + const legacyKnowledgeEnabled = Boolean(row.knowledgeShareEnabled); + const knowledgeReadEnabled = Boolean( + row.knowledgeReadEnabled ?? legacyKnowledgeEnabled, + ); + const knowledgeImportEnabled = Boolean( + row.knowledgeImportEnabled ?? legacyKnowledgeEnabled, + ); + const knowledgeMaintenanceEnabled = Boolean(row.knowledgeMaintenanceEnabled); const workflowApiEnabled = Boolean(row.workflowApiEnabled); return { apiKey: '', @@ -130,7 +142,9 @@ function createDefaultEntity(row: Partial = {}): Entity { expiredAt: null, ...row, permissionIds, - knowledgeShareEnabled, + knowledgeReadEnabled, + knowledgeImportEnabled, + knowledgeMaintenanceEnabled, workflowApiEnabled, }; } @@ -189,7 +203,9 @@ function closeDialog() { deptId: '', expiredAt: null, permissionIds: [], - knowledgeShareEnabled: false, + knowledgeReadEnabled: false, + knowledgeImportEnabled: false, + knowledgeMaintenanceEnabled: false, workflowApiEnabled: false, }; isAdd.value = true; @@ -264,10 +280,22 @@ defineExpose({ - {{ $t('sysApiKey.knowledgeSharePermission') }} + {{ $t('sysApiKey.knowledgeReadPermission') }} + + + {{ $t('sysApiKey.knowledgeImportPermission') }} + + + {{ $t('sysApiKey.knowledgeMaintenancePermission') }}