feat: 完善知识库批量导入与公共 API

- 新增批量异步导入、状态查询、失败重试与中断恢复链路

- 拆分知识库读取、导入、维护权限并完善 Public API 契约

- 补充数据库迁移、管理端交互、接口说明与相关测试
This commit is contained in:
2026-08-03 11:13:48 +08:00
parent 6df3dd9981
commit 51dbfd41b6
99 changed files with 16274 additions and 480 deletions

View File

@@ -11,8 +11,11 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
import tech.easyflow.ai.documentimport.DocumentImportDtos; import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.task.DocumentImportBatchAppService;
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService; import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
import tech.easyflow.ai.entity.Document; import tech.easyflow.ai.entity.Document;
import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.DocumentCollection;
@@ -83,6 +86,9 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
@Autowired @Autowired
private DocumentImportTaskStatusStreamService documentImportTaskStatusStreamService; private DocumentImportTaskStatusStreamService documentImportTaskStatusStreamService;
@Autowired
private DocumentImportBatchAppService documentImportBatchAppService;
@Value("${easyflow.storage.local.root:}") @Value("${easyflow.storage.local.root:}")
private String fileUploadPath; private String fileUploadPath;
@@ -312,6 +318,142 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
return documentService.retryIndexTask(request); return documentService.retryIndexTask(request);
} }
/**
* 创建文档批量上传清单。
*
* @param request 文件清单
* @return 批次与服务端文件项
*/
@PostMapping("import/batch/create")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportBatchDtos.CreateResponse> createImportBatch(
@JsonBody DocumentImportBatchDtos.CreateRequest request) {
if (request == null || request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return Result.ok(documentImportBatchAppService.createBatch(request));
}
/**
* 上传一个批次文件。
*
* @param batchId 批次 ID
* @param itemId 文件项 ID
* @param knowledgeId 知识库 ID
* @param file 上传文件
* @return 文件项状态
*/
@PostMapping(value = "import/batch/{batchId}/item/{itemId}/upload",
consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportBatchDtos.ItemResponse> uploadImportBatchItem(
@PathVariable BigInteger batchId,
@PathVariable BigInteger itemId,
@RequestParam BigInteger knowledgeId,
@RequestPart("file") MultipartFile file) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库");
return Result.ok(documentImportBatchAppService.uploadItem(
knowledgeId, batchId, itemId, file));
}
/**
* 启动手动或自动批量导入。
*
* @param request 启动请求
* @return 批次状态
*/
@PostMapping("import/batch/start")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportBatchDtos.StatusResponse> startImportBatch(
@JsonBody DocumentImportBatchDtos.StartRequest request) {
if (request == null || request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return Result.ok(documentImportBatchAppService.startBatch(request));
}
/**
* 取消一个尚未启动的上传批次。
*
* @param knowledgeId 知识库 ID
* @param batchId 批次 ID
* @return 空结果
*/
@PostMapping("import/batch/cancel")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<Void> cancelImportBatch(
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
@JsonBody(value = "batchId", required = true) BigInteger batchId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库");
documentImportBatchAppService.cancelBatch(knowledgeId, batchId);
return Result.ok();
}
/**
* 查询批次状态。
*
* @param knowledgeId 知识库 ID
* @param batchId 批次 ID
* @return 批次状态
*/
@GetMapping("import/batch/status")
@SaCheckPermission("/api/v1/documentCollection/query")
public Result<DocumentImportBatchDtos.StatusResponse> getImportBatchStatus(
@RequestParam BigInteger knowledgeId,
@RequestParam BigInteger batchId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.READ, "无权限访问知识库");
return Result.ok(documentImportBatchAppService.getBatchStatus(knowledgeId, batchId));
}
/**
* 查询知识库最近一个自动导入批次。
*
* @param knowledgeId 知识库 ID
* @return 最近批次状态
*/
@GetMapping("import/batch/current")
@SaCheckPermission("/api/v1/documentCollection/query")
public Result<DocumentImportBatchDtos.StatusResponse> getCurrentImportBatch(
@RequestParam BigInteger knowledgeId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.READ, "无权限访问知识库");
return Result.ok(documentImportBatchAppService.getLatestAutoBatch(knowledgeId));
}
/**
* 继续中断或部分失败的自动导入批次。
*
* @param knowledgeId 知识库 ID
* @param batchId 批次 ID
* @return 继续后的批次状态
*/
@PostMapping("import/batch/continue")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportBatchDtos.StatusResponse> continueImportBatch(
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
@JsonBody(value = "batchId", required = true) BigInteger batchId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库");
return Result.ok(documentImportBatchAppService.continueBatch(knowledgeId, batchId));
}
/**
* 根据解析、分块或向量化失败阶段统一重试。
*
* @param request 重试请求
* @return 重试任务状态
*/
@PostMapping("import/task/retry")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportDtos.TaskStartIndexResponse> retryImportTask(
@JsonBody DocumentImportDtos.TaskRetryRequest request) {
if (request == null || request.getKnowledgeId() == null || request.getDocumentId() == null) {
throw new BusinessException("重试信息不完整");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return documentService.retryFailedTask(request);
}
/** /**
* 更新 entity * 更新 entity
* *

View File

@@ -6,18 +6,21 @@ import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.table.TableInfo; import com.mybatisflex.core.table.TableInfo;
import com.mybatisflex.core.table.TableInfoFactory; import com.mybatisflex.core.table.TableInfoFactory;
import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletRequest;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import tech.easyflow.ai.service.KnowledgeSharePermissionService; import tech.easyflow.ai.service.KnowledgeSharePermissionService;
import tech.easyflow.ai.service.WorkflowApiPermissionService; import tech.easyflow.ai.service.WorkflowApiPermissionService;
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
import tech.easyflow.common.domain.Result; import tech.easyflow.common.domain.Result;
import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.satoken.util.SaTokenUtil; import tech.easyflow.common.satoken.util.SaTokenUtil;
import tech.easyflow.common.util.IdUtil; import tech.easyflow.common.util.IdUtil;
import tech.easyflow.common.vo.PkVo; import tech.easyflow.common.vo.PkVo;
import tech.easyflow.common.web.controller.BaseCurdController; import tech.easyflow.common.web.controller.BaseCurdController;
import tech.easyflow.common.web.jsonbody.JsonBody;
import tech.easyflow.system.entity.SysApiKey; import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.entity.SysApiKeyResourceMapping; import tech.easyflow.system.entity.SysApiKeyResourceMapping;
import tech.easyflow.system.service.SysApiKeyResourceMappingService; import tech.easyflow.system.service.SysApiKeyResourceMappingService;
@@ -29,6 +32,7 @@ import java.time.LocalDate;
import java.time.ZoneId; import java.time.ZoneId;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Set;
/** /**
* 控制层。 * 控制层。
@@ -83,12 +87,57 @@ public class SysApiKeyController extends BaseCurdController<SysApiKeyService, Sy
return Result.ok(new PkVo(pkArgs)); return Result.ok(new PkVo(pkArgs));
} }
/**
* 更新访问令牌基础信息与授权。
*
* <p>权限开关不映射数据库列,权限更新请求可能只包含主键与权限字段。
* 此时跳过主表更新,避免 MyBatis-Flex 生成空的 {@code SET} 子句。</p>
*
* @param entity 待更新的访问令牌
* @return 更新结果
*/
@Override
@PostMapping("/update")
@Transactional(rollbackFor = Exception.class)
public Result<?> update(@JsonBody SysApiKey entity) {
if (entity == null || entity.getId() == null) {
return Result.fail("访问令牌 ID 不能为空");
}
if (!hasPersistentUpdateFields(entity) && !hasPermissionUpdateFields(entity)) {
return Result.fail("没有可更新的访问令牌字段");
}
if (hasNewKnowledgePermissionFields(entity)
&& !hasCompleteKnowledgePermissionFields(entity)) {
return Result.fail("知识库读取、导入、维护权限必须同时提交");
}
if (service.getById(entity.getId()) == null) {
return Result.fail("访问令牌不存在");
}
Result<?> beforeResult = onSaveOrUpdateBefore(entity, false);
if (beforeResult != null) {
return beforeResult;
}
if (hasPersistentUpdateFields(entity)) {
service.updateById(entity);
}
onSaveOrUpdateAfter(entity, false);
return Result.ok();
}
@Override @Override
protected void onSaveOrUpdateAfter(SysApiKey entity, boolean isSave) { protected void onSaveOrUpdateAfter(SysApiKey entity, boolean isSave) {
if (entity.getPermissionIds() != null) { if (entity.getPermissionIds() != null) {
sysApiKeyResourceMappingService.authInterface(entity); sysApiKeyResourceMappingService.authInterface(entity);
} }
if (entity.getKnowledgeShareEnabled() != null) { if (hasNewKnowledgePermissionFields(entity)) {
knowledgeSharePermissionService.replaceApiPermissions(
entity.getId(),
Boolean.TRUE.equals(entity.getKnowledgeReadEnabled()),
Boolean.TRUE.equals(entity.getKnowledgeImportEnabled()),
Boolean.TRUE.equals(entity.getKnowledgeMaintenanceEnabled())
);
} else if (entity.getKnowledgeShareEnabled() != null) {
// 兼容旧客户端:开启旧总开关只授予读取和导入,维护权限保持关闭。
knowledgeSharePermissionService.replaceApiShareEnabled(entity.getId(), entity.getKnowledgeShareEnabled()); knowledgeSharePermissionService.replaceApiShareEnabled(entity.getId(), entity.getKnowledgeShareEnabled());
} }
if (entity.getWorkflowApiEnabled() != null) { if (entity.getWorkflowApiEnabled() != null) {
@@ -130,11 +179,18 @@ public class SysApiKeyController extends BaseCurdController<SysApiKeyService, Sy
List<BigInteger> resourceIds = sysApiKeyResourceMappingService.listAs(interfaceWrapper, BigInteger.class); List<BigInteger> resourceIds = sysApiKeyResourceMappingService.listAs(interfaceWrapper, BigInteger.class);
entity.setPermissionIds(resourceIds); entity.setPermissionIds(resourceIds);
QueryWrapper knowledgeWrapper = QueryWrapper.create() Set<String> knowledgeScopes =
.select(SysApiKeyResourceMapping::getId) knowledgeSharePermissionService.getApiPermissionScopes(entity.getId());
.eq(SysApiKeyResourceMapping::getApiKeyId, entity.getId()) boolean readEnabled =
.eq(SysApiKeyResourceMapping::getResourceType, "KNOWLEDGE"); knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
entity.setKnowledgeShareEnabled(sysApiKeyResourceMappingService.count(knowledgeWrapper) > 0); boolean importEnabled =
knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
boolean maintenanceEnabled =
knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
entity.setKnowledgeReadEnabled(readEnabled);
entity.setKnowledgeImportEnabled(importEnabled);
entity.setKnowledgeMaintenanceEnabled(maintenanceEnabled);
entity.setKnowledgeShareEnabled(readEnabled || importEnabled || maintenanceEnabled);
QueryWrapper workflowWrapper = QueryWrapper.create() QueryWrapper workflowWrapper = QueryWrapper.create()
.select(SysApiKeyResourceMapping::getId) .select(SysApiKeyResourceMapping::getId)
@@ -142,4 +198,57 @@ public class SysApiKeyController extends BaseCurdController<SysApiKeyService, Sy
.eq(SysApiKeyResourceMapping::getResourceType, WorkflowApiPermissionService.RESOURCE_TYPE_WORKFLOW); .eq(SysApiKeyResourceMapping::getResourceType, WorkflowApiPermissionService.RESOURCE_TYPE_WORKFLOW);
entity.setWorkflowApiEnabled(sysApiKeyResourceMappingService.count(workflowWrapper) > 0); entity.setWorkflowApiEnabled(sysApiKeyResourceMappingService.count(workflowWrapper) > 0);
} }
/**
* 判断请求是否提交了任一新版知识库权限字段。
*
* @param entity 访问令牌
* @return 是否提交新版字段
*/
private boolean hasNewKnowledgePermissionFields(SysApiKey entity) {
return entity.getKnowledgeReadEnabled() != null
|| entity.getKnowledgeImportEnabled() != null
|| entity.getKnowledgeMaintenanceEnabled() != null;
}
/**
* 判断请求是否完整提交三个新版知识库权限字段。
*
* @param entity 访问令牌
* @return 三个字段是否均已提交
*/
private boolean hasCompleteKnowledgePermissionFields(SysApiKey entity) {
return entity.getKnowledgeReadEnabled() != null
&& entity.getKnowledgeImportEnabled() != null
&& entity.getKnowledgeMaintenanceEnabled() != null;
}
/**
* 判断请求是否包含主表可持久化字段。
*
* @param entity 访问令牌
* @return 是否需要更新访问令牌主表
*/
private boolean hasPersistentUpdateFields(SysApiKey entity) {
return entity.getApiKey() != null
|| entity.getCreated() != null
|| entity.getStatus() != null
|| entity.getDeptId() != null
|| entity.getTenantId() != null
|| entity.getExpiredAt() != null
|| entity.getCreatedBy() != null;
}
/**
* 判断请求是否包含任一非主表权限字段。
*
* @param entity 访问令牌
* @return 是否需要更新权限映射
*/
private boolean hasPermissionUpdateFields(SysApiKey entity) {
return entity.getPermissionIds() != null
|| entity.getKnowledgeShareEnabled() != null
|| hasNewKnowledgePermissionFields(entity)
|| entity.getWorkflowApiEnabled() != null;
}
} }

View File

@@ -0,0 +1,142 @@
package tech.easyflow.admin.controller.system;
import org.testng.annotations.Test;
import tech.easyflow.ai.service.KnowledgeSharePermissionService;
import tech.easyflow.ai.service.WorkflowApiPermissionService;
import tech.easyflow.common.domain.Result;
import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.service.SysApiKeyResourceMappingService;
import tech.easyflow.system.service.SysApiKeyService;
import java.lang.reflect.Field;
import java.math.BigInteger;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
/**
* {@link SysApiKeyController} 更新访问令牌测试。
*/
public class SysApiKeyControllerTest {
/**
* 验证只更新权限时不会执行缺少主表更新字段的 SQL。
*/
@Test
public void updateShouldSkipMainTableForPermissionOnlyRequest() {
BigInteger apiKeyId = BigInteger.valueOf(100);
SysApiKeyService apiKeyService = mock(SysApiKeyService.class);
KnowledgeSharePermissionService knowledgePermissionService =
mock(KnowledgeSharePermissionService.class);
SysApiKeyController controller = controller(apiKeyService, knowledgePermissionService);
SysApiKey existing = new SysApiKey();
existing.setId(apiKeyId);
when(apiKeyService.getById(apiKeyId)).thenReturn(existing);
SysApiKey request = new SysApiKey();
request.setId(apiKeyId);
request.setKnowledgeReadEnabled(true);
request.setKnowledgeImportEnabled(false);
request.setKnowledgeMaintenanceEnabled(false);
Result<?> result = controller.update(request);
assertEquals(result.getErrorCode(), 0);
verify(apiKeyService, never()).updateById(request);
verify(knowledgePermissionService).replaceApiPermissions(apiKeyId, true, false, false);
}
/**
* 验证基础字段与权限同时更新时,两类数据都被保存。
*/
@Test
public void updateShouldPersistMainTableAndPermissionsTogether() {
BigInteger apiKeyId = BigInteger.valueOf(101);
SysApiKeyService apiKeyService = mock(SysApiKeyService.class);
KnowledgeSharePermissionService knowledgePermissionService =
mock(KnowledgeSharePermissionService.class);
SysApiKeyController controller = controller(apiKeyService, knowledgePermissionService);
SysApiKey existing = new SysApiKey();
existing.setId(apiKeyId);
when(apiKeyService.getById(apiKeyId)).thenReturn(existing);
SysApiKey request = new SysApiKey();
request.setId(apiKeyId);
request.setStatus(1);
request.setKnowledgeReadEnabled(true);
request.setKnowledgeImportEnabled(true);
request.setKnowledgeMaintenanceEnabled(true);
Result<?> result = controller.update(request);
assertEquals(result.getErrorCode(), 0);
verify(apiKeyService).updateById(request);
verify(knowledgePermissionService).replaceApiPermissions(apiKeyId, true, true, true);
}
/**
* 验证新版知识库权限缺少字段时拒绝更新,避免遗漏字段被隐式关闭。
*/
@Test
public void updateShouldRejectPartialKnowledgePermissions() {
SysApiKeyService apiKeyService = mock(SysApiKeyService.class);
KnowledgeSharePermissionService knowledgePermissionService =
mock(KnowledgeSharePermissionService.class);
SysApiKeyController controller = controller(apiKeyService, knowledgePermissionService);
SysApiKey request = new SysApiKey();
request.setId(BigInteger.valueOf(102));
request.setKnowledgeReadEnabled(true);
Result<?> result = controller.update(request);
assertNotEquals(result.getErrorCode(), 0);
verifyNoInteractions(apiKeyService, knowledgePermissionService);
}
/**
* 创建注入模拟依赖的控制器。
*
* @param apiKeyService 访问令牌服务
* @param knowledgePermissionService 知识库权限服务
* @return 测试控制器
*/
private SysApiKeyController controller(
SysApiKeyService apiKeyService,
KnowledgeSharePermissionService knowledgePermissionService
) {
SysApiKeyController controller = new SysApiKeyController(apiKeyService);
setField(controller, "sysApiKeyResourceMappingService", mock(SysApiKeyResourceMappingService.class));
setField(controller, "knowledgeSharePermissionService", knowledgePermissionService);
setField(controller, "workflowApiPermissionService", mock(WorkflowApiPermissionService.class));
return controller;
}
/**
* 通过反射注入控制器字段。
*
* @param target 目标对象
* @param fieldName 字段名
* @param value 字段值
*/
private void setField(Object target, String fieldName, Object value) {
Class<?> current = target.getClass();
while (current != null) {
try {
Field field = current.getDeclaredField(fieldName);
field.setAccessible(true);
field.set(target, value);
return;
} catch (NoSuchFieldException ignored) {
current = current.getSuperclass();
} catch (IllegalAccessException e) {
throw new IllegalStateException("设置测试字段失败: " + fieldName, e);
}
}
throw new IllegalArgumentException("未找到字段: " + fieldName);
}
}

View File

@@ -0,0 +1,326 @@
package tech.easyflow.publicapi.controller;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.Part;
import org.springframework.http.InvalidMediaTypeException;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.documentimport.ImportCallerContext;
import tech.easyflow.ai.documentimport.ImportCallerType;
import tech.easyflow.ai.documentimport.PublicDocumentImportDtos;
import tech.easyflow.ai.documentimport.task.KnowledgeImportBatchFacade;
import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
import tech.easyflow.ai.service.DocumentCollectionService;
import tech.easyflow.ai.service.KnowledgeShareAuditService;
import tech.easyflow.ai.service.KnowledgeSharePermissionService;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.common.web.jsonbody.JsonBody;
import tech.easyflow.publicapi.interceptor.PublicApiInterceptor;
import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.service.SysApiKeyService;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 知识库文档 Public API 批量异步导入接口。
*
* @author Codex
* @since 2026-08-02
*/
@RestController
@RequestMapping(
value = "/public-api/knowledge-share/document/import/batch",
produces = MediaType.APPLICATION_JSON_VALUE
)
public class PublicKnowledgeDocumentImportController {
private static final long MAX_METADATA_BYTES = 1024L * 1024L;
private final SysApiKeyService sysApiKeyService;
private final KnowledgeSharePermissionService permissionService;
private final KnowledgeShareAuditService auditService;
private final DocumentCollectionService documentCollectionService;
private final KnowledgeImportBatchFacade importFacade;
/**
* 创建 Public API 批量导入控制器。
*
* @param sysApiKeyService 访问令牌服务
* @param permissionService 知识库权限服务
* @param auditService 审计服务
* @param documentCollectionService 知识库服务
* @param importFacade 批量导入门面
*/
public PublicKnowledgeDocumentImportController(
SysApiKeyService sysApiKeyService,
KnowledgeSharePermissionService permissionService,
KnowledgeShareAuditService auditService,
DocumentCollectionService documentCollectionService,
KnowledgeImportBatchFacade importFacade) {
this.sysApiKeyService = sysApiKeyService;
this.permissionService = permissionService;
this.auditService = auditService;
this.documentCollectionService = documentCollectionService;
this.importFacade = importFacade;
}
/**
* 接收多文件并创建异步导入任务。
*
* @param apiKey 访问令牌
* @param metadataPart JSON 元数据 Part
* @param files 多个文件 Part
* @param servletRequest Servlet 请求
* @return HTTP 202 异步任务响应
*/
@PostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity<Result<PublicDocumentImportDtos.SubmitResponse>> submit(
@RequestHeader("ApiKey") String apiKey,
@RequestPart("metadata") Part metadataPart,
@RequestPart("files") List<MultipartFile> files,
HttpServletRequest servletRequest) {
PublicDocumentImportDtos.BatchMetadata metadata =
parseMetadata(metadataPart);
SysApiKey token = resolveAuthenticatedApiKey(servletRequest, apiKey);
assertImportPermission(
token,
servletRequest.getRequestURI(),
metadata.getKnowledgeId()
);
requireDocumentKnowledge(metadata.getKnowledgeId());
ImportCallerContext caller =
new ImportCallerContext(ImportCallerType.PUBLIC_API, token.getId());
PublicDocumentImportDtos.SubmitResponse response =
importFacade.submit(caller, metadata, files);
audit(
token,
"API批量导入文档",
servletRequest.getRequestURI(),
Map.of(
"knowledgeId", metadata.getKnowledgeId(),
"taskId", response.getTaskId(),
"totalCount", response.getTotalCount()
)
);
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Result.ok(response));
}
/**
* 查询异步导入任务状态。
*
* @param apiKey 访问令牌
* @param taskId 批次任务 ID
* @param itemStatus 可选文件状态
* @param pageNumber 页码
* @param pageSize 每页数量
* @param servletRequest Servlet 请求
* @return 任务状态
*/
@GetMapping("/status")
public Result<PublicDocumentImportDtos.StatusResponse> status(
@RequestHeader("ApiKey") String apiKey,
@RequestParam BigInteger taskId,
@RequestParam(required = false) String itemStatus,
@RequestParam(defaultValue = "1") long pageNumber,
@RequestParam(defaultValue = "20") long pageSize,
HttpServletRequest servletRequest) {
SysApiKey token = resolveAuthenticatedApiKey(servletRequest, apiKey);
ImportCallerContext caller =
new ImportCallerContext(ImportCallerType.PUBLIC_API, token.getId());
BigInteger knowledgeId =
importFacade.getOwnedKnowledgeId(caller, taskId);
assertImportPermission(token, servletRequest.getRequestURI(), knowledgeId);
PublicDocumentImportDtos.StatusResponse response =
importFacade.getStatus(
caller,
taskId,
itemStatus,
pageNumber,
pageSize
);
return Result.ok(response);
}
/**
* 对异常任务执行断点重试。
*
* @param apiKey 访问令牌
* @param request 重试请求
* @param servletRequest Servlet 请求
* @return 重试结果
*/
@PostMapping(value = "/retry", consumes = MediaType.APPLICATION_JSON_VALUE)
public Result<PublicDocumentImportDtos.RetryResponse> retry(
@RequestHeader("ApiKey") String apiKey,
@JsonBody PublicDocumentImportDtos.RetryRequest request,
HttpServletRequest servletRequest) {
SysApiKey token = resolveAuthenticatedApiKey(servletRequest, apiKey);
ImportCallerContext caller =
new ImportCallerContext(ImportCallerType.PUBLIC_API, token.getId());
BigInteger taskId = request == null ? null : request.getTaskId();
if (taskId == null) {
throw new BusinessException("taskId 不能为空");
}
BigInteger knowledgeId =
importFacade.getOwnedKnowledgeId(caller, taskId);
assertImportPermission(token, servletRequest.getRequestURI(), knowledgeId);
PublicDocumentImportDtos.RetryResponse response =
importFacade.retry(caller, request);
audit(
token,
"API重试批量导入任务",
servletRequest.getRequestURI(),
Map.of(
"knowledgeId", knowledgeId,
"taskId", response.getTaskId(),
"retriedCount", response.getRetriedCount()
)
);
return Result.ok(response);
}
/**
* 解析并校验 metadata JSON Part。
*
* @param metadataPart metadata Part
* @return 批量元数据
*/
private PublicDocumentImportDtos.BatchMetadata parseMetadata(
Part metadataPart) {
if (metadataPart == null || metadataPart.getSize() <= 0) {
throw new BusinessException("metadata 不能为空");
}
if (metadataPart.getSize() > MAX_METADATA_BYTES) {
throw new BusinessException(413, 41304, "metadata 不能超过1MiB");
}
String contentType = metadataPart.getContentType();
try {
if (contentType == null
|| !MediaType.APPLICATION_JSON.includes(
MediaType.parseMediaType(contentType))) {
throw new BusinessException(415, 41503,
"metadata Part 必须使用 application/json");
}
} catch (InvalidMediaTypeException error) {
throw new BusinessException(415, 41503,
"metadata Part Content-Type 无效", error);
}
try {
String json = new String(
metadataPart.getInputStream().readAllBytes(),
StandardCharsets.UTF_8
);
PublicDocumentImportDtos.BatchMetadata metadata =
JSON.parseObject(
json,
PublicDocumentImportDtos.BatchMetadata.class
);
if (metadata == null) {
throw new BusinessException("metadata 不能为空");
}
return metadata;
} catch (JSONException error) {
throw new BusinessException(400, 40021,
"metadata JSON 格式无效", error);
} catch (java.io.IOException error) {
throw new BusinessException(500, 50023,
"读取 metadata 失败", error);
}
}
/**
* 断言访问令牌具有知识导入权限。
*
* @param token 访问令牌
* @param requestUri 请求 URI
* @param knowledgeId 知识库 ID
*/
private void assertImportPermission(SysApiKey token,
String requestUri,
BigInteger knowledgeId) {
permissionService.assertApiShare(
token.getId(),
requestUri,
knowledgeId,
KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name()
);
}
/**
* 复用拦截器已经完成认证的访问令牌。
*
* @param request Servlet 请求
* @param apiKey 访问令牌明文
* @return 已认证访问令牌
*/
private SysApiKey resolveAuthenticatedApiKey(
HttpServletRequest request,
String apiKey) {
Object authenticated =
request.getAttribute(
PublicApiInterceptor.AUTHENTICATED_API_KEY_ATTRIBUTE
);
if (authenticated instanceof SysApiKey token) {
return token;
}
// 兼容控制器单测和绕过 MVC 拦截器的内部直接调用。
return sysApiKeyService.getSysApiKey(apiKey);
}
/**
* 断言知识库存在且为文档类型。
*
* @param knowledgeId 知识库 ID
*/
private void requireDocumentKnowledge(BigInteger knowledgeId) {
DocumentCollection knowledge =
knowledgeId == null ? null : documentCollectionService.getById(knowledgeId);
if (knowledge == null) {
throw new BusinessException(404, 404, "知识库不存在");
}
if (!knowledge.isDocumentCollection()) {
throw new BusinessException("当前知识库类型不支持文档导入");
}
}
/**
* 记录不含令牌明文的 Public API 审计。
*
* @param token 访问令牌
* @param actionName 操作名称
* @param actionUrl 操作 URI
* @param detail 业务详情
*/
private void audit(SysApiKey token,
String actionName,
String actionUrl,
Map<String, Object> detail) {
Map<String, Object> payload = new HashMap<>(detail);
payload.put("apiKeyId", token.getId());
payload.put("channel", "API");
auditService.log(
null,
actionName,
"KNOWLEDGE_API_SHARE_WRITE",
actionUrl,
payload
);
}
}

View File

@@ -18,13 +18,12 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.documentimport.DocumentImportDtos; import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
import tech.easyflow.ai.entity.Document; import tech.easyflow.ai.entity.Document;
import tech.easyflow.ai.entity.DocumentChunk; import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.ai.entity.DocumentCollection; import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.entity.FaqItem; import tech.easyflow.ai.entity.FaqItem;
import tech.easyflow.ai.entity.Model; import tech.easyflow.ai.entity.Model;
import tech.easyflow.ai.enums.KnowledgeShareActionScope; import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
import tech.easyflow.ai.rag.KnowledgeRetrievalModes; import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest; import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
import tech.easyflow.ai.service.DocumentChunkService; import tech.easyflow.ai.service.DocumentChunkService;
@@ -42,6 +41,8 @@ import tech.easyflow.common.domain.Result;
import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.common.web.jsonbody.JsonBody; import tech.easyflow.common.web.jsonbody.JsonBody;
import tech.easyflow.publicapi.dto.PublicKnowledgeDetailResponse;
import tech.easyflow.publicapi.dto.PublicKnowledgeSearchResultItem;
import tech.easyflow.system.entity.SysApiKey; import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.service.SysApiKeyService; import tech.easyflow.system.service.SysApiKeyService;
@@ -89,28 +90,43 @@ public class PublicKnowledgeShareController {
* 获取知识库详情。 * 获取知识库详情。
*/ */
@GetMapping("/detail") @GetMapping("/detail")
public Result<DocumentCollection> detail( public Result<PublicKnowledgeDetailResponse> detail(
@RequestHeader("ApiKey") String apiKey, @RequestHeader("ApiKey") String apiKey,
@RequestParam BigInteger knowledgeId, @RequestParam BigInteger knowledgeId,
@RequestParam(defaultValue = "1") int pageNumber,
@RequestParam(defaultValue = "50") int pageSize,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
validateDocumentPage(pageNumber, pageSize);
DocumentCollection knowledge = documentCollectionService.getDetail(knowledgeId.toString());
if (knowledge == null) {
throw new BusinessException("知识库不存在");
}
Page<Document> documents = knowledge.isDocumentCollection()
? documentService.getDocumentList(
knowledgeId.toString(),
pageSize,
pageNumber,
null
)
: new Page<>(Collections.emptyList(), pageNumber, pageSize, 0L);
audit(apiKey, "API读取知识库详情", "KNOWLEDGE_API_SHARE_ACCESS", request.getRequestURI(), Map.of("knowledgeId", knowledgeId)); audit(apiKey, "API读取知识库详情", "KNOWLEDGE_API_SHARE_ACCESS", request.getRequestURI(), Map.of("knowledgeId", knowledgeId));
return Result.ok(documentCollectionService.getDetail(knowledgeId.toString())); return Result.ok(new PublicKnowledgeDetailResponse(knowledge, documents));
} }
/** /**
* 检索知识库。 * 检索知识库。
*/ */
@GetMapping("/search") @GetMapping("/search")
public Result<List<KnowledgeSearchResultItem>> search( public Result<List<PublicKnowledgeSearchResultItem>> search(
@RequestHeader("ApiKey") String apiKey, @RequestHeader("ApiKey") String apiKey,
@RequestParam BigInteger knowledgeId, @RequestParam BigInteger knowledgeId,
@RequestParam String keyword, @RequestParam String keyword,
@RequestParam(required = false) String retrievalMode, @RequestParam(required = false) String retrievalMode,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.SEARCH.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
KnowledgeRetrievalRequest retrievalRequest = new KnowledgeRetrievalRequest(); KnowledgeRetrievalRequest retrievalRequest = new KnowledgeRetrievalRequest();
retrievalRequest.setKnowledgeId(knowledgeId); retrievalRequest.setKnowledgeId(knowledgeId);
retrievalRequest.setQuery(keyword); retrievalRequest.setQuery(keyword);
@@ -128,14 +144,19 @@ public class PublicKnowledgeShareController {
public Result<Page<Document>> documentPage( public Result<Page<Document>> documentPage(
@RequestHeader("ApiKey") String apiKey, @RequestHeader("ApiKey") String apiKey,
@RequestParam BigInteger knowledgeId, @RequestParam BigInteger knowledgeId,
@RequestParam(required = false) String title, @RequestParam(required = false) BigInteger documentId,
@RequestParam(defaultValue = "10") int pageSize, @RequestParam(defaultValue = "10") int pageSize,
@RequestParam(defaultValue = "1") int pageNumber, @RequestParam(defaultValue = "1") int pageNumber,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
requireDocumentKnowledge(knowledgeId); requireDocumentKnowledge(knowledgeId);
return Result.ok(documentService.getDocumentList(knowledgeId.toString(), pageSize, pageNumber, title)); return Result.ok(documentService.getDocumentListById(
knowledgeId.toString(),
pageSize,
pageNumber,
documentId
));
} }
/** /**
@@ -149,7 +170,7 @@ public class PublicKnowledgeShareController {
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response HttpServletResponse response
) throws Exception { ) throws Exception {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
requireDocumentKnowledge(knowledgeId); requireDocumentKnowledge(knowledgeId);
Document document = requireDocument(documentId, knowledgeId); Document document = requireDocument(documentId, knowledgeId);
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
@@ -169,11 +190,11 @@ public class PublicKnowledgeShareController {
@PostMapping("/document/remove") @PostMapping("/document/remove")
public Result<?> removeDocument( public Result<?> removeDocument(
@RequestHeader("ApiKey") String apiKey, @RequestHeader("ApiKey") String apiKey,
@RequestParam BigInteger knowledgeId, @JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
@JsonBody("id") String id, @JsonBody(value = "id", required = true) String id,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_DELETE.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
requireDocumentKnowledge(knowledgeId); requireDocumentKnowledge(knowledgeId);
requireDocument(new BigInteger(id), knowledgeId); requireDocument(new BigInteger(id), knowledgeId);
audit(apiKey, "API删除文档", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "documentId", id)); audit(apiKey, "API删除文档", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "documentId", id));
@@ -189,7 +210,7 @@ public class PublicKnowledgeShareController {
@JsonBody DocumentImportDtos.AnalyzeRequest request, @JsonBody DocumentImportDtos.AnalyzeRequest request,
HttpServletRequest servletRequest HttpServletRequest servletRequest
) { ) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(request.getKnowledgeId()); requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API分析文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); audit(apiKey, "API分析文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.analyzeImport(request); return documentService.analyzeImport(request);
@@ -204,7 +225,7 @@ public class PublicKnowledgeShareController {
@JsonBody DocumentImportDtos.PreviewRequest request, @JsonBody DocumentImportDtos.PreviewRequest request,
HttpServletRequest servletRequest HttpServletRequest servletRequest
) { ) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(request.getKnowledgeId()); requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API预览文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); audit(apiKey, "API预览文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.previewImport(request); return documentService.previewImport(request);
@@ -219,7 +240,7 @@ public class PublicKnowledgeShareController {
@JsonBody DocumentImportDtos.CommitRequest request, @JsonBody DocumentImportDtos.CommitRequest request,
HttpServletRequest servletRequest HttpServletRequest servletRequest
) { ) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(request.getKnowledgeId()); requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API提交文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); audit(apiKey, "API提交文档导入", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.commitImport(request); return documentService.commitImport(request);
@@ -231,7 +252,7 @@ public class PublicKnowledgeShareController {
@JsonBody DocumentImportDtos.TaskCreateRequest request, @JsonBody DocumentImportDtos.TaskCreateRequest request,
HttpServletRequest servletRequest HttpServletRequest servletRequest
) { ) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(request.getKnowledgeId()); requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API创建文档导入任务", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); audit(apiKey, "API创建文档导入任务", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.createImportTask(request); return documentService.createImportTask(request);
@@ -244,7 +265,7 @@ public class PublicKnowledgeShareController {
@RequestParam BigInteger taskId, @RequestParam BigInteger taskId,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(knowledgeId); requireDocumentKnowledge(knowledgeId);
Result<DocumentImportDtos.TaskDetailResponse> result = documentService.getImportTaskDetail(taskId); Result<DocumentImportDtos.TaskDetailResponse> result = documentService.getImportTaskDetail(taskId);
if (result.getData() == null || result.getData().getKnowledgeId() == null if (result.getData() == null || result.getData().getKnowledgeId() == null
@@ -260,7 +281,7 @@ public class PublicKnowledgeShareController {
@JsonBody DocumentImportDtos.PreviewRequest request, @JsonBody DocumentImportDtos.PreviewRequest request,
HttpServletRequest servletRequest HttpServletRequest servletRequest
) { ) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(request.getKnowledgeId()); requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API预览文档分块", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); audit(apiKey, "API预览文档分块", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.previewImportTask(request); return documentService.previewImportTask(request);
@@ -272,7 +293,7 @@ public class PublicKnowledgeShareController {
@JsonBody DocumentImportDtos.TaskStartIndexRequest request, @JsonBody DocumentImportDtos.TaskStartIndexRequest request,
HttpServletRequest servletRequest HttpServletRequest servletRequest
) { ) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(request.getKnowledgeId()); requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API启动文档向量化", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); audit(apiKey, "API启动文档向量化", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.startIndexTask(request); return documentService.startIndexTask(request);
@@ -284,7 +305,7 @@ public class PublicKnowledgeShareController {
@JsonBody DocumentImportDtos.TaskRetryRequest request, @JsonBody DocumentImportDtos.TaskRetryRequest request,
HttpServletRequest servletRequest HttpServletRequest servletRequest
) { ) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(request.getKnowledgeId()); requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API重试文档解析", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); audit(apiKey, "API重试文档解析", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.retryParseTask(request); return documentService.retryParseTask(request);
@@ -296,7 +317,7 @@ public class PublicKnowledgeShareController {
@JsonBody DocumentImportDtos.TaskRetryRequest request, @JsonBody DocumentImportDtos.TaskRetryRequest request,
HttpServletRequest servletRequest HttpServletRequest servletRequest
) { ) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireDocumentKnowledge(request.getKnowledgeId()); requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API重试文档向量化", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId())); audit(apiKey, "API重试文档向量化", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.retryIndexTask(request); return documentService.retryIndexTask(request);
@@ -314,7 +335,7 @@ public class PublicKnowledgeShareController {
@RequestParam(defaultValue = "10") long pageSize, @RequestParam(defaultValue = "10") long pageSize,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
requireDocumentKnowledge(knowledgeId); requireDocumentKnowledge(knowledgeId);
requireDocument(documentId, knowledgeId); requireDocument(documentId, knowledgeId);
QueryWrapper wrapper = QueryWrapper.create() QueryWrapper wrapper = QueryWrapper.create()
@@ -329,11 +350,11 @@ public class PublicKnowledgeShareController {
@PostMapping("/documentChunk/update") @PostMapping("/documentChunk/update")
public Result<?> updateDocumentChunk( public Result<?> updateDocumentChunk(
@RequestHeader("ApiKey") String apiKey, @RequestHeader("ApiKey") String apiKey,
@RequestParam BigInteger knowledgeId, @JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
@JsonBody DocumentChunk documentChunk, @JsonBody DocumentChunk documentChunk,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_UPDATE.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
requireDocumentKnowledge(knowledgeId); requireDocumentKnowledge(knowledgeId);
DocumentChunk current = requireDocumentChunk(documentChunk.getId(), knowledgeId); DocumentChunk current = requireDocumentChunk(documentChunk.getId(), knowledgeId);
boolean success = documentChunkService.updateById(documentChunk); boolean success = documentChunkService.updateById(documentChunk);
@@ -369,11 +390,11 @@ public class PublicKnowledgeShareController {
@PostMapping("/documentChunk/remove") @PostMapping("/documentChunk/remove")
public Result<?> removeDocumentChunk( public Result<?> removeDocumentChunk(
@RequestHeader("ApiKey") String apiKey, @RequestHeader("ApiKey") String apiKey,
@RequestParam BigInteger knowledgeId, @JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
@JsonBody("id") BigInteger chunkId, @JsonBody(value = "id", required = true) BigInteger chunkId,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_DELETE.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
requireDocumentKnowledge(knowledgeId); requireDocumentKnowledge(knowledgeId);
requireDocumentChunk(chunkId, knowledgeId); requireDocumentChunk(chunkId, knowledgeId);
DocumentCollection knowledge = documentCollectionService.getById(knowledgeId); DocumentCollection knowledge = documentCollectionService.getById(knowledgeId);
@@ -410,7 +431,7 @@ public class PublicKnowledgeShareController {
@RequestParam(defaultValue = "10") long pageSize, @RequestParam(defaultValue = "10") long pageSize,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
requireFaqKnowledge(knowledgeId); requireFaqKnowledge(knowledgeId);
faqCategoryService.ensureDefaultCategory(knowledgeId); faqCategoryService.ensureDefaultCategory(knowledgeId);
QueryWrapper queryWrapper = QueryWrapper.create() QueryWrapper queryWrapper = QueryWrapper.create()
@@ -447,7 +468,7 @@ public class PublicKnowledgeShareController {
@RequestParam String id, @RequestParam String id,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
requireFaqKnowledge(knowledgeId); requireFaqKnowledge(knowledgeId);
FaqItem faqItem = requireFaq(new BigInteger(id), knowledgeId); FaqItem faqItem = requireFaq(new BigInteger(id), knowledgeId);
return Result.ok(faqItem); return Result.ok(faqItem);
@@ -462,7 +483,7 @@ public class PublicKnowledgeShareController {
@JsonBody FaqItem entity, @JsonBody FaqItem entity,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), entity.getCollectionId(), KnowledgeShareActionScope.CONTENT_CREATE.name()); assertApiShare(apiKey, request.getRequestURI(), entity.getCollectionId(), KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
requireFaqKnowledge(entity.getCollectionId()); requireFaqKnowledge(entity.getCollectionId());
audit(apiKey, "API新增FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", entity.getCollectionId())); audit(apiKey, "API新增FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", entity.getCollectionId()));
return Result.ok(faqItemService.saveFaqItem(entity)); return Result.ok(faqItemService.saveFaqItem(entity));
@@ -478,7 +499,7 @@ public class PublicKnowledgeShareController {
@JsonBody FaqItem entity, @JsonBody FaqItem entity,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_UPDATE.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
requireFaqKnowledge(knowledgeId); requireFaqKnowledge(knowledgeId);
requireFaq(entity.getId(), knowledgeId); requireFaq(entity.getId(), knowledgeId);
audit(apiKey, "API更新FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "faqId", entity.getId())); audit(apiKey, "API更新FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "faqId", entity.getId()));
@@ -495,7 +516,7 @@ public class PublicKnowledgeShareController {
@JsonBody("id") BigInteger id, @JsonBody("id") BigInteger id,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.CONTENT_DELETE.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
requireFaqKnowledge(knowledgeId); requireFaqKnowledge(knowledgeId);
requireFaq(id, knowledgeId); requireFaq(id, knowledgeId);
audit(apiKey, "API删除FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "faqId", id)); audit(apiKey, "API删除FAQ", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "faqId", id));
@@ -512,7 +533,7 @@ public class PublicKnowledgeShareController {
BigInteger collectionId, BigInteger collectionId,
HttpServletRequest request HttpServletRequest request
) { ) {
assertApiShare(apiKey, request.getRequestURI(), collectionId, KnowledgeShareActionScope.IMPORT_EXPORT.name()); assertApiShare(apiKey, request.getRequestURI(), collectionId, KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireFaqKnowledge(collectionId); requireFaqKnowledge(collectionId);
audit(apiKey, "API导入FAQ Excel", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", collectionId)); audit(apiKey, "API导入FAQ Excel", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", collectionId));
return Result.ok(faqItemService.importFromExcel(collectionId, file)); return Result.ok(faqItemService.importFromExcel(collectionId, file));
@@ -528,7 +549,7 @@ public class PublicKnowledgeShareController {
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response HttpServletResponse response
) throws Exception { ) throws Exception {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.IMPORT_EXPORT.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
requireFaqKnowledge(knowledgeId); requireFaqKnowledge(knowledgeId);
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader( response.setHeader(
@@ -550,7 +571,7 @@ public class PublicKnowledgeShareController {
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response HttpServletResponse response
) throws Exception { ) throws Exception {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.IMPORT_EXPORT.name()); assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
requireFaqKnowledge(knowledgeId); requireFaqKnowledge(knowledgeId);
String fileName = "faq_export_" + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + ".xlsx"; String fileName = "faq_export_" + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + ".xlsx";
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
@@ -610,6 +631,22 @@ public class PublicKnowledgeShareController {
return knowledge; return knowledge;
} }
/**
* 校验详情接口中的文档分页参数。
*
* @param pageNumber 页码
* @param pageSize 每页条数
* @throws BusinessException 页码小于 1 或每页条数不在 1 到 100 之间时抛出
*/
private void validateDocumentPage(int pageNumber, int pageSize) {
if (pageNumber < 1) {
throw new BusinessException("pageNumber 必须大于等于 1");
}
if (pageSize < 1 || pageSize > 100) {
throw new BusinessException("pageSize 必须在 1 到 100 之间");
}
}
private Document requireDocument(BigInteger documentId, BigInteger knowledgeId) { private Document requireDocument(BigInteger documentId, BigInteger knowledgeId) {
Document document = documentService.getById(documentId); Document document = documentService.getById(documentId);
if (document == null || document.getCollectionId() == null || document.getCollectionId().compareTo(knowledgeId) != 0) { if (document == null || document.getCollectionId() == null || document.getCollectionId().compareTo(knowledgeId) != 0) {
@@ -642,15 +679,38 @@ public class PublicKnowledgeShareController {
knowledgeShareAuditService.log(null, actionName, actionType, actionUrl, payload); knowledgeShareAuditService.log(null, actionName, actionType, actionUrl, payload);
} }
private List<KnowledgeSearchResultItem> toKnowledgeSearchResult(List<com.easyagents.core.document.Document> documents) { private List<PublicKnowledgeSearchResultItem> toKnowledgeSearchResult(
List<KnowledgeSearchResultItem> result = new java.util.ArrayList<>(); List<com.easyagents.core.document.Document> documents
) {
List<PublicKnowledgeSearchResultItem> result = new java.util.ArrayList<>();
for (com.easyagents.core.document.Document document : documents) { for (com.easyagents.core.document.Document document : documents) {
KnowledgeSearchResultItem item = new KnowledgeSearchResultItem(); PublicKnowledgeSearchResultItem item =
new PublicKnowledgeSearchResultItem();
item.setContent(document.getContent()); item.setContent(document.getContent());
String resultType =
asString(document.getMetadata("resultType"));
item.setResultType(resultType);
Object renderMarkdown = document.getMetadata("renderMarkdown"); Object renderMarkdown = document.getMetadata("renderMarkdown");
item.setRenderMarkdown(renderMarkdown == null ? null : String.valueOf(renderMarkdown)); item.setRenderMarkdown(renderMarkdown == null ? null : String.valueOf(renderMarkdown));
Object sourceFileName = document.getMetadata("sourceFileName"); Object sourceFileName = document.getMetadata("sourceFileName");
item.setSourceFileName(sourceFileName == null ? null : String.valueOf(sourceFileName)); String documentName =
sourceFileName == null ? null : String.valueOf(sourceFileName);
item.setSourceFileName(documentName);
if (DocumentCollection.TYPE_FAQ.equalsIgnoreCase(resultType)) {
item.setFaqId(asBigInteger(document.getMetadata("faqId")));
item.setQuestion(asString(document.getMetadata("question")));
item.setAnswerText(
asString(document.getMetadata("answerText"))
);
item.setCategoryId(
asBigInteger(document.getMetadata("categoryId"))
);
} else {
item.setDocumentName(documentName);
item.setDocumentId(
asBigInteger(document.getMetadata("documentId"))
);
}
item.setScore(document.getScore()); item.setScore(document.getScore());
Object hitSource = document.getMetadata("hitSource"); Object hitSource = document.getMetadata("hitSource");
item.setHitSource(hitSource == null ? null : String.valueOf(hitSource)); item.setHitSource(hitSource == null ? null : String.valueOf(hitSource));
@@ -661,6 +721,36 @@ public class PublicKnowledgeShareController {
return result; return result;
} }
/**
* 将检索元数据转换为字符串。
*
* @param value 元数据值
* @return 字符串值;原值为空时返回 {@code null}
*/
private String asString(Object value) {
return value == null ? null : String.valueOf(value);
}
/**
* 将检索元数据转换为大整数 ID。
*
* @param value 元数据值
* @return 大整数 ID原值为空或格式无效时返回 {@code null}
*/
private BigInteger asBigInteger(Object value) {
if (value == null) {
return null;
}
if (value instanceof BigInteger) {
return (BigInteger) value;
}
try {
return new BigInteger(String.valueOf(value));
} catch (NumberFormatException ignored) {
return null;
}
}
private Double asDouble(Object value) { private Double asDouble(Object value) {
if (value == null) { if (value == null) {
return null; return null;

View File

@@ -0,0 +1,217 @@
package tech.easyflow.publicapi.dto;
import cn.hutool.core.bean.BeanUtil;
import com.mybatisflex.core.paginate.Page;
import tech.easyflow.ai.entity.Document;
import tech.easyflow.ai.entity.DocumentCollection;
import java.math.BigInteger;
import java.util.Collections;
import java.util.Date;
import java.util.List;
/**
* 公开知识库详情响应。
*
* <p>知识库原有字段保持在响应顶层,文档摘要通过 {@code documents} 分页返回。</p>
*/
public class PublicKnowledgeDetailResponse extends DocumentCollection {
/**
* 已上传文档的分页摘要。
*/
private final Page<DocumentSummary> documents;
/**
* 创建公开知识库详情响应。
*
* @param knowledge 知识库基本信息
* @param documentPage 文档实体分页FAQ 知识库可传 {@code null}
*/
public PublicKnowledgeDetailResponse(
DocumentCollection knowledge,
Page<Document> documentPage
) {
BeanUtil.copyProperties(knowledge, this);
this.documents = toDocumentSummaryPage(documentPage);
}
/**
* 获取文档分页摘要。
*
* @return 文档分页摘要
*/
public Page<DocumentSummary> getDocuments() {
return documents;
}
/**
* 将文档实体分页映射为公开摘要分页。
*
* @param source 文档实体分页
* @return 不包含内部路径、正文和配置的公开摘要分页
*/
private static Page<DocumentSummary> toDocumentSummaryPage(Page<Document> source) {
if (source == null) {
return new Page<>(Collections.emptyList(), 1, 50, 0L);
}
List<DocumentSummary> records = source.getRecords() == null
? Collections.emptyList()
: source.getRecords().stream().map(DocumentSummary::new).toList();
return new Page<>(
records,
source.getPageNumber(),
source.getPageSize(),
source.getTotalRow()
);
}
/**
* 公开文档摘要。
*/
public static class DocumentSummary {
/**
* 文档 ID。
*/
private final BigInteger id;
/**
* 文档标题。
*/
private final String title;
/**
* 文档类型。
*/
private final String documentType;
/**
* 文档内容类型。
*/
private final String contentType;
/**
* 文档处理状态。
*/
private final String processStatus;
/**
* 文档分块数。
*/
private final long chunkCount;
/**
* 处理进度百分比。
*/
private final Integer progressPercent;
/**
* 创建时间。
*/
private final Date created;
/**
* 最后修改时间。
*/
private final Date modified;
/**
* 从文档实体创建公开摘要。
*
* @param document 文档实体
*/
public DocumentSummary(Document document) {
this.id = document.getId();
this.title = document.getTitle();
this.documentType = document.getDocumentType();
this.contentType = document.getContentType();
this.processStatus = document.getProcessStatus();
this.chunkCount = document.getDisplayChunkCount();
this.progressPercent = document.getProgressPercent();
this.created = document.getCreated();
this.modified = document.getModified();
}
/**
* 获取文档 ID。
*
* @return 文档 ID
*/
public BigInteger getId() {
return id;
}
/**
* 获取文档标题。
*
* @return 文档标题
*/
public String getTitle() {
return title;
}
/**
* 获取文档类型。
*
* @return 文档类型
*/
public String getDocumentType() {
return documentType;
}
/**
* 获取文档内容类型。
*
* @return 文档内容类型
*/
public String getContentType() {
return contentType;
}
/**
* 获取文档处理状态。
*
* @return 文档处理状态
*/
public String getProcessStatus() {
return processStatus;
}
/**
* 获取文档分块数。
*
* @return 文档分块数
*/
public long getChunkCount() {
return chunkCount;
}
/**
* 获取处理进度百分比。
*
* @return 处理进度百分比
*/
public Integer getProgressPercent() {
return progressPercent;
}
/**
* 获取创建时间。
*
* @return 创建时间
*/
public Date getCreated() {
return created;
}
/**
* 获取最后修改时间。
*
* @return 最后修改时间
*/
public Date getModified() {
return modified;
}
}
}

View File

@@ -0,0 +1,176 @@
package tech.easyflow.publicapi.dto;
import com.fasterxml.jackson.annotation.JsonInclude;
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
import java.math.BigInteger;
/**
* 公开知识库检索结果。
*
* <p>根据命中来源补充文档或 FAQ 标识,便于调用方继续查询对应详情。</p>
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class PublicKnowledgeSearchResultItem extends KnowledgeSearchResultItem {
/**
* 命中来源类型。
*/
private String resultType;
/**
* 来源文档 ID。
*/
private BigInteger documentId;
/**
* 来源文档名称。
*/
private String documentName;
/**
* 来源 FAQ ID。
*/
private BigInteger faqId;
/**
* 来源 FAQ 问题。
*/
private String question;
/**
* 来源 FAQ 纯文本答案。
*/
private String answerText;
/**
* 来源 FAQ 分类 ID。
*/
private BigInteger categoryId;
/**
* 获取命中来源类型。
*
* @return {@code DOCUMENT} 或 {@code FAQ}
*/
public String getResultType() {
return resultType;
}
/**
* 设置命中来源类型。
*
* @param resultType 命中来源类型
*/
public void setResultType(String resultType) {
this.resultType = resultType;
}
/**
* 获取来源文档 ID。
*
* @return 来源文档 ID
*/
public BigInteger getDocumentId() {
return documentId;
}
/**
* 设置来源文档 ID。
*
* @param documentId 来源文档 ID
*/
public void setDocumentId(BigInteger documentId) {
this.documentId = documentId;
}
/**
* 获取来源文档名称。
*
* @return 来源文档名称
*/
public String getDocumentName() {
return documentName;
}
/**
* 设置来源文档名称。
*
* @param documentName 来源文档名称
*/
public void setDocumentName(String documentName) {
this.documentName = documentName;
}
/**
* 获取来源 FAQ ID。
*
* @return 来源 FAQ ID
*/
public BigInteger getFaqId() {
return faqId;
}
/**
* 设置来源 FAQ ID。
*
* @param faqId 来源 FAQ ID
*/
public void setFaqId(BigInteger faqId) {
this.faqId = faqId;
}
/**
* 获取来源 FAQ 问题。
*
* @return 来源 FAQ 问题
*/
public String getQuestion() {
return question;
}
/**
* 设置来源 FAQ 问题。
*
* @param question 来源 FAQ 问题
*/
public void setQuestion(String question) {
this.question = question;
}
/**
* 获取来源 FAQ 纯文本答案。
*
* @return 来源 FAQ 纯文本答案
*/
public String getAnswerText() {
return answerText;
}
/**
* 设置来源 FAQ 纯文本答案。
*
* @param answerText 来源 FAQ 纯文本答案
*/
public void setAnswerText(String answerText) {
this.answerText = answerText;
}
/**
* 获取来源 FAQ 分类 ID。
*
* @return 来源 FAQ 分类 ID
*/
public BigInteger getCategoryId() {
return categoryId;
}
/**
* 设置来源 FAQ 分类 ID。
*
* @param categoryId 来源 FAQ 分类 ID
*/
public void setCategoryId(BigInteger categoryId) {
this.categoryId = categoryId;
}
}

View File

@@ -9,11 +9,21 @@ import org.springframework.stereotype.Component;
import org.springframework.web.servlet.HandlerInterceptor; import org.springframework.web.servlet.HandlerInterceptor;
import tech.easyflow.common.domain.Result; import tech.easyflow.common.domain.Result;
import tech.easyflow.common.util.ResponseUtil; import tech.easyflow.common.util.ResponseUtil;
import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.service.SysApiKeyService; import tech.easyflow.system.service.SysApiKeyService;
/**
* Public API 访问令牌与接口权限拦截器。
*/
@Component @Component
public class PublicApiInterceptor implements HandlerInterceptor { public class PublicApiInterceptor implements HandlerInterceptor {
/**
* 请求中已完成认证的访问令牌属性名,供后续资源级鉴权复用。
*/
public static final String AUTHENTICATED_API_KEY_ATTRIBUTE =
PublicApiInterceptor.class.getName() + ".authenticatedApiKey";
private static final Logger log = LoggerFactory.getLogger(PublicApiInterceptor.class); private static final Logger log = LoggerFactory.getLogger(PublicApiInterceptor.class);
@Resource @Resource
@@ -31,7 +41,12 @@ public class PublicApiInterceptor implements HandlerInterceptor {
ResponseUtil.renderJson(response, failed); ResponseUtil.renderJson(response, failed);
return false; return false;
} }
sysApiKeyService.checkApikeyPermission(apiKey, requestURI); SysApiKey authenticatedApiKey =
sysApiKeyService.checkApikeyPermission(apiKey, requestURI);
request.setAttribute(
AUTHENTICATED_API_KEY_ATTRIBUTE,
authenticatedApiKey
);
return true; return true;
} }
} }

View File

@@ -0,0 +1,64 @@
package tech.easyflow.publicapi.controller;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.ai.documentimport.PublicDocumentImportDtos;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.publicapi.interceptor.PublicApiInterceptor;
import tech.easyflow.system.entity.SysApiKey;
import java.lang.reflect.Proxy;
import java.math.BigInteger;
/**
* {@link PublicKnowledgeDocumentImportController} 请求边界测试。
*
* @author Codex
* @since 2026-08-02
*/
public class PublicKnowledgeDocumentImportControllerTest {
/**
* 验证重试请求缺少 taskId 时返回明确参数错误。
*/
@Test
public void retryShouldRejectMissingTaskId() {
SysApiKey token = new SysApiKey();
token.setId(BigInteger.ONE);
HttpServletRequest servletRequest = (HttpServletRequest) Proxy.newProxyInstance(
HttpServletRequest.class.getClassLoader(),
new Class<?>[]{HttpServletRequest.class},
(instance, method, args) -> {
if ("getAttribute".equals(method.getName())) {
Assert.assertEquals(
PublicApiInterceptor.AUTHENTICATED_API_KEY_ATTRIBUTE,
args[0]
);
return token;
}
throw new AssertionError(
"测试路径不应调用 HttpServletRequest." + method.getName()
);
}
);
PublicKnowledgeDocumentImportController controller =
new PublicKnowledgeDocumentImportController(
null,
null,
null,
null,
null
);
PublicDocumentImportDtos.RetryRequest request =
new PublicDocumentImportDtos.RetryRequest();
try {
controller.retry("test-key", request, servletRequest);
Assert.fail("Expected missing taskId rejection");
} catch (BusinessException expected) {
Assert.assertEquals(400, expected.getHttpStatus());
Assert.assertTrue(expected.getMessage().contains("taskId 不能为空"));
}
}
}

View File

@@ -0,0 +1,192 @@
package tech.easyflow.publicapi.controller;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.web.bind.annotation.RequestParam;
import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.common.web.jsonbody.JsonBody;
import tech.easyflow.publicapi.dto.PublicKnowledgeSearchResultItem;
import java.lang.reflect.Method;
import java.lang.reflect.Parameter;
import java.math.BigInteger;
import java.util.List;
/**
* {@link PublicKnowledgeShareController} 公开接口参数契约测试。
*/
public class PublicKnowledgeShareControllerContractTest {
/**
* 验证文档删除接口的 knowledgeId 和 id 均从 JSON 请求体读取。
*
* @throws Exception 反射失败
*/
@Test
public void documentRemoveShouldReadIdentifiersFromJsonBody()
throws Exception {
Method method = PublicKnowledgeShareController.class.getDeclaredMethod(
"removeDocument",
String.class,
BigInteger.class,
String.class,
HttpServletRequest.class
);
Parameter knowledgeId = method.getParameters()[1];
Parameter documentId = method.getParameters()[2];
assertRequiredJsonField(knowledgeId, "knowledgeId");
assertRequiredJsonField(documentId, "id");
Assert.assertNull(knowledgeId.getAnnotation(RequestParam.class));
}
/**
* 验证文档分块更新接口从 JSON 请求体读取 knowledgeId。
*
* @throws Exception 反射失败
*/
@Test
public void documentChunkUpdateShouldReadKnowledgeIdFromJsonBody()
throws Exception {
Method method = PublicKnowledgeShareController.class.getDeclaredMethod(
"updateDocumentChunk",
String.class,
BigInteger.class,
DocumentChunk.class,
HttpServletRequest.class
);
Parameter knowledgeId = method.getParameters()[1];
Parameter chunk = method.getParameters()[2];
assertRequiredJsonField(knowledgeId, "knowledgeId");
Assert.assertNull(knowledgeId.getAnnotation(RequestParam.class));
JsonBody chunkBody = chunk.getAnnotation(JsonBody.class);
Assert.assertNotNull(chunkBody);
Assert.assertEquals("", chunkBody.value());
}
/**
* 验证文档分块删除接口从 JSON 请求体读取 knowledgeId 和 id。
*
* @throws Exception 反射失败
*/
@Test
public void documentChunkRemoveShouldReadIdentifiersFromJsonBody()
throws Exception {
Method method = PublicKnowledgeShareController.class.getDeclaredMethod(
"removeDocumentChunk",
String.class,
BigInteger.class,
BigInteger.class,
HttpServletRequest.class
);
Parameter knowledgeId = method.getParameters()[1];
Parameter chunkId = method.getParameters()[2];
assertRequiredJsonField(knowledgeId, "knowledgeId");
assertRequiredJsonField(chunkId, "id");
Assert.assertNull(knowledgeId.getAnnotation(RequestParam.class));
}
/**
* 验证文档检索结果包含来源文档信息且不混入 FAQ 字段。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void searchShouldExposeDocumentIdentity() throws Exception {
BigInteger documentId = BigInteger.valueOf(2002);
com.easyagents.core.document.Document hit =
com.easyagents.core.document.Document.of("命中文本");
hit.addMetadata("resultType", DocumentCollection.TYPE_DOCUMENT);
hit.addMetadata("documentId", documentId);
hit.addMetadata("sourceFileName", "manual.pdf");
List<PublicKnowledgeSearchResultItem> results =
mapSearchResult(hit);
Assert.assertEquals(1, results.size());
PublicKnowledgeSearchResultItem item = results.get(0);
Assert.assertEquals(DocumentCollection.TYPE_DOCUMENT, item.getResultType());
Assert.assertEquals(documentId, item.getDocumentId());
Assert.assertEquals("manual.pdf", item.getDocumentName());
Assert.assertEquals("manual.pdf", item.getSourceFileName());
Assert.assertNull(item.getFaqId());
Assert.assertNull(item.getQuestion());
String json = new ObjectMapper().writeValueAsString(item);
Assert.assertFalse(json.contains("\"faqId\""));
Assert.assertFalse(json.contains("\"question\""));
}
/**
* 验证 FAQ 检索结果包含 FAQ 信息且不混入文档字段。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void searchShouldExposeFaqIdentity() throws Exception {
BigInteger faqId = BigInteger.valueOf(3003);
BigInteger categoryId = BigInteger.valueOf(4004);
com.easyagents.core.document.Document hit =
com.easyagents.core.document.Document.of("FAQ 命中文本");
hit.addMetadata("resultType", DocumentCollection.TYPE_FAQ);
hit.addMetadata("faqId", faqId);
hit.addMetadata("question", "如何申请账号?");
hit.addMetadata("answerText", "请联系管理员。");
hit.addMetadata("categoryId", categoryId);
List<PublicKnowledgeSearchResultItem> results =
mapSearchResult(hit);
Assert.assertEquals(1, results.size());
PublicKnowledgeSearchResultItem item = results.get(0);
Assert.assertEquals(DocumentCollection.TYPE_FAQ, item.getResultType());
Assert.assertEquals(faqId, item.getFaqId());
Assert.assertEquals("如何申请账号?", item.getQuestion());
Assert.assertEquals("请联系管理员。", item.getAnswerText());
Assert.assertEquals(categoryId, item.getCategoryId());
Assert.assertNull(item.getDocumentId());
Assert.assertNull(item.getDocumentName());
String json = new ObjectMapper().writeValueAsString(item);
Assert.assertFalse(json.contains("\"documentId\""));
Assert.assertFalse(json.contains("\"documentName\""));
Assert.assertFalse(json.contains("\"sourceFileName\""));
}
/**
* 调用公开检索结果映射。
*
* @param hit 检索命中
* @return 公开检索结果
* @throws Exception 反射调用失败
*/
@SuppressWarnings("unchecked")
private List<PublicKnowledgeSearchResultItem> mapSearchResult(
com.easyagents.core.document.Document hit
) throws Exception {
Method method = PublicKnowledgeShareController.class
.getDeclaredMethod(
"toKnowledgeSearchResult",
List.class
);
method.setAccessible(true);
return (List<PublicKnowledgeSearchResultItem>) method.invoke(
new PublicKnowledgeShareController(),
List.of(hit)
);
}
/**
* 断言参数绑定到指定的必填 JSON 字段。
*
* @param parameter 方法参数
* @param field JSON 字段名
*/
private void assertRequiredJsonField(Parameter parameter, String field) {
JsonBody jsonBody = parameter.getAnnotation(JsonBody.class);
Assert.assertNotNull(jsonBody);
Assert.assertEquals(field, jsonBody.value());
Assert.assertTrue(jsonBody.required());
}
}

View File

@@ -0,0 +1,78 @@
package tech.easyflow.publicapi.dto;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.mybatisflex.core.paginate.Page;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.ai.entity.Document;
import tech.easyflow.ai.entity.DocumentCollection;
import java.math.BigInteger;
import java.util.Collections;
import java.util.List;
/**
* {@link PublicKnowledgeDetailResponse} 响应结构测试。
*/
public class PublicKnowledgeDetailResponseTest {
/**
* 验证知识库字段保持在顶层,文档只暴露公开摘要。
*
*/
@Test
public void shouldKeepKnowledgeFieldsAtTopLevelAndExposeDocumentSummary() {
DocumentCollection knowledge = new DocumentCollection();
knowledge.setId(BigInteger.valueOf(100));
knowledge.setTitle("测试知识库");
knowledge.setCollectionType(DocumentCollection.TYPE_DOCUMENT);
Document document = new Document();
document.setId(BigInteger.valueOf(200));
document.setTitle("manual.pdf");
document.setDocumentType("pdf");
document.setContentType("application/pdf");
document.setDocumentPath("private/path/manual.pdf");
document.setContent("内部正文");
document.setProcessStatus("INDEXED");
document.setTotalChunks(12);
document.setProgressPercent(100);
Page<Document> source = new Page<>(List.of(document), 1, 10, 1L);
PublicKnowledgeDetailResponse response =
new PublicKnowledgeDetailResponse(knowledge, source);
JSONObject json = JSON.parseObject(JSON.toJSONString(response));
Assert.assertEquals("100", json.getString("id"));
Assert.assertEquals("测试知识库", json.getString("title"));
Assert.assertFalse(json.containsKey("knowledge"));
JSONObject summary = json.getJSONObject("documents")
.getJSONArray("records")
.getJSONObject(0);
Assert.assertEquals("manual.pdf", summary.getString("title"));
Assert.assertEquals(12L, summary.getLongValue("chunkCount"));
Assert.assertFalse(summary.containsKey("documentPath"));
Assert.assertFalse(summary.containsKey("content"));
Assert.assertFalse(summary.containsKey("options"));
}
/**
* 验证空文档分页保留调用方请求的分页参数。
*/
@Test
public void shouldKeepRequestedPaginationForEmptyDocumentPage() {
DocumentCollection knowledge = new DocumentCollection();
knowledge.setId(BigInteger.valueOf(300));
knowledge.setCollectionType(DocumentCollection.TYPE_FAQ);
Page<Document> source = new Page<>(Collections.emptyList(), 3, 7, 0L);
PublicKnowledgeDetailResponse response =
new PublicKnowledgeDetailResponse(knowledge, source);
Assert.assertEquals(3L, response.getDocuments().getPageNumber());
Assert.assertEquals(7L, response.getDocuments().getPageSize());
Assert.assertEquals(0L, response.getDocuments().getTotalRow());
Assert.assertTrue(response.getDocuments().getRecords().isEmpty());
}
}

View File

@@ -4,11 +4,15 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse; import jakarta.servlet.http.HttpServletResponse;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.service.SysApiKeyService;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.io.StringWriter; import java.io.StringWriter;
import java.lang.reflect.Proxy; import java.lang.reflect.Proxy;
import java.lang.reflect.Field;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/** /**
* {@link PublicApiInterceptor} 鉴权响应测试。 * {@link PublicApiInterceptor} 鉴权响应测试。
@@ -66,6 +70,65 @@ public class PublicApiInterceptorTest {
Assert.assertTrue(body.toString().contains("密钥不正确")); Assert.assertTrue(body.toString().contains("密钥不正确"));
} }
/**
* 验证通过接口权限校验的访问令牌会写入请求,供资源级鉴权复用。
*
* @throws Exception 拦截器处理失败时抛出
*/
@Test
public void shouldExposeAuthenticatedApiKeyToController() throws Exception {
SysApiKey authenticated = new SysApiKey();
AtomicReference<Object> requestAttribute = new AtomicReference<>();
HttpServletRequest request = proxy(
HttpServletRequest.class,
(instance, method, args) -> {
if ("getRequestURI".equals(method.getName())) {
return "/public-api/knowledge-share/detail";
}
if ("getHeader".equals(method.getName())) {
return "test-key";
}
if ("setAttribute".equals(method.getName())) {
Assert.assertEquals(
PublicApiInterceptor.AUTHENTICATED_API_KEY_ATTRIBUTE,
args[0]);
requestAttribute.set(args[1]);
return null;
}
throw new AssertionError(
"测试路径不应调用 HttpServletRequest."
+ method.getName());
});
HttpServletResponse response = proxy(
HttpServletResponse.class,
(instance, method, args) -> {
throw new AssertionError(
"测试路径不应调用 HttpServletResponse."
+ method.getName());
});
SysApiKeyService service = proxy(
SysApiKeyService.class,
(instance, method, args) -> {
if ("checkApikeyPermission".equals(method.getName())) {
return authenticated;
}
throw new AssertionError(
"测试路径不应调用 SysApiKeyService."
+ method.getName());
});
PublicApiInterceptor interceptor = new PublicApiInterceptor();
Field serviceField = PublicApiInterceptor.class
.getDeclaredField("sysApiKeyService");
serviceField.setAccessible(true);
serviceField.set(interceptor, service);
boolean allowed =
interceptor.preHandle(request, response, new Object());
Assert.assertTrue(allowed);
Assert.assertSame(authenticated, requestAttribute.get());
}
/** /**
* 创建接口代理。 * 创建接口代理。
* *

View File

@@ -108,7 +108,7 @@ public class LocalFileStorageServiceImpl implements FileStorageService {
} }
/** /**
* 删除旧版路径对应的本地文件。 * 幂等删除旧版路径对应的本地文件。
* *
* @param path 文件路径 * @param path 文件路径
*/ */
@@ -116,7 +116,7 @@ public class LocalFileStorageServiceImpl implements FileStorageService {
public void delete(String path) { public void delete(String path) {
try { try {
File file = getLocalFile(path); File file = getLocalFile(path);
Files.delete(file.toPath()); Files.deleteIfExists(file.toPath());
} catch (IOException e) { } catch (IOException e) {
LOG.error("删除本地文件出错: {}", path, e); LOG.error("删除本地文件出错: {}", path, e);
throw new RuntimeException("删除本地文件出错:",e); throw new RuntimeException("删除本地文件出错:",e);

View File

@@ -80,6 +80,22 @@ public class LocalFileStorageServiceImplTest {
assertFalse(Files.exists(part)); assertFalse(Files.exists(part));
} }
/**
* 验证旧版路径删除在文件已不存在时仍可幂等成功。
*
* @throws Exception 测试目录或反射配置失败
*/
@Test
public void deleteMissingLegacyPathIsIdempotent() throws Exception {
File root = temporaryFolder.newFolder("legacy-delete-root");
LocalFileStorageServiceImpl service = createService(root, "/files");
service.delete("/files/missing.bin");
service.delete("/files/missing.bin");
assertFalse(Files.exists(root.toPath().resolve("missing.bin")));
}
/** /**
* 验证句柄路径中的符号链接不会被跟随到存储根目录外。 * 验证句柄路径中的符号链接不会被跟随到存储根目录外。
* *

View File

@@ -5,11 +5,13 @@ import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan;
import tech.easyflow.ai.documentimport.task.DocumentImportParseMonitorProperties; import tech.easyflow.ai.documentimport.task.DocumentImportParseMonitorProperties;
import tech.easyflow.ai.documentimport.task.DocumentImportBulkProperties;
import tech.easyflow.ai.documentimport.task.DocumentImportStatusBroadcastProperties; import tech.easyflow.ai.documentimport.task.DocumentImportStatusBroadcastProperties;
@MapperScan("tech.easyflow.ai.mapper") @MapperScan("tech.easyflow.ai.mapper")
@ComponentScan("tech.easyflow.ai") @ComponentScan("tech.easyflow.ai")
@EnableConfigurationProperties({ @EnableConfigurationProperties({
DocumentImportBulkProperties.class,
DocumentImportParseMonitorProperties.class, DocumentImportParseMonitorProperties.class,
DocumentImportStatusBroadcastProperties.class, DocumentImportStatusBroadcastProperties.class,
RagHealthProperties.class RagHealthProperties.class

View File

@@ -54,6 +54,19 @@ public interface DocumentParseBridgeService {
*/ */
DocumentParsedResult queryResult(String taskId); DocumentParsedResult queryResult(String taskId);
/**
* 按提交文档的源信息获取异步任务最终结果。
*
* <p>源信息仅用于精确选择提交任务时使用的解析服务,不会重新读取文档内容。</p>
*
* @param taskId 任务 ID
* @param source 提交任务时的文档源信息
* @return 标准化解析结果
*/
default DocumentParsedResult queryResult(String taskId, DocumentSourceRef source) {
return queryResult(taskId);
}
/** /**
* 聚合查询异步任务信息。 * 聚合查询异步任务信息。
* *
@@ -64,4 +77,17 @@ public interface DocumentParseBridgeService {
* @return 聚合任务信息 * @return 聚合任务信息
*/ */
DocumentParseTaskInfo queryTaskInfo(String taskId); DocumentParseTaskInfo queryTaskInfo(String taskId);
/**
* 按提交文档的源信息聚合查询异步任务信息。
*
* <p>源信息仅用于精确选择提交任务时使用的解析服务,不会重新读取文档内容。</p>
*
* @param taskId 任务 ID
* @param source 提交任务时的文档源信息
* @return 聚合任务信息
*/
default DocumentParseTaskInfo queryTaskInfo(String taskId, DocumentSourceRef source) {
return queryTaskInfo(taskId);
}
} }

View File

@@ -150,12 +150,24 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
*/ */
@Override @Override
public DocumentParsedResult queryResult(String taskId) { public DocumentParsedResult queryResult(String taskId) {
return queryResult(taskId, null);
}
/**
* {@inheritDoc}
*/
@Override
public DocumentParsedResult queryResult(String taskId, @Nullable DocumentSourceRef source) {
if (!StringUtils.hasText(taskId)) { if (!StringUtils.hasText(taskId)) {
throw DocumentParseBridgeException.resultFetchFailed("taskId 不能为空"); throw DocumentParseBridgeException.resultFetchFailed("taskId 不能为空");
} }
try { try {
LOG.info("桥接服务开始获取异步解析结果: providerTaskId={}", taskId); LOG.info("桥接服务开始获取异步解析结果: providerTaskId={}", taskId);
ParseResponse response = executeAgainstTaskService(taskId, service -> service.queryResult(taskId)); ParseResponse response = executeAgainstTaskService(
taskId,
source,
service -> service.queryResult(taskId)
);
DocumentParsedResult result = parseResultMapper.map(extractSingleResult(response, true)); DocumentParsedResult result = parseResultMapper.map(extractSingleResult(response, true));
LOG.info("桥接服务获取异步解析结果完成: providerTaskId={}, preferredTextLength={}", LOG.info("桥接服务获取异步解析结果完成: providerTaskId={}, preferredTextLength={}",
taskId, resolveTextLength(result)); taskId, resolveTextLength(result));
@@ -174,11 +186,23 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
*/ */
@Override @Override
public DocumentParseTaskInfo queryTaskInfo(String taskId) { public DocumentParseTaskInfo queryTaskInfo(String taskId) {
return queryTaskInfo(taskId, null);
}
/**
* {@inheritDoc}
*/
@Override
public DocumentParseTaskInfo queryTaskInfo(String taskId, @Nullable DocumentSourceRef source) {
if (!StringUtils.hasText(taskId)) { if (!StringUtils.hasText(taskId)) {
throw DocumentParseBridgeException.taskFailed("taskId 不能为空"); throw DocumentParseBridgeException.taskFailed("taskId 不能为空");
} }
try { try {
ParseTaskInfo taskInfo = executeAgainstTaskService(taskId, service -> service.queryTaskInfo(taskId)); ParseTaskInfo taskInfo = executeAgainstTaskService(
taskId,
source,
service -> service.queryTaskInfo(taskId)
);
DocumentParseTaskInfo mappedTaskInfo = parseResultMapper.map(taskInfo); DocumentParseTaskInfo mappedTaskInfo = parseResultMapper.map(taskInfo);
LOG.info("桥接服务查询异步解析任务状态: providerTaskId={}, status={}, hasResult={}", LOG.info("桥接服务查询异步解析任务状态: providerTaskId={}, status={}, hasResult={}",
taskId, taskId,
@@ -223,6 +247,16 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
private DocumentParseService resolveService(LoadedDocumentSource loadedSource) { private DocumentParseService resolveService(LoadedDocumentSource loadedSource) {
DocumentParseSourceType sourceType = DocumentParseSourceType.resolve(loadedSource.getFileName(), loadedSource.getContentType()); DocumentParseSourceType sourceType = DocumentParseSourceType.resolve(loadedSource.getFileName(), loadedSource.getContentType());
return resolveService(sourceType);
}
/**
* 按文档源类型选择解析服务。
*
* @param sourceType 文档源类型
* @return 对应解析服务
*/
private DocumentParseService resolveService(DocumentParseSourceType sourceType) {
switch (sourceType) { switch (sourceType) {
case PDF: case PDF:
return requireSpecificService(pdfDocumentParseService, defaultDocumentParseService, "PDF"); return requireSpecificService(pdfDocumentParseService, defaultDocumentParseService, "PDF");
@@ -249,6 +283,28 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
throw DocumentParseBridgeException.serviceNotEnabled("未启用 " + sourceType + " 文档解析服务"); throw DocumentParseBridgeException.serviceNotEnabled("未启用 " + sourceType + " 文档解析服务");
} }
/**
* 在已知任务源信息时精确查询对应服务,缺少源信息时保留旧版兼容遍历。
*
* @param taskId 任务 ID
* @param source 提交任务时的文档源信息
* @param action 查询操作
* @param <T> 查询结果类型
* @return 查询结果
*/
private <T> T executeAgainstTaskService(String taskId,
@Nullable DocumentSourceRef source,
Function<DocumentParseService, T> action) {
if (source == null) {
return executeAgainstTaskService(taskId, action);
}
DocumentParseSourceType sourceType = DocumentParseSourceType.resolve(
source.getFileName(),
source.getContentType()
);
return action.apply(resolveService(sourceType));
}
private <T> T executeAgainstTaskService(String taskId, Function<DocumentParseService, T> action) { private <T> T executeAgainstTaskService(String taskId, Function<DocumentParseService, T> action) {
List<DocumentParseService> services = availableServices(); List<DocumentParseService> services = availableServices();
if (services.isEmpty()) { if (services.isEmpty()) {

View File

@@ -0,0 +1,82 @@
package tech.easyflow.ai.documentimport;
/**
* 批量导入建单上下文。
*
* @author Codex
* @since 2026-08-02
*/
public final class DocumentImportBatchCreateContext {
private final ImportCallerContext caller;
private final String idempotencyKeyHash;
private final String requestDigest;
private final String duplicatePolicy;
private final String requestedStrategyJson;
/**
* 创建批量导入建单上下文。
*
* @param caller 调用者上下文
* @param idempotencyKeyHash 幂等键哈希
* @param requestDigest 请求摘要
* @param duplicatePolicy 重复文件策略
* @param requestedStrategyJson 请求分块策略 JSON
*/
public DocumentImportBatchCreateContext(ImportCallerContext caller,
String idempotencyKeyHash,
String requestDigest,
String duplicatePolicy,
String requestedStrategyJson) {
this.caller = caller;
this.idempotencyKeyHash = idempotencyKeyHash;
this.requestDigest = requestDigest;
this.duplicatePolicy = duplicatePolicy;
this.requestedStrategyJson = requestedStrategyJson;
}
/**
* 获取调用者上下文。
*
* @return 调用者上下文
*/
public ImportCallerContext getCaller() {
return caller;
}
/**
* 获取幂等键哈希。
*
* @return 幂等键哈希
*/
public String getIdempotencyKeyHash() {
return idempotencyKeyHash;
}
/**
* 获取请求摘要。
*
* @return 请求摘要
*/
public String getRequestDigest() {
return requestDigest;
}
/**
* 获取重复文件策略。
*
* @return 重复文件策略
*/
public String getDuplicatePolicy() {
return duplicatePolicy;
}
/**
* 获取请求分块策略 JSON。
*
* @return 分块策略 JSON
*/
public String getRequestedStrategyJson() {
return requestedStrategyJson;
}
}

View File

@@ -0,0 +1,388 @@
package tech.easyflow.ai.documentimport;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* 文档批量导入接口数据对象。
*
* @author Codex
* @since 2026-07-31
*/
public final class DocumentImportBatchDtos {
private DocumentImportBatchDtos() {
}
/**
* 客户端文件清单项。
*/
public static class ManifestItem implements Serializable {
private String clientFileKey;
private String fileName;
private String relativePath;
private Long fileSize;
public String getClientFileKey() {
return clientFileKey;
}
public void setClientFileKey(String clientFileKey) {
this.clientFileKey = clientFileKey;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getRelativePath() {
return relativePath;
}
public void setRelativePath(String relativePath) {
this.relativePath = relativePath;
}
public Long getFileSize() {
return fileSize;
}
public void setFileSize(Long fileSize) {
this.fileSize = fileSize;
}
}
/**
* 创建批次请求。
*/
public static class CreateRequest implements Serializable {
private BigInteger knowledgeId;
private List<ManifestItem> files = new ArrayList<ManifestItem>();
public BigInteger getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
public List<ManifestItem> getFiles() {
return files;
}
public void setFiles(List<ManifestItem> files) {
this.files = files;
}
}
/**
* 服务端文件项。
*/
public static class ItemResponse implements Serializable {
private BigInteger itemId;
private BigInteger documentId;
private String clientFileKey;
private String fileName;
private String relativePath;
private Long fileSize;
private String stage;
private String status;
private String errorSummary;
public BigInteger getItemId() {
return itemId;
}
public void setItemId(BigInteger itemId) {
this.itemId = itemId;
}
public BigInteger getDocumentId() {
return documentId;
}
public void setDocumentId(BigInteger documentId) {
this.documentId = documentId;
}
public String getClientFileKey() {
return clientFileKey;
}
public void setClientFileKey(String clientFileKey) {
this.clientFileKey = clientFileKey;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getRelativePath() {
return relativePath;
}
public void setRelativePath(String relativePath) {
this.relativePath = relativePath;
}
public Long getFileSize() {
return fileSize;
}
public void setFileSize(Long fileSize) {
this.fileSize = fileSize;
}
public String getStage() {
return stage;
}
public void setStage(String stage) {
this.stage = stage;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public String getErrorSummary() {
return errorSummary;
}
public void setErrorSummary(String errorSummary) {
this.errorSummary = errorSummary;
}
}
/**
* 创建批次响应。
*/
public static class CreateResponse implements Serializable {
private BigInteger batchId;
private Integer uploadConcurrency;
private List<ItemResponse> items = new ArrayList<ItemResponse>();
public BigInteger getBatchId() {
return batchId;
}
public void setBatchId(BigInteger batchId) {
this.batchId = batchId;
}
public Integer getUploadConcurrency() {
return uploadConcurrency;
}
public void setUploadConcurrency(Integer uploadConcurrency) {
this.uploadConcurrency = uploadConcurrency;
}
public List<ItemResponse> getItems() {
return items;
}
public void setItems(List<ItemResponse> items) {
this.items = items;
}
}
/**
* 启动批次请求。
*/
public static class StartRequest implements Serializable {
private BigInteger knowledgeId;
private BigInteger batchId;
private String importMode;
private String duplicatePolicy;
public BigInteger getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
public BigInteger getBatchId() {
return batchId;
}
public void setBatchId(BigInteger batchId) {
this.batchId = batchId;
}
public String getImportMode() {
return importMode;
}
public void setImportMode(String importMode) {
this.importMode = importMode;
}
public String getDuplicatePolicy() {
return duplicatePolicy;
}
public void setDuplicatePolicy(String duplicatePolicy) {
this.duplicatePolicy = duplicatePolicy;
}
}
/**
* 批次状态响应。
*/
public static class StatusResponse implements Serializable {
private BigInteger batchId;
private String importMode;
private String status;
private Integer totalCount;
private Long totalBytes;
private Integer completedCount;
private Integer processingCount;
private Integer failedCount;
private Integer pendingCount;
private Integer skippedCount;
private Integer cancelledCount;
private Integer retryableFailedCount;
private Integer progressPercent;
private Date startedAt;
private Date finishedAt;
public BigInteger getBatchId() {
return batchId;
}
public void setBatchId(BigInteger batchId) {
this.batchId = batchId;
}
public String getImportMode() {
return importMode;
}
public void setImportMode(String importMode) {
this.importMode = importMode;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Integer getTotalCount() {
return totalCount;
}
public void setTotalCount(Integer totalCount) {
this.totalCount = totalCount;
}
public Long getTotalBytes() {
return totalBytes;
}
public void setTotalBytes(Long totalBytes) {
this.totalBytes = totalBytes;
}
public Integer getCompletedCount() {
return completedCount;
}
public void setCompletedCount(Integer completedCount) {
this.completedCount = completedCount;
}
public Integer getProcessingCount() {
return processingCount;
}
public void setProcessingCount(Integer processingCount) {
this.processingCount = processingCount;
}
public Integer getFailedCount() {
return failedCount;
}
public void setFailedCount(Integer failedCount) {
this.failedCount = failedCount;
}
public Integer getPendingCount() {
return pendingCount;
}
public void setPendingCount(Integer pendingCount) {
this.pendingCount = pendingCount;
}
public Integer getSkippedCount() {
return skippedCount;
}
public void setSkippedCount(Integer skippedCount) {
this.skippedCount = skippedCount;
}
public Integer getCancelledCount() {
return cancelledCount;
}
public void setCancelledCount(Integer cancelledCount) {
this.cancelledCount = cancelledCount;
}
public Integer getRetryableFailedCount() {
return retryableFailedCount;
}
public void setRetryableFailedCount(Integer retryableFailedCount) {
this.retryableFailedCount = retryableFailedCount;
}
public Integer getProgressPercent() {
return progressPercent;
}
public void setProgressPercent(Integer progressPercent) {
this.progressPercent = progressPercent;
}
public Date getStartedAt() {
return startedAt;
}
public void setStartedAt(Date startedAt) {
this.startedAt = startedAt;
}
public Date getFinishedAt() {
return finishedAt;
}
public void setFinishedAt(Date finishedAt) {
this.finishedAt = finishedAt;
}
}
}

View File

@@ -0,0 +1,71 @@
package tech.easyflow.ai.documentimport;
import java.math.BigInteger;
/**
* 文档批量导入重试的稳定响应快照。
*
* @author Codex
* @since 2026-08-02
*/
public class DocumentImportBatchRetryResult {
private final BigInteger taskId;
private final String status;
private final Integer retryGeneration;
private final Integer retriedCount;
/**
* 创建重试响应快照。
*
* @param taskId 批次任务 ID
* @param status 领取重试时的任务状态
* @param retryGeneration 重试代次
* @param retriedCount 本次领取的文件数
*/
public DocumentImportBatchRetryResult(BigInteger taskId,
String status,
Integer retryGeneration,
Integer retriedCount) {
this.taskId = taskId;
this.status = status;
this.retryGeneration = retryGeneration;
this.retriedCount = retriedCount;
}
/**
* 获取批次任务 ID。
*
* @return 批次任务 ID
*/
public BigInteger getTaskId() {
return taskId;
}
/**
* 获取领取重试时的任务状态。
*
* @return 任务状态
*/
public String getStatus() {
return status;
}
/**
* 获取重试代次。
*
* @return 重试代次
*/
public Integer getRetryGeneration() {
return retryGeneration;
}
/**
* 获取本次领取的文件数。
*
* @return 本次领取的文件数
*/
public Integer getRetriedCount() {
return retriedCount;
}
}

View File

@@ -15,6 +15,7 @@ public final class DocumentImportKeys {
public static final String KEY_DOCUMENT_STRATEGY_CODE = "splitter.strategyCode"; public static final String KEY_DOCUMENT_STRATEGY_CODE = "splitter.strategyCode";
public static final String KEY_DOCUMENT_STRATEGY_LABEL = "splitter.strategyLabel"; public static final String KEY_DOCUMENT_STRATEGY_LABEL = "splitter.strategyLabel";
public static final String KEY_DOCUMENT_STRATEGY_SNAPSHOT = "splitter.strategySnapshot"; public static final String KEY_DOCUMENT_STRATEGY_SNAPSHOT = "splitter.strategySnapshot";
public static final String KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH = "splitter.chunkSnapshotPath";
public static final String KEY_DOCUMENT_ANALYSIS_SUMMARY = "splitter.analysisSummary"; public static final String KEY_DOCUMENT_ANALYSIS_SUMMARY = "splitter.analysisSummary";
public static final String KEY_DOCUMENT_SOURCE_FILE_EXT = "splitter.sourceFileExt"; public static final String KEY_DOCUMENT_SOURCE_FILE_EXT = "splitter.sourceFileExt";
public static final String KEY_DOCUMENT_PREVIEW_VERSION = "splitter.previewVersion"; public static final String KEY_DOCUMENT_PREVIEW_VERSION = "splitter.previewVersion";
@@ -30,6 +31,7 @@ public final class DocumentImportKeys {
public static final String KEY_DOCUMENT_PARSE_PROCESSED_ITEMS = "parse.processedItems"; public static final String KEY_DOCUMENT_PARSE_PROCESSED_ITEMS = "parse.processedItems";
public static final String KEY_DOCUMENT_PARSE_TOTAL_ITEMS = "parse.totalItems"; public static final String KEY_DOCUMENT_PARSE_TOTAL_ITEMS = "parse.totalItems";
public static final String KEY_DOCUMENT_PARSE_STATUS_MESSAGE = "parse.statusMessage"; public static final String KEY_DOCUMENT_PARSE_STATUS_MESSAGE = "parse.statusMessage";
public static final String KEY_DOCUMENT_TASK_ERROR_CODE = "task.errorCode";
public static final String KEY_DOCUMENT_RENDER_MARKDOWN = "renderMarkdown"; public static final String KEY_DOCUMENT_RENDER_MARKDOWN = "renderMarkdown";
public static final String KEY_DOCUMENT_PAGE_INDEX = "pageIndex"; public static final String KEY_DOCUMENT_PAGE_INDEX = "pageIndex";
public static final String KEY_DOCUMENT_SHEET_NAME = "sheetName"; public static final String KEY_DOCUMENT_SHEET_NAME = "sheetName";
@@ -37,4 +39,8 @@ public final class DocumentImportKeys {
public static final String KEY_DOCUMENT_ROW_END = "rowEnd"; public static final String KEY_DOCUMENT_ROW_END = "rowEnd";
public static final String KEY_DOCUMENT_IMAGE_REFS = "imageRefs"; public static final String KEY_DOCUMENT_IMAGE_REFS = "imageRefs";
public static final String KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY = "parseArtifactSummary"; public static final String KEY_DOCUMENT_PARSE_ARTIFACT_SUMMARY = "parseArtifactSummary";
public static final String KEY_DOCUMENT_IMPORT_MODE = "import.mode";
public static final String KEY_DOCUMENT_IMPORT_BATCH_ID = "import.batchId";
public static final String KEY_DOCUMENT_IMPORT_BATCH_ITEM_ID = "import.batchItemId";
public static final String KEY_DOCUMENT_IMPORT_RELATIVE_PATH = "import.relativePath";
} }

View File

@@ -0,0 +1,50 @@
package tech.easyflow.ai.documentimport;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.math.BigInteger;
/**
* 文档批量导入调用者上下文。
*
* @author Codex
* @since 2026-08-02
*/
public final class ImportCallerContext {
private final ImportCallerType callerType;
private final BigInteger callerId;
/**
* 创建调用者上下文。
*
* @param callerType 调用者类型
* @param callerId 调用者 ID
* @throws BusinessException 调用者信息不完整时抛出
*/
public ImportCallerContext(ImportCallerType callerType, BigInteger callerId) {
if (callerType == null || callerId == null) {
throw new BusinessException("导入调用者信息不完整");
}
this.callerType = callerType;
this.callerId = callerId;
}
/**
* 获取调用者类型。
*
* @return 调用者类型
*/
public ImportCallerType getCallerType() {
return callerType;
}
/**
* 获取调用者 ID。
*
* @return 调用者 ID
*/
public BigInteger getCallerId() {
return callerId;
}
}

View File

@@ -0,0 +1,20 @@
package tech.easyflow.ai.documentimport;
/**
* 文档批量导入调用者类型。
*
* @author Codex
* @since 2026-08-02
*/
public enum ImportCallerType {
/**
* 管理端登录用户。
*/
ADMIN,
/**
* Public API 访问令牌。
*/
PUBLIC_API
}

View File

@@ -0,0 +1,455 @@
package tech.easyflow.ai.documentimport;
import com.easyagents.rag.ingestion.model.StrategyConfig;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* 知识库 Public API 批量导入数据对象。
*
* @author Codex
* @since 2026-08-02
*/
public final class PublicDocumentImportDtos {
private PublicDocumentImportDtos() {
}
/**
* Multipart 元数据。
*/
public static class BatchMetadata implements Serializable {
private BigInteger knowledgeId;
private StrategyConfig chunkStrategy = StrategyConfig.defaults();
private String duplicatePolicy = "SKIP";
private List<DocumentImportBatchDtos.ManifestItem> files = new ArrayList<>();
public BigInteger getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
public StrategyConfig getChunkStrategy() {
return chunkStrategy;
}
public void setChunkStrategy(StrategyConfig chunkStrategy) {
this.chunkStrategy = chunkStrategy == null
? StrategyConfig.defaults()
: chunkStrategy;
}
public String getDuplicatePolicy() {
return duplicatePolicy;
}
public void setDuplicatePolicy(String duplicatePolicy) {
this.duplicatePolicy = duplicatePolicy;
}
public List<DocumentImportBatchDtos.ManifestItem> getFiles() {
return files;
}
public void setFiles(List<DocumentImportBatchDtos.ManifestItem> files) {
this.files = files;
}
}
/**
* 批量提交响应。
*/
public static class SubmitResponse implements Serializable {
private BigInteger taskId;
private String status;
private Integer totalCount;
private Long totalBytes;
private Date createdAt;
public BigInteger getTaskId() {
return taskId;
}
public void setTaskId(BigInteger taskId) {
this.taskId = taskId;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Integer getTotalCount() {
return totalCount;
}
public void setTotalCount(Integer totalCount) {
this.totalCount = totalCount;
}
public Long getTotalBytes() {
return totalBytes;
}
public void setTotalBytes(Long totalBytes) {
this.totalBytes = totalBytes;
}
public Date getCreatedAt() {
return createdAt;
}
public void setCreatedAt(Date createdAt) {
this.createdAt = createdAt;
}
}
/**
* 批次计数。
*/
public static class Counts implements Serializable {
private Integer total;
private Integer completed;
private Integer processing;
private Integer pending;
private Integer failed;
private Integer skipped;
private Integer retryableFailed;
public Integer getTotal() {
return total;
}
public void setTotal(Integer total) {
this.total = total;
}
public Integer getCompleted() {
return completed;
}
public void setCompleted(Integer completed) {
this.completed = completed;
}
public Integer getProcessing() {
return processing;
}
public void setProcessing(Integer processing) {
this.processing = processing;
}
public Integer getPending() {
return pending;
}
public void setPending(Integer pending) {
this.pending = pending;
}
public Integer getFailed() {
return failed;
}
public void setFailed(Integer failed) {
this.failed = failed;
}
public Integer getSkipped() {
return skipped;
}
public void setSkipped(Integer skipped) {
this.skipped = skipped;
}
public Integer getRetryableFailed() {
return retryableFailed;
}
public void setRetryableFailed(Integer retryableFailed) {
this.retryableFailed = retryableFailed;
}
}
/**
* 文件失败信息。
*/
public static class ItemError implements Serializable {
private String code;
private String message;
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
}
/**
* 批次文件状态记录。
*/
public static class ItemRecord implements Serializable {
private String fileKey;
private String relativePath;
private BigInteger documentId;
private String stage;
private String status;
private Integer attemptCount;
private Boolean retryable;
private ItemError error;
public String getFileKey() {
return fileKey;
}
public void setFileKey(String fileKey) {
this.fileKey = fileKey;
}
public String getRelativePath() {
return relativePath;
}
public void setRelativePath(String relativePath) {
this.relativePath = relativePath;
}
public BigInteger getDocumentId() {
return documentId;
}
public void setDocumentId(BigInteger documentId) {
this.documentId = documentId;
}
public String getStage() {
return stage;
}
public void setStage(String stage) {
this.stage = stage;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Integer getAttemptCount() {
return attemptCount;
}
public void setAttemptCount(Integer attemptCount) {
this.attemptCount = attemptCount;
}
public Boolean getRetryable() {
return retryable;
}
public void setRetryable(Boolean retryable) {
this.retryable = retryable;
}
public ItemError getError() {
return error;
}
public void setError(ItemError error) {
this.error = error;
}
}
/**
* 批次文件状态分页。
*/
public static class ItemPage implements Serializable {
private Long pageNumber;
private Long pageSize;
private Long total;
private List<ItemRecord> records = new ArrayList<>();
public Long getPageNumber() {
return pageNumber;
}
public void setPageNumber(Long pageNumber) {
this.pageNumber = pageNumber;
}
public Long getPageSize() {
return pageSize;
}
public void setPageSize(Long pageSize) {
this.pageSize = pageSize;
}
public Long getTotal() {
return total;
}
public void setTotal(Long total) {
this.total = total;
}
public List<ItemRecord> getRecords() {
return records;
}
public void setRecords(List<ItemRecord> records) {
this.records = records;
}
}
/**
* 批次状态响应。
*/
public static class StatusResponse implements Serializable {
private BigInteger taskId;
private BigInteger knowledgeId;
private String status;
private Integer progressPercent;
private Counts counts;
private Boolean canRetry;
private ItemPage items;
public BigInteger getTaskId() {
return taskId;
}
public void setTaskId(BigInteger taskId) {
this.taskId = taskId;
}
public BigInteger getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Integer getProgressPercent() {
return progressPercent;
}
public void setProgressPercent(Integer progressPercent) {
this.progressPercent = progressPercent;
}
public Counts getCounts() {
return counts;
}
public void setCounts(Counts counts) {
this.counts = counts;
}
public Boolean getCanRetry() {
return canRetry;
}
public void setCanRetry(Boolean canRetry) {
this.canRetry = canRetry;
}
public ItemPage getItems() {
return items;
}
public void setItems(ItemPage items) {
this.items = items;
}
}
/**
* 异常任务重试请求。
*/
public static class RetryRequest implements Serializable {
private BigInteger taskId;
private List<String> fileKeys;
public BigInteger getTaskId() {
return taskId;
}
public void setTaskId(BigInteger taskId) {
this.taskId = taskId;
}
public List<String> getFileKeys() {
return fileKeys;
}
public void setFileKeys(List<String> fileKeys) {
this.fileKeys = fileKeys;
}
}
/**
* 异常任务重试响应。
*/
public static class RetryResponse implements Serializable {
private BigInteger taskId;
private String status;
private Integer retriedCount;
public BigInteger getTaskId() {
return taskId;
}
public void setTaskId(BigInteger taskId) {
this.taskId = taskId;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Integer getRetriedCount() {
return retriedCount;
}
public void setRetriedCount(Integer retriedCount) {
this.retriedCount = retriedCount;
}
}
}

View File

@@ -0,0 +1,498 @@
package tech.easyflow.ai.documentimport.task;
import com.mybatisflex.core.query.QueryWrapper;
import org.springframework.stereotype.Service;
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.enums.DocumentImportMode;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
import tech.easyflow.ai.service.DocumentImportBatchService;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.math.BigInteger;
import java.util.Date;
/**
* 批量导入文件项与批次汇总状态跟踪器。
*
* @author Codex
* @since 2026-07-31
*/
@Service
public class DocumentImportBatchTracker {
private final DocumentImportBatchService batchService;
private final DocumentImportBatchItemService itemService;
private final DocumentImportBatchMapper batchMapper;
private final DocumentImportBatchItemMapper itemMapper;
/**
* 创建批次状态跟踪器。
*
* @param batchService 批次服务
* @param itemService 批次项服务
* @param batchMapper 批次 Mapper
* @param itemMapper 批次项 Mapper
*/
public DocumentImportBatchTracker(DocumentImportBatchService batchService,
DocumentImportBatchItemService itemService,
DocumentImportBatchMapper batchMapper,
DocumentImportBatchItemMapper itemMapper) {
this.batchService = batchService;
this.itemService = itemService;
this.batchMapper = batchMapper;
this.itemMapper = itemMapper;
}
/**
* 查询批次。
*
* @param batchId 批次 ID
* @return 批次实体
*/
public DocumentImportBatch requireBatch(BigInteger batchId) {
DocumentImportBatch batch = batchId == null ? null : batchService.getById(batchId);
if (batch == null) {
throw new BusinessException("导入批次不存在");
}
return batch;
}
/**
* 查询批次文件项。
*
* @param itemId 文件项 ID
* @return 文件项实体
*/
public DocumentImportBatchItem requireItem(BigInteger itemId) {
DocumentImportBatchItem item = itemId == null ? null : itemService.getById(itemId);
if (item == null) {
throw new BusinessException("导入文件不存在");
}
return item;
}
/**
* 判断指定批次是否采用自动导入。
*
* @param batchId 批次 ID
* @return 是否自动导入
*/
public boolean isAutoBatch(BigInteger batchId) {
if (batchId == null) {
return false;
}
return DocumentImportMode.AUTO.name().equals(requireBatch(batchId).getImportMode());
}
/**
* 更新文件项阶段与状态。
*
* @param itemId 文件项 ID
* @param stage 处理阶段
* @param status 处理状态
* @param errorSummary 错误摘要
*/
@org.springframework.transaction.annotation.Transactional
public boolean updateItem(BigInteger itemId,
DocumentImportBatchItemStage stage,
DocumentImportBatchItemStatus status,
String errorSummary) {
if (itemId == null) {
return false;
}
DocumentImportBatchItem current = requireItem(itemId);
int attemptDelta = (status == DocumentImportBatchItemStatus.PENDING
|| status == DocumentImportBatchItemStatus.RUNNING)
&& DocumentImportBatchItemStatus.FAILED.name().equals(current.getStatus())
? 1
: 0;
return transitionItem(itemId, stage, status, errorSummary,
status == DocumentImportBatchItemStatus.FAILED, attemptDelta, null);
}
/**
* 按预期旧状态原子迁移文件项,并同步更新批次计数。
*
* @param itemId 文件项 ID
* @param stage 新阶段
* @param status 新状态
* @param errorSummary 错误摘要
* @param retryable 是否允许批量重试
* @param attemptDelta 重试次数增量
* @return 状态迁移或同状态刷新成功时返回 {@code true}
*/
@org.springframework.transaction.annotation.Transactional
public boolean transitionItem(BigInteger itemId,
DocumentImportBatchItemStage stage,
DocumentImportBatchItemStatus status,
String errorSummary,
boolean retryable,
int attemptDelta) {
return transitionItem(itemId, stage, status, errorSummary,
retryable, attemptDelta, null);
}
/**
* 按预期旧状态原子迁移文件项,并记录稳定失败码。
*
* @param itemId 文件项 ID
* @param stage 新阶段
* @param status 新状态
* @param errorSummary 错误摘要
* @param retryable 是否允许批量重试
* @param attemptDelta 重试次数增量
* @param failureCode 稳定失败码
* @return 状态迁移或同状态刷新成功时返回 {@code true}
*/
@org.springframework.transaction.annotation.Transactional
public boolean transitionItem(BigInteger itemId,
DocumentImportBatchItemStage stage,
DocumentImportBatchItemStatus status,
String errorSummary,
boolean retryable,
int attemptDelta,
String failureCode) {
if (itemId == null) {
return false;
}
for (int attempt = 0; attempt < 3; attempt++) {
DocumentImportBatchItem current = requireItem(itemId);
DocumentImportBatchItemStatus currentStatus =
DocumentImportBatchItemStatus.valueOf(current.getStatus());
if (!isAllowedTransition(currentStatus, status)) {
return false;
}
String expectedStatus = current.getStatus();
Date now = new Date();
int updated = itemMapper.transitionStatus(
itemId,
expectedStatus,
stage.name(),
status.name(),
errorSummary,
failureCode,
retryable,
Math.max(0, attemptDelta),
now
);
if (updated <= 0) {
continue;
}
CounterDelta delta = CounterDelta.between(current, status, retryable);
if (!delta.isZero()) {
batchMapper.adjustCounters(
current.getBatchId(),
delta.completed,
delta.processing,
delta.failed,
delta.pending,
delta.uploaded,
delta.skipped,
delta.cancelled,
delta.retryableFailed,
now
);
}
refreshBatch(current.getBatchId());
return true;
}
return false;
}
/**
* 校验文件项状态机,拒绝迟到任务覆盖终态。
*
* @param current 当前状态
* @param next 目标状态
* @return 是否允许迁移
*/
private boolean isAllowedTransition(DocumentImportBatchItemStatus current,
DocumentImportBatchItemStatus next) {
if (current == next) {
return true;
}
return switch (next) {
case UPLOADING -> current == DocumentImportBatchItemStatus.PENDING;
case UPLOADED -> current == DocumentImportBatchItemStatus.UPLOADING;
case RUNNING -> current == DocumentImportBatchItemStatus.PENDING
|| current == DocumentImportBatchItemStatus.FAILED;
case PENDING -> current == DocumentImportBatchItemStatus.RUNNING
|| current == DocumentImportBatchItemStatus.FAILED;
case FAILED, COMPLETED -> current == DocumentImportBatchItemStatus.RUNNING
|| current == DocumentImportBatchItemStatus.PENDING;
case SKIPPED -> current == DocumentImportBatchItemStatus.UPLOADED;
case CANCELLED -> current == DocumentImportBatchItemStatus.PENDING
|| current == DocumentImportBatchItemStatus.UPLOADING
|| current == DocumentImportBatchItemStatus.UPLOADED;
};
}
/**
* 将批次文件项绑定到创建后的文档。
*
* @param itemId 文件项 ID
* @param documentId 文档 ID
*/
@org.springframework.transaction.annotation.Transactional
public void bindDocument(BigInteger itemId, BigInteger documentId) {
DocumentImportBatchItem item = requireItem(itemId);
if (documentId.equals(item.getDocumentId())
&& DocumentImportBatchItemStatus.PENDING.name().equals(item.getStatus())) {
return;
}
Date now = new Date();
int updated = itemMapper.bindDocument(itemId, documentId, now);
if (updated <= 0) {
throw new BusinessException("导入文件状态已变化,请刷新后重试");
}
batchMapper.adjustCounters(item.getBatchId(),
0, 0, 0, 0, -1, 0, 0, 0, now);
refreshBatch(item.getBatchId());
}
/**
* 原子完成文件上传并增量更新批次上传数。
*
* @param itemId 文件项 ID
* @param filePath 存储路径
* @param storageLocator 可恢复存储定位符
* @return 文件项仍处于上传中且完成成功时返回 {@code true}
*/
@org.springframework.transaction.annotation.Transactional
public boolean completeUpload(BigInteger itemId,
String filePath,
String storageLocator) {
DocumentImportBatchItem item = requireItem(itemId);
if (DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus())) {
return filePath.equals(item.getFilePath())
&& storageLocator.equals(item.getStorageLocator());
}
if (!DocumentImportBatchItemStatus.UPLOADING.name().equals(item.getStatus())) {
return false;
}
Date now = new Date();
int updated = itemMapper.completeUpload(
itemId,
filePath,
storageLocator,
now
);
if (updated <= 0) {
return false;
}
batchMapper.adjustCounters(item.getBatchId(),
0, 0, 0, 0, 1, 0, 0, 0, now);
refreshBatch(item.getBatchId());
return true;
}
/**
* 记录文件项成功后需要清理的历史文档。
*
* @param itemId 文件项 ID
* @param replacedDocumentId 历史文档 ID
*/
@org.springframework.transaction.annotation.Transactional
public void markReplacement(BigInteger itemId, BigInteger replacedDocumentId) {
Date now = new Date();
int updated = itemMapper.markReplacement(itemId, replacedDocumentId, now);
if (updated <= 0) {
throw new BusinessException("重复文件状态已变化,请刷新后重试");
}
}
/**
* 清除已完成的历史文档覆盖标记。
*
* @param itemId 文件项 ID
* @param replacedDocumentId 历史文档 ID
*/
@org.springframework.transaction.annotation.Transactional
public void clearReplacement(BigInteger itemId, BigInteger replacedDocumentId) {
itemMapper.clearReplacement(itemId, replacedDocumentId, new Date());
}
/**
* 汇总并持久化批次状态。
*
* @param batchId 批次 ID
* @return 最新状态
*/
public DocumentImportBatchDtos.StatusResponse refreshBatch(BigInteger batchId) {
DocumentImportBatch batch = requireBatch(batchId);
Date now = new Date();
String nextStatus = batch.getStatus();
Date nextFinishedAt = batch.getFinishedAt();
if (batch.getImportMode() == null) {
if (!DocumentImportBatchStatus.CANCELLED.name().equals(batch.getStatus())) {
nextStatus = valueOrZero(batch.getUploadedCount()) == valueOrZero(batch.getTotalCount())
? DocumentImportBatchStatus.READY.name()
: DocumentImportBatchStatus.UPLOADING.name();
}
} else if (!DocumentImportBatchStatus.CANCELLED.name().equals(batch.getStatus())
&& !DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())) {
int terminalCount = valueOrZero(batch.getCompletedCount())
+ valueOrZero(batch.getFailedCount())
+ valueOrZero(batch.getSkippedCount())
+ valueOrZero(batch.getCancelledCount());
if (terminalCount >= valueOrZero(batch.getTotalCount())
&& valueOrZero(batch.getProcessingCount()) == 0
&& valueOrZero(batch.getPendingCount()) == 0) {
nextStatus = valueOrZero(batch.getFailedCount()) > 0
? DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name()
: DocumentImportBatchStatus.COMPLETED.name();
nextFinishedAt = now;
} else {
nextStatus = DocumentImportBatchStatus.RUNNING.name();
}
}
if (!java.util.Objects.equals(nextStatus, batch.getStatus())
|| !java.util.Objects.equals(nextFinishedAt, batch.getFinishedAt())) {
batch.setStatus(nextStatus);
batch.setFinishedAt(nextFinishedAt);
batch.setModified(now);
batchService.updateById(batch, false);
}
return toStatusResponse(batch);
}
/**
* 将批次标记为已中断,保留已完成文件并允许批量继续。
*
* @param batchId 批次 ID
*/
public void markInterrupted(BigInteger batchId) {
if (batchId == null) {
return;
}
Date now = new Date();
DocumentImportBatch update = new DocumentImportBatch();
update.setStatus(DocumentImportBatchStatus.INTERRUPTED.name());
update.setModified(now);
batchMapper.updateByQuery(update,
QueryWrapper.create()
.eq(DocumentImportBatch::getId, batchId)
.eq(DocumentImportBatch::getStatus, DocumentImportBatchStatus.RUNNING.name()));
}
/**
* 将批次转换为状态响应。
*
* @param batch 批次实体
* @return 状态响应
*/
public DocumentImportBatchDtos.StatusResponse toStatusResponse(DocumentImportBatch batch) {
DocumentImportBatchDtos.StatusResponse response = new DocumentImportBatchDtos.StatusResponse();
response.setBatchId(batch.getId());
response.setImportMode(batch.getImportMode());
response.setStatus(batch.getStatus());
response.setTotalCount(valueOrZero(batch.getTotalCount()));
response.setTotalBytes(batch.getTotalBytes() == null ? 0L : batch.getTotalBytes());
response.setCompletedCount(valueOrZero(batch.getCompletedCount()));
response.setProcessingCount(valueOrZero(batch.getProcessingCount()));
response.setFailedCount(valueOrZero(batch.getFailedCount()));
response.setPendingCount(valueOrZero(batch.getPendingCount()));
response.setSkippedCount(valueOrZero(batch.getSkippedCount()));
response.setCancelledCount(valueOrZero(batch.getCancelledCount()));
response.setRetryableFailedCount(valueOrZero(batch.getRetryableFailedCount()));
int total = Math.max(1, valueOrZero(batch.getTotalCount()));
int terminalCount = valueOrZero(batch.getCompletedCount())
+ valueOrZero(batch.getFailedCount())
+ valueOrZero(batch.getSkippedCount())
+ valueOrZero(batch.getCancelledCount());
response.setProgressPercent(Math.min(100, terminalCount * 100 / total));
response.setStartedAt(batch.getStartedAt());
response.setFinishedAt(batch.getFinishedAt());
return response;
}
private int valueOrZero(Integer value) {
return value == null ? 0 : value;
}
/**
* 文件项状态变化对应的批次计数增量。
*/
private static final class CounterDelta {
private int completed;
private int processing;
private int failed;
private int pending;
private int uploaded;
private int skipped;
private int cancelled;
private int retryableFailed;
/**
* 计算文件项状态迁移前后的计数差值。
*
* @param current 当前文件项
* @param nextStatus 新状态
* @param nextRetryable 新状态是否允许重试
* @return 计数差值
*/
private static CounterDelta between(DocumentImportBatchItem current,
DocumentImportBatchItemStatus nextStatus,
boolean nextRetryable) {
CounterDelta delta = new CounterDelta();
apply(delta, DocumentImportBatchItemStatus.valueOf(current.getStatus()),
Boolean.TRUE.equals(current.getRetryable()), -1);
apply(delta, nextStatus, nextRetryable, 1);
return delta;
}
/**
* 将一个状态映射到对应计数桶。
*
* @param delta 待修改的差值
* @param status 文件项状态
* @param retryable 是否允许重试
* @param direction 增加或减少方向
*/
private static void apply(CounterDelta delta,
DocumentImportBatchItemStatus status,
boolean retryable,
int direction) {
switch (status) {
case COMPLETED -> delta.completed += direction;
case RUNNING -> delta.processing += direction;
case FAILED -> {
delta.failed += direction;
if (retryable) {
delta.retryableFailed += direction;
}
}
case SKIPPED -> delta.skipped += direction;
case CANCELLED -> delta.cancelled += direction;
case PENDING, UPLOADING, UPLOADED -> delta.pending += direction;
default -> throw new IllegalStateException("未知批次文件状态: " + status);
}
if (status == DocumentImportBatchItemStatus.UPLOADED) {
delta.uploaded += direction;
}
}
/**
* 判断所有增量是否均为零。
*
* @return 是否没有计数变化
*/
private boolean isZero() {
return completed == 0
&& processing == 0
&& failed == 0
&& pending == 0
&& uploaded == 0
&& skipped == 0
&& cancelled == 0
&& retryableFailed == 0;
}
}
}

View File

@@ -0,0 +1,192 @@
package tech.easyflow.ai.documentimport.task;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.util.unit.DataSize;
import java.time.Duration;
/**
* 文档批量导入容量与并发缺省配置。
*
* @author Codex
* @since 2026-07-31
*/
@ConfigurationProperties(prefix = "easyflow.ai.document-import.bulk")
public class DocumentImportBulkProperties {
private int maxFileCount = 2000;
private DataSize maxTotalSize = DataSize.ofGigabytes(1);
private DataSize maxFileSize = DataSize.ofMegabytes(100);
private int uploadConcurrency = 3;
private int parseMaxRunning = 2;
private int splitMaxRunning = 2;
private int indexMaxRunning = 2;
private int perBatchParseMaxRunning = 2;
private int pendingDispatchBatchSize = 100;
private Duration pendingDispatchInterval = Duration.ofSeconds(2);
private Duration pendingRedispatchDelay = Duration.ofSeconds(5);
private Duration pendingTimeout = Duration.ofHours(24);
private Duration parseSubmitTimeout = Duration.ofSeconds(120);
private Duration interruptionTimeout = Duration.ofMinutes(10);
private int maxTaskAttempts = 3;
public int getMaxFileCount() {
return maxFileCount;
}
public void setMaxFileCount(int maxFileCount) {
this.maxFileCount = maxFileCount;
}
public DataSize getMaxTotalSize() {
return maxTotalSize;
}
public void setMaxTotalSize(DataSize maxTotalSize) {
this.maxTotalSize = maxTotalSize;
}
public DataSize getMaxFileSize() {
return maxFileSize;
}
public void setMaxFileSize(DataSize maxFileSize) {
this.maxFileSize = maxFileSize;
}
public int getUploadConcurrency() {
return uploadConcurrency;
}
public void setUploadConcurrency(int uploadConcurrency) {
this.uploadConcurrency = uploadConcurrency;
}
public int getParseMaxRunning() {
return parseMaxRunning;
}
public void setParseMaxRunning(int parseMaxRunning) {
this.parseMaxRunning = parseMaxRunning;
}
/**
* 获取全局分块任务并发上限。
*
* @return 分块任务并发上限
*/
public int getSplitMaxRunning() {
return splitMaxRunning;
}
/**
* 设置全局分块任务并发上限。
*
* @param splitMaxRunning 分块任务并发上限
*/
public void setSplitMaxRunning(int splitMaxRunning) {
this.splitMaxRunning = splitMaxRunning;
}
public int getIndexMaxRunning() {
return indexMaxRunning;
}
public void setIndexMaxRunning(int indexMaxRunning) {
this.indexMaxRunning = indexMaxRunning;
}
public int getPerBatchParseMaxRunning() {
return perBatchParseMaxRunning;
}
public void setPerBatchParseMaxRunning(int perBatchParseMaxRunning) {
this.perBatchParseMaxRunning = perBatchParseMaxRunning;
}
public int getPendingDispatchBatchSize() {
return pendingDispatchBatchSize;
}
public void setPendingDispatchBatchSize(int pendingDispatchBatchSize) {
this.pendingDispatchBatchSize = pendingDispatchBatchSize;
}
public Duration getPendingDispatchInterval() {
return pendingDispatchInterval;
}
public void setPendingDispatchInterval(Duration pendingDispatchInterval) {
this.pendingDispatchInterval = pendingDispatchInterval;
}
public Duration getPendingRedispatchDelay() {
return pendingRedispatchDelay;
}
public void setPendingRedispatchDelay(Duration pendingRedispatchDelay) {
this.pendingRedispatchDelay = pendingRedispatchDelay;
}
/**
* 获取待处理任务最长排队时间。
*
* @return 最长排队时间
*/
public Duration getPendingTimeout() {
return pendingTimeout;
}
/**
* 设置待处理任务最长排队时间。
*
* @param pendingTimeout 最长排队时间
*/
public void setPendingTimeout(Duration pendingTimeout) {
this.pendingTimeout = pendingTimeout;
}
/**
* 获取解析服务任务提交超时时间。
*
* @return 任务提交超时时间
*/
public Duration getParseSubmitTimeout() {
return parseSubmitTimeout;
}
/**
* 设置解析服务任务提交超时时间。
*
* @param parseSubmitTimeout 任务提交超时时间
*/
public void setParseSubmitTimeout(Duration parseSubmitTimeout) {
this.parseSubmitTimeout = parseSubmitTimeout;
}
public Duration getInterruptionTimeout() {
return interruptionTimeout;
}
public void setInterruptionTimeout(Duration interruptionTimeout) {
this.interruptionTimeout = interruptionTimeout;
}
/**
* 获取单文件每阶段最大执行次数。
*
* @return 最大执行次数
*/
public int getMaxTaskAttempts() {
return maxTaskAttempts;
}
/**
* 设置单文件每阶段最大执行次数。
*
* @param maxTaskAttempts 最大执行次数
*/
public void setMaxTaskAttempts(int maxTaskAttempts) {
this.maxTaskAttempts = maxTaskAttempts;
}
}

View File

@@ -0,0 +1,94 @@
package tech.easyflow.ai.documentimport.task;
import com.alibaba.fastjson2.JSON;
import org.springframework.stereotype.Service;
import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.easyagents.CustomMultipartFile;
import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.util.StringUtil;
import tech.easyflow.common.web.exceptions.BusinessException;
import javax.annotation.Resource;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
/**
* 自动导入分块快照持久化服务。
*
* <p>快照写入对象存储,向量化任务只保存稳定路径,避免依赖短期预览缓存。</p>
*
* @author Codex
* @since 2026-07-31
*/
@Service
public class DocumentImportChunkSnapshotService {
private static final long MAX_SNAPSHOT_BYTES = 256L * 1024L * 1024L;
@Resource(name = "default")
private FileStorageService storageService;
/**
* 持久化预览会话及其最终分块。
*
* @param session 预览会话
* @return 快照存储路径
*/
public String save(DocumentImportDtos.PreviewSession session) {
if (session == null || session.getKnowledgeId() == null || session.getDocumentId() == null
|| session.getDocumentChunks() == null || session.getDocumentChunks().isEmpty()) {
throw new BusinessException("分块快照内容不完整");
}
String fileName = session.getDocumentId() + "-chunks.json";
byte[] payload = JSON.toJSONBytes(session);
if (payload.length > MAX_SNAPSHOT_BYTES) {
throw new BusinessException("分块快照过大,请调整文档后重试");
}
CustomMultipartFile file = new CustomMultipartFile(
payload, fileName, fileName, "application/json");
String path = storageService.save(file,
"knowledge-import-snapshots/" + session.getKnowledgeId() + "/" + session.getDocumentId());
if (!StringUtil.hasText(path)) {
throw new BusinessException("分块快照保存失败");
}
return path;
}
/**
* 从稳定存储恢复预览会话。
*
* @param path 快照路径
* @return 预览会话
*/
public DocumentImportDtos.PreviewSession load(String path) {
if (!StringUtil.hasText(path)) {
throw new BusinessException("分块快照不存在,请重试");
}
try (InputStream inputStream = storageService.readStream(path)) {
byte[] payload = DocumentInputStreamSupport.readBytes(inputStream, MAX_SNAPSHOT_BYTES);
String json = new String(payload, StandardCharsets.UTF_8);
DocumentImportDtos.PreviewSession session =
JSON.parseObject(json, DocumentImportDtos.PreviewSession.class);
if (session == null || session.getDocumentChunks() == null
|| session.getDocumentChunks().isEmpty()) {
throw new BusinessException("分块快照无有效内容,请重试");
}
return session;
} catch (IOException error) {
throw new BusinessException("分块快照读取失败,请重试");
}
}
/**
* 删除已完成向量化的分块快照。
*
* @param path 快照路径
*/
public void delete(String path) {
if (StringUtil.hasText(path)) {
storageService.delete(path);
}
}
}

View File

@@ -0,0 +1,59 @@
package tech.easyflow.ai.documentimport.task;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tech.easyflow.common.cache.DistributedScheduledLock;
/**
* 文档导入待处理任务恢复调度器。
*
* @author Codex
* @since 2026-07-31
*/
@Component
public class DocumentImportPendingTaskMonitor {
private final KnowledgeDocumentImportTaskAppService appService;
/**
* 创建文档导入恢复调度器。
*
* @param appService 文档导入任务服务
*/
public DocumentImportPendingTaskMonitor(KnowledgeDocumentImportTaskAppService appService) {
this.appService = appService;
}
/**
* 周期性投递仍处于等待状态的解析与向量化任务。
*/
@Scheduled(
fixedDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}",
initialDelayString = "${easyflow.ai.document-import.bulk.pending-dispatch-interval:2s}"
)
@DistributedScheduledLock(
key = "easyflow:schedule:document-import:pending-dispatch",
leaseSeconds = 30L
)
public void dispatchPendingTasks() {
appService.dispatchPendingTasks();
}
/**
* 检测排队超时与长时间无心跳的任务。
*/
@Scheduled(
fixedDelayString = "${easyflow.ai.document-import.bulk.interruption-scan-interval:60s}",
initialDelayString = "${easyflow.ai.document-import.bulk.interruption-scan-interval:60s}"
)
@DistributedScheduledLock(
key = "easyflow:schedule:document-import:interruption-scan",
leaseSeconds = 30L
)
public void recoverTimedOutTasks() {
appService.expireTimedOutParseSubmissions();
appService.recoverInterruptedTasks();
appService.expireTimedOutPendingTasks();
appService.cleanupCompletedReplacements();
}
}

View File

@@ -0,0 +1,94 @@
package tech.easyflow.ai.documentimport.task;
import com.alibaba.fastjson2.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.common.mq.config.MQProperties;
import tech.easyflow.common.mq.core.MQConsumerHandler;
import tech.easyflow.common.mq.core.MQMessage;
import tech.easyflow.common.mq.core.MQSubscription;
import java.util.List;
/**
* 文档分块任务消费者。
*
* @author Codex
* @since 2026-08-02
*/
@Component
public class DocumentImportSplitTaskConsumer implements MQConsumerHandler {
private static final Logger LOG =
LoggerFactory.getLogger(DocumentImportSplitTaskConsumer.class);
private final KnowledgeDocumentImportTaskAppService appService;
private final MQProperties mqProperties;
/**
* 创建分块任务消费者。
*
* @param appService 文档导入应用服务
* @param mqProperties MQ 配置
*/
public DocumentImportSplitTaskConsumer(
KnowledgeDocumentImportTaskAppService appService,
MQProperties mqProperties) {
this.appService = appService;
this.mqProperties = mqProperties;
}
/**
* 获取分块任务订阅。
*
* @return MQ 订阅信息
*/
@Override
public MQSubscription subscription() {
MQSubscription subscription = new MQSubscription();
subscription.setTopic(DocumentImportTaskMqConstants.SPLIT_TOPIC);
subscription.setConsumerGroup(DocumentImportTaskMqConstants.SPLIT_GROUP);
subscription.setShardCount(resolveShardCount());
return subscription;
}
/**
* 处理一批分块任务消息。
*
* @param messages MQ 消息
*/
@Override
public void handle(List<MQMessage> messages) {
LOG.info("文档分块消费者收到消息批次: count={}",
messages == null ? 0 : messages.size());
for (MQMessage message : messages) {
DocumentImportTaskMessage event =
JSON.parseObject(message.getBody(), DocumentImportTaskMessage.class);
if (event == null || event.getTaskId() == null) {
LOG.warn("文档分块消费者跳过非法消息: streamMessageId={}, messageId={}",
message == null ? null : message.getStreamMessageId(),
message == null ? null : message.getMessageId());
continue;
}
try {
appService.handleSplitTask(event.getTaskId());
} catch (Exception error) {
LOG.error("文档分块消费者处理失败: taskId={}, messageId={}, streamMessageId={}",
event.getTaskId(), message.getMessageId(),
message.getStreamMessageId(), error);
throw error;
}
}
}
/**
* 获取当前 Redis Stream 分片数。
*
* @return 分片数
*/
private int resolveShardCount() {
return Math.max(
mqProperties.getRedis().getChatPersistShardCount(), 1);
}
}

View File

@@ -0,0 +1,58 @@
package tech.easyflow.ai.documentimport.task;
import com.alibaba.fastjson2.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import tech.easyflow.common.mq.core.MQMessage;
import tech.easyflow.common.mq.core.MQProducer;
import java.math.BigInteger;
import java.util.Date;
/**
* 文档分块任务消息生产者。
*
* @author Codex
* @since 2026-08-02
*/
@Service
public class DocumentImportSplitTaskProducer {
private static final Logger LOG =
LoggerFactory.getLogger(DocumentImportSplitTaskProducer.class);
private final MQProducer mqProducer;
/**
* 创建分块任务消息生产者。
*
* @param mqProducer MQ 生产者
*/
public DocumentImportSplitTaskProducer(MQProducer mqProducer) {
this.mqProducer = mqProducer;
}
/**
* 发送分块任务消息。
*
* @param taskId 任务 ID
*/
public void send(BigInteger taskId) {
DocumentImportTaskMessage event = new DocumentImportTaskMessage();
event.setTaskId(taskId);
event.setOccurredAt(new Date());
MQMessage message = new MQMessage();
message.setMessageId("split-" + taskId);
message.setTopic(DocumentImportTaskMqConstants.SPLIT_TOPIC);
message.setKey(String.valueOf(taskId));
message.setCreatedAt(event.getOccurredAt());
message.setBody(JSON.toJSONString(event));
LOG.info("准备投递文档分块 MQ 消息: topic={}, taskId={}, messageId={}",
message.getTopic(), taskId, message.getMessageId());
String recordId = mqProducer.send(message);
LOG.info("文档分块 MQ 消息投递完成: topic={}, taskId={}, messageId={}, recordId={}",
message.getTopic(), taskId, message.getMessageId(), recordId);
}
}

View File

@@ -0,0 +1,120 @@
package tech.easyflow.ai.documentimport.task;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tech.easyflow.ai.documentimport.ImportCallerContext;
import tech.easyflow.ai.documentimport.ImportCallerType;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.common.cache.DistributedScheduledLock;
import tech.easyflow.common.util.StringUtil;
import java.time.Duration;
import java.util.Date;
import java.util.List;
/**
* 文档导入未完成批次与取消对象清理调度器。
*
* @author Codex
* @since 2026-08-02
*/
@Component
public class DocumentImportStaleBatchMonitor {
private static final Logger LOG =
LoggerFactory.getLogger(DocumentImportStaleBatchMonitor.class);
private static final Duration INCOMPLETE_TIMEOUT = Duration.ofMinutes(30);
private static final int BATCH_SIZE = 100;
private final DocumentImportBatchMapper batchMapper;
private final DocumentImportBatchAppService batchAppService;
/**
* 创建未完成批次清理调度器。
*
* @param batchMapper 批次 Mapper
* @param batchAppService 批次应用服务
*/
public DocumentImportStaleBatchMonitor(
DocumentImportBatchMapper batchMapper,
DocumentImportBatchAppService batchAppService) {
this.batchMapper = batchMapper;
this.batchAppService = batchAppService;
}
/**
* 分批回收长时间无进展的上传批次,并重试对象清理。
*/
@Scheduled(
fixedDelayString =
"${easyflow.ai.document-import.bulk.stale-batch-scan-interval:60s}",
initialDelayString =
"${easyflow.ai.document-import.bulk.stale-batch-scan-interval:60s}"
)
@DistributedScheduledLock(
key = "easyflow:schedule:document-import:stale-batch-cleanup",
leaseSeconds = 50L
)
public void cleanupStaleBatches() {
Date cutoff = new Date(
System.currentTimeMillis() - INCOMPLETE_TIMEOUT.toMillis()
);
List<DocumentImportBatch> candidates =
batchMapper.selectStaleIncompleteBatches(cutoff, BATCH_SIZE);
for (DocumentImportBatch batch : candidates) {
cancelCandidate(batch, cutoff);
}
batchAppService.cleanupCancelledStoredObjects(BATCH_SIZE);
}
/**
* 通过数据库条件更新尝试取消单个候选批次。
*
* @param batch 候选批次
* @param cutoff 最后进展截止时间
*/
private void cancelCandidate(DocumentImportBatch batch, Date cutoff) {
if (batch.getCallerType() == null || batch.getCallerId() == null) {
LOG.warn("跳过调用者信息不完整的超时导入批次: batchId={}", batch.getId());
return;
}
try {
ImportCallerContext caller = new ImportCallerContext(
ImportCallerType.valueOf(batch.getCallerType()),
batch.getCallerId()
);
boolean cancelled = batchAppService.cancelStaleBatch(
batch.getKnowledgeId(),
batch.getId(),
caller,
cutoff
);
if (cancelled && StringUtil.hasText(batch.getIdempotencyKeyHash())) {
int released = batchMapper.releaseSubmissionFingerprint(
batch.getId(),
batch.getIdempotencyKeyHash(),
cutoff,
new Date()
);
if (released <= 0) {
LOG.warn(
"超时导入批次已取消但提交指纹未释放: batchId={}",
batch.getId()
);
}
}
} catch (IllegalArgumentException error) {
LOG.error(
"超时导入批次调用者类型无效: batchId={}, callerType={}",
batch.getId(),
batch.getCallerType(),
error
);
} catch (RuntimeException error) {
LOG.error("回收超时导入批次失败: batchId={}", batch.getId(), error);
}
}
}

View File

@@ -13,6 +13,8 @@ public final class DocumentImportTaskMqConstants {
public static final String PARSE_TOPIC = "knowledge-document-parse"; public static final String PARSE_TOPIC = "knowledge-document-parse";
public static final String PARSE_GROUP = "knowledge-document-parse-group"; public static final String PARSE_GROUP = "knowledge-document-parse-group";
public static final String SPLIT_TOPIC = "knowledge-document-split";
public static final String SPLIT_GROUP = "knowledge-document-split-group";
public static final String INDEX_TOPIC = "knowledge-document-index"; public static final String INDEX_TOPIC = "knowledge-document-index";
public static final String INDEX_GROUP = "knowledge-document-index-group"; public static final String INDEX_GROUP = "knowledge-document-index-group";
} }

View File

@@ -147,6 +147,7 @@ public class DocumentImportTaskStatusStreamService {
payload.put("parseCurrentStage", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_PARSE_CURRENT_STAGE)); payload.put("parseCurrentStage", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_PARSE_CURRENT_STAGE));
payload.put("parseStatusMessage", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_PARSE_STATUS_MESSAGE)); payload.put("parseStatusMessage", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_PARSE_STATUS_MESSAGE));
payload.put("lastTaskError", document.getLastTaskError()); payload.put("lastTaskError", document.getLastTaskError());
payload.put("lastTaskErrorCode", readOptionAsString(document, DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE));
payload.put("taskModifiedAt", document.getTaskModifiedAt()); payload.put("taskModifiedAt", document.getTaskModifiedAt());
return payload; return payload;
} }

View File

@@ -0,0 +1,333 @@
package tech.easyflow.ai.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
/**
* 知识库文档批量导入批次。
*
* @author Codex
* @since 2026-07-31
*/
@Table(value = "tb_document_import_batch", comment = "知识库文档批量导入批次")
public class DocumentImportBatch extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "知识库ID")
private BigInteger knowledgeId;
@Column(comment = "调用者类型")
private String callerType;
@Column(comment = "调用者ID")
private BigInteger callerId;
@Column(comment = "幂等键哈希")
private String idempotencyKeyHash;
@Column(comment = "请求摘要")
private String requestDigest;
@Column(comment = "重复文件策略")
private String duplicatePolicy;
@Column(comment = "请求分块策略")
private String requestedStrategyJson;
@Column(comment = "重试代次")
private Integer retryGeneration;
@Column(comment = "乐观锁版本")
private Integer version;
@Column(comment = "导入模式")
private String importMode;
@Column(comment = "批次状态")
private String status;
@Column(comment = "文件总数")
private Integer totalCount;
@Column(comment = "文件总字节数")
private Long totalBytes;
@Column(comment = "完成数")
private Integer completedCount;
@Column(comment = "处理中数量")
private Integer processingCount;
@Column(comment = "失败数")
private Integer failedCount;
@Column(comment = "等待数")
private Integer pendingCount;
@Column(comment = "已上传数")
private Integer uploadedCount;
@Column(comment = "跳过数")
private Integer skippedCount;
@Column(comment = "取消数")
private Integer cancelledCount;
@Column(comment = "可重试失败数")
private Integer retryableFailedCount;
@Column(comment = "开始时间")
private Date startedAt;
@Column(comment = "结束时间")
private Date finishedAt;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() {
return id;
}
public void setId(BigInteger id) {
this.id = id;
}
public BigInteger getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
public String getCallerType() {
return callerType;
}
public void setCallerType(String callerType) {
this.callerType = callerType;
}
public BigInteger getCallerId() {
return callerId;
}
public void setCallerId(BigInteger callerId) {
this.callerId = callerId;
}
public String getIdempotencyKeyHash() {
return idempotencyKeyHash;
}
public void setIdempotencyKeyHash(String idempotencyKeyHash) {
this.idempotencyKeyHash = idempotencyKeyHash;
}
public String getRequestDigest() {
return requestDigest;
}
public void setRequestDigest(String requestDigest) {
this.requestDigest = requestDigest;
}
public String getDuplicatePolicy() {
return duplicatePolicy;
}
public void setDuplicatePolicy(String duplicatePolicy) {
this.duplicatePolicy = duplicatePolicy;
}
public String getRequestedStrategyJson() {
return requestedStrategyJson;
}
public void setRequestedStrategyJson(String requestedStrategyJson) {
this.requestedStrategyJson = requestedStrategyJson;
}
public Integer getRetryGeneration() {
return retryGeneration;
}
public void setRetryGeneration(Integer retryGeneration) {
this.retryGeneration = retryGeneration;
}
public Integer getVersion() {
return version;
}
public void setVersion(Integer version) {
this.version = version;
}
public String getImportMode() {
return importMode;
}
public void setImportMode(String importMode) {
this.importMode = importMode;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Integer getTotalCount() {
return totalCount;
}
public void setTotalCount(Integer totalCount) {
this.totalCount = totalCount;
}
public Long getTotalBytes() {
return totalBytes;
}
public void setTotalBytes(Long totalBytes) {
this.totalBytes = totalBytes;
}
public Integer getCompletedCount() {
return completedCount;
}
public void setCompletedCount(Integer completedCount) {
this.completedCount = completedCount;
}
public Integer getProcessingCount() {
return processingCount;
}
public void setProcessingCount(Integer processingCount) {
this.processingCount = processingCount;
}
public Integer getFailedCount() {
return failedCount;
}
public void setFailedCount(Integer failedCount) {
this.failedCount = failedCount;
}
public Integer getPendingCount() {
return pendingCount;
}
public void setPendingCount(Integer pendingCount) {
this.pendingCount = pendingCount;
}
public Integer getUploadedCount() {
return uploadedCount;
}
public void setUploadedCount(Integer uploadedCount) {
this.uploadedCount = uploadedCount;
}
public Integer getSkippedCount() {
return skippedCount;
}
public void setSkippedCount(Integer skippedCount) {
this.skippedCount = skippedCount;
}
public Integer getCancelledCount() {
return cancelledCount;
}
public void setCancelledCount(Integer cancelledCount) {
this.cancelledCount = cancelledCount;
}
public Integer getRetryableFailedCount() {
return retryableFailedCount;
}
public void setRetryableFailedCount(Integer retryableFailedCount) {
this.retryableFailedCount = retryableFailedCount;
}
public Date getStartedAt() {
return startedAt;
}
public void setStartedAt(Date startedAt) {
this.startedAt = startedAt;
}
public Date getFinishedAt() {
return finishedAt;
}
public void setFinishedAt(Date finishedAt) {
this.finishedAt = finishedAt;
}
@Override
public Date getCreated() {
return created;
}
@Override
public void setCreated(Date created) {
this.created = created;
}
public BigInteger getCreatedBy() {
return createdBy;
}
public void setCreatedBy(BigInteger createdBy) {
this.createdBy = createdBy;
}
@Override
public Date getModified() {
return modified;
}
@Override
public void setModified(Date modified) {
this.modified = modified;
}
public BigInteger getModifiedBy() {
return modifiedBy;
}
public void setModifiedBy(BigInteger modifiedBy) {
this.modifiedBy = modifiedBy;
}
}

View File

@@ -0,0 +1,300 @@
package tech.easyflow.ai.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
/**
* 知识库文档批量导入文件项。
*
* @author Codex
* @since 2026-07-31
*/
@Table(value = "tb_document_import_batch_item", comment = "知识库文档批量导入文件项")
public class DocumentImportBatchItem extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "批次ID")
private BigInteger batchId;
@Column(comment = "知识库ID")
private BigInteger knowledgeId;
@Column(comment = "文档ID")
private BigInteger documentId;
@Column(comment = "待覆盖的历史文档ID")
private BigInteger replacedDocumentId;
@Column(comment = "客户端文件键")
private String clientFileKey;
@Column(comment = "文件名")
private String fileName;
@Column(comment = "文件夹相对路径")
private String relativePath;
@Column(comment = "文件大小")
private Long fileSize;
@Column(comment = "存储路径")
private String filePath;
@Column(comment = "可恢复存储定位符")
private String storageLocator;
@Column(comment = "是否等待清理存储对象")
private Boolean cleanupPending;
@Column(comment = "文件内容SHA-256")
private String contentSha256;
@Column(comment = "当前阶段")
private String stage;
@Column(comment = "当前状态")
private String status;
@Column(comment = "错误摘要")
private String errorSummary;
@Column(comment = "稳定失败码")
private String failureCode;
@Column(comment = "实际分块策略编码")
private String appliedStrategyCode;
@Column(comment = "分块策略快照")
private String strategySnapshotJson;
@Column(comment = "是否允许批量重试")
private Boolean retryable;
@Column(comment = "重试次数")
private Integer attemptCount;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() {
return id;
}
public void setId(BigInteger id) {
this.id = id;
}
public BigInteger getBatchId() {
return batchId;
}
public void setBatchId(BigInteger batchId) {
this.batchId = batchId;
}
public BigInteger getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(BigInteger knowledgeId) {
this.knowledgeId = knowledgeId;
}
public BigInteger getDocumentId() {
return documentId;
}
public void setDocumentId(BigInteger documentId) {
this.documentId = documentId;
}
public BigInteger getReplacedDocumentId() {
return replacedDocumentId;
}
public void setReplacedDocumentId(BigInteger replacedDocumentId) {
this.replacedDocumentId = replacedDocumentId;
}
public String getClientFileKey() {
return clientFileKey;
}
public void setClientFileKey(String clientFileKey) {
this.clientFileKey = clientFileKey;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getRelativePath() {
return relativePath;
}
public void setRelativePath(String relativePath) {
this.relativePath = relativePath;
}
public Long getFileSize() {
return fileSize;
}
public void setFileSize(Long fileSize) {
this.fileSize = fileSize;
}
public String getFilePath() {
return filePath;
}
public void setFilePath(String filePath) {
this.filePath = filePath;
}
public String getStorageLocator() {
return storageLocator;
}
public void setStorageLocator(String storageLocator) {
this.storageLocator = storageLocator;
}
public Boolean getCleanupPending() {
return cleanupPending;
}
public void setCleanupPending(Boolean cleanupPending) {
this.cleanupPending = cleanupPending;
}
public String getContentSha256() {
return contentSha256;
}
public void setContentSha256(String contentSha256) {
this.contentSha256 = contentSha256;
}
public String getStage() {
return stage;
}
public void setStage(String stage) {
this.stage = stage;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public String getErrorSummary() {
return errorSummary;
}
public void setErrorSummary(String errorSummary) {
this.errorSummary = errorSummary;
}
public String getFailureCode() {
return failureCode;
}
public void setFailureCode(String failureCode) {
this.failureCode = failureCode;
}
public String getAppliedStrategyCode() {
return appliedStrategyCode;
}
public void setAppliedStrategyCode(String appliedStrategyCode) {
this.appliedStrategyCode = appliedStrategyCode;
}
public String getStrategySnapshotJson() {
return strategySnapshotJson;
}
public void setStrategySnapshotJson(String strategySnapshotJson) {
this.strategySnapshotJson = strategySnapshotJson;
}
public Boolean getRetryable() {
return retryable;
}
public void setRetryable(Boolean retryable) {
this.retryable = retryable;
}
public Integer getAttemptCount() {
return attemptCount;
}
public void setAttemptCount(Integer attemptCount) {
this.attemptCount = attemptCount;
}
@Override
public Date getCreated() {
return created;
}
@Override
public void setCreated(Date created) {
this.created = created;
}
public BigInteger getCreatedBy() {
return createdBy;
}
public void setCreatedBy(BigInteger createdBy) {
this.createdBy = createdBy;
}
@Override
public Date getModified() {
return modified;
}
@Override
public void setModified(Date modified) {
this.modified = modified;
}
public BigInteger getModifiedBy() {
return modifiedBy;
}
public void setModifiedBy(BigInteger modifiedBy) {
this.modifiedBy = modifiedBy;
}
}

View File

@@ -31,6 +31,12 @@ public class DocumentImportTask extends DateEntity implements Serializable {
@Column(comment = "知识库ID") @Column(comment = "知识库ID")
private BigInteger knowledgeId; private BigInteger knowledgeId;
@Column(comment = "批次ID")
private BigInteger batchId;
@Column(comment = "批次文件项ID")
private BigInteger batchItemId;
@Column(comment = "任务阶段") @Column(comment = "任务阶段")
private String phase; private String phase;
@@ -46,6 +52,21 @@ public class DocumentImportTask extends DateEntity implements Serializable {
@Column(comment = "错误摘要") @Column(comment = "错误摘要")
private String errorSummary; private String errorSummary;
@Column(comment = "稳定失败码")
private String failureCode;
@Column(comment = "执行尝试次数")
private Integer attemptNo;
@Column(comment = "执行令牌")
private String executionToken;
@Column(comment = "租约到期时间")
private Date leaseUntil;
@Column(comment = "乐观锁版本")
private Integer version;
@Column(comment = "开始时间") @Column(comment = "开始时间")
private Date startedAt; private Date startedAt;
@@ -88,6 +109,22 @@ public class DocumentImportTask extends DateEntity implements Serializable {
this.knowledgeId = knowledgeId; this.knowledgeId = knowledgeId;
} }
public BigInteger getBatchId() {
return batchId;
}
public void setBatchId(BigInteger batchId) {
this.batchId = batchId;
}
public BigInteger getBatchItemId() {
return batchItemId;
}
public void setBatchItemId(BigInteger batchItemId) {
this.batchItemId = batchItemId;
}
public String getPhase() { public String getPhase() {
return phase; return phase;
} }
@@ -128,6 +165,46 @@ public class DocumentImportTask extends DateEntity implements Serializable {
this.errorSummary = errorSummary; this.errorSummary = errorSummary;
} }
public String getFailureCode() {
return failureCode;
}
public void setFailureCode(String failureCode) {
this.failureCode = failureCode;
}
public Integer getAttemptNo() {
return attemptNo;
}
public void setAttemptNo(Integer attemptNo) {
this.attemptNo = attemptNo;
}
public String getExecutionToken() {
return executionToken;
}
public void setExecutionToken(String executionToken) {
this.executionToken = executionToken;
}
public Date getLeaseUntil() {
return leaseUntil;
}
public void setLeaseUntil(Date leaseUntil) {
this.leaseUntil = leaseUntil;
}
public Integer getVersion() {
return version;
}
public void setVersion(Integer version) {
this.version = version;
}
public Date getStartedAt() { public Date getStartedAt() {
return startedAt; return startedAt;
} }

View File

@@ -0,0 +1,25 @@
package tech.easyflow.ai.enums;
/**
* 文档批量导入项当前阶段。
*
* @author Codex
* @since 2026-07-31
*/
public enum DocumentImportBatchItemStage {
/** 上传阶段。 */
UPLOAD,
/** 解析阶段。 */
PARSE,
/** 分块阶段。 */
SPLIT,
/** 向量化阶段。 */
INDEX,
/** 全流程结束。 */
DONE
}

View File

@@ -0,0 +1,34 @@
package tech.easyflow.ai.enums;
/**
* 文档批量导入项状态。
*
* @author Codex
* @since 2026-07-31
*/
public enum DocumentImportBatchItemStatus {
/** 等待处理。 */
PENDING,
/** 文件正在上传。 */
UPLOADING,
/** 正在处理。 */
RUNNING,
/** 文件已上传。 */
UPLOADED,
/** 处理失败。 */
FAILED,
/** 处理完成。 */
COMPLETED,
/** 因重复而跳过。 */
SKIPPED,
/** 文件随未启动批次一并取消。 */
CANCELLED
}

View File

@@ -0,0 +1,31 @@
package tech.easyflow.ai.enums;
/**
* 文档批量导入状态。
*
* @author Codex
* @since 2026-07-31
*/
public enum DocumentImportBatchStatus {
/** 文件上传中。 */
UPLOADING,
/** 文件已上传,等待选择导入方式。 */
READY,
/** 批次处理中。 */
RUNNING,
/** 批次处理已中断,可继续。 */
INTERRUPTED,
/** 部分文件失败,可继续失败项。 */
PARTIAL_SUCCEEDED,
/** 未启动的上传批次已取消。 */
CANCELLED,
/** 批次全部完成。 */
COMPLETED
}

View File

@@ -0,0 +1,20 @@
package tech.easyflow.ai.enums;
/**
* 知识库文档导入模式。
*
* @author Codex
* @since 2026-07-31
*/
public enum DocumentImportMode {
/**
* 解析完成后由用户确认分块策略。
*/
MANUAL,
/**
* 自动完成解析、分块、向量化和入库。
*/
AUTO
}

View File

@@ -13,6 +13,11 @@ public enum DocumentImportTaskPhase {
*/ */
PARSE, PARSE,
/**
* 文档分块阶段。
*/
SPLIT,
/** /**
* 向量化阶段。 * 向量化阶段。
*/ */

View File

@@ -28,6 +28,16 @@ public enum DocumentProcessStatus {
*/ */
READY_FOR_SEGMENT, READY_FOR_SEGMENT,
/**
* 自动分块处理中。
*/
SPLITTING,
/**
* 自动分块失败。
*/
SPLIT_FAILED,
/** /**
* 已确认分块,可开始向量化。 * 已确认分块,可开始向量化。
*/ */
@@ -54,6 +64,18 @@ public enum DocumentProcessStatus {
* @return 是否运行中 * @return 是否运行中
*/ */
public boolean isProcessing() { public boolean isProcessing() {
return this == PARSING || this == INDEXING; return this == PARSING || this == SPLITTING || this == INDEXING;
}
/**
* 判断状态名称是否属于运行中状态。
*
* @param status 状态名称
* @return 是否运行中
*/
public static boolean isProcessing(String status) {
return PARSING.name().equals(status)
|| SPLITTING.name().equals(status)
|| INDEXING.name().equals(status);
} }
} }

View File

@@ -0,0 +1,52 @@
package tech.easyflow.ai.enums;
import java.util.LinkedHashSet;
import java.util.Set;
/**
* 知识库 Public API 产品权限范围。
*
* @author Codex
* @since 2026-08-02
*/
public enum KnowledgeApiPermissionScope {
/**
* 知识库读取权限。
*/
KNOWLEDGE_READ,
/**
* 知识库导入权限。
*/
KNOWLEDGE_IMPORT,
/**
* 知识库维护权限。
*/
KNOWLEDGE_MAINTENANCE;
/**
* 根据三个权限开关构造稳定 Scope 集合。
*
* @param readEnabled 是否开启读取
* @param importEnabled 是否开启导入
* @param maintenanceEnabled 是否开启维护
* @return 已开启的权限 Scope
*/
public static Set<String> enabledScopes(boolean readEnabled,
boolean importEnabled,
boolean maintenanceEnabled) {
Set<String> scopes = new LinkedHashSet<>();
if (readEnabled) {
scopes.add(KNOWLEDGE_READ.name());
}
if (importEnabled) {
scopes.add(KNOWLEDGE_IMPORT.name());
}
if (maintenanceEnabled) {
scopes.add(KNOWLEDGE_MAINTENANCE.name());
}
return scopes;
}
}

View File

@@ -0,0 +1,292 @@
package tech.easyflow.ai.mapper;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import java.math.BigInteger;
import java.util.Date;
/**
* 文档批量导入文件项映射层。
*
* @author Codex
* @since 2026-07-31
*/
public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImportBatchItem> {
/**
* 原子领取文件上传权并刷新批次进度时间。
*
* <p>文件项与批次在同一条 MySQL 多表更新中加锁,确保上传领取和
* 超时取消之间不存在旧快照窗口。</p>
*
* @param batchId 批次 ID
* @param itemId 文件项 ID
* @param knowledgeId 知识库 ID
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch batch "
+ "INNER JOIN tb_document_import_batch_item item "
+ "ON item.batch_id=batch.id SET "
+ "item.stage='UPLOAD', item.status='UPLOADING', "
+ "item.error_summary=NULL, item.failure_code=NULL, "
+ "item.retryable=0, item.modified=#{modified}, "
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
+ "WHERE batch.id=#{batchId} AND batch.knowledge_id=#{knowledgeId} "
+ "AND batch.status IN ('UPLOADING','READY') "
+ "AND item.id=#{itemId} AND item.status='PENDING' "
+ "AND item.cleanup_pending=0 AND item.storage_locator IS NULL")
int claimUpload(
@Param("batchId") BigInteger batchId,
@Param("itemId") BigInteger itemId,
@Param("knowledgeId") BigInteger knowledgeId,
@Param("modified") Date modified
);
/**
* 按预期状态原子迁移文件项。
*
* @param id 文件项 ID
* @param expectedStatus 预期状态
* @param stage 新阶段
* @param status 新状态
* @param errorSummary 错误摘要
* @param failureCode 稳定失败码
* @param retryable 是否允许批量重试
* @param attemptDelta 重试次数增量
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "stage=#{stage}, status=#{status}, error_summary=#{errorSummary}, "
+ "failure_code=#{failureCode}, "
+ "retryable=#{retryable}, attempt_count=attempt_count + #{attemptDelta}, "
+ "modified=#{modified} WHERE id=#{id} AND status=#{expectedStatus}")
int transitionStatus(
@Param("id") BigInteger id,
@Param("expectedStatus") String expectedStatus,
@Param("stage") String stage,
@Param("status") String status,
@Param("errorSummary") String errorSummary,
@Param("failureCode") String failureCode,
@Param("retryable") boolean retryable,
@Param("attemptDelta") int attemptDelta,
@Param("modified") Date modified
);
/**
* 将上传项原子绑定到创建后的文档。
*
* @param id 文件项 ID
* @param documentId 文档 ID
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "document_id=#{documentId}, stage='PARSE', status='PENDING', "
+ "error_summary=NULL, retryable=0, modified=#{modified} "
+ "WHERE id=#{id} AND status='UPLOADED' AND document_id IS NULL")
int bindDocument(
@Param("id") BigInteger id,
@Param("documentId") BigInteger documentId,
@Param("modified") Date modified
);
/**
* 在物理写入前持久化可恢复存储定位符。
*
* @param id 文件项 ID
* @param storageLocator 可恢复存储定位符
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "storage_locator=#{storageLocator}, cleanup_pending=0, "
+ "modified=#{modified} "
+ "WHERE id=#{id} AND status='UPLOADING' "
+ "AND storage_locator IS NULL AND cleanup_pending=0")
int registerUploadWriteIntent(
@Param("id") BigInteger id,
@Param("storageLocator") String storageLocator,
@Param("modified") Date modified
);
/**
* 在物理写入前原子撤销上传领取与写意图登记。
*
* <p>同时兼容登记未提交和提交结果未知两种情况;仅清除空定位符或
* 本次预期定位符,避免覆盖其他请求的新写入意图。</p>
*
* @param id 文件项 ID
* @param storageLocator 本次预期存储定位符,可为空
* @param errorSummary 上传准备失败摘要
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch batch "
+ "INNER JOIN tb_document_import_batch_item item "
+ "ON item.batch_id=batch.id SET "
+ "item.status='PENDING', item.storage_locator=NULL, "
+ "item.file_path=NULL, item.cleanup_pending=0, "
+ "item.error_summary=#{errorSummary}, item.failure_code=NULL, "
+ "item.retryable=0, item.modified=#{modified}, "
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
+ "WHERE item.id=#{id} AND item.status='UPLOADING' "
+ "AND item.cleanup_pending=0 "
+ "AND (item.storage_locator IS NULL "
+ "OR item.storage_locator=#{storageLocator})")
int abortUploadBeforeWrite(
@Param("id") BigInteger id,
@Param("storageLocator") String storageLocator,
@Param("errorSummary") String errorSummary,
@Param("modified") Date modified
);
/**
* 在文件仍处于上传中时原子绑定存储路径并完成上传。
*
* @param id 文件项 ID
* @param filePath 存储路径
* @param storageLocator 可恢复存储定位符
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "file_path=#{filePath}, status='UPLOADED', error_summary=NULL, "
+ "cleanup_pending=0, retryable=0, modified=#{modified} "
+ "WHERE id=#{id} AND status='UPLOADING' "
+ "AND storage_locator=#{storageLocator}")
int completeUpload(
@Param("id") BigInteger id,
@Param("filePath") String filePath,
@Param("storageLocator") String storageLocator,
@Param("modified") Date modified
);
/**
* 将指定可恢复写意图标记为等待清理。
*
* @param id 文件项 ID
* @param storageLocator 可恢复存储定位符
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "cleanup_pending=1, modified=#{modified} "
+ "WHERE id=#{id} AND status IN ('UPLOADING','CANCELLED') "
+ "AND storage_locator=#{storageLocator}")
int markUploadCleanupPending(
@Param("id") BigInteger id,
@Param("storageLocator") String storageLocator,
@Param("modified") Date modified
);
/**
* 将已取消文件项的现有存储引用标记为等待清理。
*
* @param id 文件项 ID
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "cleanup_pending=1, modified=#{modified} "
+ "WHERE id=#{id} AND status='CANCELLED' "
+ "AND (storage_locator IS NOT NULL OR file_path IS NOT NULL)")
int markCancelledCleanupPending(
@Param("id") BigInteger id,
@Param("modified") Date modified
);
/**
* 为新导入项记录待覆盖的历史文档。
*
* @param id 文件项 ID
* @param replacedDocumentId 历史文档 ID
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "replaced_document_id=#{replacedDocumentId}, modified=#{modified} "
+ "WHERE id=#{id} AND status='UPLOADED' AND replaced_document_id IS NULL")
int markReplacement(
@Param("id") BigInteger id,
@Param("replacedDocumentId") BigInteger replacedDocumentId,
@Param("modified") Date modified
);
/**
* 完成历史文档清理后原子清除覆盖标记。
*
* @param id 文件项 ID
* @param replacedDocumentId 历史文档 ID
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch_item SET "
+ "replaced_document_id=NULL, modified=#{modified} "
+ "WHERE id=#{id} AND replaced_document_id=#{replacedDocumentId}")
int clearReplacement(
@Param("id") BigInteger id,
@Param("replacedDocumentId") BigInteger replacedDocumentId,
@Param("modified") Date modified
);
/**
* 对象删除成功后原子清理上传中项并恢复待上传状态。
*
* @param id 文件项 ID
* @param storageLocator 已删除对象的恢复定位符
* @param filePath 已删除对象的兼容存储路径
* @param recoveryMessage 恢复为待上传时的提示
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch batch "
+ "INNER JOIN tb_document_import_batch_item item "
+ "ON item.batch_id=batch.id SET "
+ "item.stage='UPLOAD', item.error_summary=#{recoveryMessage}, "
+ "item.failure_code=NULL, item.retryable=0, item.status='PENDING', "
+ "item.cleanup_pending=0, item.storage_locator=NULL, "
+ "item.file_path=NULL, item.modified=#{modified}, "
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
+ "WHERE item.id=#{id} AND item.cleanup_pending=1 "
+ "AND item.status='UPLOADING' "
+ "AND item.storage_locator <=> #{storageLocator} "
+ "AND item.file_path <=> #{filePath}")
int completeUploadingStorageCleanup(
@Param("id") BigInteger id,
@Param("storageLocator") String storageLocator,
@Param("filePath") String filePath,
@Param("recoveryMessage") String recoveryMessage,
@Param("modified") Date modified
);
/**
* 对象删除成功后原子清理已取消项并保持取消状态。
*
* @param id 文件项 ID
* @param storageLocator 已删除对象的恢复定位符
* @param filePath 已删除对象的兼容存储路径
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch batch "
+ "INNER JOIN tb_document_import_batch_item item "
+ "ON item.batch_id=batch.id SET "
+ "item.cleanup_pending=0, item.storage_locator=NULL, "
+ "item.file_path=NULL, item.modified=#{modified}, "
+ "batch.modified=#{modified}, batch.version=batch.version + 1 "
+ "WHERE item.id=#{id} AND item.cleanup_pending=1 "
+ "AND item.status='CANCELLED' "
+ "AND item.storage_locator <=> #{storageLocator} "
+ "AND item.file_path <=> #{filePath}")
int completeCancelledStorageCleanup(
@Param("id") BigInteger id,
@Param("storageLocator") String storageLocator,
@Param("filePath") String filePath,
@Param("modified") Date modified
);
}

View File

@@ -0,0 +1,167 @@
package tech.easyflow.ai.mapper;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.ai.entity.DocumentImportBatch;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
/**
* 文档批量导入批次映射层。
*
* @author Codex
* @since 2026-07-31
*/
public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatch> {
/**
* 锁定并读取调用者拥有的批次,保证后续重试基于最新状态。
*
* @param batchId 批次 ID
* @param callerType 调用者类型
* @param callerId 调用者 ID
* @return 批次;不存在时返回 null
*/
@Select("SELECT * FROM tb_document_import_batch "
+ "WHERE id=#{batchId} AND caller_type=#{callerType} "
+ "AND caller_id=#{callerId} FOR UPDATE")
DocumentImportBatch selectOwnedForUpdate(
@Param("batchId") BigInteger batchId,
@Param("callerType") String callerType,
@Param("callerId") BigInteger callerId
);
/**
* 按文件项状态迁移增量更新批次计数。
*
* @param batchId 批次 ID
* @param completedDelta 完成数增量
* @param processingDelta 处理中数量增量
* @param failedDelta 失败数增量
* @param pendingDelta 等待数增量
* @param uploadedDelta 已上传数增量
* @param skippedDelta 跳过数增量
* @param cancelledDelta 取消数增量
* @param retryableFailedDelta 可重试失败数增量
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch SET "
+ "completed_count=GREATEST(0, completed_count + #{completedDelta}), "
+ "processing_count=GREATEST(0, processing_count + #{processingDelta}), "
+ "failed_count=GREATEST(0, failed_count + #{failedDelta}), "
+ "pending_count=GREATEST(0, pending_count + #{pendingDelta}), "
+ "uploaded_count=GREATEST(0, uploaded_count + #{uploadedDelta}), "
+ "skipped_count=GREATEST(0, skipped_count + #{skippedDelta}), "
+ "cancelled_count=GREATEST(0, cancelled_count + #{cancelledDelta}), "
+ "retryable_failed_count=GREATEST(0, retryable_failed_count + #{retryableFailedDelta}), "
+ "modified=#{modified} WHERE id=#{batchId}")
int adjustCounters(
@Param("batchId") BigInteger batchId,
@Param("completedDelta") int completedDelta,
@Param("processingDelta") int processingDelta,
@Param("failedDelta") int failedDelta,
@Param("pendingDelta") int pendingDelta,
@Param("uploadedDelta") int uploadedDelta,
@Param("skippedDelta") int skippedDelta,
@Param("cancelledDelta") int cancelledDelta,
@Param("retryableFailedDelta") int retryableFailedDelta,
@Param("modified") Date modified
);
/**
* 原子领取 Public API 重试代次。
*
* @param batchId 批次 ID
* @param callerType 调用者类型
* @param callerId 调用者 ID
* @param expectedGeneration 预期重试代次
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch SET "
+ "status='RUNNING', finished_at=NULL, "
+ "retry_generation=retry_generation + 1, "
+ "version=version + 1, modified=#{modified} "
+ "WHERE id=#{batchId} AND caller_type=#{callerType} AND caller_id=#{callerId} "
+ "AND retry_generation=#{expectedGeneration} "
+ "AND status IN ('PARTIAL_SUCCEEDED','INTERRUPTED')")
int claimRetry(
@Param("batchId") BigInteger batchId,
@Param("callerType") String callerType,
@Param("callerId") BigInteger callerId,
@Param("expectedGeneration") int expectedGeneration,
@Param("modified") Date modified
);
/**
* 原子领取无进展的未完成批次取消权。
*
* @param batchId 批次 ID
* @param knowledgeId 知识库 ID
* @param callerType 调用者类型
* @param callerId 调用者 ID
* @param incompleteCutoff 最后进展截止时间
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch SET "
+ "status='CANCELLED', finished_at=#{modified}, "
+ "version=version + 1, modified=#{modified} "
+ "WHERE id=#{batchId} AND knowledge_id=#{knowledgeId} "
+ "AND caller_type=#{callerType} AND caller_id=#{callerId} "
+ "AND status IN ('UPLOADING','READY') "
+ "AND ((modified IS NOT NULL AND modified < #{incompleteCutoff}) "
+ "OR (modified IS NULL AND created < #{incompleteCutoff}))")
int claimStaleCancellation(
@Param("batchId") BigInteger batchId,
@Param("knowledgeId") BigInteger knowledgeId,
@Param("callerType") String callerType,
@Param("callerId") BigInteger callerId,
@Param("incompleteCutoff") Date incompleteCutoff,
@Param("modified") Date modified
);
/**
* 分批查询无进展的未完成批次。
*
* @param incompleteCutoff 最后进展截止时间
* @param limit 最大返回数量
* @return 待回收批次
*/
@Select("SELECT * FROM tb_document_import_batch "
+ "WHERE status IN ('UPLOADING','READY') "
+ "AND ((modified IS NOT NULL AND modified < #{incompleteCutoff}) "
+ "OR (modified IS NULL AND created < #{incompleteCutoff})) "
+ "ORDER BY COALESCE(modified, created), id LIMIT #{limit}")
List<DocumentImportBatch> selectStaleIncompleteBatches(
@Param("incompleteCutoff") Date incompleteCutoff,
@Param("limit") int limit
);
/**
* 释放已取消或超过去重窗口的服务端提交指纹。
*
* @param batchId 批次 ID
* @param submissionFingerprint 当前服务端提交指纹
* @param deduplicationCutoff 去重窗口起点
* @param modified 修改时间
* @return 更新行数
*/
@Update("UPDATE tb_document_import_batch SET "
+ "idempotency_key_hash=NULL, modified=#{modified}, version=version + 1 "
+ "WHERE id=#{batchId} AND idempotency_key_hash=#{submissionFingerprint} "
+ "AND (status='CANCELLED' OR ("
+ "COALESCE(finished_at, modified, created) < #{deduplicationCutoff} "
+ "AND status IN ('INTERRUPTED','PARTIAL_SUCCEEDED','COMPLETED')))")
int releaseSubmissionFingerprint(
@Param("batchId") BigInteger batchId,
@Param("submissionFingerprint") String submissionFingerprint,
@Param("deduplicationCutoff") Date deduplicationCutoff,
@Param("modified") Date modified
);
}

View File

@@ -1,8 +1,15 @@
package tech.easyflow.ai.mapper; package tech.easyflow.ai.mapper;
import com.mybatisflex.core.BaseMapper; import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.ai.entity.DocumentImportTask; import tech.easyflow.ai.entity.DocumentImportTask;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
/** /**
* 文档导入任务映射层。 * 文档导入任务映射层。
* *
@@ -10,4 +17,169 @@ import tech.easyflow.ai.entity.DocumentImportTask;
* @since 2026-04-14 * @since 2026-04-14
*/ */
public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask> { public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask> {
/**
* 按任务阶段和批次轮转查询待投递任务,避免大批次独占投递窗口。
*
* @param redispatchBefore 允许重新投递的修改时间边界
* @param limit 最大任务数
* @return 公平排序后的待投递任务
*/
@Select("SELECT task.* FROM tb_document_import_task task JOIN ("
+ "SELECT task.id, ROW_NUMBER() OVER ("
+ "PARTITION BY task.phase, COALESCE(task.batch_id, task.id) "
+ "ORDER BY task.created, task.id) AS lane_row "
+ "FROM tb_document_import_task task "
+ "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore}"
+ ") ranked ON ranked.id=task.id "
+ "ORDER BY ranked.lane_row, task.created, task.id LIMIT #{limit}")
List<DocumentImportTask> selectPendingFairly(
@Param("redispatchBefore") Date redispatchBefore,
@Param("limit") int limit
);
/**
* 仅在任务仍待处理时原子更新时间戳,取得本轮重新投递资格。
*
* @param id 任务 ID
* @param redispatchBefore 允许重新投递的修改时间边界
* @param now 当前时间
* @param operatorId 操作人 ID
* @return 更新行数
*/
@Update("UPDATE tb_document_import_task SET modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='PENDING' "
+ "AND modified <= #{redispatchBefore}")
int touchPendingForDispatch(
@Param("id") BigInteger id,
@Param("redispatchBefore") Date redispatchBefore,
@Param("now") Date now,
@Param("operatorId") BigInteger operatorId
);
/**
* 查询租约已过期的运行任务,并兼容迁移前没有租约的历史任务。
*
* @param now 当前时间
* @param legacyCutoff 历史任务失联时间边界
* @param limit 最大任务数
* @return 已失去执行租约的运行任务
*/
@Select("SELECT * FROM tb_document_import_task "
+ "WHERE status='RUNNING' AND ("
+ "(lease_until IS NOT NULL AND lease_until <= #{now}) OR "
+ "(lease_until IS NULL AND modified <= #{legacyCutoff})) "
+ "ORDER BY COALESCE(lease_until, modified), id LIMIT #{limit}")
List<DocumentImportTask> selectExpiredRunningTasks(
@Param("now") Date now,
@Param("legacyCutoff") Date legacyCutoff,
@Param("limit") int limit
);
/**
* 使用执行令牌原子领取一个待处理任务。
*
* @param id 任务 ID
* @param executionToken 本轮执行令牌
* @param leaseUntil 租约到期时间
* @param now 当前时间
* @param operatorId 操作人 ID
* @return 更新行数
*/
@Update("UPDATE tb_document_import_task SET status='RUNNING', "
+ "attempt_no=COALESCE(attempt_no, 0) + 1, "
+ "execution_token=#{executionToken}, lease_until=#{leaseUntil}, "
+ "started_at=COALESCE(started_at, #{now}), finished_at=NULL, "
+ "error_summary=NULL, failure_code=NULL, modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='PENDING'")
int claimPending(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@Param("leaseUntil") Date leaseUntil,
@Param("now") Date now,
@Param("operatorId") BigInteger operatorId
);
/**
* 在仍持有执行令牌时续租任务。
*
* @param id 任务 ID
* @param executionToken 本轮执行令牌
* @param leaseUntil 新租约到期时间
* @param now 当前时间
* @param operatorId 操作人 ID
* @return 更新行数
*/
@Update("UPDATE tb_document_import_task SET lease_until=#{leaseUntil}, "
+ "modified=#{now}, modified_by=#{operatorId}, "
+ "version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='RUNNING' "
+ "AND execution_token=#{executionToken}")
int renewLease(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@Param("leaseUntil") Date leaseUntil,
@Param("now") Date now,
@Param("operatorId") BigInteger operatorId
);
/**
* 在仍持有执行令牌时写入任务终态。
*
* @param id 任务 ID
* @param executionToken 本轮执行令牌
* @param status 任务终态
* @param errorSummary 错误摘要
* @param failureCode 稳定失败码
* @param now 当前时间
* @param operatorId 操作人 ID
* @return 更新行数
*/
@Update("UPDATE tb_document_import_task SET status=#{status}, "
+ "error_summary=#{errorSummary}, failure_code=#{failureCode}, "
+ "lease_until=NULL, finished_at=#{now}, modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='RUNNING' "
+ "AND execution_token=#{executionToken}")
int finishOwned(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@Param("status") String status,
@Param("errorSummary") String errorSummary,
@Param("failureCode") String failureCode,
@Param("now") Date now,
@Param("operatorId") BigInteger operatorId
);
/**
* 仅在当前执行令牌仍持有已过期租约时将任务标记为失败。
*
* @param id 任务 ID
* @param executionToken 本轮执行令牌
* @param errorSummary 错误摘要
* @param failureCode 稳定失败码
* @param now 当前时间
* @param legacyCutoff 历史任务失联时间边界
* @param operatorId 操作人 ID
* @return 更新行数
*/
@Update("UPDATE tb_document_import_task SET status='FAILED', "
+ "error_summary=#{errorSummary}, failure_code=#{failureCode}, "
+ "lease_until=NULL, finished_at=#{now}, modified=#{now}, "
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
+ "WHERE id=#{id} AND status='RUNNING' "
+ "AND execution_token <=> #{executionToken} AND ("
+ "(lease_until IS NOT NULL AND lease_until <= #{now}) OR "
+ "(lease_until IS NULL AND modified <= #{legacyCutoff}))")
int failExpiredOwned(
@Param("id") BigInteger id,
@Param("executionToken") String executionToken,
@Param("errorSummary") String errorSummary,
@Param("failureCode") String failureCode,
@Param("now") Date now,
@Param("legacyCutoff") Date legacyCutoff,
@Param("operatorId") BigInteger operatorId
);
} }

View File

@@ -0,0 +1,13 @@
package tech.easyflow.ai.service;
import com.mybatisflex.core.service.IService;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
/**
* 文档批量导入文件项服务。
*
* @author Codex
* @since 2026-07-31
*/
public interface DocumentImportBatchItemService extends IService<DocumentImportBatchItem> {
}

View File

@@ -0,0 +1,13 @@
package tech.easyflow.ai.service;
import com.mybatisflex.core.service.IService;
import tech.easyflow.ai.entity.DocumentImportBatch;
/**
* 文档批量导入批次服务。
*
* @author Codex
* @since 2026-07-31
*/
public interface DocumentImportBatchService extends IService<DocumentImportBatch> {
}

View File

@@ -19,7 +19,37 @@ import java.util.List;
*/ */
public interface DocumentService extends IService<Document> { public interface DocumentService extends IService<Document> {
Page<Document> getDocumentList(String knowledgeId , int pageSize, int pageNum, String fileName); /**
* 按知识库和文件标题查询文档分页。
*
* @param knowledgeId 知识库 ID
* @param pageSize 每页条数
* @param pageNum 页码
* @param fileName 可选的文件标题筛选
* @return 文档分页
*/
Page<Document> getDocumentList(
String knowledgeId,
int pageSize,
int pageNum,
String fileName
);
/**
* 按知识库和文档 ID 查询文档分页。
*
* @param knowledgeId 知识库 ID
* @param pageSize 每页条数
* @param pageNum 页码
* @param documentId 可选的文档 ID
* @return 文档分页
*/
Page<Document> getDocumentListById(
String knowledgeId,
int pageSize,
int pageNum,
BigInteger documentId
);
boolean removeDoc(String id); boolean removeDoc(String id);
@@ -44,4 +74,6 @@ public interface DocumentService extends IService<Document> {
Result<DocumentImportDtos.TaskStartIndexResponse> retryParseTask(DocumentImportDtos.TaskRetryRequest request); Result<DocumentImportDtos.TaskStartIndexResponse> retryParseTask(DocumentImportDtos.TaskRetryRequest request);
Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(DocumentImportDtos.TaskRetryRequest request); Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(DocumentImportDtos.TaskRetryRequest request);
Result<DocumentImportDtos.TaskStartIndexResponse> retryFailedTask(DocumentImportDtos.TaskRetryRequest request);
} }

View File

@@ -27,6 +27,27 @@ public interface KnowledgeSharePermissionService {
*/ */
void replaceApiShareEnabled(BigInteger apiKeyId, boolean enabled); void replaceApiShareEnabled(BigInteger apiKeyId, boolean enabled);
/**
* 替换访问令牌的三类知识库 Public API 权限。
*
* @param apiKeyId 系统访问令牌 ID
* @param readEnabled 是否开启读取
* @param importEnabled 是否开启导入
* @param maintenanceEnabled 是否开启维护
*/
void replaceApiPermissions(BigInteger apiKeyId,
boolean readEnabled,
boolean importEnabled,
boolean maintenanceEnabled);
/**
* 查询访问令牌已开启的知识库 Public API 权限。
*
* @param apiKeyId 系统访问令牌 ID
* @return 权限 Scope 集合
*/
Set<String> getApiPermissionScopes(BigInteger apiKeyId);
/** /**
* 断言当前令牌具备知识库分享权限。 * 断言当前令牌具备知识库分享权限。
* *

View File

@@ -486,6 +486,10 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
} }
item.setContent(content); item.setContent(content);
item.addMetadata("chunkId", chunkId); item.addMetadata("chunkId", chunkId);
item.addMetadata(
"resultType",
DocumentCollection.TYPE_DOCUMENT
);
Object sourceDocumentId = hitSnapshot.findSourceDocumentId(item.getId()); Object sourceDocumentId = hitSnapshot.findSourceDocumentId(item.getId());
if (sourceDocumentId != null) { if (sourceDocumentId != null) {
item.addMetadata("documentId", sourceDocumentId); item.addMetadata("documentId", sourceDocumentId);
@@ -605,6 +609,11 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
.collect(Collectors.toList()); .collect(Collectors.toList());
metadataMap.put("chunkId", item.getId()); metadataMap.put("chunkId", item.getId());
metadataMap.put("documentId", item.getId()); metadataMap.put("documentId", item.getId());
metadataMap.put("resultType", DocumentCollection.TYPE_FAQ);
metadataMap.put("faqId", faqItem.getId());
metadataMap.put("question", faqItem.getQuestion());
metadataMap.put("answerText", faqItem.getAnswerText());
metadataMap.put("categoryId", faqItem.getCategoryId());
metadataMap.put("imageUrls", imageUrls); metadataMap.put("imageUrls", imageUrls);
item.setMetadataMap(metadataMap); item.setMetadataMap(metadataMap);
}); });

View File

@@ -0,0 +1,19 @@
package tech.easyflow.ai.service.impl;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
/**
* 文档批量导入文件项服务实现。
*
* @author Codex
* @since 2026-07-31
*/
@Service
public class DocumentImportBatchItemServiceImpl
extends ServiceImpl<DocumentImportBatchItemMapper, DocumentImportBatchItem>
implements DocumentImportBatchItemService {
}

View File

@@ -0,0 +1,19 @@
package tech.easyflow.ai.service.impl;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.service.DocumentImportBatchService;
/**
* 文档批量导入批次服务实现。
*
* @author Codex
* @since 2026-07-31
*/
@Service
public class DocumentImportBatchServiceImpl
extends ServiceImpl<DocumentImportBatchMapper, DocumentImportBatch>
implements DocumentImportBatchService {
}

View File

@@ -106,6 +106,45 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
@Override @Override
public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) { public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) {
return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null);
}
/**
* 按知识库和文档 ID 查询文档分页。
*
* @param knowledgeId 知识库 ID
* @param pageSize 每页条数
* @param pageNum 页码
* @param documentId 可选的文档 ID
* @return 文档分页
*/
@Override
public Page<Document> getDocumentListById(
String knowledgeId,
int pageSize,
int pageNum,
BigInteger documentId
) {
return queryDocumentList(knowledgeId, pageSize, pageNum, null, documentId);
}
/**
* 执行文档分页查询。
*
* @param knowledgeId 知识库 ID
* @param pageSize 每页条数
* @param pageNum 页码
* @param fileName 可选的文件标题筛选
* @param documentId 可选的文档 ID
* @return 文档分页
*/
private Page<Document> queryDocumentList(
String knowledgeId,
int pageSize,
int pageNum,
String fileName,
BigInteger documentId
) {
QueryWrapper queryWrapper=QueryWrapper.create() QueryWrapper queryWrapper=QueryWrapper.create()
.select( .select(
DOCUMENT.ALL_COLUMNS, DOCUMENT.ALL_COLUMNS,
@@ -120,19 +159,23 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
if (fileName != null && !fileName.trim().isEmpty()) { if (fileName != null && !fileName.trim().isEmpty()) {
queryWrapper.and(DOCUMENT.TITLE.like(fileName)); queryWrapper.and(DOCUMENT.TITLE.like(fileName));
} }
if (documentId != null) {
queryWrapper.and(DOCUMENT.ID.eq(documentId));
}
// 分组 // 分组
queryWrapper.groupBy(DOCUMENT.ID); queryWrapper.groupBy(DOCUMENT.ID);
Page<Document> documentVoPage = documentMapper.paginateAs(pageNum, pageSize, queryWrapper, Document.class); return documentMapper.paginateAs(pageNum, pageSize, queryWrapper, Document.class);
return documentVoPage;
} }
/** /**
* 根据文档id删除文件 * 删除文档的向量、搜索索引、分块、存储文件和主记录。
* *
* @param id 文档id * @param id 文档 ID
* @return * @return 全部数据库清理成功时返回 true
* @throws BusinessException 文档仍在处理中时抛出
*/ */
@Override @Override
@Transactional
public boolean removeDoc(String id) { public boolean removeDoc(String id) {
// 查询该文档对应哪些分割的字段,先删除 // 查询该文档对应哪些分割的字段,先删除
QueryWrapper queryWrapperDocument = QueryWrapper.create().eq(Document::getId, id); QueryWrapper queryWrapperDocument = QueryWrapper.create().eq(Document::getId, id);
@@ -140,8 +183,7 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
if (oneByQuery == null) { if (oneByQuery == null) {
return false; return false;
} }
if (DocumentProcessStatus.PARSING.name().equals(oneByQuery.getProcessStatus()) if (DocumentProcessStatus.isProcessing(oneByQuery.getProcessStatus())) {
|| DocumentProcessStatus.INDEXING.name().equals(oneByQuery.getProcessStatus())) {
throw new BusinessException("文档处理中,暂不允许删除"); throw new BusinessException("文档处理中,暂不允许删除");
} }
DocumentCollection knowledge = knowledgeService.getById(oneByQuery.getCollectionId()); DocumentCollection knowledge = knowledgeService.getById(oneByQuery.getCollectionId());
@@ -149,28 +191,40 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
return false; return false;
} }
// 存储到知识库 QueryWrapper queryWrapper = QueryWrapper.create()
DocumentStore documentStore = knowledge.toDocumentStore(); .select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id);
if (documentStore == null) { List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(
return false; queryWrapper,
} BigInteger.class
);
DocumentStore documentStore = null;
try { try {
Model model = modelService.getById(knowledge.getVectorEmbedModelId()); if (!chunkIds.isEmpty()) {
if (model == null) { documentStore = knowledge.toDocumentStore();
return false; if (documentStore == null) {
return false;
}
Model model = modelService.getById(
knowledge.getVectorEmbedModelId()
);
if (model == null) {
return false;
}
StoreOptions options = StoreOptions.ofCollectionName(
knowledge.getVectorStoreCollection()
);
EmbeddingOptions embeddingOptions = new EmbeddingOptions();
embeddingOptions.setModel(model.getModelName());
options.setEmbeddingOptions(embeddingOptions);
StoreResult deleteResult = documentStore.delete(chunkIds, options);
if (deleteResult == null || !deleteResult.isSuccess()) {
String failReason = deleteResult == null
? "未返回结果"
: deleteResult.getFailReason();
Log.error("删除文档向量失败: documentId={}, reason={}", id, failReason);
throw new BusinessException("文档向量删除失败");
}
} }
// 设置向量模型
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
EmbeddingOptions embeddingOptions = new EmbeddingOptions();
embeddingOptions.setModel(model.getModelName());
options.setEmbeddingOptions(embeddingOptions);
options.setCollectionName(knowledge.getVectorStoreCollection());
// 查询文本分割表tb_document_chunk中对应的有哪些数据找出来删除
QueryWrapper queryWrapper = QueryWrapper.create()
.select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id);
List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class);
documentStore.delete(chunkIds, options);
// 删除搜索引擎中的数据 // 删除搜索引擎中的数据
DocumentSearcher searcher = searcherFactory.getSearcher(); DocumentSearcher searcher = searcherFactory.getSearcher();
if (searcher != null) { if (searcher != null) {
@@ -181,9 +235,16 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
return false; return false;
} }
// 再删除指定路径下的文件 // 再删除指定路径下的文件
Document document = documentMapper.selectOneByQuery(queryWrapperDocument); String chunkSnapshotPath = oneByQuery.getOptions() == null
storageService.delete(document.getDocumentPath()); ? null
return true; : asString(oneByQuery.getOptions().get(
DocumentImportKeys.KEY_DOCUMENT_CHUNK_SNAPSHOT_PATH));
if (StringUtil.hasText(chunkSnapshotPath)) {
storageService.delete(chunkSnapshotPath);
}
storageService.delete(oneByQuery.getDocumentPath());
// 主记录必须最后删除;否则接口返回成功后文档仍会出现在列表中。
return documentMapper.deleteById(oneByQuery.getId()) > 0;
} finally { } finally {
DocumentStoreLifecycleSupport.closeQuietly(documentStore); DocumentStoreLifecycleSupport.closeQuietly(documentStore);
} }
@@ -1012,4 +1073,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
public Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(DocumentImportDtos.TaskRetryRequest request) { public Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(DocumentImportDtos.TaskRetryRequest request) {
return importTaskAppService.retryIndexTask(request); return importTaskAppService.retryIndexTask(request);
} }
@Override
public Result<DocumentImportDtos.TaskStartIndexResponse> retryFailedTask(DocumentImportDtos.TaskRetryRequest request) {
return importTaskAppService.retryFailedTask(request);
}
} }

View File

@@ -3,8 +3,13 @@ package tech.easyflow.ai.service.impl;
import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.query.QueryWrapper;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
import tech.easyflow.ai.enums.KnowledgeShareActionScope; import tech.easyflow.ai.enums.KnowledgeShareActionScope;
import tech.easyflow.ai.service.KnowledgeSharePermissionService; import tech.easyflow.ai.service.KnowledgeSharePermissionService;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.common.cache.RedisLockExecutor.LockHandle;
import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.system.entity.SysApiKey; import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.entity.SysApiKeyResource; import tech.easyflow.system.entity.SysApiKeyResource;
@@ -15,9 +20,9 @@ import tech.easyflow.system.service.SysApiKeyService;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.math.BigInteger; import java.math.BigInteger;
import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@@ -30,10 +35,51 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
public static final String RESOURCE_TYPE_KNOWLEDGE = "KNOWLEDGE"; public static final String RESOURCE_TYPE_KNOWLEDGE = "KNOWLEDGE";
private static final Map<String, List<String>> URI_SCOPE_MAPPING = new LinkedHashMap<>(); private static final String PERMISSION_LOCK_PREFIX =
"easyflow:lock:knowledge-api-permission:";
private static final Duration PERMISSION_LOCK_WAIT = Duration.ofSeconds(2);
private static final Duration PERMISSION_LOCK_LEASE = Duration.ofSeconds(15);
private static final Map<String, List<String>> API_SCOPE_URI_MAPPING = new LinkedHashMap<>();
private static final Map<String, List<String>> LEGACY_ACTION_URI_MAPPING = new LinkedHashMap<>();
static { static {
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.VIEW.name(), List.of( API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_READ.name(), List.of(
"/public-api/knowledge-share/detail",
"/public-api/knowledge-share/search",
"/public-api/knowledge-share/document/page",
"/public-api/knowledge-share/document/download",
"/public-api/knowledge-share/documentChunk/page",
"/public-api/knowledge-share/faq/page",
"/public-api/knowledge-share/faq/detail",
"/public-api/knowledge-share/faq/exportExcel"
));
API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name(), List.of(
"/public-api/knowledge-share/document/import/batch",
"/public-api/knowledge-share/document/import/batch/status",
"/public-api/knowledge-share/document/import/batch/retry",
"/public-api/knowledge-share/document/import/analyze",
"/public-api/knowledge-share/document/import/preview",
"/public-api/knowledge-share/document/import/commit",
"/public-api/knowledge-share/document/import/task/create",
"/public-api/knowledge-share/document/import/task/detail",
"/public-api/knowledge-share/document/import/task/preview",
"/public-api/knowledge-share/document/import/task/startIndex",
"/public-api/knowledge-share/document/import/task/retryParse",
"/public-api/knowledge-share/document/import/task/retryIndex",
"/public-api/knowledge-share/faq/importExcel",
"/public-api/knowledge-share/faq/downloadImportTemplate"
));
API_SCOPE_URI_MAPPING.put(KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name(), List.of(
"/public-api/knowledge-share/document/remove",
"/public-api/knowledge-share/documentChunk/update",
"/public-api/knowledge-share/documentChunk/remove",
"/public-api/knowledge-share/faq/save",
"/public-api/knowledge-share/faq/update",
"/public-api/knowledge-share/faq/remove"
));
LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.VIEW.name(), List.of(
"/public-api/knowledge-share/detail", "/public-api/knowledge-share/detail",
"/public-api/knowledge-share/document/page", "/public-api/knowledge-share/document/page",
"/public-api/knowledge-share/document/download", "/public-api/knowledge-share/document/download",
@@ -42,10 +88,10 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
"/public-api/knowledge-share/faq/page", "/public-api/knowledge-share/faq/page",
"/public-api/knowledge-share/faq/detail" "/public-api/knowledge-share/faq/detail"
)); ));
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.SEARCH.name(), List.of( LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.SEARCH.name(), List.of(
"/public-api/knowledge-share/search" "/public-api/knowledge-share/search"
)); ));
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_CREATE.name(), List.of( LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_CREATE.name(), List.of(
"/public-api/knowledge-share/document/import/analyze", "/public-api/knowledge-share/document/import/analyze",
"/public-api/knowledge-share/document/import/preview", "/public-api/knowledge-share/document/import/preview",
"/public-api/knowledge-share/document/import/commit", "/public-api/knowledge-share/document/import/commit",
@@ -54,18 +100,21 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
"/public-api/knowledge-share/document/import/task/startIndex", "/public-api/knowledge-share/document/import/task/startIndex",
"/public-api/knowledge-share/document/import/task/retryParse", "/public-api/knowledge-share/document/import/task/retryParse",
"/public-api/knowledge-share/document/import/task/retryIndex", "/public-api/knowledge-share/document/import/task/retryIndex",
"/public-api/knowledge-share/document/import/batch",
"/public-api/knowledge-share/document/import/batch/status",
"/public-api/knowledge-share/document/import/batch/retry",
"/public-api/knowledge-share/faq/save" "/public-api/knowledge-share/faq/save"
)); ));
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_UPDATE.name(), List.of( LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_UPDATE.name(), List.of(
"/public-api/knowledge-share/documentChunk/update", "/public-api/knowledge-share/documentChunk/update",
"/public-api/knowledge-share/faq/update" "/public-api/knowledge-share/faq/update"
)); ));
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.CONTENT_DELETE.name(), List.of( LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.CONTENT_DELETE.name(), List.of(
"/public-api/knowledge-share/document/remove", "/public-api/knowledge-share/document/remove",
"/public-api/knowledge-share/documentChunk/remove", "/public-api/knowledge-share/documentChunk/remove",
"/public-api/knowledge-share/faq/remove" "/public-api/knowledge-share/faq/remove"
)); ));
URI_SCOPE_MAPPING.put(KnowledgeShareActionScope.IMPORT_EXPORT.name(), List.of( LEGACY_ACTION_URI_MAPPING.put(KnowledgeShareActionScope.IMPORT_EXPORT.name(), List.of(
"/public-api/knowledge-share/faq/importExcel", "/public-api/knowledge-share/faq/importExcel",
"/public-api/knowledge-share/faq/exportExcel", "/public-api/knowledge-share/faq/exportExcel",
"/public-api/knowledge-share/faq/downloadImportTemplate" "/public-api/knowledge-share/faq/downloadImportTemplate"
@@ -78,6 +127,8 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
private SysApiKeyResourceService resourceService; private SysApiKeyResourceService resourceService;
@Resource @Resource
private SysApiKeyResourceMappingService mappingService; private SysApiKeyResourceMappingService mappingService;
@Resource
private RedisLockExecutor redisLockExecutor;
@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
@@ -97,32 +148,48 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
throw new BusinessException("动作范围不能为空"); throw new BusinessException("动作范围不能为空");
} }
mappingService.removeScopedMappings(apiKeyId, RESOURCE_TYPE_KNOWLEDGE, knowledgeId); Runnable releaseLock = acquirePermissionMutationLock(apiKeyId);
List<SysApiKeyResourceMapping> rows = new ArrayList<>(); try {
for (String scope : normalizedScopes) { mappingService.removeScopedMappings(
List<String> uris = URI_SCOPE_MAPPING.get(scope); apiKeyId,
if (uris == null || uris.isEmpty()) { RESOURCE_TYPE_KNOWLEDGE,
continue; knowledgeId
);
List<SysApiKeyResourceMapping> rows = new ArrayList<>();
for (String legacyScope : normalizedScopes) {
List<String> uris = LEGACY_ACTION_URI_MAPPING.get(legacyScope);
if (uris == null || uris.isEmpty()) {
continue;
}
for (String uri : uris) {
rows.add(buildMapping(
apiKeyId,
knowledgeId,
uri,
requireApiScope(uri)
));
}
} }
for (String uri : uris) { if (!rows.isEmpty()) {
SysApiKeyResource resource = ensureResource(uri); mappingService.saveBatch(rows);
SysApiKeyResourceMapping row = new SysApiKeyResourceMapping();
row.setApiKeyId(apiKeyId);
row.setApiKeyResourceId(resource.getId());
row.setResourceType(RESOURCE_TYPE_KNOWLEDGE);
row.setResourceTargetId(knowledgeId);
row.setActionScope(scope);
rows.add(row);
} }
} } finally {
if (!rows.isEmpty()) { releaseLock.run();
mappingService.saveBatch(rows);
} }
} }
@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public void replaceApiShareEnabled(BigInteger apiKeyId, boolean enabled) { public void replaceApiShareEnabled(BigInteger apiKeyId, boolean enabled) {
replaceApiPermissions(apiKeyId, enabled, enabled, false);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void replaceApiPermissions(BigInteger apiKeyId,
boolean readEnabled,
boolean importEnabled,
boolean maintenanceEnabled) {
if (apiKeyId == null) { if (apiKeyId == null) {
throw new BusinessException("系统访问令牌不能为空"); throw new BusinessException("系统访问令牌不能为空");
} }
@@ -130,30 +197,73 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
if (apiKey == null) { if (apiKey == null) {
throw new BusinessException("系统访问令牌不存在"); throw new BusinessException("系统访问令牌不存在");
} }
mappingService.removeScopedMappings(apiKeyId, RESOURCE_TYPE_KNOWLEDGE); Runnable releaseLock = acquirePermissionMutationLock(apiKeyId);
if (!enabled) { try {
return; // 全局开关只替换全局授权,保留分享页配置的指定知识库权限。
} mappingService.remove(
QueryWrapper.create()
.eq(SysApiKeyResourceMapping::getApiKeyId, apiKeyId)
.eq(
SysApiKeyResourceMapping::getResourceType,
RESOURCE_TYPE_KNOWLEDGE
)
.isNull(
SysApiKeyResourceMapping::getResourceTargetId
)
);
Set<String> enabledScopes = KnowledgeApiPermissionScope.enabledScopes(
readEnabled,
importEnabled,
maintenanceEnabled
);
if (enabledScopes.isEmpty()) {
return;
}
List<SysApiKeyResourceMapping> rows = new ArrayList<>(); List<SysApiKeyResourceMapping> rows = new ArrayList<>();
for (String scope : KnowledgeShareActionScope.defaultApiScopes()) { for (String scope : enabledScopes) {
List<String> uris = URI_SCOPE_MAPPING.get(scope); List<String> uris = API_SCOPE_URI_MAPPING.get(scope);
if (uris == null || uris.isEmpty()) { if (uris == null || uris.isEmpty()) {
continue; continue;
}
for (String uri : uris) {
rows.add(buildMapping(apiKeyId, null, uri, scope));
}
} }
for (String uri : uris) { if (!rows.isEmpty()) {
SysApiKeyResource resource = ensureResource(uri); mappingService.saveBatch(rows);
SysApiKeyResourceMapping row = new SysApiKeyResourceMapping(); }
row.setApiKeyId(apiKeyId); } finally {
row.setApiKeyResourceId(resource.getId()); releaseLock.run();
row.setResourceType(RESOURCE_TYPE_KNOWLEDGE); }
row.setActionScope(scope); }
rows.add(row);
@Override
public Set<String> getApiPermissionScopes(BigInteger apiKeyId) {
if (apiKeyId == null) {
return Set.of();
}
List<SysApiKeyResourceMapping> mappings = mappingService.list(
QueryWrapper.create()
.select(SysApiKeyResourceMapping::getActionScope)
.eq(SysApiKeyResourceMapping::getApiKeyId, apiKeyId)
.eq(SysApiKeyResourceMapping::getResourceType, RESOURCE_TYPE_KNOWLEDGE)
.isNull(SysApiKeyResourceMapping::getResourceTargetId)
.in(
SysApiKeyResourceMapping::getActionScope,
API_SCOPE_URI_MAPPING.keySet()
)
);
if (mappings == null || mappings.isEmpty()) {
return Set.of();
}
Set<String> scopes = new java.util.LinkedHashSet<>();
for (SysApiKeyResourceMapping mapping : mappings) {
if (mapping.getActionScope() != null) {
scopes.add(mapping.getActionScope());
} }
} }
if (!rows.isEmpty()) { return scopes;
mappingService.saveBatch(rows);
}
} }
@Override @Override
@@ -161,9 +271,56 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
if (apiKeyId == null || knowledgeId == null) { if (apiKeyId == null || knowledgeId == null) {
throw new BusinessException("API 分享鉴权参数不完整"); throw new BusinessException("API 分享鉴权参数不完整");
} }
if (!API_SCOPE_URI_MAPPING.containsKey(actionScope)) {
throw new IllegalArgumentException("未知的知识库 API 权限范围");
}
sysApiKeyService.checkResourceScope(apiKeyId, requestUri, RESOURCE_TYPE_KNOWLEDGE, knowledgeId, actionScope); sysApiKeyService.checkResourceScope(apiKeyId, requestUri, RESOURCE_TYPE_KNOWLEDGE, knowledgeId, actionScope);
} }
/**
* 构造单条知识库 API 权限映射。
*
* @param apiKeyId 访问令牌 ID
* @param knowledgeId 知识库 ID为空表示全局授权
* @param uri 请求 URI
* @param scope 产品权限 Scope
* @return 权限映射
*/
private SysApiKeyResourceMapping buildMapping(BigInteger apiKeyId,
BigInteger knowledgeId,
String uri,
String scope) {
SysApiKeyResource resource = ensureResource(uri);
SysApiKeyResourceMapping row = new SysApiKeyResourceMapping();
row.setApiKeyId(apiKeyId);
row.setApiKeyResourceId(resource.getId());
row.setResourceType(RESOURCE_TYPE_KNOWLEDGE);
row.setResourceTargetId(knowledgeId);
row.setActionScope(scope);
return row;
}
/**
* 按 URI 获取唯一的产品权限 Scope。
*
* @param uri 请求 URI
* @return 产品权限 Scope
*/
private String requireApiScope(String uri) {
for (Map.Entry<String, List<String>> entry : API_SCOPE_URI_MAPPING.entrySet()) {
if (entry.getValue().contains(uri)) {
return entry.getKey();
}
}
throw new IllegalArgumentException("知识库接口未归类: " + uri);
}
/**
* 获取或创建固定 URI 对应的 API 资源。
*
* @param requestInterface 请求 URI
* @return API 资源
*/
private SysApiKeyResource ensureResource(String requestInterface) { private SysApiKeyResource ensureResource(String requestInterface) {
QueryWrapper wrapper = QueryWrapper.create() QueryWrapper wrapper = QueryWrapper.create()
.eq(SysApiKeyResource::getRequestInterface, requestInterface); .eq(SysApiKeyResource::getRequestInterface, requestInterface);
@@ -177,4 +334,35 @@ public class KnowledgeSharePermissionServiceImpl implements KnowledgeSharePermis
resourceService.save(resource); resourceService.save(resource);
return resource; return resource;
} }
/**
* 获取访问令牌知识库权限变更锁,并在事务完成后释放。
*
* @param apiKeyId 访问令牌 ID
* @return 非事务直接调用时使用的释放动作
*/
private Runnable acquirePermissionMutationLock(BigInteger apiKeyId) {
LockHandle handle = redisLockExecutor.tryAcquire(
PERMISSION_LOCK_PREFIX + apiKeyId,
PERMISSION_LOCK_WAIT,
PERMISSION_LOCK_LEASE
);
if (handle == null) {
throw new BusinessException("访问令牌权限正在更新,请稍后重试");
}
if (TransactionSynchronizationManager.isSynchronizationActive()
&& TransactionSynchronizationManager.isActualTransactionActive()) {
TransactionSynchronizationManager.registerSynchronization(
new TransactionSynchronization() {
@Override
public void afterCompletion(int status) {
handle.release();
}
}
);
return () -> {
};
}
return handle::release;
}
} }

View File

@@ -108,6 +108,26 @@ public class DocumentParseBridgeServiceImplTest {
Assert.assertEquals("# demo", taskInfo.getResult().getPreferredText()); Assert.assertEquals("# demo", taskInfo.getResult().getPreferredText());
} }
/**
* 验证带 PDF 源信息的任务查询会直达 PDF 服务,不试探无关的 PPTX 服务。
*/
@Test
public void shouldQueryPdfTaskAgainstResolvedService() {
FakePdfDocumentParseService pdfService = new FakePdfDocumentParseService();
pdfService.taskStatusValue = "completed";
FakePptxDocumentParseService pptxService = new FakePptxDocumentParseService();
DocumentParseBridgeServiceImpl bridgeService =
buildBridgeService(pdfService, pptxService, null, pdfService);
DocumentParseTaskInfo taskInfo = bridgeService.queryTaskInfo("task-1", buildSource());
DocumentParsedResult result = bridgeService.queryResult("task-1", buildSource());
Assert.assertEquals("completed", taskInfo.getStatus());
Assert.assertEquals("# demo", result.getPreferredText());
Assert.assertEquals(0, pptxService.queryTaskInfoCallCount);
Assert.assertEquals(0, pptxService.queryResultCallCount);
}
/** /**
* 验证缺少底层服务时抛出稳定错误码。 * 验证缺少底层服务时抛出稳定错误码。
*/ */
@@ -390,6 +410,8 @@ public class DocumentParseBridgeServiceImplTest {
private static class FakePptxDocumentParseService implements PptxDocumentParseService { private static class FakePptxDocumentParseService implements PptxDocumentParseService {
private int parseCallCount; private int parseCallCount;
private int queryTaskInfoCallCount;
private int queryResultCallCount;
@Override @Override
public ParseResponse parse(ParseRequest request) { public ParseResponse parse(ParseRequest request) {
@@ -415,6 +437,13 @@ public class DocumentParseBridgeServiceImplTest {
@Override @Override
public ParseResponse queryResult(String taskId) { public ParseResponse queryResult(String taskId) {
queryResultCallCount++;
throw new UnsupportedOperationException();
}
@Override
public ParseTaskInfo queryTaskInfo(String taskId) {
queryTaskInfoCallCount++;
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
} }

View File

@@ -0,0 +1,913 @@
package tech.easyflow.ai.documentimport.task;
import com.mybatisflex.core.query.QueryWrapper;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult;
import tech.easyflow.ai.documentimport.ImportCallerContext;
import tech.easyflow.ai.documentimport.ImportCallerType;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
import tech.easyflow.ai.service.DocumentImportBatchService;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.filestorage.FileStorageWriteHandle;
import tech.easyflow.common.filestorage.FileStorageWriteResult;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
import java.util.Set;
/**
* {@link DocumentImportBatchAppService} 批次启动与重复策略回归测试。
*
* @author Codex
* @since 2026-07-31
*/
public class DocumentImportBatchAppServiceTest {
/**
* 验证上传领取通过单条多表更新同步刷新批次进度时间。
*/
@Test
public void uploadShouldAtomicallyClaimItemAndTouchBatch() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(41));
DocumentImportBatchItem uploaded = pendingUploadItem(BigInteger.valueOf(41));
uploaded.setStatus(DocumentImportBatchItemStatus.UPLOADED.name());
uploaded.setFilePath("/stored/demo.docx");
FileStorageWriteHandle handle = writeHandle(pending);
String locator = handle.encodeLocator();
uploaded.setStorageLocator(locator);
uploaded.setCleanupPending(false);
Mockito.when(context.batchTracker.requireItem(pending.getId()))
.thenReturn(pending, uploaded);
Mockito.when(context.itemMapper.claimUpload(
Mockito.eq(batch.getId()),
Mockito.eq(pending.getId()),
Mockito.eq(batch.getKnowledgeId()),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.itemMapper.registerUploadWriteIntent(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.batchTracker.completeUpload(
pending.getId(),
"/stored/demo.docx",
locator
)).thenReturn(true);
FileStorageService storage = Mockito.mock(FileStorageService.class);
Mockito.when(storage.prepareRecoverableWrite(
Mockito.anyString(),
Mockito.anyString()
)).thenReturn(handle);
Mockito.when(storage.saveRecoverable(
Mockito.any(MultipartFile.class),
Mockito.eq(handle)
)).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator));
setStorageService(context.service, storage);
MultipartFile file = uploadFile("demo.docx", pending.getFileSize());
DocumentImportBatchDtos.ItemResponse response = context.service.uploadItem(
batch.getKnowledgeId(),
batch.getId(),
pending.getId(),
file,
publicCaller()
);
Assert.assertEquals(
DocumentImportBatchItemStatus.UPLOADED.name(),
response.getStatus()
);
org.mockito.InOrder order = Mockito.inOrder(context.itemMapper, storage);
order.verify(context.itemMapper).claimUpload(
Mockito.eq(batch.getId()),
Mockito.eq(pending.getId()),
Mockito.eq(batch.getKnowledgeId()),
Mockito.any(Date.class)
);
order.verify(context.itemMapper).registerUploadWriteIntent(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
);
order.verify(storage).saveRecoverable(file, handle);
}
/**
* 验证对象绑定失败后先清理,再恢复为可重新上传状态。
*/
@Test
public void uploadBindFailureShouldCleanupAndAllowSameItemRetry() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(42));
FileStorageWriteHandle handle = writeHandle(pending);
String locator = handle.encodeLocator();
DocumentImportBatchItem cleanup = pendingUploadItem(pending.getId());
cleanup.setStatus(DocumentImportBatchItemStatus.UPLOADING.name());
cleanup.setStorageLocator(locator);
cleanup.setFilePath("/stored/demo.docx");
cleanup.setCleanupPending(true);
DocumentImportBatchItem retryPending = pendingUploadItem(pending.getId());
DocumentImportBatchItem uploaded = pendingUploadItem(pending.getId());
uploaded.setStatus(DocumentImportBatchItemStatus.UPLOADED.name());
uploaded.setStorageLocator(locator);
uploaded.setFilePath("/stored/demo.docx");
Mockito.when(context.batchTracker.requireItem(pending.getId()))
.thenReturn(pending, pending, cleanup, retryPending, uploaded);
Mockito.when(context.itemMapper.claimUpload(
Mockito.eq(batch.getId()),
Mockito.eq(pending.getId()),
Mockito.eq(batch.getKnowledgeId()),
Mockito.any(Date.class)
)).thenReturn(1, 1);
Mockito.when(context.itemMapper.registerUploadWriteIntent(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
)).thenReturn(1, 1);
Mockito.when(context.batchTracker.completeUpload(
pending.getId(),
"/stored/demo.docx",
locator
)).thenReturn(false, true);
Mockito.when(context.itemMapper.markUploadCleanupPending(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.itemMapper.completeUploadingStorageCleanup(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.eq("/stored/demo.docx"),
Mockito.anyString(),
Mockito.any(Date.class)
)).thenReturn(1);
FileStorageService storage = Mockito.mock(FileStorageService.class);
Mockito.when(storage.prepareRecoverableWrite(
Mockito.anyString(),
Mockito.anyString()
)).thenReturn(handle);
Mockito.when(storage.saveRecoverable(
Mockito.any(MultipartFile.class),
Mockito.eq(handle)
)).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator));
setStorageService(context.service, storage);
MultipartFile file = uploadFile("demo.docx", pending.getFileSize());
try {
context.service.uploadItem(
batch.getKnowledgeId(),
batch.getId(),
pending.getId(),
file,
publicCaller()
);
Assert.fail("Expected cancelled upload rejection");
} catch (BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("上传批次已取消"));
}
DocumentImportBatchDtos.ItemResponse retried = context.service.uploadItem(
batch.getKnowledgeId(),
batch.getId(),
pending.getId(),
file,
publicCaller()
);
Assert.assertEquals(
DocumentImportBatchItemStatus.UPLOADED.name(),
retried.getStatus()
);
org.mockito.InOrder order = Mockito.inOrder(context.itemMapper, storage);
order.verify(context.itemMapper).registerUploadWriteIntent(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
);
order.verify(storage).saveRecoverable(file, handle);
order.verify(context.itemMapper).markUploadCleanupPending(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
);
order.verify(storage).deleteRecoverable(handle);
order.verify(context.itemMapper).completeUploadingStorageCleanup(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.eq("/stored/demo.docx"),
Mockito.anyString(),
Mockito.any(Date.class)
);
Mockito.verify(context.itemMapper, Mockito.times(2)).claimUpload(
Mockito.eq(batch.getId()),
Mockito.eq(pending.getId()),
Mockito.eq(batch.getKnowledgeId()),
Mockito.any(Date.class)
);
Mockito.verify(context.batchTracker, Mockito.never()).transitionItem(
Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(),
Mockito.anyBoolean(), Mockito.anyInt()
);
}
/**
* 验证批次并发取消后,对象清理不会把文件项恢复为待上传。
*/
@Test
public void uploadCleanupShouldPreserveConcurrentCancellation() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(44));
FileStorageWriteHandle handle = writeHandle(pending);
String locator = handle.encodeLocator();
DocumentImportBatchItem cancelled = pendingUploadItem(pending.getId());
cancelled.setStatus(DocumentImportBatchItemStatus.CANCELLED.name());
cancelled.setStorageLocator(locator);
cancelled.setFilePath("/stored/demo.docx");
cancelled.setCleanupPending(true);
Mockito.when(context.batchTracker.requireItem(pending.getId()))
.thenReturn(pending, pending, cancelled);
Mockito.when(context.itemMapper.claimUpload(
Mockito.eq(batch.getId()),
Mockito.eq(pending.getId()),
Mockito.eq(batch.getKnowledgeId()),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.itemMapper.registerUploadWriteIntent(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.batchTracker.completeUpload(
pending.getId(),
"/stored/demo.docx",
locator
)).thenReturn(false);
Mockito.when(context.itemMapper.markUploadCleanupPending(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.itemMapper.completeCancelledStorageCleanup(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.eq("/stored/demo.docx"),
Mockito.any(Date.class)
)).thenReturn(1);
FileStorageService storage = Mockito.mock(FileStorageService.class);
Mockito.when(storage.prepareRecoverableWrite(
Mockito.anyString(),
Mockito.anyString()
)).thenReturn(handle);
Mockito.when(storage.saveRecoverable(
Mockito.any(MultipartFile.class),
Mockito.eq(handle)
)).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator));
setStorageService(context.service, storage);
try {
context.service.uploadItem(
batch.getKnowledgeId(),
batch.getId(),
pending.getId(),
uploadFile("demo.docx", pending.getFileSize()),
publicCaller()
);
Assert.fail("Expected cancelled upload rejection");
} catch (BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("上传批次已取消"));
}
Mockito.verify(storage).deleteRecoverable(handle);
Mockito.verify(context.itemMapper).completeCancelledStorageCleanup(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.eq("/stored/demo.docx"),
Mockito.any(Date.class)
);
}
/**
* 验证对象删除后的数据库恢复失败会保留待办,并由下一轮幂等完成。
*/
@Test
public void cleanupRecoveryFailureShouldRemainRetryable() {
TestContext context = createContext();
DocumentImportBatchItem cleanup = pendingUploadItem(BigInteger.valueOf(45));
FileStorageWriteHandle handle = writeHandle(cleanup);
String locator = handle.encodeLocator();
cleanup.setStatus(DocumentImportBatchItemStatus.UPLOADING.name());
cleanup.setStorageLocator(locator);
cleanup.setFilePath("/stored/demo.docx");
cleanup.setCleanupPending(true);
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(cleanup));
Mockito.when(context.itemMapper.completeUploadingStorageCleanup(
Mockito.eq(cleanup.getId()),
Mockito.eq(locator),
Mockito.eq("/stored/demo.docx"),
Mockito.anyString(),
Mockito.any(Date.class)
)).thenThrow(new RuntimeException("database unavailable"))
.thenReturn(1);
FileStorageService storage = Mockito.mock(FileStorageService.class);
setStorageService(context.service, storage);
Assert.assertEquals(0, context.service.cleanupCancelledStoredObjects(10));
Assert.assertEquals(1, context.service.cleanupCancelledStoredObjects(10));
Mockito.verify(storage, Mockito.times(2)).deleteRecoverable(handle);
Mockito.verify(context.itemMapper, Mockito.times(2))
.completeUploadingStorageCleanup(
Mockito.eq(cleanup.getId()),
Mockito.eq(locator),
Mockito.eq("/stored/demo.docx"),
Mockito.anyString(),
Mockito.any(Date.class)
);
}
/**
* 验证写意图登记已提交但调用抛异常时,写入前撤销会清理定位符。
*/
@Test
public void uploadIntentUnknownCommitShouldAbortBeforePhysicalWrite() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(46));
FileStorageWriteHandle handle = writeHandle(pending);
String locator = handle.encodeLocator();
Mockito.when(context.batchTracker.requireItem(pending.getId()))
.thenReturn(pending);
Mockito.when(context.itemMapper.claimUpload(
Mockito.eq(batch.getId()),
Mockito.eq(pending.getId()),
Mockito.eq(batch.getKnowledgeId()),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.itemMapper.registerUploadWriteIntent(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
)).thenThrow(new RuntimeException("commit result unknown"));
Mockito.when(context.itemMapper.abortUploadBeforeWrite(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.anyString(),
Mockito.any(Date.class)
)).thenReturn(1);
FileStorageService storage = Mockito.mock(FileStorageService.class);
Mockito.when(storage.prepareRecoverableWrite(
Mockito.anyString(),
Mockito.anyString()
)).thenReturn(handle);
setStorageService(context.service, storage);
try {
context.service.uploadItem(
batch.getKnowledgeId(),
batch.getId(),
pending.getId(),
uploadFile("demo.docx", pending.getFileSize()),
publicCaller()
);
Assert.fail("Expected unknown commit exception");
} catch (RuntimeException expected) {
Assert.assertEquals("commit result unknown", expected.getMessage());
}
Mockito.verify(context.itemMapper).abortUploadBeforeWrite(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.anyString(),
Mockito.any(Date.class)
);
Mockito.verify(storage, Mockito.never()).saveRecoverable(
Mockito.any(MultipartFile.class),
Mockito.any(FileStorageWriteHandle.class)
);
}
/**
* 验证提交响应异常但数据库已完成上传时不会误删有效对象。
*/
@Test
public void uploadCommitUnknownShouldKeepConfirmedUploadedObject() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
DocumentImportBatchItem pending = pendingUploadItem(BigInteger.valueOf(43));
FileStorageWriteHandle handle = writeHandle(pending);
String locator = handle.encodeLocator();
DocumentImportBatchItem committed = pendingUploadItem(pending.getId());
committed.setStatus(DocumentImportBatchItemStatus.UPLOADED.name());
committed.setFilePath("/stored/demo.docx");
committed.setStorageLocator(locator);
committed.setCleanupPending(false);
Mockito.when(context.batchTracker.requireItem(pending.getId()))
.thenReturn(pending, committed);
Mockito.when(context.itemMapper.claimUpload(
Mockito.eq(batch.getId()),
Mockito.eq(pending.getId()),
Mockito.eq(batch.getKnowledgeId()),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.itemMapper.registerUploadWriteIntent(
Mockito.eq(pending.getId()),
Mockito.eq(locator),
Mockito.any(Date.class)
)).thenReturn(1);
Mockito.when(context.batchTracker.completeUpload(
pending.getId(),
"/stored/demo.docx",
locator
)).thenThrow(new RuntimeException("commit result unknown"));
FileStorageService storage = Mockito.mock(FileStorageService.class);
Mockito.when(storage.prepareRecoverableWrite(
Mockito.anyString(),
Mockito.anyString()
)).thenReturn(handle);
Mockito.when(storage.saveRecoverable(
Mockito.any(MultipartFile.class),
Mockito.eq(handle)
)).thenReturn(new FileStorageWriteResult("/stored/demo.docx", locator));
setStorageService(context.service, storage);
DocumentImportBatchDtos.ItemResponse response = context.service.uploadItem(
batch.getKnowledgeId(),
batch.getId(),
pending.getId(),
uploadFile("demo.docx", pending.getFileSize()),
publicCaller()
);
Assert.assertEquals(
DocumentImportBatchItemStatus.UPLOADED.name(),
response.getStatus()
);
Mockito.verify(context.itemMapper, Mockito.never())
.markUploadCleanupPending(
Mockito.any(),
Mockito.anyString(),
Mockito.any(Date.class)
);
Mockito.verify(storage, Mockito.never())
.deleteRecoverable(Mockito.any(FileStorageWriteHandle.class));
}
/**
* 验证同一知识库已有运行中的自动批次时拒绝再次启动。
*/
@Test
public void startAutoShouldRejectOtherRunningBatch() {
TestContext context = createContext();
Mockito.when(context.batchService.count(Mockito.any(QueryWrapper.class))).thenReturn(1L);
beginTransactionSynchronization();
try {
context.service.startBatch(startRequest("AUTO", "SKIP"));
Assert.fail("Expected active automatic batch rejection");
} catch (BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("已有自动导入批次"));
} finally {
completeTransactionSynchronization(TransactionSynchronization.STATUS_ROLLED_BACK);
}
Mockito.verify(context.batchMapper, Mockito.never())
.updateByQuery(Mockito.any(DocumentImportBatch.class), Mockito.any());
Mockito.verify(context.lockHandle).release();
}
/**
* 验证覆盖策略会持久化历史文档 ID并继续创建新文档任务。
*/
@Test
public void startOverwriteShouldRecordHistoricalDocument() {
TestContext context = createContext();
Mockito.when(context.batchService.count(Mockito.any(QueryWrapper.class))).thenReturn(0L);
DocumentImportBatchItem uploaded = uploadedItem(BigInteger.valueOf(11), BigInteger.ONE);
DocumentImportBatchItem historical = uploadedItem(BigInteger.valueOf(21), BigInteger.valueOf(22));
historical.setBatchId(BigInteger.valueOf(99));
historical.setDocumentId(BigInteger.valueOf(23));
historical.setStatus(DocumentImportBatchItemStatus.COMPLETED.name());
historical.setCreated(new Date());
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(uploaded), List.of(historical));
tech.easyflow.ai.entity.Document historicalDocument = new tech.easyflow.ai.entity.Document();
historicalDocument.setId(historical.getDocumentId());
historicalDocument.setCollectionId(BigInteger.TWO);
Mockito.when(context.documentMapper.selectListByQuery(Mockito.any()))
.thenReturn(List.of(historicalDocument));
Mockito.when(context.batchMapper.updateByQuery(
Mockito.any(DocumentImportBatch.class), Mockito.any()
)).thenReturn(1);
Mockito.when(context.batchTracker.refreshBatch(BigInteger.ONE))
.thenReturn(new DocumentImportBatchDtos.StatusResponse());
beginTransactionSynchronization();
try {
context.service.startBatch(startRequest("AUTO", "OVERWRITE"));
} finally {
completeTransactionSynchronization(TransactionSynchronization.STATUS_COMMITTED);
}
Mockito.verify(context.batchTracker)
.markReplacement(uploaded.getId(), historical.getDocumentId());
Mockito.verify(context.taskAppService)
.createBatchImportTasks(Mockito.any(DocumentImportBatch.class), Mockito.anyList());
Mockito.verify(context.lockHandle).release();
}
/**
* 验证重试代次由服务端读取并原子领取。
*/
@Test
public void retryShouldClaimCurrentGeneration() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
batch.setRetryGeneration(0);
DocumentImportBatchItem failed = uploadedItem(
BigInteger.valueOf(31),
batch.getId()
);
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
failed.setRetryable(true);
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(failed));
Mockito.when(context.batchMapper.claimRetry(
Mockito.any(),
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
Mockito.any(Date.class)
)).thenReturn(1);
beginTransactionSynchronization();
DocumentImportBatchRetryResult first;
try {
first = context.service.retryOwnedBatch(
batch.getId(),
publicCaller(),
Set.of("file-key")
);
} finally {
completeTransactionSynchronization(
TransactionSynchronization.STATUS_ROLLED_BACK
);
}
Assert.assertEquals(Integer.valueOf(1), first.getRetryGeneration());
Assert.assertEquals(Integer.valueOf(1), first.getRetriedCount());
Mockito.verify(context.batchMapper).claimRetry(
Mockito.any(),
Mockito.anyString(),
Mockito.any(),
Mockito.eq(0),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
}
/**
* 验证修复前的可恢复失败项会在重试领取事务中恢复重试资格和批次计数。
*/
@Test
public void retryShouldRestoreLegacyRecoverableItem() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
batch.setRetryGeneration(0);
DocumentImportBatchItem failed = uploadedItem(
BigInteger.valueOf(32),
batch.getId()
);
failed.setStage(DocumentImportBatchItemStage.PARSE.name());
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
failed.setRetryable(false);
failed.setFailureCode("parse_failed");
failed.setErrorSummary("历史代码异常");
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
.thenReturn(List.of(failed));
Mockito.when(context.taskAppService.isRecoverableBatchFailure(failed))
.thenReturn(true);
Mockito.when(context.batchTracker.transitionItem(
Mockito.eq(failed.getId()),
Mockito.eq(DocumentImportBatchItemStage.PARSE),
Mockito.eq(DocumentImportBatchItemStatus.FAILED),
Mockito.eq("历史代码异常"),
Mockito.eq(true),
Mockito.eq(0),
Mockito.eq("parse_failed")
)).thenReturn(true);
Mockito.when(context.batchMapper.claimRetry(
Mockito.any(),
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
Mockito.any(Date.class)
)).thenReturn(1);
beginTransactionSynchronization();
DocumentImportBatchRetryResult result;
try {
result = context.service.retryOwnedBatch(
batch.getId(),
publicCaller(),
Set.of()
);
} finally {
completeTransactionSynchronization(
TransactionSynchronization.STATUS_ROLLED_BACK
);
}
Assert.assertEquals(Integer.valueOf(1), result.getRetriedCount());
Assert.assertTrue(failed.getRetryable());
Mockito.verify(context.batchTracker).transitionItem(
failed.getId(),
DocumentImportBatchItemStage.PARSE,
DocumentImportBatchItemStatus.FAILED,
"历史代码异常",
true,
0,
"parse_failed"
);
Mockito.verify(context.batchMapper).claimRetry(
Mockito.eq(batch.getId()),
Mockito.anyString(),
Mockito.any(),
Mockito.eq(0),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
}
/**
* 验证运行中的任务拒绝再次领取,调用方可继续查询原 taskId。
*/
@Test
public void retryShouldRejectTaskThatIsAlreadyRunning() {
TestContext context = createContext();
DocumentImportBatch batch = context.batchService.getOne(
QueryWrapper.create()
);
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
beginTransactionSynchronization();
try {
context.service.retryOwnedBatch(
BigInteger.ONE,
publicCaller(),
Set.of()
);
Assert.fail("Expected running task rejection");
} catch (BusinessException expected) {
Assert.assertTrue(
expected.getMessage().contains("当前任务状态不允许重试")
);
} finally {
completeTransactionSynchronization(
TransactionSynchronization.STATUS_ROLLED_BACK
);
}
Mockito.verify(context.batchMapper, Mockito.never()).claimRetry(
Mockito.any(),
Mockito.anyString(),
Mockito.any(),
Mockito.anyInt(),
Mockito.any(Date.class)
);
Mockito.verify(context.lockHandle).release();
}
/**
* 验证候选快照过期后若任务已恢复进展,不会继续取消或清理文件项。
*/
@Test
public void staleCancellationShouldStopWhenConditionalClaimMisses() {
TestContext context = createContext();
Mockito.when(context.batchMapper.claimStaleCancellation(
Mockito.any(),
Mockito.any(),
Mockito.anyString(),
Mockito.any(),
Mockito.any(Date.class),
Mockito.any(Date.class)
)).thenReturn(0);
beginTransactionSynchronization();
boolean cancelled;
try {
cancelled = context.service.cancelStaleBatch(
BigInteger.TWO,
BigInteger.ONE,
publicCaller(),
new Date(System.currentTimeMillis() - 30L * 60L * 1000L)
);
} finally {
completeTransactionSynchronization(
TransactionSynchronization.STATUS_ROLLED_BACK
);
}
Assert.assertFalse(cancelled);
Mockito.verify(context.itemService, Mockito.never())
.list(Mockito.any(QueryWrapper.class));
Mockito.verifyNoInteractions(context.itemMapper);
Mockito.verify(context.lockHandle).release();
}
private TestContext createContext() {
DocumentImportBatchService batchService = Mockito.mock(DocumentImportBatchService.class);
DocumentImportBatchItemService itemService = Mockito.mock(DocumentImportBatchItemService.class);
DocumentImportBatchTracker batchTracker = Mockito.mock(DocumentImportBatchTracker.class);
KnowledgeDocumentImportTaskAppService taskAppService =
Mockito.mock(KnowledgeDocumentImportTaskAppService.class);
DocumentImportBatchMapper batchMapper = Mockito.mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper =
Mockito.mock(DocumentImportBatchItemMapper.class);
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
Mockito.when(redisLockExecutor.tryAcquire(
Mockito.anyString(), Mockito.any(), Mockito.any()
)).thenReturn(lockHandle);
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(BigInteger.ONE);
batch.setKnowledgeId(BigInteger.TWO);
batch.setStatus(DocumentImportBatchStatus.READY.name());
Mockito.when(batchTracker.requireBatch(BigInteger.ONE)).thenReturn(batch);
Mockito.when(batchService.getOne(Mockito.any(QueryWrapper.class)))
.thenReturn(batch);
Mockito.when(batchMapper.selectOwnedForUpdate(
Mockito.any(), Mockito.anyString(), Mockito.any()
)).thenReturn(batch);
DocumentImportBatchAppService service = new DocumentImportBatchAppService(
batchService,
itemService,
batchTracker,
new DocumentImportBulkProperties(),
taskAppService,
batchMapper,
itemMapper,
documentMapper,
redisLockExecutor
);
return new TestContext(
service, batchService, itemService, batchTracker,
taskAppService, batchMapper, itemMapper, documentMapper, lockHandle
);
}
/**
* 创建 Public API 测试调用者。
*
* @return Public API 调用者
*/
private ImportCallerContext publicCaller() {
return new ImportCallerContext(
ImportCallerType.PUBLIC_API,
BigInteger.valueOf(77)
);
}
private DocumentImportBatchDtos.StartRequest startRequest(String mode, String duplicatePolicy) {
DocumentImportBatchDtos.StartRequest request = new DocumentImportBatchDtos.StartRequest();
request.setKnowledgeId(BigInteger.TWO);
request.setBatchId(BigInteger.ONE);
request.setImportMode(mode);
request.setDuplicatePolicy(duplicatePolicy);
return request;
}
private DocumentImportBatchItem uploadedItem(BigInteger itemId, BigInteger batchId) {
DocumentImportBatchItem item = new DocumentImportBatchItem();
item.setId(itemId);
item.setBatchId(batchId);
item.setKnowledgeId(BigInteger.TWO);
item.setClientFileKey("file-key");
item.setFileName("demo.docx");
item.setFilePath("/demo.docx");
item.setStatus(DocumentImportBatchItemStatus.UPLOADED.name());
return item;
}
private DocumentImportBatchItem pendingUploadItem(BigInteger itemId) {
DocumentImportBatchItem item = new DocumentImportBatchItem();
item.setId(itemId);
item.setBatchId(BigInteger.ONE);
item.setKnowledgeId(BigInteger.TWO);
item.setClientFileKey("file-key-" + itemId);
item.setFileName("demo.docx");
item.setRelativePath("demo.docx");
item.setFileSize(4L);
item.setStage("UPLOAD");
item.setStatus(DocumentImportBatchItemStatus.PENDING.name());
item.setCleanupPending(false);
return item;
}
private FileStorageWriteHandle writeHandle(DocumentImportBatchItem item) {
return new FileStorageWriteHandle(
"mock-storage",
"",
"/tmp/easyflow-test",
"knowledge-import/" + item.getBatchId() + "/" + item.getId(),
item.getId() + ".docx"
);
}
private MultipartFile uploadFile(String fileName, long size) {
MultipartFile file = Mockito.mock(MultipartFile.class);
Mockito.when(file.getOriginalFilename()).thenReturn(fileName);
Mockito.when(file.getSize()).thenReturn(size);
Mockito.when(file.isEmpty()).thenReturn(false);
return file;
}
private void setStorageService(DocumentImportBatchAppService service,
FileStorageService storageService) {
try {
Field field = DocumentImportBatchAppService.class.getDeclaredField(
"storageService"
);
field.setAccessible(true);
field.set(service, storageService);
} catch (ReflectiveOperationException error) {
throw new AssertionError("Failed to inject storage service", error);
}
}
private void beginTransactionSynchronization() {
TransactionSynchronizationManager.initSynchronization();
}
private void completeTransactionSynchronization(int status) {
List<TransactionSynchronization> synchronizations =
TransactionSynchronizationManager.getSynchronizations();
for (TransactionSynchronization synchronization : synchronizations) {
if (status == TransactionSynchronization.STATUS_COMMITTED) {
synchronization.afterCommit();
}
synchronization.afterCompletion(status);
}
TransactionSynchronizationManager.clearSynchronization();
}
/**
* 测试依赖集合。
*/
private record TestContext(
DocumentImportBatchAppService service,
DocumentImportBatchService batchService,
DocumentImportBatchItemService itemService,
DocumentImportBatchTracker batchTracker,
KnowledgeDocumentImportTaskAppService taskAppService,
DocumentImportBatchMapper batchMapper,
DocumentImportBatchItemMapper itemMapper,
DocumentMapper documentMapper,
RedisLockExecutor.LockHandle lockHandle
) {
}
}

View File

@@ -0,0 +1,186 @@
package tech.easyflow.ai.documentimport.task;
import org.junit.Test;
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.enums.DocumentImportMode;
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
import tech.easyflow.ai.service.DocumentImportBatchService;
import java.math.BigInteger;
import java.util.Date;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 文档批量导入状态汇总测试。
*/
public class DocumentImportBatchTrackerTest {
/**
* 验证运行批次的完成、处理、失败与等待数量。
*/
@Test
public void shouldAggregateRunningBatchProgress() {
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 4);
batch.setCompletedCount(1);
batch.setProcessingCount(1);
batch.setFailedCount(1);
batch.setPendingCount(1);
when(batchService.getById(batch.getId())).thenReturn(batch);
DocumentImportBatchTracker tracker =
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
DocumentImportBatchDtos.StatusResponse response = tracker.refreshBatch(batch.getId());
assertEquals(1, response.getCompletedCount().intValue());
assertEquals(1, response.getProcessingCount().intValue());
assertEquals(1, response.getFailedCount().intValue());
assertEquals(1, response.getPendingCount().intValue());
assertEquals(50, response.getProgressPercent().intValue());
assertEquals(DocumentImportBatchStatus.RUNNING.name(), response.getStatus());
verify(batchService, never()).updateById(batch, false);
}
/**
* 验证中断批次在汇总失败项后仍保留可继续状态。
*/
@Test
public void shouldPreserveInterruptedStatusUntilUserContinues() {
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
DocumentImportBatch batch = batch(DocumentImportBatchStatus.INTERRUPTED, 2);
batch.setCompletedCount(1);
batch.setFailedCount(1);
batch.setRetryableFailedCount(1);
when(batchService.getById(batch.getId())).thenReturn(batch);
DocumentImportBatchTracker tracker =
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
DocumentImportBatchDtos.StatusResponse response = tracker.refreshBatch(batch.getId());
assertEquals(DocumentImportBatchStatus.INTERRUPTED.name(), response.getStatus());
assertEquals(100, response.getProgressPercent().intValue());
assertEquals(1, response.getRetryableFailedCount().intValue());
}
/**
* 验证失败项重试通过状态 CAS 增量迁移计数。
*/
@Test
public void shouldMoveRetryableFailureBackToPendingAtomically() {
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1);
batch.setFailedCount(1);
batch.setRetryableFailedCount(1);
DocumentImportBatchItem item = new DocumentImportBatchItem();
item.setId(BigInteger.TEN);
item.setBatchId(batch.getId());
item.setStage(DocumentImportBatchItemStage.INDEX.name());
item.setStatus(DocumentImportBatchItemStatus.FAILED.name());
item.setRetryable(true);
when(itemService.getById(item.getId())).thenReturn(item);
when(itemMapper.transitionStatus(
eq(item.getId()),
eq(DocumentImportBatchItemStatus.FAILED.name()),
eq(DocumentImportBatchItemStage.INDEX.name()),
eq(DocumentImportBatchItemStatus.PENDING.name()),
eq(null),
eq(null),
eq(false),
eq(1),
any(Date.class)
)).thenReturn(1);
when(batchService.getById(batch.getId())).thenReturn(batch);
DocumentImportBatchTracker tracker =
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
assertTrue(tracker.updateItem(item.getId(),
DocumentImportBatchItemStage.INDEX,
DocumentImportBatchItemStatus.PENDING,
null));
verify(batchMapper).adjustCounters(
eq(batch.getId()),
eq(0),
eq(0),
eq(-1),
eq(1),
eq(0),
eq(0),
eq(0),
eq(-1),
any(Date.class)
);
}
/**
* 验证迟到任务不能把已完成文件重新改为处理中。
*/
@Test
public void shouldRejectLateTransitionFromCompletedToRunning() {
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
DocumentImportBatchItem item = new DocumentImportBatchItem();
item.setId(BigInteger.TEN);
item.setBatchId(BigInteger.ONE);
item.setStatus(DocumentImportBatchItemStatus.COMPLETED.name());
when(itemService.getById(item.getId())).thenReturn(item);
DocumentImportBatchTracker tracker =
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
assertFalse(tracker.updateItem(item.getId(),
DocumentImportBatchItemStage.INDEX,
DocumentImportBatchItemStatus.RUNNING,
null));
verify(itemMapper, never()).transitionStatus(
any(), any(), any(), any(), any(), any(), anyBoolean(), anyInt(), any(Date.class));
}
private DocumentImportBatch batch(DocumentImportBatchStatus status, int totalCount) {
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(BigInteger.ONE);
batch.setKnowledgeId(BigInteger.TWO);
batch.setImportMode(DocumentImportMode.AUTO.name());
batch.setStatus(status.name());
batch.setTotalCount(totalCount);
batch.setTotalBytes(100L);
batch.setCompletedCount(0);
batch.setProcessingCount(0);
batch.setFailedCount(0);
batch.setPendingCount(0);
batch.setUploadedCount(0);
batch.setSkippedCount(0);
batch.setCancelledCount(0);
batch.setRetryableFailedCount(0);
return batch;
}
}

View File

@@ -0,0 +1,68 @@
package tech.easyflow.ai.documentimport.task;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.common.filestorage.FileStorageService;
import java.io.ByteArrayInputStream;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
/**
* {@link DocumentImportChunkSnapshotService} 持久化恢复测试。
*/
public class DocumentImportChunkSnapshotServiceTest {
/**
* 验证分块快照可跨缓存写入并从文件存储完整恢复。
*
* @throws Exception 反射注入或文件读取异常
*/
@Test
public void shouldPersistAndRestoreChunkSnapshot() throws Exception {
String storedPath = "http://localhost/snapshots/9-chunks.json";
AtomicReference<byte[]> storedBytes = new AtomicReference<byte[]>();
FileStorageService storageService = Mockito.mock(FileStorageService.class);
Mockito.when(storageService.save(
Mockito.any(MultipartFile.class),
Mockito.anyString()
)).thenAnswer(invocation -> {
MultipartFile file = invocation.getArgument(0);
storedBytes.set(file.getBytes());
return storedPath;
});
Mockito.when(storageService.readStream(storedPath))
.thenAnswer(invocation -> new ByteArrayInputStream(storedBytes.get()));
DocumentImportChunkSnapshotService service = new DocumentImportChunkSnapshotService();
Field storageField = DocumentImportChunkSnapshotService.class
.getDeclaredField("storageService");
storageField.setAccessible(true);
storageField.set(service, storageService);
DocumentChunk chunk = new DocumentChunk();
chunk.setId(BigInteger.valueOf(11));
chunk.setDocumentId(BigInteger.valueOf(9));
chunk.setDocumentCollectionId(BigInteger.valueOf(7));
chunk.setContent("稳定分块");
DocumentImportDtos.PreviewSession session = new DocumentImportDtos.PreviewSession();
session.setKnowledgeId(BigInteger.valueOf(7));
session.setDocumentId(BigInteger.valueOf(9));
session.setDocumentChunks(List.of(chunk));
String path = service.save(session);
DocumentImportDtos.PreviewSession restored = service.load(path);
Assert.assertEquals(storedPath, path);
Assert.assertEquals(session.getKnowledgeId(), restored.getKnowledgeId());
Assert.assertEquals(session.getDocumentId(), restored.getDocumentId());
Assert.assertEquals(1, restored.getDocumentChunks().size());
Assert.assertEquals("稳定分块", restored.getDocumentChunks().get(0).getContent());
}
}

View File

@@ -6,11 +6,15 @@ import org.mockito.ArgumentMatchers;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import tech.easyflow.ai.documentimport.DocumentImportKeys;
import tech.easyflow.ai.entity.Document; import tech.easyflow.ai.entity.Document;
import tech.easyflow.ai.mapper.DocumentMapper; import tech.easyflow.ai.mapper.DocumentMapper;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.math.BigInteger; import java.math.BigInteger;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
/** /**
@@ -18,6 +22,33 @@ import java.util.concurrent.atomic.AtomicReference;
*/ */
public class DocumentImportTaskStatusStreamServiceTest { public class DocumentImportTaskStatusStreamServiceTest {
/**
* 验证状态流会携带稳定错误码,供前端进行本地化展示。
*
* @throws Exception 反射调用异常
*/
@Test
public void buildDocumentPayloadShouldIncludeTaskErrorCode() throws Exception {
Document document = new Document();
document.setId(BigInteger.valueOf(88));
document.setCollectionId(BigInteger.valueOf(99));
document.setOptions(new LinkedHashMap<String, Object>(Map.of(
DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE,
"parse_service_unavailable"
)));
DocumentImportTaskStatusStreamService service = new DocumentImportTaskStatusStreamService();
Method method = DocumentImportTaskStatusStreamService.class.getDeclaredMethod(
"buildDocumentPayload",
Document.class
);
method.setAccessible(true);
@SuppressWarnings("unchecked")
Map<String, Object> payload = (Map<String, Object>) method.invoke(service, document);
Assert.assertEquals("parse_service_unavailable", payload.get("lastTaskErrorCode"));
}
/** /**
* 验证文档状态变更会向 Redis 广播文档 ID。 * 验证文档状态变更会向 Redis 广播文档 ID。
* *

View File

@@ -3,28 +3,53 @@ package tech.easyflow.ai.documentimport.task;
import com.easyagents.document.core.entity.DocumentBlock; import com.easyagents.document.core.entity.DocumentBlock;
import com.easyagents.document.core.entity.DocumentImage; import com.easyagents.document.core.entity.DocumentImage;
import com.easyagents.document.core.entity.DocumentTable; import com.easyagents.document.core.entity.DocumentTable;
import com.easyagents.core.model.embedding.EmbeddingModel;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.StoreOptions;
import com.easyagents.core.store.StoreResult;
import com.easyagents.rag.ingestion.model.StrategyConfig; import com.easyagents.rag.ingestion.model.StrategyConfig;
import com.easyagents.search.engine.service.DocumentSearcher;
import org.apache.ibatis.annotations.Update;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
import tech.easyflow.ai.document.model.DocumentParseArtifacts; import tech.easyflow.ai.document.model.DocumentParseArtifacts;
import tech.easyflow.ai.document.model.DocumentParsedResult; import tech.easyflow.ai.document.model.DocumentParsedResult;
import tech.easyflow.ai.document.model.DocumentSourceRef;
import tech.easyflow.ai.documentimport.DocumentImportKeys; import tech.easyflow.ai.documentimport.DocumentImportKeys;
import tech.easyflow.ai.entity.DocumentChunk; import tech.easyflow.ai.entity.DocumentChunk;
import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.entity.DocumentImportBatchItem;
import tech.easyflow.ai.entity.DocumentImportTask; import tech.easyflow.ai.entity.DocumentImportTask;
import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
import tech.easyflow.ai.enums.DocumentImportTaskStatus; import tech.easyflow.ai.enums.DocumentImportTaskStatus;
import tech.easyflow.ai.enums.DocumentImportTaskPhase;
import tech.easyflow.ai.enums.DocumentProcessStatus; import tech.easyflow.ai.enums.DocumentProcessStatus;
import tech.easyflow.ai.mapper.DocumentImportTaskMapper;
import tech.easyflow.ai.mapper.DocumentMapper; import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
import tech.easyflow.ai.service.DocumentImportTaskService; import tech.easyflow.ai.service.DocumentImportTaskService;
import tech.easyflow.ai.service.DocumentService;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.common.filestorage.FileStorageService; import tech.easyflow.common.filestorage.FileStorageService;
import java.io.ByteArrayInputStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.lang.reflect.Proxy; import java.lang.reflect.Proxy;
import java.math.BigInteger; import java.math.BigInteger;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Base64; import java.util.Base64;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -38,6 +63,666 @@ import java.util.concurrent.atomic.AtomicReference;
*/ */
public class KnowledgeDocumentImportTaskAppServiceTest { public class KnowledgeDocumentImportTaskAppServiceTest {
/**
* 验证待处理任务重新投递只更新必要字段,避免自定义查询结果覆盖非空列。
*
* @throws Exception 反射注入异常
*/
@Test
public void dispatchPendingTasksShouldTouchPendingWithCas()
throws Exception {
BigInteger taskId = BigInteger.valueOf(29);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setPhase(DocumentImportTaskPhase.PARSE.name());
task.setStatus(DocumentImportTaskStatus.PENDING.name());
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.selectPendingFairly(
Mockito.any(Date.class), Mockito.anyInt()))
.thenReturn(List.of(task));
Mockito.when(taskMapper.touchPendingForDispatch(
Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
Mockito.any(BigInteger.class)))
.thenReturn(1);
DocumentImportParseTaskProducer producer =
Mockito.mock(DocumentImportParseTaskProducer.class);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "parseTaskProducer", producer);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
service.dispatchPendingTasks();
ArgumentCaptor<Date> selectCutoffCaptor =
ArgumentCaptor.forClass(Date.class);
Mockito.verify(taskMapper).selectPendingFairly(
selectCutoffCaptor.capture(), Mockito.anyInt());
ArgumentCaptor<Date> touchCutoffCaptor =
ArgumentCaptor.forClass(Date.class);
Mockito.verify(taskMapper).touchPendingForDispatch(
Mockito.eq(taskId), touchCutoffCaptor.capture(), Mockito.any(Date.class),
Mockito.any(BigInteger.class));
Assert.assertEquals(selectCutoffCaptor.getValue(), touchCutoffCaptor.getValue());
Mockito.verify(producer).send(taskId);
}
/**
* 验证任务已离开待处理状态时不再发送重复消息。
*
* @throws Exception 反射注入异常
*/
@Test
public void dispatchPendingTasksShouldSkipChangedTask()
throws Exception {
BigInteger taskId = BigInteger.valueOf(30);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setPhase(DocumentImportTaskPhase.PARSE.name());
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.selectPendingFairly(
Mockito.any(Date.class), Mockito.anyInt()))
.thenReturn(List.of(task));
Mockito.when(taskMapper.touchPendingForDispatch(
Mockito.eq(taskId), Mockito.any(Date.class), Mockito.any(Date.class),
Mockito.any(BigInteger.class)))
.thenReturn(0);
DocumentImportParseTaskProducer producer =
Mockito.mock(DocumentImportParseTaskProducer.class);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "parseTaskProducer", producer);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
service.dispatchPendingTasks();
Mockito.verify(producer, Mockito.never()).send(Mockito.any());
}
/**
* 验证重新投递资格更新包含待处理状态和过期时间边界,确保竞争更新仅一个成功。
*
* @throws Exception 映射方法不存在时抛出
*/
@Test
public void touchPendingForDispatchSqlShouldFenceConcurrentDispatch()
throws Exception {
Method method = DocumentImportTaskMapper.class.getMethod(
"touchPendingForDispatch",
BigInteger.class,
Date.class,
Date.class,
BigInteger.class
);
Update update = method.getAnnotation(Update.class);
String sql = String.join(" ", update.value());
Assert.assertTrue(sql.contains("status='PENDING'"));
Assert.assertTrue(sql.contains("modified <= #{redispatchBefore}"));
Assert.assertTrue(sql.contains("modified=#{now}"));
}
/**
* 验证分块策略快照使用忽略 null 的部分实体更新。
*
* @throws Exception 反射调用异常
*/
@Test
public void persistAppliedStrategyShouldKeepRequiredBatchItemFields()
throws Exception {
BigInteger itemId = BigInteger.valueOf(31);
DocumentImportBatchItemService itemService =
Mockito.mock(DocumentImportBatchItemService.class);
Mockito.when(itemService.updateById(Mockito.any()))
.thenReturn(true);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportBatchItemService", itemService);
Method method =
KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"persistAppliedStrategy",
BigInteger.class,
StrategyConfig.class
);
method.setAccessible(true);
method.invoke(service, itemId, StrategyConfig.defaults());
ArgumentCaptor<DocumentImportBatchItem> updateCaptor =
ArgumentCaptor.forClass(DocumentImportBatchItem.class);
Mockito.verify(itemService).updateById(updateCaptor.capture());
Mockito.verify(itemService, Mockito.never()).updateById(
Mockito.any(), Mockito.anyBoolean());
Assert.assertEquals(itemId, updateCaptor.getValue().getId());
Assert.assertNotNull(
updateCaptor.getValue().getStrategySnapshotJson());
}
/**
* 验证 SPLIT 待处理任务使用 execution token 和租约原子领取。
*
* @throws Exception 反射注入异常
*/
@Test
public void tryMarkSplitTaskRunningShouldCreateExecutionFence()
throws Exception {
BigInteger taskId = BigInteger.valueOf(35);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setPhase(DocumentImportTaskPhase.SPLIT.name());
task.setStatus(DocumentImportTaskStatus.PENDING.name());
DocumentImportTaskService taskService =
Mockito.mock(DocumentImportTaskService.class);
Mockito.when(taskService.getById(taskId)).thenReturn(task);
Mockito.when(taskService.count(
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
.thenReturn(0L);
DocumentImportTaskMapper taskMapper =
Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.claimPending(
Mockito.eq(taskId),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.any(Date.class),
Mockito.nullable(BigInteger.class)
)).thenReturn(1);
RedisLockExecutor lockExecutor =
Mockito.mock(RedisLockExecutor.class);
RedisLockExecutor.LockHandle lockHandle =
Mockito.mock(RedisLockExecutor.LockHandle.class);
Mockito.when(lockExecutor.tryAcquire(
Mockito.anyString(), Mockito.any(), Mockito.any()
)).thenReturn(lockHandle);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskService", taskService);
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "redisLockExecutor", lockExecutor);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
Assert.assertTrue(service.tryMarkTaskRunning(taskId));
ArgumentCaptor<String> token = ArgumentCaptor.forClass(String.class);
ArgumentCaptor<Date> lease = ArgumentCaptor.forClass(Date.class);
Mockito.verify(taskMapper).claimPending(
Mockito.eq(taskId),
token.capture(),
lease.capture(),
Mockito.any(Date.class),
Mockito.nullable(BigInteger.class)
);
Assert.assertFalse(token.getValue().isBlank());
Assert.assertTrue(lease.getValue().after(new Date()));
Mockito.verify(lockHandle).release();
}
/**
* 验证新文档完成后会清理待覆盖的历史文档并清除持久化待办。
*
* @throws Exception 反射注入异常
*/
@Test
public void cleanupCompletedReplacementShouldDeleteHistoricalDocument() throws Exception {
BigInteger itemId = BigInteger.valueOf(901);
BigInteger documentId = BigInteger.valueOf(902);
BigInteger replacedDocumentId = BigInteger.valueOf(903);
BigInteger knowledgeId = BigInteger.valueOf(904);
DocumentImportBatchItem item = new DocumentImportBatchItem();
item.setId(itemId);
item.setDocumentId(documentId);
item.setReplacedDocumentId(replacedDocumentId);
tech.easyflow.ai.entity.Document replacement = new tech.easyflow.ai.entity.Document();
replacement.setId(documentId);
replacement.setCollectionId(knowledgeId);
replacement.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
tech.easyflow.ai.entity.Document historical = new tech.easyflow.ai.entity.Document();
historical.setId(replacedDocumentId);
historical.setCollectionId(knowledgeId);
historical.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
DocumentImportBatchTracker tracker = Mockito.mock(DocumentImportBatchTracker.class);
Mockito.when(tracker.requireItem(itemId)).thenReturn(item);
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
Mockito.when(documentMapper.selectOneById(documentId)).thenReturn(replacement);
Mockito.when(documentMapper.selectOneById(replacedDocumentId)).thenReturn(historical);
DocumentService documentService = Mockito.mock(DocumentService.class);
Mockito.when(documentService.removeDoc(replacedDocumentId.toString())).thenReturn(true);
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
Mockito.when(redisLockExecutor.tryAcquire(
Mockito.anyString(), Mockito.any(), Mockito.any()
)).thenReturn(lockHandle);
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportBatchTracker", tracker);
setField(service, "documentMapper", documentMapper);
setField(service, "documentService", documentService);
setField(service, "redisLockExecutor", redisLockExecutor);
service.cleanupCompletedReplacement(itemId);
Mockito.verify(documentService).removeDoc(replacedDocumentId.toString());
Mockito.verify(documentMapper).deleteById(replacedDocumentId);
Mockito.verify(tracker).clearReplacement(itemId, replacedDocumentId);
Mockito.verify(lockHandle).release();
}
/**
* 验证失联且关联文档已删除的运行任务会被收口,避免持续占用并发名额。
*
* @throws Exception 反射注入异常
*/
@Test
public void recoverInterruptedTaskShouldFailOrphanWithoutDocument() throws Exception {
BigInteger taskId = BigInteger.valueOf(41);
Date cutoff = new Date(System.currentTimeMillis() - 60_000L);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setDocumentId(BigInteger.valueOf(42));
task.setPhase(DocumentImportTaskPhase.PARSE.name());
task.setStatus(DocumentImportTaskStatus.RUNNING.name());
task.setModified(new Date(cutoff.getTime() - 1_000L));
DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class);
Mockito.when(taskService.getById(taskId)).thenReturn(task);
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.failExpiredOwned(
Mockito.eq(taskId),
Mockito.isNull(),
Mockito.anyString(),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.eq(cutoff),
Mockito.any()
)).thenReturn(1);
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
Mockito.when(documentMapper.selectOneById(task.getDocumentId())).thenReturn(null);
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskService", taskService);
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "documentMapper", documentMapper);
service.recoverInterruptedTask(taskId, cutoff);
Mockito.verify(taskMapper).failExpiredOwned(
Mockito.eq(taskId),
Mockito.isNull(),
Mockito.eq("任务执行中断,请重试"),
Mockito.eq("execution_interrupted"),
Mockito.any(Date.class),
Mockito.eq(cutoff),
Mockito.any()
);
}
/**
* 验证运行中断收口后仍保留批次文件的人工重试资格。
*
* @throws Exception 反射调用异常
*/
@Test
public void recoveredBatchStateShouldRemainRetryableBeforeAttemptLimit()
throws Exception {
BigInteger batchId = BigInteger.valueOf(61);
BigInteger itemId = BigInteger.valueOf(62);
DocumentImportTask task = new DocumentImportTask();
task.setBatchId(batchId);
task.setBatchItemId(itemId);
DocumentImportBatchItem item = new DocumentImportBatchItem();
item.setId(itemId);
item.setAttemptCount(0);
DocumentImportBatchTracker tracker =
Mockito.mock(DocumentImportBatchTracker.class);
Mockito.when(tracker.requireItem(itemId)).thenReturn(item);
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportBatchTracker", tracker);
setField(service, "bulkProperties", new DocumentImportBulkProperties());
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod(
"finishRecoveredBatchState",
DocumentImportTask.class,
DocumentImportBatchItemStage.class,
String.class,
String.class
);
method.setAccessible(true);
method.invoke(
service,
task,
DocumentImportBatchItemStage.PARSE,
"任务执行中断,请继续批次",
"execution_interrupted"
);
Mockito.verify(tracker).transitionItem(
itemId,
DocumentImportBatchItemStage.PARSE,
DocumentImportBatchItemStatus.FAILED,
"任务执行中断,请继续批次",
true,
0,
"execution_interrupted"
);
Mockito.verify(tracker).markInterrupted(batchId);
}
/**
* 验证待处理超时按创建时间判定,即使重新投递刷新了修改时间也会正常收口。
*
* @throws Exception 反射注入异常
*/
@Test
public void expireTimedOutPendingTaskShouldUseCreatedTime() throws Exception {
BigInteger taskId = BigInteger.valueOf(51);
BigInteger documentId = BigInteger.valueOf(52);
Date cutoff = new Date(System.currentTimeMillis() - 60_000L);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setDocumentId(documentId);
task.setPhase(DocumentImportTaskPhase.PARSE.name());
task.setStatus(DocumentImportTaskStatus.PENDING.name());
task.setCreated(new Date(cutoff.getTime() - 1_000L));
task.setModified(new Date());
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
document.setId(documentId);
document.setCollectionId(BigInteger.valueOf(53));
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef =
new AtomicReference<tech.easyflow.ai.entity.Document>();
DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class);
Mockito.when(taskService.getById(taskId)).thenReturn(task);
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.updateByQuery(
Mockito.any(DocumentImportTask.class),
Mockito.any()
)).thenReturn(1);
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskService", taskService);
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "documentMapper", mockDocumentMapper(document, updatedDocumentRef));
setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService());
service.expireTimedOutPendingTask(taskId, cutoff);
tech.easyflow.ai.entity.Document updatedDocument = updatedDocumentRef.get();
Assert.assertNotNull(updatedDocument);
Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(), updatedDocument.getProcessStatus());
Assert.assertEquals("任务排队超时,请重试", updatedDocument.getLastTaskError());
Assert.assertEquals("pending_timeout",
updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE));
}
/**
* 验证未取得 MinerU 任务 ID 的超时提交会失败并释放运行状态。
*
* @throws Exception 反射注入异常
*/
@Test
public void expireTimedOutParseSubmissionShouldExposeFriendlyTimeout() throws Exception {
BigInteger taskId = BigInteger.valueOf(61);
BigInteger documentId = BigInteger.valueOf(62);
Date cutoff = new Date(System.currentTimeMillis() - 60_000L);
DocumentImportTask task = new DocumentImportTask();
task.setId(taskId);
task.setDocumentId(documentId);
task.setPhase(DocumentImportTaskPhase.PARSE.name());
task.setStatus(DocumentImportTaskStatus.RUNNING.name());
task.setStartedAt(new Date(cutoff.getTime() - 1_000L));
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
document.setId(documentId);
document.setCollectionId(BigInteger.valueOf(63));
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef =
new AtomicReference<tech.easyflow.ai.entity.Document>();
DocumentImportTaskService taskService = Mockito.mock(DocumentImportTaskService.class);
Mockito.when(taskService.getById(taskId)).thenReturn(task);
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.updateByQuery(
Mockito.any(DocumentImportTask.class),
Mockito.any()
)).thenReturn(1);
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
setField(service, "documentImportTaskService", taskService);
setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "documentMapper", mockDocumentMapper(document, updatedDocumentRef));
setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService());
service.expireTimedOutParseSubmission(taskId, cutoff);
ArgumentCaptor<DocumentImportTask> updateCaptor = ArgumentCaptor.forClass(DocumentImportTask.class);
Mockito.verify(taskMapper).updateByQuery(updateCaptor.capture(), Mockito.any());
Assert.assertEquals(DocumentImportTaskStatus.FAILED.name(), updateCaptor.getValue().getStatus());
Assert.assertEquals("文档解析服务响应超时,请重试", updateCaptor.getValue().getErrorSummary());
tech.easyflow.ai.entity.Document updatedDocument = updatedDocumentRef.get();
Assert.assertNotNull(updatedDocument);
Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(), updatedDocument.getProcessStatus());
Assert.assertEquals("文档解析服务响应超时,请重试", updatedDocument.getLastTaskError());
Assert.assertEquals("parse_service_timeout",
updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE));
}
/**
* 验证 MinerU 任意 5xx 会归一化为服务暂不可用错误。
*
* @throws Exception 反射调用异常
*/
@Test
public void resolveParseFailureCodeShouldRecognizeMineruServerErrors() throws Exception {
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"resolveParseFailureCode",
Throwable.class
);
method.setAccessible(true);
for (int statusCode : new int[] {500, 502, 503, 504, 599}) {
Object code = method.invoke(service,
new RuntimeException("MinerU request failed: path=/tasks, status=" + statusCode + ", body="));
Assert.assertEquals("status=" + statusCode, "parse_service_unavailable", code);
}
}
/**
* 验证底层读取超时会归一化为解析服务超时错误。
*
* @throws Exception 反射调用异常
*/
@Test
public void resolveParseFailureCodeShouldRecognizeSocketTimeout() throws Exception {
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"resolveParseFailureCode",
Throwable.class
);
method.setAccessible(true);
Object code = method.invoke(service,
new RuntimeException("wrapped", new SocketTimeoutException("timeout")));
Assert.assertEquals("parse_service_timeout", code);
}
/**
* 验证文档源读取失败优先于底层 UnknownHostException 分类,避免误报 MinerU 不可用。
*
* @throws Exception 反射调用异常
*/
@Test
public void resolveParseFailureCodeShouldPreferDocumentSourceFailure() throws Exception {
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"resolveParseFailureCode",
Throwable.class
);
method.setAccessible(true);
DocumentParseBridgeException sourceError = DocumentParseBridgeException.sourceLoadFailed(
"下载文档 URL 失败",
new UnknownHostException("远端文档地址不允许访问非公网目标")
);
Object code = method.invoke(service, new RuntimeException("wrapped", sourceError));
Assert.assertEquals("document_source_unavailable", code);
}
/**
* 验证解析桥接层的确定性输入错误会转换为不可重试错误码。
*
* @throws Exception 反射调用异常
*/
@Test
public void resolveParseFailureCodeShouldClassifyPermanentBridgeFailures()
throws Exception {
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("resolveParseFailureCode", Throwable.class);
method.setAccessible(true);
Object unsupported = method.invoke(
service,
DocumentParseBridgeException.unsupportedSource("不支持的文件类型")
);
Object invalidRequest = method.invoke(
service,
DocumentParseBridgeException.requestBuildFailed("解析请求无效")
);
Assert.assertEquals("unsupported_document_source", unsupported);
Assert.assertEquals("invalid_parse_request", invalidRequest);
}
/**
* 验证未识别的系统异常仍会获得稳定错误码并允许后续人工重试。
*
* @throws Exception 反射调用异常
*/
@Test
public void resolveParseFailureCodeShouldStabilizeUnknownFailures()
throws Exception {
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("resolveParseFailureCode", Throwable.class);
method.setAccessible(true);
Object code = method.invoke(service, new IllegalStateException("代码异常"));
Assert.assertEquals("parse_failed", code);
}
/**
* 验证未知系统异常和临时源读取失败允许人工重试,确定性输入错误除外。
*
* @throws Exception 反射调用异常
*/
@Test
public void parseRetryabilityShouldDefaultToRecoverable()
throws Exception {
KnowledgeDocumentImportTaskAppService service =
new KnowledgeDocumentImportTaskAppService();
Method method = KnowledgeDocumentImportTaskAppService.class
.getDeclaredMethod("isRetryableParseFailure", String.class);
method.setAccessible(true);
Assert.assertTrue((Boolean) method.invoke(service, new Object[] {null}));
Assert.assertTrue((Boolean) method.invoke(
service, "document_source_unavailable"));
Assert.assertTrue((Boolean) method.invoke(service, "parse_failed"));
Assert.assertFalse((Boolean) method.invoke(
service, "unsupported_document_source"));
Assert.assertFalse((Boolean) method.invoke(
service, "invalid_parse_request"));
}
/**
* 验证批次上传文件通过受信任存储服务读取为字节,避免内部附件 URL 进入公网 URL 校验。
*
* @throws Exception 反射调用或存储桩异常
*/
@Test
public void buildBridgeSourceRefShouldReadBatchFileFromStorage() throws Exception {
byte[] content = "batch-document".getBytes(StandardCharsets.UTF_8);
String storedUrl = "http://127.0.0.1:39000/easyflow/attachment/test.docx";
FileStorageService storageService = Mockito.mock(FileStorageService.class);
Mockito.when(storageService.readStream(storedUrl))
.thenReturn(new ByteArrayInputStream(content));
DocumentImportBulkProperties properties = new DocumentImportBulkProperties();
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
setField(service, "storageService", storageService);
setField(service, "bulkProperties", properties);
DocumentImportTask task = new DocumentImportTask();
task.setBatchId(BigInteger.valueOf(61));
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
document.setTitle("test.docx");
document.setDocumentPath(storedUrl);
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"buildBridgeSourceRef",
DocumentImportTask.class,
tech.easyflow.ai.entity.Document.class,
String.class
);
method.setAccessible(true);
DocumentSourceRef sourceRef = (DocumentSourceRef) method.invoke(service, task, document, "docx");
Assert.assertArrayEquals(content, sourceRef.getContentBytes());
Assert.assertNull(sourceRef.getFilePath());
Assert.assertEquals(Long.valueOf(content.length), sourceRef.getSize());
Mockito.verify(storageService).readStream(storedUrl);
}
/**
* 验证非批次远程文档仍保留 URL由文档源加载器继续执行 SSRF 防护。
*
* @throws Exception 反射调用异常
*/
@Test
public void buildBridgeSourceRefShouldKeepExternalUrlForSsrfValidation() throws Exception {
FileStorageService storageService = Mockito.mock(FileStorageService.class);
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
setField(service, "storageService", storageService);
DocumentImportTask task = new DocumentImportTask();
tech.easyflow.ai.entity.Document document = new tech.easyflow.ai.entity.Document();
document.setTitle("external.pdf");
document.setDocumentPath("https://example.com/external.pdf");
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"buildBridgeSourceRef",
DocumentImportTask.class,
tech.easyflow.ai.entity.Document.class,
String.class
);
method.setAccessible(true);
DocumentSourceRef sourceRef = (DocumentSourceRef) method.invoke(service, task, document, "pdf");
Assert.assertEquals("https://example.com/external.pdf", sourceRef.getFilePath());
Assert.assertNull(sourceRef.getContentBytes());
Mockito.verifyNoInteractions(storageService);
}
/** /**
* 验证向量化失败会按整文档失败语义重置进度,并刷新任务错误信息。 * 验证向量化失败会按整文档失败语义重置进度,并刷新任务错误信息。
* *
@@ -59,11 +744,16 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
persistedDocument.setLastTaskError("旧错误"); persistedDocument.setLastTaskError("旧错误");
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef = new AtomicReference<tech.easyflow.ai.entity.Document>(); AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef = new AtomicReference<tech.easyflow.ai.entity.Document>();
AtomicReference<DocumentImportTask> updatedTaskRef = new AtomicReference<DocumentImportTask>(); DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.finishOwned(
Mockito.any(), Mockito.anyString(), Mockito.anyString(),
Mockito.any(), Mockito.anyString(), Mockito.any(),
Mockito.any()
)).thenReturn(1);
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService(); KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
setField(service, "documentMapper", mockDocumentMapper(persistedDocument, updatedDocumentRef)); setField(service, "documentMapper", mockDocumentMapper(persistedDocument, updatedDocumentRef));
setField(service, "documentImportTaskService", mockDocumentImportTaskService(updatedTaskRef)); setField(service, "documentImportTaskMapper", taskMapper);
setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService()); setField(service, "documentImportTaskStatusStreamService", new NoopTaskStatusStreamService());
DocumentImportTask task = new DocumentImportTask(); DocumentImportTask task = new DocumentImportTask();
@@ -72,6 +762,7 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
task.setKnowledgeId(knowledgeId); task.setKnowledgeId(knowledgeId);
task.setStatus(DocumentImportTaskStatus.RUNNING.name()); task.setStatus(DocumentImportTaskStatus.RUNNING.name());
task.setErrorSummary("旧错误"); task.setErrorSummary("旧错误");
task.setExecutionToken("attempt-token");
tech.easyflow.ai.entity.Document inputDocument = new tech.easyflow.ai.entity.Document(); tech.easyflow.ai.entity.Document inputDocument = new tech.easyflow.ai.entity.Document();
inputDocument.setId(documentId); inputDocument.setId(documentId);
@@ -93,11 +784,62 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Assert.assertEquals(Integer.valueOf(8), updatedDocument.getFailedChunks()); Assert.assertEquals(Integer.valueOf(8), updatedDocument.getFailedChunks());
Assert.assertEquals(Integer.valueOf(0), updatedDocument.getProgressPercent()); Assert.assertEquals(Integer.valueOf(0), updatedDocument.getProgressPercent());
Assert.assertEquals("新错误", updatedDocument.getLastTaskError()); Assert.assertEquals("新错误", updatedDocument.getLastTaskError());
Assert.assertEquals("index_failed",
updatedDocument.getOptions().get(DocumentImportKeys.KEY_DOCUMENT_TASK_ERROR_CODE));
DocumentImportTask updatedTask = updatedTaskRef.get(); Mockito.verify(taskMapper).finishOwned(
Assert.assertNotNull(updatedTask); Mockito.eq(task.getId()),
Assert.assertEquals(DocumentImportTaskStatus.FAILED.name(), updatedTask.getStatus()); Mockito.eq("attempt-token"),
Assert.assertEquals("新错误", updatedTask.getErrorSummary()); Mockito.eq(DocumentImportTaskStatus.FAILED.name()),
Mockito.eq("新错误"),
Mockito.eq("index_failed"),
Mockito.any(Date.class),
Mockito.any()
);
}
/**
* 验证看门狗已经收口任务后,迟到执行者不能覆盖文档终态。
*
* @throws Exception 反射调用异常
*/
@Test
public void markIndexFailedShouldIgnoreLostTaskOwnership() throws Exception {
BigInteger documentId = BigInteger.valueOf(81);
tech.easyflow.ai.entity.Document persistedDocument = new tech.easyflow.ai.entity.Document();
persistedDocument.setId(documentId);
persistedDocument.setProcessStatus(DocumentProcessStatus.PARSE_FAILED.name());
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef =
new AtomicReference<tech.easyflow.ai.entity.Document>();
DocumentImportTaskMapper taskMapper = Mockito.mock(DocumentImportTaskMapper.class);
Mockito.when(taskMapper.updateByQuery(
Mockito.any(DocumentImportTask.class),
Mockito.any()
)).thenReturn(0);
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
setField(service, "documentMapper", mockDocumentMapper(persistedDocument, updatedDocumentRef));
setField(service, "documentImportTaskMapper", taskMapper);
DocumentImportTask task = new DocumentImportTask();
task.setId(BigInteger.valueOf(82));
task.setDocumentId(documentId);
task.setStatus(DocumentImportTaskStatus.RUNNING.name());
tech.easyflow.ai.entity.Document inputDocument = new tech.easyflow.ai.entity.Document();
inputDocument.setId(documentId);
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"markIndexFailed",
DocumentImportTask.class,
tech.easyflow.ai.entity.Document.class,
String.class
);
method.setAccessible(true);
method.invoke(service, task, inputDocument, "迟到错误");
Assert.assertNull(updatedDocumentRef.get());
Assert.assertEquals(DocumentProcessStatus.PARSE_FAILED.name(),
persistedDocument.getProcessStatus());
} }
/** /**
@@ -232,6 +974,8 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Assert.assertEquals(2, chunks.size()); Assert.assertEquals(2, chunks.size());
DocumentChunk firstChunk = chunks.get(0); DocumentChunk firstChunk = chunks.get(0);
Assert.assertNotNull(firstChunk.getId());
Assert.assertNotEquals(firstChunk.getId(), chunks.get(1).getId());
Assert.assertTrue(firstChunk.getContent().contains("Slide 1")); Assert.assertTrue(firstChunk.getContent().contains("Slide 1"));
Assert.assertTrue(firstChunk.getContent().contains("本页介绍季度目标")); Assert.assertTrue(firstChunk.getContent().contains("本页介绍季度目标"));
Assert.assertEquals("https://example.com/slides/slide-001.png", Assert.assertEquals("https://example.com/slides/slide-001.png",
@@ -352,6 +1096,61 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
Assert.assertTrue(chunks.isEmpty()); Assert.assertTrue(chunks.isEmpty());
} }
/**
* 验证向量存储回滚会去重分块 ID并在 Milvus 删除失败后停止清理搜索索引。
*
* @throws Exception 反射调用异常
*/
@Test
public void rollbackStoredChunksShouldValidateVectorDeleteResult() throws Exception {
KnowledgeDocumentImportTaskAppService service = new KnowledgeDocumentImportTaskAppService();
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class);
StoreOptions storeOptions = StoreOptions.ofCollectionName("knowledge-test");
Mockito.when(documentStore.delete(Mockito.anyCollection(), Mockito.same(storeOptions)))
.thenReturn(StoreResult.fail("milvus unavailable"));
DocumentCollection knowledge = new DocumentCollection();
knowledge.setId(BigInteger.valueOf(501));
Class<?> contextClass = Class.forName(
KnowledgeDocumentImportTaskAppService.class.getName() + "$StoreExecutionContext"
);
Constructor<?> constructor = contextClass.getDeclaredConstructor(
DocumentCollection.class,
EmbeddingModel.class,
DocumentStore.class,
StoreOptions.class,
DocumentSearcher.class
);
constructor.setAccessible(true);
Object context = constructor.newInstance(knowledge, null, documentStore, storeOptions, searcher);
DocumentChunk first = new DocumentChunk();
first.setId(BigInteger.valueOf(601));
DocumentChunk duplicate = new DocumentChunk();
duplicate.setId(first.getId());
Method method = KnowledgeDocumentImportTaskAppService.class.getDeclaredMethod(
"rollbackStoredChunks",
BigInteger.class,
BigInteger.class,
contextClass,
List.class
);
method.setAccessible(true);
method.invoke(service,
BigInteger.valueOf(701),
BigInteger.valueOf(702),
context,
List.of(first, duplicate));
@SuppressWarnings("rawtypes")
ArgumentCaptor<Collection> idsCaptor = ArgumentCaptor.forClass(Collection.class);
Mockito.verify(documentStore).delete(idsCaptor.capture(), Mockito.same(storeOptions));
Assert.assertEquals(1, idsCaptor.getValue().size());
Assert.assertTrue(idsCaptor.getValue().contains(first.getId()));
Mockito.verifyNoInteractions(searcher);
}
private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument, private static DocumentMapper mockDocumentMapper(tech.easyflow.ai.entity.Document persistedDocument,
AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef) { AtomicReference<tech.easyflow.ai.entity.Document> updatedDocumentRef) {
return (DocumentMapper) Proxy.newProxyInstance( return (DocumentMapper) Proxy.newProxyInstance(
@@ -370,20 +1169,6 @@ public class KnowledgeDocumentImportTaskAppServiceTest {
); );
} }
private static DocumentImportTaskService mockDocumentImportTaskService(AtomicReference<DocumentImportTask> updatedTaskRef) {
return (DocumentImportTaskService) Proxy.newProxyInstance(
DocumentImportTaskService.class.getClassLoader(),
new Class<?>[]{DocumentImportTaskService.class},
(proxy, method, args) -> {
if ("updateById".equals(method.getName())) {
updatedTaskRef.set((DocumentImportTask) args[0]);
return true;
}
return defaultValue(method.getReturnType());
}
);
}
private static FileStorageService mockFileStorageService(AtomicReference<String> savedPrePathRef, private static FileStorageService mockFileStorageService(AtomicReference<String> savedPrePathRef,
AtomicReference<String> savedFilenameRef) { AtomicReference<String> savedFilenameRef) {
return (FileStorageService) Proxy.newProxyInstance( return (FileStorageService) Proxy.newProxyInstance(

View File

@@ -0,0 +1,431 @@
package tech.easyflow.ai.documentimport.task;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext;
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
import tech.easyflow.ai.documentimport.ImportCallerContext;
import tech.easyflow.ai.documentimport.ImportCallerType;
import tech.easyflow.ai.documentimport.PublicDocumentImportDtos;
import tech.easyflow.ai.entity.DocumentImportBatch;
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
import tech.easyflow.ai.service.DocumentImportBatchItemService;
import tech.easyflow.ai.service.DocumentImportBatchService;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.io.ByteArrayInputStream;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* Public API 批量导入门面边界与服务端去重测试。
*
* @author Codex
* @since 2026-08-02
*/
public class KnowledgeImportBatchFacadeTest {
/**
* 验证单次文件数超过 200 时在创建批次前拒绝。
*/
@Test
public void submitShouldRejectMoreThanTwoHundredFiles() {
TestContext context = createContext();
PublicDocumentImportDtos.BatchMetadata metadata =
new PublicDocumentImportDtos.BatchMetadata();
metadata.setKnowledgeId(BigInteger.ONE);
List<DocumentImportBatchDtos.ManifestItem> manifest = new ArrayList<>();
List<MultipartFile> files = new ArrayList<>();
for (int index = 0; index < 201; index++) {
String name = "file-" + index + ".txt";
manifest.add(manifest(name, name, 1L));
files.add(file(name, new byte[]{'a'}));
}
metadata.setFiles(manifest);
try {
context.facade.submit(context.caller, metadata, files);
Assert.fail("Expected file count rejection");
} catch (BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("最多上传200个文件"));
}
Mockito.verifyNoInteractions(context.batchAppService);
}
/**
* 验证文件夹清单拒绝父目录穿越。
*/
@Test
public void submitShouldRejectRelativePathTraversal() {
TestContext context = createContext();
PublicDocumentImportDtos.BatchMetadata metadata =
metadata("demo.txt", "../demo.txt", 4L);
try {
context.facade.submit(
context.caller,
metadata,
List.of(file(
"demo.txt",
"demo".getBytes(StandardCharsets.UTF_8)
))
);
Assert.fail("Expected relative path rejection");
} catch (BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("相对路径无效"));
}
Mockito.verifyNoInteractions(context.batchAppService);
}
/**
* 验证 Public API 可省略 fileSize内部批次使用服务端实测大小。
*/
@Test
public void submitShouldUseMeasuredSizeWhenMetadataOmitsFileSize() {
TestContext context = createContext();
PublicDocumentImportDtos.BatchMetadata metadata =
metadata("demo.txt", "folder/demo.txt", 4L);
metadata.getFiles().get(0).setFileSize(null);
byte[] content = "demo".getBytes(StandardCharsets.UTF_8);
DocumentImportBatchDtos.CreateResponse created =
new DocumentImportBatchDtos.CreateResponse();
created.setBatchId(BigInteger.valueOf(91));
DocumentImportBatchDtos.ItemResponse createdItem =
new DocumentImportBatchDtos.ItemResponse();
createdItem.setItemId(BigInteger.valueOf(92));
created.setItems(List.of(createdItem));
Mockito.when(context.batchAppService.createBatch(
Mockito.any(), Mockito.any()
)).thenReturn(created);
Mockito.when(context.itemService.updateById(Mockito.any()))
.thenReturn(true);
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(created.getBatchId());
batch.setKnowledgeId(BigInteger.ONE);
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
batch.setTotalCount(1);
batch.setTotalBytes((long) content.length);
batch.setCreated(new Date());
Mockito.when(context.batchAppService.requireOwnedBatch(
BigInteger.ONE, created.getBatchId(), context.caller
)).thenReturn(batch);
context.facade.submit(
context.caller,
metadata,
List.of(file("demo.txt", content))
);
ArgumentCaptor<DocumentImportBatchDtos.CreateRequest> request =
ArgumentCaptor.forClass(DocumentImportBatchDtos.CreateRequest.class);
Mockito.verify(context.batchAppService).createBatch(
request.capture(), Mockito.any());
Assert.assertEquals(
Long.valueOf(content.length),
request.getValue().getFiles().get(0).getFileSize()
);
Assert.assertNull(metadata.getFiles().get(0).getFileSize());
}
/**
* 验证相同提交由服务端指纹复用,包含仍在上传的并发请求。
*/
@Test
public void submitShouldReuseTaskByServerFingerprint() {
TestContext context = createContext();
PublicDocumentImportDtos.BatchMetadata metadata =
metadata("demo.txt", "folder/demo.txt", 4L);
List<MultipartFile> files = List.of(file(
"demo.txt",
"demo".getBytes(StandardCharsets.UTF_8)
));
DocumentImportBatchDtos.CreateResponse created =
new DocumentImportBatchDtos.CreateResponse();
created.setBatchId(BigInteger.valueOf(91));
DocumentImportBatchDtos.ItemResponse createdItem =
new DocumentImportBatchDtos.ItemResponse();
createdItem.setItemId(BigInteger.valueOf(92));
created.setItems(List.of(createdItem));
Mockito.when(context.batchAppService.createBatch(
Mockito.any(), Mockito.any()
)).thenReturn(created);
Mockito.when(context.itemService.updateById(Mockito.any()))
.thenReturn(true);
DocumentImportBatch batch = new DocumentImportBatch();
batch.setId(created.getBatchId());
batch.setKnowledgeId(BigInteger.ONE);
batch.setStatus(DocumentImportBatchStatus.RUNNING.name());
batch.setTotalCount(1);
batch.setTotalBytes(4L);
batch.setCreated(new Date());
Mockito.when(context.batchAppService.requireOwnedBatch(
BigInteger.ONE, created.getBatchId(), context.caller
)).thenReturn(batch);
context.facade.submit(context.caller, metadata, files);
ArgumentCaptor<tech.easyflow.ai.entity.DocumentImportBatchItem> hashUpdate =
ArgumentCaptor.forClass(tech.easyflow.ai.entity.DocumentImportBatchItem.class);
Mockito.verify(context.itemService).updateById(hashUpdate.capture());
Assert.assertNotNull(hashUpdate.getValue().getContentSha256());
Assert.assertEquals(createdItem.getItemId(), hashUpdate.getValue().getId());
ArgumentCaptor<DocumentImportBatchCreateContext> createContext =
ArgumentCaptor.forClass(DocumentImportBatchCreateContext.class);
Mockito.verify(context.batchAppService).createBatch(
Mockito.any(), createContext.capture());
batch.setRequestDigest(createContext.getValue().getRequestDigest());
Mockito.when(context.batchService.getOne(
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
.thenReturn(batch);
PublicDocumentImportDtos.SubmitResponse repeated =
context.facade.submit(
context.caller, metadata, files);
Assert.assertEquals(batch.getId(), repeated.getTaskId());
batch.setStatus(DocumentImportBatchStatus.UPLOADING.name());
PublicDocumentImportDtos.SubmitResponse uploading =
context.facade.submit(context.caller, metadata, files);
Assert.assertEquals(batch.getId(), uploading.getTaskId());
Mockito.verify(context.batchAppService, Mockito.times(1))
.createBatch(Mockito.any(), Mockito.any());
}
/**
* 验证长时间无进展的上传任务会先取消并释放指纹,再创建新任务。
*/
@Test
public void submitShouldReplaceStaleIncompleteTask() {
TestContext context = createContext();
PublicDocumentImportDtos.BatchMetadata metadata =
metadata("demo.txt", "folder/demo.txt", 4L);
List<MultipartFile> files = List.of(file(
"demo.txt",
"demo".getBytes(StandardCharsets.UTF_8)
));
DocumentImportBatch stale = new DocumentImportBatch();
stale.setId(BigInteger.valueOf(81));
stale.setKnowledgeId(BigInteger.ONE);
stale.setStatus(DocumentImportBatchStatus.UPLOADING.name());
stale.setModified(new Date(
System.currentTimeMillis() - 31L * 60L * 1000L
));
Mockito.when(context.batchService.getOne(
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
.thenReturn(stale)
.thenReturn(null);
Mockito.when(context.batchAppService.cancelStaleBatch(
Mockito.eq(BigInteger.ONE),
Mockito.eq(stale.getId()),
Mockito.eq(context.caller),
Mockito.any(Date.class)
)).thenReturn(true);
Mockito.when(context.batchMapper.releaseSubmissionFingerprint(
Mockito.eq(stale.getId()),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.any(Date.class)
)).thenReturn(1);
DocumentImportBatchDtos.CreateResponse created =
new DocumentImportBatchDtos.CreateResponse();
created.setBatchId(BigInteger.valueOf(91));
DocumentImportBatchDtos.ItemResponse item =
new DocumentImportBatchDtos.ItemResponse();
item.setItemId(BigInteger.valueOf(92));
created.setItems(List.of(item));
Mockito.when(context.batchAppService.createBatch(
Mockito.any(), Mockito.any()
)).thenReturn(created);
Mockito.when(context.itemService.updateById(Mockito.any()))
.thenReturn(true);
DocumentImportBatch replacement = new DocumentImportBatch();
replacement.setId(created.getBatchId());
replacement.setKnowledgeId(BigInteger.ONE);
replacement.setStatus(DocumentImportBatchStatus.RUNNING.name());
replacement.setTotalCount(1);
replacement.setTotalBytes(4L);
replacement.setCreated(new Date());
Mockito.when(context.batchAppService.requireOwnedBatch(
BigInteger.ONE, created.getBatchId(), context.caller
)).thenReturn(replacement);
PublicDocumentImportDtos.SubmitResponse response =
context.facade.submit(context.caller, metadata, files);
Assert.assertEquals(replacement.getId(), response.getTaskId());
Mockito.verify(context.batchAppService).cancelStaleBatch(
Mockito.eq(BigInteger.ONE),
Mockito.eq(stale.getId()),
Mockito.eq(context.caller),
Mockito.any(Date.class)
);
Mockito.verify(context.batchMapper).releaseSubmissionFingerprint(
Mockito.eq(stale.getId()),
Mockito.anyString(),
Mockito.any(Date.class),
Mockito.any(Date.class)
);
}
/**
* 验证显式空文件键数组会被拒绝,省略时由服务端重试全部异常文件。
*/
@Test
public void retryShouldDistinguishOmittedAndEmptyFileKeys() {
TestContext context = createContext();
PublicDocumentImportDtos.RetryRequest empty =
new PublicDocumentImportDtos.RetryRequest();
empty.setTaskId(BigInteger.ONE);
empty.setFileKeys(List.of());
try {
context.facade.retry(context.caller, empty);
Assert.fail("Expected empty fileKeys rejection");
} catch (BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("不能为空数组"));
}
PublicDocumentImportDtos.RetryRequest omitted =
new PublicDocumentImportDtos.RetryRequest();
omitted.setTaskId(BigInteger.ONE);
tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult result =
new tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult(
BigInteger.ONE,
DocumentImportBatchStatus.RUNNING.name(),
1,
2
);
Mockito.when(context.batchAppService.retryOwnedBatch(
BigInteger.ONE,
context.caller,
java.util.Set.of()
)).thenReturn(result);
PublicDocumentImportDtos.RetryResponse response =
context.facade.retry(context.caller, omitted);
Assert.assertEquals(Integer.valueOf(2), response.getRetriedCount());
}
/**
* 创建单文件元数据。
*
* @param fileName 文件名
* @param relativePath 相对路径
* @param fileSize 文件大小
* @return 元数据
*/
private PublicDocumentImportDtos.BatchMetadata metadata(
String fileName,
String relativePath,
long fileSize) {
PublicDocumentImportDtos.BatchMetadata metadata =
new PublicDocumentImportDtos.BatchMetadata();
metadata.setKnowledgeId(BigInteger.ONE);
metadata.setFiles(List.of(
manifest(fileName, relativePath, fileSize)));
return metadata;
}
/**
* 创建清单项。
*
* @param fileName 文件名
* @param relativePath 相对路径
* @param fileSize 文件大小
* @return 清单项
*/
private DocumentImportBatchDtos.ManifestItem manifest(
String fileName,
String relativePath,
long fileSize) {
DocumentImportBatchDtos.ManifestItem item =
new DocumentImportBatchDtos.ManifestItem();
item.setClientFileKey(relativePath);
item.setFileName(fileName);
item.setRelativePath(relativePath);
item.setFileSize(fileSize);
return item;
}
/**
* 创建可重复读取的 MultipartFile。
*
* @param fileName 文件名
* @param bytes 文件内容
* @return Multipart 文件
*/
private MultipartFile file(String fileName, byte[] bytes) {
MultipartFile file = Mockito.mock(MultipartFile.class);
try {
Mockito.when(file.getInputStream())
.thenAnswer(invocation -> new ByteArrayInputStream(bytes));
} catch (java.io.IOException error) {
throw new IllegalStateException(error);
}
Mockito.when(file.getOriginalFilename()).thenReturn(fileName);
Mockito.when(file.getSize()).thenReturn((long) bytes.length);
Mockito.when(file.isEmpty()).thenReturn(bytes.length == 0);
return file;
}
/**
* 创建测试依赖。
*
* @return 测试上下文
*/
private TestContext createContext() {
DocumentImportBatchAppService batchAppService =
Mockito.mock(DocumentImportBatchAppService.class);
DocumentImportBatchTracker batchTracker =
Mockito.mock(DocumentImportBatchTracker.class);
DocumentImportBatchService batchService =
Mockito.mock(DocumentImportBatchService.class);
DocumentImportBatchItemService itemService =
Mockito.mock(DocumentImportBatchItemService.class);
DocumentImportBatchMapper batchMapper =
Mockito.mock(DocumentImportBatchMapper.class);
KnowledgeImportBatchFacade facade = new KnowledgeImportBatchFacade(
batchAppService,
batchTracker,
batchService,
itemService,
batchMapper
);
return new TestContext(
facade,
batchAppService,
batchService,
itemService,
batchMapper,
new ImportCallerContext(
ImportCallerType.PUBLIC_API,
BigInteger.valueOf(77)
)
);
}
/**
* 测试依赖集合。
*/
private record TestContext(
KnowledgeImportBatchFacade facade,
DocumentImportBatchAppService batchAppService,
DocumentImportBatchService batchService,
DocumentImportBatchItemService itemService,
DocumentImportBatchMapper batchMapper,
ImportCallerContext caller
) {
}
}

View File

@@ -10,6 +10,7 @@ import tech.easyflow.ai.config.SearcherFactory;
import tech.easyflow.ai.enums.DocumentProcessStatus; import tech.easyflow.ai.enums.DocumentProcessStatus;
import tech.easyflow.ai.mapper.DocumentChunkMapper; import tech.easyflow.ai.mapper.DocumentChunkMapper;
import tech.easyflow.ai.mapper.DocumentMapper; import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.ai.mapper.FaqItemMapper;
import java.io.Serializable; import java.io.Serializable;
import java.lang.reflect.Field; import java.lang.reflect.Field;
@@ -92,6 +93,89 @@ public class DocumentCollectionServiceImplTest {
Assert.assertEquals(completedChunkId, result.get(0).getId()); Assert.assertEquals(completedChunkId, result.get(0).getId());
Assert.assertEquals("completed chunk", result.get(0).getContent()); Assert.assertEquals("completed chunk", result.get(0).getContent());
Assert.assertEquals(String.valueOf(knowledgeId), searcher.lastKnowledgeId); Assert.assertEquals(String.valueOf(knowledgeId), searcher.lastKnowledgeId);
Assert.assertEquals(
tech.easyflow.ai.entity.DocumentCollection.TYPE_DOCUMENT,
result.get(0).getMetadata("resultType")
);
Assert.assertEquals(
completedDocumentId,
result.get(0).getMetadata("documentId")
);
Assert.assertEquals(
completedDocument.getTitle(),
result.get(0).getMetadata("sourceFileName")
);
}
/**
* 验证 FAQ 检索使用当前数据库记录回填稳定的 FAQ 来源信息。
*
* @throws Exception 反射注入异常
*/
@Test
public void searchShouldFillFaqSourceMetadataFromDatabase() throws Exception {
BigInteger knowledgeId = BigInteger.ONE;
BigInteger faqId = BigInteger.valueOf(2001);
BigInteger categoryId = BigInteger.valueOf(3001);
tech.easyflow.ai.entity.DocumentCollection collection =
new tech.easyflow.ai.entity.DocumentCollection();
collection.setId(knowledgeId);
collection.setCollectionType(
tech.easyflow.ai.entity.DocumentCollection.TYPE_FAQ
);
collection.setOptions(new HashMap<String, Object>() {{
put(KEY_DOC_RECALL_MAX_NUM, 5);
put(KEY_SIMILARITY_THRESHOLD, BigDecimal.ZERO);
}});
tech.easyflow.ai.entity.FaqItem faqItem =
new tech.easyflow.ai.entity.FaqItem();
faqItem.setId(faqId);
faqItem.setCollectionId(knowledgeId);
faqItem.setCategoryId(categoryId);
faqItem.setQuestion("如何申请账号?");
faqItem.setAnswerText("请联系管理员。");
TestKeywordSearcher searcher = new TestKeywordSearcher(
List.of(buildHit(faqId, 0.85D))
);
DocumentCollectionServiceImpl service =
new TestDocumentCollectionService(collection);
setField(
service,
"searcherFactory",
new SearcherFactory(
new StaticObjectProvider<DocumentSearcher>(searcher)
)
);
setField(service, "faqItemMapper", mockFaqItemMapper(faqItem));
tech.easyflow.ai.rag.KnowledgeRetrievalRequest request =
new tech.easyflow.ai.rag.KnowledgeRetrievalRequest();
request.setKnowledgeId(knowledgeId);
request.setQuery("账号");
request.setRetrievalMode(
com.easyagents.rag.retrieval.RetrievalMode.KEYWORD
);
List<Document> result = service.search(request);
Assert.assertEquals(1, result.size());
Document item = result.get(0);
Assert.assertEquals(
tech.easyflow.ai.entity.DocumentCollection.TYPE_FAQ,
item.getMetadata("resultType")
);
Assert.assertEquals(faqId, item.getMetadata("faqId"));
Assert.assertEquals(
faqItem.getQuestion(),
item.getMetadata("question")
);
Assert.assertEquals(
faqItem.getAnswerText(),
item.getMetadata("answerText")
);
Assert.assertEquals(categoryId, item.getMetadata("categoryId"));
} }
private static Document buildHit(BigInteger id, double score) { private static Document buildHit(BigInteger id, double score) {
@@ -132,6 +216,27 @@ public class DocumentCollectionServiceImplTest {
); );
} }
/**
* 创建返回固定 FAQ 的 Mapper 桩。
*
* @param faqItem FAQ 数据
* @return Mapper 桩
*/
private static FaqItemMapper mockFaqItemMapper(
tech.easyflow.ai.entity.FaqItem faqItem
) {
return (FaqItemMapper) Proxy.newProxyInstance(
FaqItemMapper.class.getClassLoader(),
new Class<?>[]{FaqItemMapper.class},
(proxy, method, args) -> {
if ("selectListByQuery".equals(method.getName())) {
return List.of(faqItem);
}
return defaultValue(method.getReturnType());
}
);
}
private static void setField(Object target, String fieldName, Object value) throws Exception { private static void setField(Object target, String fieldName, Object value) throws Exception {
Field field = DocumentCollectionServiceImpl.class.getDeclaredField(fieldName); Field field = DocumentCollectionServiceImpl.class.getDeclaredField(fieldName);
field.setAccessible(true); field.setAccessible(true);

View File

@@ -0,0 +1,255 @@
package tech.easyflow.ai.service.impl;
import com.easyagents.core.store.DocumentStore;
import com.easyagents.core.store.StoreResult;
import com.mybatisflex.core.query.QueryWrapper;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import tech.easyflow.ai.config.SearcherFactory;
import tech.easyflow.ai.entity.Document;
import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.entity.Model;
import tech.easyflow.ai.enums.DocumentProcessStatus;
import tech.easyflow.ai.mapper.DocumentChunkMapper;
import tech.easyflow.ai.mapper.DocumentMapper;
import tech.easyflow.ai.service.DocumentCollectionService;
import tech.easyflow.ai.service.ModelService;
import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.List;
/**
* {@link DocumentServiceImpl} 文档维护回归测试。
*
* @author Codex
* @since 2026-08-02
*/
public class DocumentServiceImplTest {
/**
* 验证删除链路在外部索引和分块清理后删除文档主记录。
*
* @throws Exception 反射注入异常
*/
@Test
public void removeDocShouldDeleteDocumentRecord() throws Exception {
BigInteger documentId = BigInteger.valueOf(101);
BigInteger knowledgeId = BigInteger.valueOf(102);
BigInteger modelId = BigInteger.valueOf(103);
Document document = new Document();
document.setId(documentId);
document.setCollectionId(knowledgeId);
document.setDocumentPath("storage://document.txt");
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
Mockito.when(documentStore.delete(
Mockito.anyList(), Mockito.any())).thenReturn(StoreResult.success());
DocumentCollection knowledge = Mockito.mock(DocumentCollection.class);
Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId);
Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test");
Mockito.when(knowledge.toDocumentStore()).thenReturn(documentStore);
Model model = new Model();
model.setModelName("embedding-test");
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
Mockito.when(documentMapper.selectOneByQuery(
Mockito.any(QueryWrapper.class))).thenReturn(document);
Mockito.when(documentMapper.deleteById(documentId)).thenReturn(1);
DocumentChunkMapper chunkMapper =
Mockito.mock(DocumentChunkMapper.class);
Mockito.when(chunkMapper.selectListByQueryAs(
Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class)))
.thenReturn(List.of(BigInteger.valueOf(201)));
Mockito.when(chunkMapper.deleteByQuery(
Mockito.any(QueryWrapper.class))).thenReturn(1);
DocumentCollectionService knowledgeService =
Mockito.mock(DocumentCollectionService.class);
Mockito.when(knowledgeService.getById(knowledgeId))
.thenReturn(knowledge);
ModelService modelService = Mockito.mock(ModelService.class);
Mockito.when(modelService.getById(modelId)).thenReturn(model);
FileStorageService storageService =
Mockito.mock(FileStorageService.class);
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
DocumentServiceImpl service = new DocumentServiceImpl();
setField(service, "documentMapper", documentMapper);
setField(service, "documentChunkMapper", chunkMapper);
setField(service, "knowledgeService", knowledgeService);
setField(service, "modelService", modelService);
setField(service, "storageService", storageService);
setField(service, "searcherFactory", searcherFactory);
Assert.assertTrue(service.removeDoc(documentId.toString()));
Mockito.verify(documentMapper).deleteById(documentId);
Mockito.verify(storageService).delete(document.getDocumentPath());
}
/**
* 验证最后一个分块已单独删除时,文档删除跳过空向量请求并完成清理。
*
* @throws Exception 反射注入异常
*/
@Test
public void removeDocShouldHandleDocumentWithoutChunks() throws Exception {
BigInteger documentId = BigInteger.valueOf(202);
BigInteger knowledgeId = BigInteger.valueOf(203);
BigInteger modelId = BigInteger.valueOf(204);
Document document = new Document();
document.setId(documentId);
document.setCollectionId(knowledgeId);
document.setDocumentPath("storage://empty-document.txt");
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
DocumentCollection knowledge = Mockito.mock(DocumentCollection.class);
Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId);
Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test");
Mockito.when(knowledge.toDocumentStore()).thenReturn(null);
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
Mockito.when(documentMapper.selectOneByQuery(
Mockito.any(QueryWrapper.class))).thenReturn(document);
Mockito.when(documentMapper.deleteById(documentId)).thenReturn(1);
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
Mockito.when(chunkMapper.selectListByQueryAs(
Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class)))
.thenReturn(List.of());
Mockito.when(chunkMapper.deleteByQuery(
Mockito.any(QueryWrapper.class))).thenReturn(0);
DocumentCollectionService knowledgeService =
Mockito.mock(DocumentCollectionService.class);
Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge);
ModelService modelService = Mockito.mock(ModelService.class);
FileStorageService storageService = Mockito.mock(FileStorageService.class);
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
DocumentServiceImpl service = new DocumentServiceImpl();
setField(service, "documentMapper", documentMapper);
setField(service, "documentChunkMapper", chunkMapper);
setField(service, "knowledgeService", knowledgeService);
setField(service, "modelService", modelService);
setField(service, "storageService", storageService);
setField(service, "searcherFactory", searcherFactory);
Assert.assertTrue(service.removeDoc(documentId.toString()));
Mockito.verify(knowledge, Mockito.never()).toDocumentStore();
Mockito.verifyNoInteractions(modelService);
Mockito.verify(documentMapper).deleteById(documentId);
Mockito.verify(storageService).delete(document.getDocumentPath());
}
/**
* 验证分块中的文档禁止删除,且不会触发任何外部或数据库清理。
*
* @throws Exception 反射注入异常
*/
@Test
public void removeDocShouldRejectSplittingDocumentWithoutSideEffects()
throws Exception {
BigInteger documentId = BigInteger.valueOf(301);
Document document = new Document();
document.setId(documentId);
document.setProcessStatus(DocumentProcessStatus.SPLITTING.name());
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
Mockito.when(documentMapper.selectOneByQuery(
Mockito.any(QueryWrapper.class))).thenReturn(document);
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
DocumentCollectionService knowledgeService =
Mockito.mock(DocumentCollectionService.class);
DocumentServiceImpl service = new DocumentServiceImpl();
setField(service, "documentMapper", documentMapper);
setField(service, "documentChunkMapper", chunkMapper);
setField(service, "knowledgeService", knowledgeService);
try {
service.removeDoc(documentId.toString());
Assert.fail("分块中的文档应拒绝删除");
} catch (BusinessException expected) {
Assert.assertEquals("文档处理中,暂不允许删除", expected.getMessage());
}
Mockito.verifyNoInteractions(chunkMapper, knowledgeService);
Mockito.verify(documentMapper, Mockito.never()).deleteById(Mockito.any());
}
/**
* 验证向量删除失败时停止后续清理,避免接口返回虚假成功。
*
* @throws Exception 反射注入异常
*/
@Test
public void removeDocShouldStopWhenVectorDeleteFails() throws Exception {
BigInteger documentId = BigInteger.valueOf(401);
BigInteger knowledgeId = BigInteger.valueOf(402);
BigInteger modelId = BigInteger.valueOf(403);
Document document = new Document();
document.setId(documentId);
document.setCollectionId(knowledgeId);
document.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
DocumentStore documentStore = Mockito.mock(DocumentStore.class);
Mockito.when(documentStore.delete(
Mockito.anyList(), Mockito.any())).thenReturn(StoreResult.fail("测试失败"));
DocumentCollection knowledge = Mockito.mock(DocumentCollection.class);
Mockito.when(knowledge.getVectorEmbedModelId()).thenReturn(modelId);
Mockito.when(knowledge.getVectorStoreCollection()).thenReturn("kb-test");
Mockito.when(knowledge.toDocumentStore()).thenReturn(documentStore);
Model model = new Model();
model.setModelName("embedding-test");
DocumentMapper documentMapper = Mockito.mock(DocumentMapper.class);
Mockito.when(documentMapper.selectOneByQuery(
Mockito.any(QueryWrapper.class))).thenReturn(document);
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
Mockito.when(chunkMapper.selectListByQueryAs(
Mockito.any(QueryWrapper.class), Mockito.eq(BigInteger.class)))
.thenReturn(List.of(BigInteger.valueOf(404)));
DocumentCollectionService knowledgeService =
Mockito.mock(DocumentCollectionService.class);
Mockito.when(knowledgeService.getById(knowledgeId)).thenReturn(knowledge);
ModelService modelService = Mockito.mock(ModelService.class);
Mockito.when(modelService.getById(modelId)).thenReturn(model);
FileStorageService storageService = Mockito.mock(FileStorageService.class);
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
DocumentServiceImpl service = new DocumentServiceImpl();
setField(service, "documentMapper", documentMapper);
setField(service, "documentChunkMapper", chunkMapper);
setField(service, "knowledgeService", knowledgeService);
setField(service, "modelService", modelService);
setField(service, "storageService", storageService);
setField(service, "searcherFactory", searcherFactory);
try {
service.removeDoc(documentId.toString());
Assert.fail("向量删除失败时应停止文档删除");
} catch (BusinessException expected) {
Assert.assertEquals("文档向量删除失败", expected.getMessage());
}
Mockito.verify(chunkMapper, Mockito.never()).deleteByQuery(Mockito.any());
Mockito.verify(documentMapper, Mockito.never()).deleteById(Mockito.any());
Mockito.verifyNoInteractions(storageService);
}
/**
* 通过反射注入测试依赖。
*
* @param target 目标对象
* @param fieldName 字段名
* @param value 字段值
* @throws Exception 字段不存在或无法访问时抛出
*/
private static void setField(Object target,
String fieldName,
Object value) throws Exception {
Field field = DocumentServiceImpl.class.getDeclaredField(fieldName);
field.setAccessible(true);
field.set(target, value);
}
}

View File

@@ -0,0 +1,137 @@
package tech.easyflow.ai.service.impl;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.entity.SysApiKeyResource;
import tech.easyflow.system.entity.SysApiKeyResourceMapping;
import tech.easyflow.system.service.SysApiKeyResourceMappingService;
import tech.easyflow.system.service.SysApiKeyResourceService;
import tech.easyflow.system.service.SysApiKeyService;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
/**
* 知识库 Public API 产品权限映射测试。
*
* @author Codex
* @since 2026-08-02
*/
public class KnowledgeSharePermissionServiceImplTest {
/**
* 验证读取、导入和维护权限分别只生成各自接口映射。
*
* @throws Exception 依赖注入失败
*/
@Test
public void replacePermissionsShouldKeepReadImportMaintenanceSeparated()
throws Exception {
BigInteger apiKeyId = BigInteger.valueOf(701);
SysApiKeyService apiKeyService = Mockito.mock(SysApiKeyService.class);
SysApiKeyResourceService resourceService =
Mockito.mock(SysApiKeyResourceService.class);
SysApiKeyResourceMappingService mappingService =
Mockito.mock(SysApiKeyResourceMappingService.class);
RedisLockExecutor redisLockExecutor =
Mockito.mock(RedisLockExecutor.class);
RedisLockExecutor.LockHandle lockHandle =
Mockito.mock(RedisLockExecutor.LockHandle.class);
Mockito.when(redisLockExecutor.tryAcquire(
Mockito.anyString(),
Mockito.any(),
Mockito.any()
)).thenReturn(lockHandle);
Mockito.when(apiKeyService.getById(apiKeyId)).thenReturn(new SysApiKey());
AtomicLong resourceId = new AtomicLong(800);
Mockito.when(resourceService.getOne(
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)))
.thenAnswer(invocation -> {
SysApiKeyResource resource = new SysApiKeyResource();
resource.setId(BigInteger.valueOf(resourceId.incrementAndGet()));
return resource;
});
KnowledgeSharePermissionServiceImpl service =
new KnowledgeSharePermissionServiceImpl();
setField(service, "sysApiKeyService", apiKeyService);
setField(service, "resourceService", resourceService);
setField(service, "mappingService", mappingService);
setField(service, "redisLockExecutor", redisLockExecutor);
service.replaceApiPermissions(apiKeyId, true, false, false);
service.replaceApiPermissions(apiKeyId, false, true, false);
service.replaceApiPermissions(apiKeyId, false, false, true);
@SuppressWarnings("unchecked")
ArgumentCaptor<List<SysApiKeyResourceMapping>> mappings =
ArgumentCaptor.forClass(List.class);
Mockito.verify(mappingService, Mockito.times(3)).remove(
Mockito.any(com.mybatisflex.core.query.QueryWrapper.class)
);
Mockito.verify(mappingService, Mockito.times(3))
.saveBatch(mappings.capture());
Mockito.verify(lockHandle, Mockito.times(3)).release();
Assert.assertEquals(3, mappings.getAllValues().size());
assertScopeMappings(
mappings.getAllValues().get(0),
KnowledgeApiPermissionScope.KNOWLEDGE_READ,
8
);
assertScopeMappings(
mappings.getAllValues().get(1),
KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT,
14
);
assertScopeMappings(
mappings.getAllValues().get(2),
KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE,
6
);
}
/**
* 断言一组全局知识库接口映射只包含指定权限。
*
* @param mappings 接口映射
* @param expectedScope 预期权限
* @param expectedSize 预期接口数量
*/
private void assertScopeMappings(
List<SysApiKeyResourceMapping> mappings,
KnowledgeApiPermissionScope expectedScope,
int expectedSize) {
Assert.assertEquals(expectedSize, mappings.size());
for (SysApiKeyResourceMapping mapping : mappings) {
Assert.assertEquals(
expectedScope.name(),
mapping.getActionScope()
);
Assert.assertNull(mapping.getResourceTargetId());
}
}
/**
* 反射注入测试依赖。
*
* @param target 目标对象
* @param fieldName 字段名
* @param value 字段值
* @throws Exception 字段不存在或不可访问
*/
private static void setField(
Object target,
String fieldName,
Object value) throws Exception {
Field field = target.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
field.set(target, value);
}
}

View File

@@ -26,6 +26,15 @@ public class SysApiKey extends SysApiKeyBase {
@Column(ignore = true) @Column(ignore = true)
private Boolean knowledgeShareEnabled; private Boolean knowledgeShareEnabled;
@Column(ignore = true)
private Boolean knowledgeReadEnabled;
@Column(ignore = true)
private Boolean knowledgeImportEnabled;
@Column(ignore = true)
private Boolean knowledgeMaintenanceEnabled;
@Column(ignore = true) @Column(ignore = true)
private Boolean workflowApiEnabled; private Boolean workflowApiEnabled;
@@ -56,6 +65,60 @@ public class SysApiKey extends SysApiKeyBase {
this.knowledgeShareEnabled = knowledgeShareEnabled; this.knowledgeShareEnabled = knowledgeShareEnabled;
} }
/**
* 获取知识库读取权限开关。
*
* @return 是否开启读取权限
*/
public Boolean getKnowledgeReadEnabled() {
return knowledgeReadEnabled;
}
/**
* 设置知识库读取权限开关。
*
* @param knowledgeReadEnabled 是否开启读取权限
*/
public void setKnowledgeReadEnabled(Boolean knowledgeReadEnabled) {
this.knowledgeReadEnabled = knowledgeReadEnabled;
}
/**
* 获取知识库导入权限开关。
*
* @return 是否开启导入权限
*/
public Boolean getKnowledgeImportEnabled() {
return knowledgeImportEnabled;
}
/**
* 设置知识库导入权限开关。
*
* @param knowledgeImportEnabled 是否开启导入权限
*/
public void setKnowledgeImportEnabled(Boolean knowledgeImportEnabled) {
this.knowledgeImportEnabled = knowledgeImportEnabled;
}
/**
* 获取知识库维护权限开关。
*
* @return 是否开启维护权限
*/
public Boolean getKnowledgeMaintenanceEnabled() {
return knowledgeMaintenanceEnabled;
}
/**
* 设置知识库维护权限开关。
*
* @param knowledgeMaintenanceEnabled 是否开启维护权限
*/
public void setKnowledgeMaintenanceEnabled(Boolean knowledgeMaintenanceEnabled) {
this.knowledgeMaintenanceEnabled = knowledgeMaintenanceEnabled;
}
public Boolean getWorkflowApiEnabled() { public Boolean getWorkflowApiEnabled() {
return workflowApiEnabled; return workflowApiEnabled;
} }

View File

@@ -13,7 +13,14 @@ import java.math.BigInteger;
*/ */
public interface SysApiKeyService extends IService<SysApiKey> { public interface SysApiKeyService extends IService<SysApiKey> {
void checkApikeyPermission(String apiKey, String requestURI); /**
* 校验访问令牌是否具有接口权限。
*
* @param apiKey 访问令牌明文
* @param requestURI 请求 URI
* @return 已通过身份和接口权限校验的访问令牌
*/
SysApiKey checkApikeyPermission(String apiKey, String requestURI);
SysApiKey getSysApiKey(String apiKey); SysApiKey getSysApiKey(String apiKey);

View File

@@ -36,7 +36,7 @@ public class SysApiKeyServiceImpl extends ServiceImpl<SysApiKeyMapper, SysApiKey
private SysApiKeyResourceService resourceService; private SysApiKeyResourceService resourceService;
@Override @Override
public void checkApikeyPermission(String apiKey, String requestURI) { public SysApiKey checkApikeyPermission(String apiKey, String requestURI) {
SysApiKey sysApiKey = getSysApiKey(apiKey); SysApiKey sysApiKey = getSysApiKey(apiKey);
List<String> candidateRequestUris = getCandidateRequestUris(requestURI); List<String> candidateRequestUris = getCandidateRequestUris(requestURI);
QueryWrapper w = QueryWrapper.create(); QueryWrapper w = QueryWrapper.create();
@@ -55,6 +55,7 @@ public class SysApiKeyServiceImpl extends ServiceImpl<SysApiKeyMapper, SysApiKey
if (count == 0) { if (count == 0) {
throw new BusinessException(403, 403, "该apiKey无权限访问该接口"); throw new BusinessException(403, 403, "该apiKey无权限访问该接口");
} }
return sysApiKey;
} }
private List<String> getCandidateRequestUris(String requestURI) { private List<String> getCandidateRequestUris(String requestURI) {

View File

@@ -47,7 +47,7 @@ easyflow:
max-retry: 16 max-retry: 16
consumer-executor: consumer-executor:
core-size: 4 core-size: 4
max-size: 12 max-size: 32
queue-capacity: 64 queue-capacity: 64
keep-alive-seconds: 60 keep-alive-seconds: 60
pool: pool:

View File

@@ -10,6 +10,9 @@ server:
enabled: true enabled: true
charset: UTF-8 # 必须设置 UTF-8避免 WebFlux 流式返回AI 场景)会乱码问题 charset: UTF-8 # 必须设置 UTF-8避免 WebFlux 流式返回AI 场景)会乱码问题
force: true force: true
tomcat:
# Public API 单次最多 200 个文件,额外为 metadata 与表单边界预留 Part。
max-part-count: 205
spring: spring:
profiles: profiles:
@@ -44,8 +47,8 @@ spring:
servlet: servlet:
multipart: multipart:
max-file-size: 100MB max-file-size: 100MB
# 为 multipart 边界和请求头预留空间,文件本身仍由 M18 的 100 MiB 硬上限约束 # Public API 业务上限为 200 MiB为 metadata 与 multipart 边界预留空间
max-request-size: 105MB max-request-size: 220MB
web: web:
resources: resources:
# 示例windows【file: C:\easyflow\attachment】 linux【file: /www/easyflow/attachment】 # 示例windows【file: C:\easyflow\attachment】 linux【file: /www/easyflow/attachment】
@@ -127,7 +130,7 @@ easyflow:
max-retry: 16 max-retry: 16
consumer-executor: consumer-executor:
core-size: 16 core-size: 16
max-size: 24 max-size: 32
queue-capacity: 64 queue-capacity: 64
keep-alive-seconds: 60 keep-alive-seconds: 60
pool: pool:
@@ -198,6 +201,22 @@ easyflow:
health: health:
cache-ttl: 5s cache-ttl: 5s
document-import: document-import:
bulk:
max-file-count: 2000
max-total-size: 1GB
max-file-size: 100MB
upload-concurrency: 3
parse-max-running: 2
split-max-running: 2
index-max-running: 2
per-batch-parse-max-running: 2
pending-dispatch-batch-size: 100
pending-dispatch-interval: 2s
pending-redispatch-delay: 5s
pending-timeout: 24h
parse-submit-timeout: 120s
interruption-timeout: 10m
max-task-attempts: 3
status-broadcast-channel: easyflow:document-import:status status-broadcast-channel: easyflow:document-import:status
parse-monitor: parse-monitor:
fixed-delay: 10000 fixed-delay: 10000
@@ -249,7 +268,8 @@ easy-agents:
provider: mineru provider: mineru
mineru: mineru:
# 统一文档解析桥接层直接复用 easy-agents 的 provider 配置,不在 easyflow 再复制一套配置体系 # 统一文档解析桥接层直接复用 easy-agents 的 provider 配置,不在 easyflow 再复制一套配置体系
base-url: https://hub.wust.edu.cn/modelServer/mineru-api base-url: https://ontoweb.wust.edu.cn/mineru-api
submit-timeout-ms: 120000
default-lang-list: default-lang-list:
- ch - ch

View File

@@ -0,0 +1,52 @@
CREATE TABLE `tb_document_import_batch`
(
`id` bigint UNSIGNED NOT NULL COMMENT '主键',
`knowledge_id` bigint UNSIGNED NOT NULL COMMENT '知识库ID',
`import_mode` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '导入模式',
`status` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '批次状态',
`total_count` int NOT NULL DEFAULT 0 COMMENT '文件总数',
`total_bytes` bigint NOT NULL DEFAULT 0 COMMENT '文件总字节数',
`completed_count` int NOT NULL DEFAULT 0 COMMENT '完成数',
`processing_count` int NOT NULL DEFAULT 0 COMMENT '处理中数量',
`failed_count` int NOT NULL DEFAULT 0 COMMENT '失败数',
`pending_count` int NOT NULL DEFAULT 0 COMMENT '等待数',
`started_at` datetime NULL DEFAULT NULL COMMENT '开始时间',
`finished_at` datetime NULL DEFAULT NULL COMMENT '结束时间',
`created` datetime NULL DEFAULT NULL COMMENT '创建时间',
`created_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '创建人',
`modified` datetime NULL DEFAULT NULL COMMENT '修改时间',
`modified_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '修改人',
PRIMARY KEY (`id`) USING BTREE,
INDEX `idx_document_import_batch_knowledge_status` (`knowledge_id`, `status`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '知识库文档批量导入批次' ROW_FORMAT = DYNAMIC;
CREATE TABLE `tb_document_import_batch_item`
(
`id` bigint UNSIGNED NOT NULL COMMENT '主键',
`batch_id` bigint UNSIGNED NOT NULL COMMENT '批次ID',
`knowledge_id` bigint UNSIGNED NOT NULL COMMENT '知识库ID',
`document_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '文档ID',
`client_file_key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '客户端文件键',
`file_name` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '文件名',
`relative_path` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文件夹相对路径',
`file_size` bigint NOT NULL DEFAULT 0 COMMENT '文件大小',
`file_path` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '存储路径',
`stage` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '当前阶段',
`status` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '当前状态',
`error_summary` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '错误摘要',
`created` datetime NULL DEFAULT NULL COMMENT '创建时间',
`created_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '创建人',
`modified` datetime NULL DEFAULT NULL COMMENT '修改时间',
`modified_by` bigint UNSIGNED NULL DEFAULT NULL COMMENT '修改人',
PRIMARY KEY (`id`) USING BTREE,
UNIQUE INDEX `uk_document_import_batch_item_key` (`batch_id`, `client_file_key`) USING BTREE,
INDEX `idx_document_import_batch_item_batch_status` (`batch_id`, `status`) USING BTREE,
INDEX `idx_document_import_batch_item_knowledge_key` (`knowledge_id`, `client_file_key`) USING BTREE,
INDEX `idx_document_import_batch_item_document` (`document_id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '知识库文档批量导入文件项' ROW_FORMAT = DYNAMIC;
ALTER TABLE `tb_document_import_task`
ADD COLUMN `batch_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '批次ID' AFTER `knowledge_id`,
ADD COLUMN `batch_item_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '批次文件项ID' AFTER `batch_id`,
ADD INDEX `idx_document_import_task_batch_phase_status` (`batch_id`, `phase`, `status`) USING BTREE,
ADD INDEX `idx_document_import_task_batch_item` (`batch_item_id`) USING BTREE;

View File

@@ -0,0 +1,42 @@
ALTER TABLE `tb_document_import_batch`
ADD COLUMN `uploaded_count` int NOT NULL DEFAULT 0 COMMENT '已上传数' AFTER `pending_count`,
ADD COLUMN `skipped_count` int NOT NULL DEFAULT 0 COMMENT '跳过数' AFTER `uploaded_count`,
ADD COLUMN `cancelled_count` int NOT NULL DEFAULT 0 COMMENT '取消数' AFTER `skipped_count`,
ADD COLUMN `retryable_failed_count` int NOT NULL DEFAULT 0 COMMENT '可重试失败数' AFTER `cancelled_count`;
ALTER TABLE `tb_document_import_batch_item`
ADD COLUMN `retryable` tinyint(1) NOT NULL DEFAULT 0 COMMENT '是否允许批量重试' AFTER `error_summary`,
ADD COLUMN `attempt_count` int NOT NULL DEFAULT 0 COMMENT '重试次数' AFTER `retryable`,
DROP INDEX `idx_document_import_batch_item_batch_status`,
ADD INDEX `idx_document_import_batch_item_batch_status` (`batch_id`, `status`, `modified`) USING BTREE;
ALTER TABLE `tb_document_import_task`
ADD INDEX `idx_document_import_task_phase_status_modified`
(`phase`, `status`, `modified`, `batch_id`, `created`) USING BTREE;
UPDATE `tb_document_import_batch` batch
SET batch.`uploaded_count` = (
SELECT COUNT(*)
FROM `tb_document_import_batch_item` item
WHERE item.`batch_id` = batch.`id`
AND item.`status` = 'UPLOADED'
),
batch.`skipped_count` = (
SELECT COUNT(*)
FROM `tb_document_import_batch_item` item
WHERE item.`batch_id` = batch.`id`
AND item.`status` = 'SKIPPED'
),
batch.`cancelled_count` = (
SELECT COUNT(*)
FROM `tb_document_import_batch_item` item
WHERE item.`batch_id` = batch.`id`
AND item.`status` = 'CANCELLED'
),
batch.`retryable_failed_count` = (
SELECT COUNT(*)
FROM `tb_document_import_batch_item` item
WHERE item.`batch_id` = batch.`id`
AND item.`status` = 'FAILED'
AND item.`retryable` = 1
);

View File

@@ -0,0 +1,5 @@
ALTER TABLE `tb_document_import_batch_item`
ADD COLUMN `replaced_document_id` bigint UNSIGNED NULL DEFAULT NULL
COMMENT '待覆盖的历史文档ID' AFTER `document_id`,
ADD INDEX `idx_document_import_batch_item_replacement`
(`replaced_document_id`, `status`, `modified`) USING BTREE;

View File

@@ -0,0 +1,15 @@
UPDATE `tb_document_import_batch_item`
SET `retryable` = 1
WHERE `status` = 'FAILED'
AND `stage` IN ('PARSE', 'SPLIT', 'INDEX')
AND `document_id` IS NOT NULL
AND `retryable` = 0;
UPDATE `tb_document_import_batch` batch
SET batch.`retryable_failed_count` = (
SELECT COUNT(*)
FROM `tb_document_import_batch_item` item
WHERE item.`batch_id` = batch.`id`
AND item.`status` = 'FAILED'
AND item.`retryable` = 1
);

View File

@@ -0,0 +1,191 @@
SET NAMES utf8mb4;
INSERT INTO `tb_sys_api_key_resource` (`id`, `request_interface`, `title`)
VALUES
(366700000000000101, '/public-api/knowledge-share/detail', '知识库读取'),
(366700000000000102, '/public-api/knowledge-share/search', '知识库读取'),
(366700000000000103, '/public-api/knowledge-share/document/page', '知识库读取'),
(366700000000000104, '/public-api/knowledge-share/document/download', '知识库读取'),
(366700000000000105, '/public-api/knowledge-share/documentChunk/page', '知识库读取'),
(366700000000000106, '/public-api/knowledge-share/faq/page', '知识库读取'),
(366700000000000107, '/public-api/knowledge-share/faq/detail', '知识库读取'),
(366700000000000108, '/public-api/knowledge-share/faq/exportExcel', '知识库读取'),
(366700000000000109, '/public-api/knowledge-share/document/import/batch', '知识导入'),
(366700000000000110, '/public-api/knowledge-share/document/import/batch/status', '知识导入'),
(366700000000000111, '/public-api/knowledge-share/document/import/batch/retry', '知识导入'),
(366700000000000112, '/public-api/knowledge-share/document/import/analyze', '知识导入'),
(366700000000000113, '/public-api/knowledge-share/document/import/preview', '知识导入'),
(366700000000000114, '/public-api/knowledge-share/document/import/commit', '知识导入'),
(366700000000000115, '/public-api/knowledge-share/document/import/task/create', '知识导入'),
(366700000000000116, '/public-api/knowledge-share/document/import/task/detail', '知识导入'),
(366700000000000117, '/public-api/knowledge-share/document/import/task/preview', '知识导入'),
(366700000000000118, '/public-api/knowledge-share/document/import/task/startIndex', '知识导入'),
(366700000000000119, '/public-api/knowledge-share/document/import/task/retryParse', '知识导入'),
(366700000000000120, '/public-api/knowledge-share/document/import/task/retryIndex', '知识导入'),
(366700000000000121, '/public-api/knowledge-share/faq/importExcel', '知识导入'),
(366700000000000122, '/public-api/knowledge-share/faq/downloadImportTemplate', '知识导入'),
(366700000000000123, '/public-api/knowledge-share/document/remove', '知识库维护'),
(366700000000000124, '/public-api/knowledge-share/documentChunk/update', '知识库维护'),
(366700000000000125, '/public-api/knowledge-share/documentChunk/remove', '知识库维护'),
(366700000000000126, '/public-api/knowledge-share/faq/save', '知识库维护'),
(366700000000000127, '/public-api/knowledge-share/faq/update', '知识库维护'),
(366700000000000128, '/public-api/knowledge-share/faq/remove', '知识库维护')
ON DUPLICATE KEY UPDATE `title` = VALUES(`title`);
SET @migrated_knowledge_api_key_count = (
SELECT COUNT(DISTINCT mapping.`api_key_id`)
FROM `tb_sys_api_key_resource_mapping` mapping
WHERE mapping.`resource_type` = 'KNOWLEDGE'
AND mapping.`resource_target_id` IS NULL
);
UPDATE `tb_sys_api_key_resource_mapping` mapping
JOIN `tb_sys_api_key_resource` resource
ON resource.`id` = mapping.`api_key_resource_id`
SET mapping.`action_scope` = CASE
WHEN resource.`request_interface` IN (
'/public-api/knowledge-share/detail',
'/public-api/knowledge-share/search',
'/public-api/knowledge-share/document/page',
'/public-api/knowledge-share/document/download',
'/public-api/knowledge-share/documentChunk/page',
'/public-api/knowledge-share/faq/page',
'/public-api/knowledge-share/faq/detail',
'/public-api/knowledge-share/faq/exportExcel'
) THEN 'KNOWLEDGE_READ'
WHEN resource.`request_interface` IN (
'/public-api/knowledge-share/document/import/analyze',
'/public-api/knowledge-share/document/import/preview',
'/public-api/knowledge-share/document/import/commit',
'/public-api/knowledge-share/document/import/task/create',
'/public-api/knowledge-share/document/import/task/detail',
'/public-api/knowledge-share/document/import/task/preview',
'/public-api/knowledge-share/document/import/task/startIndex',
'/public-api/knowledge-share/document/import/task/retryParse',
'/public-api/knowledge-share/document/import/task/retryIndex',
'/public-api/knowledge-share/faq/importExcel',
'/public-api/knowledge-share/faq/downloadImportTemplate'
) THEN 'KNOWLEDGE_IMPORT'
WHEN resource.`request_interface` IN (
'/public-api/knowledge-share/document/remove',
'/public-api/knowledge-share/documentChunk/update',
'/public-api/knowledge-share/documentChunk/remove',
'/public-api/knowledge-share/faq/save',
'/public-api/knowledge-share/faq/update',
'/public-api/knowledge-share/faq/remove'
) THEN 'KNOWLEDGE_MAINTENANCE'
ELSE mapping.`action_scope`
END
WHERE mapping.`resource_type` = 'KNOWLEDGE';
SET @revoked_maintenance_mapping_count = (
SELECT COUNT(*)
FROM `tb_sys_api_key_resource_mapping`
WHERE `resource_type` = 'KNOWLEDGE'
AND `resource_target_id` IS NULL
AND `action_scope` = 'KNOWLEDGE_MAINTENANCE'
);
DELETE FROM `tb_sys_api_key_resource_mapping`
WHERE `resource_type` = 'KNOWLEDGE'
AND `resource_target_id` IS NULL
AND `action_scope` = 'KNOWLEDGE_MAINTENANCE';
INSERT INTO `tb_sys_api_key_resource_mapping`
(`id`, `api_key_id`, `api_key_resource_id`, `resource_type`,
`resource_target_id`, `action_scope`)
SELECT
CAST(
366900000000000000
+ ROW_NUMBER() OVER (
ORDER BY token.`api_key_id`,
COALESCE(token.`resource_target_id`, 0),
resource.`id`
)
AS UNSIGNED
),
token.`api_key_id`,
resource.`id`,
'KNOWLEDGE',
token.`resource_target_id`,
'KNOWLEDGE_IMPORT'
FROM (
SELECT DISTINCT mapping.`api_key_id`, mapping.`resource_target_id`
FROM `tb_sys_api_key_resource_mapping` mapping
WHERE mapping.`resource_type` = 'KNOWLEDGE'
AND mapping.`action_scope` = 'KNOWLEDGE_IMPORT'
) token
JOIN `tb_sys_api_key_resource` resource
ON resource.`request_interface` IN (
'/public-api/knowledge-share/document/import/batch',
'/public-api/knowledge-share/document/import/batch/status',
'/public-api/knowledge-share/document/import/batch/retry'
)
LEFT JOIN `tb_sys_api_key_resource_mapping` existing
ON existing.`api_key_id` = token.`api_key_id`
AND existing.`api_key_resource_id` = resource.`id`
AND existing.`resource_type` = 'KNOWLEDGE'
AND existing.`resource_target_id` <=> token.`resource_target_id`
AND existing.`action_scope` = 'KNOWLEDGE_IMPORT'
WHERE existing.`id` IS NULL;
SELECT
@migrated_knowledge_api_key_count AS `migrated_knowledge_api_key_count`,
@revoked_maintenance_mapping_count AS `revoked_maintenance_mapping_count`;
ALTER TABLE `tb_document_import_batch`
ADD COLUMN `caller_type` varchar(16) NULL DEFAULT NULL
COMMENT '调用者类型' AFTER `knowledge_id`,
ADD COLUMN `caller_id` bigint UNSIGNED NULL DEFAULT NULL
COMMENT '调用者ID' AFTER `caller_type`,
ADD COLUMN `idempotency_key_hash` char(64) NULL DEFAULT NULL
COMMENT '提交幂等键SHA-256' AFTER `caller_id`,
ADD COLUMN `request_digest` char(64) NULL DEFAULT NULL
COMMENT '请求摘要SHA-256' AFTER `idempotency_key_hash`,
ADD COLUMN `duplicate_policy` varchar(16) NULL DEFAULT NULL
COMMENT '重复文件策略' AFTER `request_digest`,
ADD COLUMN `requested_strategy_json` text NULL
COMMENT '请求分块策略' AFTER `duplicate_policy`,
ADD COLUMN `retry_generation` int NOT NULL DEFAULT 0
COMMENT '重试代次' AFTER `requested_strategy_json`,
ADD COLUMN `last_retry_key_hash` char(64) NULL DEFAULT NULL
COMMENT '最近重试幂等键SHA-256' AFTER `retry_generation`,
ADD COLUMN `last_retry_generation` int NOT NULL DEFAULT 0
COMMENT '最近重试响应代次' AFTER `last_retry_key_hash`,
ADD COLUMN `version` int NOT NULL DEFAULT 0
COMMENT '乐观锁版本' AFTER `last_retry_generation`,
ADD UNIQUE INDEX `uk_document_import_batch_caller_idempotency`
(`caller_type`, `caller_id`, `idempotency_key_hash`) USING BTREE;
UPDATE `tb_document_import_batch`
SET `caller_type` = 'ADMIN',
`caller_id` = COALESCE(`created_by`, 0),
`duplicate_policy` = COALESCE(`duplicate_policy`, 'SKIP')
WHERE `caller_type` IS NULL;
ALTER TABLE `tb_document_import_batch_item`
ADD COLUMN `content_sha256` char(64) NULL DEFAULT NULL
COMMENT '文件内容SHA-256' AFTER `file_path`,
ADD COLUMN `failure_code` varchar(64) NULL DEFAULT NULL
COMMENT '稳定失败码' AFTER `error_summary`,
ADD COLUMN `applied_strategy_code` varchar(64) NULL DEFAULT NULL
COMMENT '实际分块策略编码' AFTER `failure_code`,
ADD COLUMN `strategy_snapshot_json` text NULL
COMMENT '分块策略快照' AFTER `applied_strategy_code`,
DROP INDEX `idx_document_import_batch_item_batch_status`,
ADD INDEX `idx_document_import_batch_item_batch_status`
(`batch_id`, `status`, `modified`, `id`) USING BTREE;
ALTER TABLE `tb_document_import_task`
ADD COLUMN `failure_code` varchar(64) NULL DEFAULT NULL
COMMENT '稳定失败码' AFTER `error_summary`,
ADD COLUMN `attempt_no` int NOT NULL DEFAULT 0
COMMENT '执行尝试次数' AFTER `failure_code`,
ADD COLUMN `execution_token` varchar(64) NULL DEFAULT NULL
COMMENT '执行令牌' AFTER `attempt_no`,
ADD COLUMN `lease_until` datetime NULL DEFAULT NULL
COMMENT '租约到期时间' AFTER `execution_token`,
ADD COLUMN `version` int NOT NULL DEFAULT 0
COMMENT '乐观锁版本' AFTER `lease_until`,
ADD INDEX `idx_document_import_task_phase_status_lease`
(`phase`, `status`, `lease_until`, `created`, `id`) USING BTREE;

View File

@@ -0,0 +1,15 @@
CREATE TABLE `tb_document_import_batch_retry_request`
(
`id` bigint UNSIGNED NOT NULL COMMENT '主键',
`batch_id` bigint UNSIGNED NOT NULL COMMENT '批次ID',
`retry_key_hash` char(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '重试幂等键SHA-256',
`request_digest` char(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL COMMENT '重试请求摘要SHA-256',
`response_status` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '首次领取后的响应状态',
`retry_generation` int NOT NULL COMMENT '首次领取后的重试代次',
`created` datetime NULL DEFAULT NULL COMMENT '创建时间',
`modified` datetime NULL DEFAULT NULL COMMENT '修改时间',
PRIMARY KEY (`id`) USING BTREE,
UNIQUE INDEX `uk_document_import_retry_batch_key` (`batch_id`, `retry_key_hash`) USING BTREE,
INDEX `idx_document_import_retry_batch_created` (`batch_id`, `created`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci
COMMENT = '知识库文档批量导入重试幂等请求' ROW_FORMAT = DYNAMIC;

View File

@@ -0,0 +1,7 @@
DROP TABLE IF EXISTS `tb_document_import_batch_retry_request`;
ALTER TABLE `tb_document_import_batch`
DROP COLUMN `last_retry_key_hash`,
DROP COLUMN `last_retry_generation`,
ADD INDEX `idx_document_import_batch_status_modified`
(`status`, `modified`, `id`) USING BTREE;

View File

@@ -0,0 +1,3 @@
ALTER TABLE `tb_document_import_batch_item`
ADD INDEX `idx_document_import_batch_item_cleanup`
(`stage`, `status`, `modified`, `id`) USING BTREE;

View File

@@ -0,0 +1,16 @@
ALTER TABLE `tb_document_import_batch_item`
ADD COLUMN `storage_locator` varchar(2048) NULL DEFAULT NULL
COMMENT '可恢复存储定位符' AFTER `file_path`,
ADD COLUMN `cleanup_pending` tinyint(1) NOT NULL DEFAULT 0
COMMENT '是否等待清理存储对象' AFTER `storage_locator`;
UPDATE `tb_document_import_batch_item`
SET `cleanup_pending` = 1
WHERE `stage` = 'UPLOAD'
AND `status` IN ('UPLOADING', 'CANCELLED')
AND `file_path` IS NOT NULL;
ALTER TABLE `tb_document_import_batch_item`
DROP INDEX `idx_document_import_batch_item_cleanup`,
ADD INDEX `idx_document_import_batch_item_cleanup_pending`
(`cleanup_pending`, `modified`, `id`) USING BTREE;

View File

@@ -78,6 +78,50 @@
"fileName": "File Name", "fileName": "File Name",
"progressUpload": "Progress of file upload", "progressUpload": "Progress of file upload",
"fileSize": "File size", "fileSize": "File size",
"batchUploadTitle": "Select files or drop them here",
"batchUploadDescription": "TXT, PDF, DOCX, MD, PPTX and XLSX. Up to 100MB per file and 1GB per folder.",
"batchUploadTip": "When upload completes, choose manual or automatic import.",
"selectFolder": "Select Folder",
"manualImport": "Manual Import",
"autoImport": "Auto Import",
"autoImportTip": "Automatically parse, chunk, vectorize and store every document.",
"manualImportStarted": "Documents are parsing and will be ready for manual chunking",
"autoImportStarted": "Automatic import started",
"uploadCompleteFirst": "Wait for every file to finish uploading",
"uploaded": "Uploaded",
"uploading": "Uploading",
"retryUpload": "Retry Upload",
"retry": "Retry",
"singleFileLimit": "Each file must be no larger than 100MB",
"folderSizeLimit": "The folder must be no larger than 1GB",
"fileCountLimit": "A batch can contain up to 2000 files",
"noSupportedFiles": "No supported documents found",
"unsupportedSkipped": "Unsupported files were skipped",
"createBatchFailed": "Failed to create upload batch",
"cancelBatchFailed": "Failed to cancel the upload batch. Please retry.",
"uploadFailed": "Upload failed. Please retry.",
"duplicatePolicy": "Duplicates",
"skipDuplicates": "Skip",
"overwriteDuplicates": "Replace",
"reimportDuplicates": "Reimport",
"batchStatus": "Automatic import status",
"batchRunning": "Running",
"batchInterrupted": "Interrupted",
"batchPartial": "Partially failed",
"batchFailed": "Import failed",
"batchCompleted": "Completed",
"completedCount": "Completed",
"processingCount": "Processing",
"failedCount": "Failed",
"pendingCount": "Pending",
"skippedCount": "Skipped",
"continueBatch": "Continue",
"splitOrIndexFailed": "Chunking or indexing failed. Please retry.",
"documentSourceUnavailable": "Unable to read the document file. Please contact the administrator.",
"parseServiceUnavailable": "The document parsing service is temporarily unavailable. Please retry later.",
"parseServiceTimeout": "The document parsing service timed out. Please retry later.",
"taskPendingTimeout": "The task waited too long in the queue. Please retry.",
"taskExecutionInterrupted": "The task was interrupted. Please retry.",
"uploadCreateTip": "After upload, the document appears in the list first and is parsed asynchronously. Continue with chunking after parsing finishes.", "uploadCreateTip": "After upload, the document appears in the list first and is parsed asynchronously. Continue with chunking after parsing finishes.",
"analysisTip": "The system analyzes multilingual structure first and recommends a splitting strategy. You can still adjust each file manually.", "analysisTip": "The system analyzes multilingual structure first and recommends a splitting strategy. You can still adjust each file manually.",
"manualStrategyTip": "The preview refreshes automatically when the chunking strategy changes. Start indexing after it looks right.", "manualStrategyTip": "The preview refreshes automatically when the chunking strategy changes. Start indexing after it looks right.",
@@ -134,6 +178,8 @@
"PARSING": "Parsing", "PARSING": "Parsing",
"PARSE_FAILED": "Parse Failed", "PARSE_FAILED": "Parse Failed",
"READY_FOR_SEGMENT": "Ready for Chunking", "READY_FOR_SEGMENT": "Ready for Chunking",
"SPLITTING": "Chunking",
"SPLIT_FAILED": "Chunking Failed",
"READY_FOR_INDEX": "Ready for Indexing", "READY_FOR_INDEX": "Ready for Indexing",
"INDEXING": "Indexing", "INDEXING": "Indexing",
"INDEX_FAILED": "Index Failed", "INDEX_FAILED": "Index Failed",

View File

@@ -14,7 +14,9 @@
"failure": "Failure" "failure": "Failure"
}, },
"permissions": "AuthInterface", "permissions": "AuthInterface",
"knowledgeSharePermission": "Knowledge Share", "knowledgeReadPermission": "Knowledge Read",
"knowledgeImportPermission": "Knowledge Import",
"knowledgeMaintenancePermission": "Knowledge Maintenance",
"workflowApiPermission": "Workflow API", "workflowApiPermission": "Workflow API",
"addApiKeyNotice": "This operation will generate an API key. Please confirm whether to proceed" "addApiKeyNotice": "This operation will generate an API key. Please confirm whether to proceed"
} }

View File

@@ -78,6 +78,50 @@
"fileName": "文件名称", "fileName": "文件名称",
"progressUpload": "文件上传进度", "progressUpload": "文件上传进度",
"fileSize": "文件大小", "fileSize": "文件大小",
"batchUploadTitle": "点击选择文件,或将文件拖到这里上传",
"batchUploadDescription": "支持 TXT、PDF、DOCX、MD、PPTX、XLSX单个文件不超过 100MB文件夹总大小不超过 1GB。",
"batchUploadTip": "上传完成后,可选择手动导入或自动导入。",
"selectFolder": "选择文件夹",
"manualImport": "手动导入",
"autoImport": "自动导入",
"autoImportTip": "自动完成解析、分块、向量化和入库,无需逐篇确认。",
"manualImportStarted": "文档已进入列表,解析完成后可继续选择分块策略",
"autoImportStarted": "自动导入已开始",
"uploadCompleteFirst": "请等待所有文件上传完成",
"uploaded": "已上传",
"uploading": "上传中",
"retryUpload": "重试上传",
"retry": "重试",
"singleFileLimit": "单个文件不能超过 100MB",
"folderSizeLimit": "文件夹总大小不能超过 1GB",
"fileCountLimit": "单批次文件数不能超过 2000",
"noSupportedFiles": "未找到支持的文档",
"unsupportedSkipped": "已跳过不支持的文件",
"createBatchFailed": "创建上传批次失败",
"cancelBatchFailed": "取消上传批次失败,请重试",
"uploadFailed": "文件上传失败,请重试",
"duplicatePolicy": "重复文件",
"skipDuplicates": "跳过",
"overwriteDuplicates": "覆盖",
"reimportDuplicates": "重新导入",
"batchStatus": "自动导入状态",
"batchRunning": "处理中",
"batchInterrupted": "已中断",
"batchPartial": "部分失败",
"batchFailed": "导入失败",
"batchCompleted": "已完成",
"completedCount": "完成",
"processingCount": "处理中",
"failedCount": "失败",
"pendingCount": "等待",
"skippedCount": "跳过",
"continueBatch": "继续",
"splitOrIndexFailed": "分块或向量化失败,请重试",
"documentSourceUnavailable": "文档文件读取失败,请联系管理员",
"parseServiceUnavailable": "文档解析服务暂不可用,请稍后重试",
"parseServiceTimeout": "文档解析服务响应超时,请稍后重试",
"taskPendingTimeout": "任务排队超时,请重试",
"taskExecutionInterrupted": "任务执行中断,请重试",
"uploadCreateTip": "上传完成后,文档会先进入列表并异步解析,解析完成后再继续分块和向量化。", "uploadCreateTip": "上传完成后,文档会先进入列表并异步解析,解析完成后再继续分块和向量化。",
"analysisTip": "系统会先基于文档结构做中英文规则分析,再推荐拆分策略,你也可以逐个文件手动调整。", "analysisTip": "系统会先基于文档结构做中英文规则分析,再推荐拆分策略,你也可以逐个文件手动调整。",
"manualStrategyTip": "调整分块策略后会自动刷新预览,确认效果后再启动向量化。", "manualStrategyTip": "调整分块策略后会自动刷新预览,确认效果后再启动向量化。",
@@ -134,6 +178,8 @@
"PARSING": "解析中", "PARSING": "解析中",
"PARSE_FAILED": "解析失败", "PARSE_FAILED": "解析失败",
"READY_FOR_SEGMENT": "待分块", "READY_FOR_SEGMENT": "待分块",
"SPLITTING": "分块中",
"SPLIT_FAILED": "分块失败",
"READY_FOR_INDEX": "待向量化", "READY_FOR_INDEX": "待向量化",
"INDEXING": "向量化中", "INDEXING": "向量化中",
"INDEX_FAILED": "向量化失败", "INDEX_FAILED": "向量化失败",

View File

@@ -14,7 +14,9 @@
"failure": "已失效" "failure": "已失效"
}, },
"permissions": "授权接口", "permissions": "授权接口",
"knowledgeSharePermission": "知识库分享授权", "knowledgeReadPermission": "知识库读取",
"knowledgeImportPermission": "知识导入",
"knowledgeMaintenancePermission": "知识库维护",
"workflowApiPermission": "工作流 API 调用授权", "workflowApiPermission": "工作流 API 调用授权",
"addApiKeyNotice": "该操作会生成一个apiKey,请确认是否生成" "addApiKeyNotice": "该操作会生成一个apiKey,请确认是否生成"
} }

View File

@@ -13,6 +13,7 @@ import { api } from '#/api/request';
import bookIcon from '#/assets/ai/knowledge/book.svg'; import bookIcon from '#/assets/ai/knowledge/book.svg';
import HeaderSearch from '#/components/headerSearch/HeaderSearch.vue'; import HeaderSearch from '#/components/headerSearch/HeaderSearch.vue';
import { createLazyComponentController } from '#/utils/lazy-component'; import { createLazyComponentController } from '#/utils/lazy-component';
import DocumentImportBatchStatus from '#/views/ai/documentCollection/DocumentImportBatchStatus.vue';
const ChunkDocumentTable = defineAsyncComponent( const ChunkDocumentTable = defineAsyncComponent(
() => import('#/views/ai/documentCollection/ChunkDocumentTable.vue'), () => import('#/views/ai/documentCollection/ChunkDocumentTable.vue'),
@@ -188,6 +189,7 @@ const headerButtons = [
]; ];
const panelMode = ref<'chunk' | 'list' | 'process'>('list'); const panelMode = ref<'chunk' | 'list' | 'process'>('list');
const documentTableRef = ref(); const documentTableRef = ref();
const batchStatusRefreshKey = ref(0);
const documentTitle = ref(''); const documentTitle = ref('');
const handleSearch = (searchParams: string) => { const handleSearch = (searchParams: string) => {
documentTableRef.value?.search?.(searchParams); documentTableRef.value?.search?.(searchParams);
@@ -227,6 +229,7 @@ const backDoc = async () => {
documentTitle.value = ''; documentTitle.value = '';
await nextTick(); await nextTick();
documentTableRef.value?.reload?.(); documentTableRef.value?.reload?.();
batchStatusRefreshKey.value += 1;
}; };
</script> </script>
@@ -273,7 +276,16 @@ const backDoc = async () => {
:buttons="canManageCurrentKnowledge ? headerButtons : []" :buttons="canManageCurrentKnowledge ? headerButtons : []"
@search="handleSearch" @search="handleSearch"
@button-click="handleButtonClick" @button-click="handleButtonClick"
/> >
<template #middle>
<DocumentImportBatchStatus
:knowledge-id="knowledgeId"
:manageable="canManageCurrentKnowledge"
:refresh-key="batchStatusRefreshKey"
@continued="backDoc"
/>
</template>
</HeaderSearch>
</div> </div>
<div v-if="panelMode === 'chunk'" class="doc-sub-back"> <div v-if="panelMode === 'chunk'" class="doc-sub-back">
<ElButton @click="backDoc"> <ElButton @click="backDoc">
@@ -343,6 +355,7 @@ const backDoc = async () => {
:is="ImportKnowledgeDocFileComponent" :is="ImportKnowledgeDocFileComponent"
v-if="ImportKnowledgeDocFileComponent" v-if="ImportKnowledgeDocFileComponent"
ref="importDocModalRef" ref="importDocModalRef"
enable-bulk-auto
:knowledge-id-prop="String(knowledgeId)" :knowledge-id-prop="String(knowledgeId)"
@imported="backDoc" @imported="backDoc"
/> />
@@ -386,6 +399,10 @@ const backDoc = async () => {
margin: 0 auto; margin: 0 auto;
} }
.doc-header :deep(.search-middle) {
flex: 1 1 360px;
}
.doc-content { .doc-content {
display: flex; display: flex;
flex-direction: column; flex-direction: column;

View File

@@ -284,7 +284,8 @@ const actions: ActionButton[] = [
text: $t('button.offline'), text: $t('button.offline'),
permission: '/api/v1/documentCollection/save', permission: '/api/v1/documentCollection/save',
placement: 'menu', placement: 'menu',
visible: (row) => canAiResourceOffline(row.displayPublishStatus, row.publishStatus), visible: (row) =>
canAiResourceOffline(row.displayPublishStatus, row.publishStatus),
onClick(row) { onClick(row) {
if (!ensureManageKnowledgeItem(row)) { if (!ensureManageKnowledgeItem(row)) {
return; return;
@@ -298,7 +299,8 @@ const actions: ActionButton[] = [
tone: 'danger', tone: 'danger',
permission: '/api/v1/documentCollection/remove', permission: '/api/v1/documentCollection/remove',
placement: 'menu', placement: 'menu',
visible: (row) => canAiResourceDelete(row.displayPublishStatus, row.publishStatus), visible: (row) =>
canAiResourceDelete(row.displayPublishStatus, row.publishStatus),
onClick(row) { onClick(row) {
if (!ensureManageKnowledgeItem(row)) { if (!ensureManageKnowledgeItem(row)) {
return; return;
@@ -325,8 +327,8 @@ const submitPublishAction = async (item: any) => {
const confirmation = await confirmPublishSubmission({ const confirmation = await confirmPublishSubmission({
api, api,
confirmMessage: isRepublishAction(item) confirmMessage: isRepublishAction(item)
? $t('documentCollection.submitRepublishApprovalConfirm') ? $t('documentCollection.submitRepublishApprovalConfirm')
: $t('documentCollection.submitPublishApprovalConfirm'), : $t('documentCollection.submitPublishApprovalConfirm'),
id: item.id, id: item.id,
resourcePath: '/api/v1/documentCollection', resourcePath: '/api/v1/documentCollection',
title: $t('message.noticeTitle'), title: $t('message.noticeTitle'),
@@ -356,12 +358,9 @@ const submitOfflineAction = async (item: any) => {
const impactRes = await api.get<{ const impactRes = await api.get<{
data: OfflineImpactCheck; data: OfflineImpactCheck;
errorCode: number; errorCode: number;
}>( }>('/api/v1/documentCollection/offlineImpactCheck', {
'/api/v1/documentCollection/offlineImpactCheck', params: { id: item.id },
{ });
params: { id: item.id },
},
);
if (impactRes.errorCode !== 0) { if (impactRes.errorCode !== 0) {
return; return;
} }
@@ -399,9 +398,12 @@ const submitOfflineAction = async (item: any) => {
} catch { } catch {
return; return;
} }
const res = await api.post('/api/v1/documentCollection/submitOfflineApproval', { const res = await api.post(
id: item.id, '/api/v1/documentCollection/submitOfflineApproval',
}); {
id: item.id,
},
);
if (res.errorCode === 0) { if (res.errorCode === 0) {
ElMessage.success(res.message || $t('message.saveOkMessage')); ElMessage.success(res.message || $t('message.saveOkMessage'));
reloadKnowledgeList(); reloadKnowledgeList();

View File

@@ -0,0 +1,82 @@
import { flushPromises, mount } from '@vue/test-utils';
import { afterEach, describe, expect, it, vi } from 'vitest';
import DocumentImportBatchStatus from './DocumentImportBatchStatus.vue';
const apiMocks = vi.hoisted(() => ({
get: vi.fn(),
post: vi.fn(),
}));
vi.mock('#/api/request', () => ({ api: apiMocks }));
vi.mock('@easyflow/locales', () => ({
$t: (key: string) => key,
}));
function createBatch(status: 'COMPLETED' | 'RUNNING', completedCount: number) {
return {
batchId: 'batch-1',
completedCount,
failedCount: 0,
importMode: 'AUTO',
pendingCount: status === 'RUNNING' ? 1 : 0,
processingCount: 0,
progressPercent: status === 'COMPLETED' ? 100 : 50,
skippedCount: 0,
status,
totalCount: 2,
};
}
describe('documentImportBatchStatus', () => {
afterEach(() => {
vi.clearAllMocks();
vi.useRealTimers();
});
it('页面首次进入时不恢复已完成批次', async () => {
apiMocks.get.mockResolvedValue({
data: createBatch('COMPLETED', 2),
errorCode: 0,
});
const wrapper = mount(DocumentImportBatchStatus, {
props: { knowledgeId: 'knowledge-1' },
});
await flushPromises();
expect(wrapper.find('.batch-status').exists()).toBe(false);
wrapper.unmount();
});
it('当前页面跟踪的批次完成后继续展示结果', async () => {
vi.useFakeTimers();
apiMocks.get
.mockResolvedValueOnce({
data: createBatch('RUNNING', 1),
errorCode: 0,
})
.mockResolvedValueOnce({
data: createBatch('COMPLETED', 2),
errorCode: 0,
});
const wrapper = mount(DocumentImportBatchStatus, {
props: { knowledgeId: 'knowledge-1' },
});
await flushPromises();
expect(wrapper.find('.batch-status').exists()).toBe(true);
await vi.advanceTimersByTimeAsync(3000);
await flushPromises();
expect(wrapper.find('.batch-status').exists()).toBe(true);
expect(wrapper.text()).toContain(
'documentCollection.importDoc.batchCompleted',
);
wrapper.unmount();
});
});

View File

@@ -0,0 +1,312 @@
<script setup lang="ts">
import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue';
import { $t } from '@easyflow/locales';
import { ElButton, ElProgress } from 'element-plus';
import { api } from '#/api/request';
interface BatchStatus {
batchId: string;
completedCount: number;
failedCount: number;
importMode: 'AUTO' | 'MANUAL';
pendingCount: number;
processingCount: number;
progressPercent: number;
retryableFailedCount?: number;
skippedCount: number;
status:
| 'CANCELLED'
| 'COMPLETED'
| 'INTERRUPTED'
| 'PARTIAL_SUCCEEDED'
| 'RUNNING';
totalCount: number;
}
const props = defineProps({
knowledgeId: {
type: String,
required: true,
},
refreshKey: {
type: Number,
default: 0,
},
manageable: {
type: Boolean,
default: false,
},
});
const emit = defineEmits(['continued']);
const batch = ref<BatchStatus>();
const continuing = ref(false);
let pollTimer: null | ReturnType<typeof setTimeout> = null;
let disposed = false;
let refreshGeneration = 0;
const canContinue = computed(
() =>
props.manageable &&
(batch.value?.status === 'INTERRUPTED' ||
batch.value?.status === 'PARTIAL_SUCCEEDED') &&
(Number(batch.value?.retryableFailedCount || 0) > 0 ||
Number(batch.value?.failedCount || 0) > 0),
);
const processedCount = computed(
() =>
Number(batch.value?.completedCount || 0) +
Number(batch.value?.processingCount || 0),
);
const allFailed = computed(
() =>
Number(batch.value?.totalCount || 0) > 0 &&
Number(batch.value?.failedCount || 0) ===
Number(batch.value?.totalCount || 0),
);
const statusLabel = computed(() => {
const status = batch.value?.status;
if (status === 'COMPLETED') {
return $t('documentCollection.importDoc.batchCompleted');
}
if (status === 'INTERRUPTED') {
return $t('documentCollection.importDoc.batchInterrupted');
}
if (status === 'PARTIAL_SUCCEEDED') {
return $t(
allFailed.value
? 'documentCollection.importDoc.batchFailed'
: 'documentCollection.importDoc.batchPartial',
);
}
return $t('documentCollection.importDoc.batchRunning');
});
async function refresh(hideCompletedOnRestore = false) {
if (!props.knowledgeId) return;
const currentGeneration = ++refreshGeneration;
try {
const response = await api.get('/api/v1/document/import/batch/current', {
params: { knowledgeId: props.knowledgeId },
});
if (disposed || currentGeneration !== refreshGeneration) {
return;
}
const restoredBatch =
response.errorCode === 0 ? response.data || undefined : undefined;
batch.value =
hideCompletedOnRestore && restoredBatch?.status === 'COMPLETED'
? undefined
: restoredBatch;
} finally {
if (!disposed && currentGeneration === refreshGeneration) {
schedulePoll();
}
}
}
function schedulePoll() {
if (pollTimer) clearTimeout(pollTimer);
pollTimer = null;
if (
disposed ||
!batch.value ||
batch.value.status === 'COMPLETED' ||
batch.value.status === 'PARTIAL_SUCCEEDED' ||
batch.value.status === 'INTERRUPTED' ||
batch.value.status === 'CANCELLED'
) {
return;
}
pollTimer = setTimeout(refresh, 3000);
}
async function continueBatch() {
if (!batch.value || continuing.value) return;
continuing.value = true;
try {
const response = await api.post('/api/v1/document/import/batch/continue', {
batchId: batch.value.batchId,
knowledgeId: props.knowledgeId,
});
if (response.errorCode === 0) {
batch.value = response.data;
emit('continued');
schedulePoll();
}
} finally {
continuing.value = false;
}
}
onMounted(() => {
disposed = false;
refresh(true);
});
onBeforeUnmount(() => {
disposed = true;
refreshGeneration += 1;
if (pollTimer) clearTimeout(pollTimer);
});
watch(
() => [props.knowledgeId, props.refreshKey] as const,
([knowledgeId], [previousKnowledgeId]) => {
const knowledgeChanged = knowledgeId !== previousKnowledgeId;
if (knowledgeChanged) {
batch.value = undefined;
}
refresh(knowledgeChanged);
},
);
</script>
<template>
<section
v-if="batch"
class="batch-status"
:aria-label="$t('documentCollection.importDoc.batchStatus')"
>
<div class="batch-status__summary">
<div class="batch-status__headline">
<span class="batch-status__title">
{{ $t('documentCollection.importDoc.autoImport') }}
</span>
<span class="batch-status__count">
{{ processedCount }} / {{ batch.totalCount }}
</span>
<span class="batch-status__state">{{ statusLabel }}</span>
</div>
<ElProgress
class="batch-status__progress"
:percentage="Number(batch.progressPercent || 0)"
:show-text="false"
:stroke-width="7"
:status="
batch.status === 'COMPLETED'
? 'success'
: allFailed
? 'exception'
: undefined
"
/>
<div class="batch-status__metrics">
<span>
{{ $t('documentCollection.importDoc.completedCount') }}
{{ batch.completedCount }}
</span>
<span>
{{ $t('documentCollection.importDoc.processingCount') }}
{{ batch.processingCount }}
</span>
<span
:class="{ 'batch-status__metric--danger': batch.failedCount > 0 }"
>
{{ $t('documentCollection.importDoc.failedCount') }}
{{ batch.failedCount }}
</span>
<span>
{{ $t('documentCollection.importDoc.pendingCount') }}
{{ batch.pendingCount }}
</span>
<span v-if="batch.skippedCount > 0">
{{ $t('documentCollection.importDoc.skippedCount') }}
{{ batch.skippedCount }}
</span>
</div>
</div>
<ElButton
v-if="canContinue"
class="batch-status__continue"
type="primary"
link
:loading="continuing"
@click="continueBatch"
>
{{ $t('documentCollection.importDoc.continueBatch') }}
</ElButton>
</section>
</template>
<style scoped>
.batch-status {
display: flex;
flex: 1 1 360px;
gap: 12px;
align-items: center;
min-width: min(360px, 100%);
max-width: 620px;
padding: 8px 12px;
background: hsl(var(--surface-contrast-soft) / 74%);
border: 1px solid var(--el-border-color-lighter);
border-radius: 12px;
}
.batch-status__summary {
flex: 1;
min-width: 0;
}
.batch-status__headline,
.batch-status__metrics {
display: flex;
flex-wrap: wrap;
gap: 4px 10px;
align-items: center;
}
.batch-status__headline {
margin-bottom: 5px;
font-size: 12px;
}
.batch-status__title {
font-weight: 600;
color: var(--el-text-color-primary);
}
.batch-status__count {
font-variant-numeric: tabular-nums;
color: var(--el-color-primary);
}
.batch-status__state {
color: var(--el-text-color-secondary);
}
.batch-status__progress {
width: 100%;
}
.batch-status__metrics {
margin-top: 4px;
overflow: hidden;
font-size: 11px;
line-height: 16px;
color: var(--el-text-color-secondary);
white-space: nowrap;
}
.batch-status__metric--danger {
color: var(--el-color-danger);
}
.batch-status__continue {
flex-shrink: 0;
min-width: 48px;
}
@media (max-width: 900px) {
.batch-status {
flex-basis: 100%;
max-width: none;
}
}
</style>

View File

@@ -32,6 +32,7 @@ import { buildKnowledgeShareUrl } from '#/api/knowledge-share';
import { api, SseClient } from '#/api/request'; import { api, SseClient } from '#/api/request';
import documentIcon from '#/assets/ai/knowledge/document.svg'; import documentIcon from '#/assets/ai/knowledge/document.svg';
import PageData from '#/components/page/PageData.vue'; import PageData from '#/components/page/PageData.vue';
import { resolveDocumentTaskErrorText } from '#/views/ai/documentCollection/document-import-error';
import { buildKnowledgePath } from '#/views/ai/documentCollection/share-path'; import { buildKnowledgePath } from '#/views/ai/documentCollection/share-path';
interface DocumentStatusPayload { interface DocumentStatusPayload {
@@ -40,6 +41,7 @@ interface DocumentStatusPayload {
failedChunks?: number; failedChunks?: number;
knowledgeId?: number | string; knowledgeId?: number | string;
lastTaskError?: string; lastTaskError?: string;
lastTaskErrorCode?: string;
parseCurrentStage?: string; parseCurrentStage?: string;
parseStatusMessage?: string; parseStatusMessage?: string;
processStatus?: string; processStatus?: string;
@@ -86,6 +88,7 @@ const STREAM_RECONNECT_DELAY = 1500;
const STREAM_RELOAD_DELAY = 250; const STREAM_RELOAD_DELAY = 250;
const pageDataRef = ref(); const pageDataRef = ref();
const retryingDocumentIds = ref<Set<string>>(new Set());
const taskStatusStreamClient = new SseClient(); const taskStatusStreamClient = new SseClient();
let reconnectTimer: null | ReturnType<typeof setTimeout> = null; let reconnectTimer: null | ReturnType<typeof setTimeout> = null;
let reloadTimer: null | ReturnType<typeof setTimeout> = null; let reloadTimer: null | ReturnType<typeof setTimeout> = null;
@@ -102,7 +105,7 @@ defineExpose({
}, },
}); });
const processingStatuses = new Set(['INDEXING', 'PARSING']); const processingStatuses = new Set(['INDEXING', 'PARSING', 'SPLITTING']);
const isProcessingStatus = (status?: string) => const isProcessingStatus = (status?: string) =>
processingStatuses.has(status || ''); processingStatuses.has(status || '');
@@ -113,7 +116,8 @@ const resolvedPermissions = computed(() => ({
canDownloadContent: props.permissions?.canDownloadContent ?? true, canDownloadContent: props.permissions?.canDownloadContent ?? true,
})); }));
const hasPermission = (key: PermissionKey) => Boolean(resolvedPermissions.value[key]); const hasPermission = (key: PermissionKey) =>
Boolean(resolvedPermissions.value[key]);
const statusMetaMap: Record< const statusMetaMap: Record<
string, string,
@@ -142,6 +146,14 @@ const statusMetaMap: Record<
icon: Loading, icon: Loading,
toneClass: 'status-pill--warning', toneClass: 'status-pill--warning',
}, },
SPLIT_FAILED: {
icon: CloseBold,
toneClass: 'status-pill--danger',
},
SPLITTING: {
icon: Loading,
toneClass: 'status-pill--warning',
},
READY_FOR_INDEX: { READY_FOR_INDEX: {
icon: Opportunity, icon: Opportunity,
toneClass: 'status-pill--primary', toneClass: 'status-pill--primary',
@@ -204,9 +216,21 @@ const parseStageLabels: Record<string, string> = {
}; };
const getProcessingHint = (row: any) => const getProcessingHint = (row: any) =>
row.parseStatusMessage || row.parseStatusMessage || parseStageLabels[row.parseCurrentStage || ''] || '';
parseStageLabels[row.parseCurrentStage || ''] ||
''; const getErrorText = (row: any) => {
const taskError = resolveDocumentTaskErrorText(row, $t);
if (taskError) {
return taskError;
}
if (
row.processStatus === 'SPLIT_FAILED' ||
row.processStatus === 'INDEX_FAILED'
) {
return $t('documentCollection.importDoc.splitOrIndexFailed');
}
return '';
};
const clearReconnectTimer = () => { const clearReconnectTimer = () => {
if (!reconnectTimer) { if (!reconnectTimer) {
@@ -242,6 +266,7 @@ const patchDocumentRow = (payload: DocumentStatusPayload) => {
completedChunks: payload.completedChunks, completedChunks: payload.completedChunks,
failedChunks: payload.failedChunks, failedChunks: payload.failedChunks,
lastTaskError: payload.lastTaskError, lastTaskError: payload.lastTaskError,
lastTaskErrorCode: payload.lastTaskErrorCode,
parseCurrentStage: payload.parseCurrentStage, parseCurrentStage: payload.parseCurrentStage,
parseStatusMessage: payload.parseStatusMessage, parseStatusMessage: payload.parseStatusMessage,
processStatus: payload.processStatus, processStatus: payload.processStatus,
@@ -363,17 +388,33 @@ const handleContinue = (row: any) => {
emits('continueProcess', row); emits('continueProcess', row);
}; };
const handleRetryParse = async (row: any) => { const handleRetry = async (row: any) => {
await requestTaskAction( const documentId = String(row.id);
'/api/v1/document/import/task/retryParse', if (retryingDocumentIds.value.has(documentId)) {
{ return;
knowledgeId: props.knowledgeId, }
documentId: row.id, retryingDocumentIds.value = new Set([
}, documentId,
getStatusLabel('PARSING'), ...retryingDocumentIds.value,
); ]);
try {
await requestTaskAction(
'/api/v1/document/import/task/retry',
{
knowledgeId: props.knowledgeId,
documentId: row.id,
},
getStatusLabel('PARSING'),
);
} finally {
const next = new Set(retryingDocumentIds.value);
next.delete(documentId);
retryingDocumentIds.value = next;
}
}; };
const isRetrying = (row: any) => retryingDocumentIds.value.has(String(row.id));
const handleView = (row: any) => { const handleView = (row: any) => {
emits('viewDoc', row.id); emits('viewDoc', row.id);
}; };
@@ -428,13 +469,18 @@ const primaryActionConfigs: Record<
label: () => $t('button.viewSegmentation'), label: () => $t('button.viewSegmentation'),
}, },
INDEX_FAILED: { INDEX_FAILED: {
handler: handleContinue, handler: handleRetry,
label: () => $t('button.continueProcess'), label: () => $t('documentCollection.importDoc.retry'),
permission: 'canCreateContent', permission: 'canCreateContent',
}, },
PARSE_FAILED: { PARSE_FAILED: {
handler: handleRetryParse, handler: handleRetry,
label: () => $t('button.retryParse'), label: () => $t('documentCollection.importDoc.retry'),
permission: 'canCreateContent',
},
SPLIT_FAILED: {
handler: handleRetry,
label: () => $t('documentCollection.importDoc.retry'),
permission: 'canCreateContent', permission: 'canCreateContent',
}, },
READY_FOR_INDEX: { READY_FOR_INDEX: {
@@ -583,7 +629,8 @@ watch(
<div <div
v-if=" v-if="
row.processStatus === 'INDEXING' || row.processStatus === 'INDEXING' ||
row.processStatus === 'PARSING' row.processStatus === 'PARSING' ||
row.processStatus === 'SPLITTING'
" "
class="status-progress" class="status-progress"
> >
@@ -595,19 +642,24 @@ watch(
{{ getProgressText(row) }} {{ getProgressText(row) }}
</span> </span>
<span <span
v-if="row.processStatus === 'PARSING' && getProcessingHint(row)" v-if="
row.processStatus === 'PARSING' && getProcessingHint(row)
"
class="status-progress__hint" class="status-progress__hint"
> >
{{ getProcessingHint(row) }} {{ getProcessingHint(row) }}
</span> </span>
</div> </div>
<div <ElTooltip
v-else-if="row.lastTaskError" v-else-if="getErrorText(row)"
class="status-error" :content="getErrorText(row)"
:title="row.lastTaskError" placement="top"
:show-after="300"
> >
{{ row.lastTaskError }} <div class="status-error">
</div> {{ getErrorText(row) }}
</div>
</ElTooltip>
</div> </div>
</template> </template>
</ElTableColumn> </ElTableColumn>
@@ -624,6 +676,8 @@ watch(
v-if="getPrimaryActionLabel(row)" v-if="getPrimaryActionLabel(row)"
link link
type="primary" type="primary"
:disabled="isRetrying(row)"
:loading="isRetrying(row)"
@click="handlePrimaryAction(row)" @click="handlePrimaryAction(row)"
> >
{{ getPrimaryActionLabel(row) }} {{ getPrimaryActionLabel(row) }}

View File

@@ -5,7 +5,15 @@ import { useRoute } from 'vue-router';
import { EasyFlowFormModal } from '@easyflow/common-ui'; import { EasyFlowFormModal } from '@easyflow/common-ui';
import { $t } from '@easyflow/locales'; import { $t } from '@easyflow/locales';
import { ElMessage } from 'element-plus'; import { InfoFilled } from '@element-plus/icons-vue';
import {
ElButton,
ElIcon,
ElMessage,
ElRadioButton,
ElRadioGroup,
ElTooltip,
} from 'element-plus';
import { api } from '#/api/request'; import { api } from '#/api/request';
import ImportKnowledgeFileContainer from '#/views/ai/documentCollection/ImportKnowledgeFileContainer.vue'; import ImportKnowledgeFileContainer from '#/views/ai/documentCollection/ImportKnowledgeFileContainer.vue';
@@ -24,6 +32,10 @@ const props = defineProps({
type: String, type: String,
default: '', default: '',
}, },
enableBulkAuto: {
type: Boolean,
default: false,
},
}); });
const emits = defineEmits(['imported']); const emits = defineEmits(['imported']);
@@ -32,22 +44,43 @@ const route = useRoute();
const fileUploadRef = ref<InstanceType<typeof ImportKnowledgeFileContainer>>(); const fileUploadRef = ref<InstanceType<typeof ImportKnowledgeFileContainer>>();
const dialogVisible = ref(false); const dialogVisible = ref(false);
const submitting = ref(false); const submitting = ref(false);
const batchReady = ref(false);
const duplicatePolicy = ref<'OVERWRITE' | 'REIMPORT' | 'SKIP'>('SKIP');
const knowledgeId = computed( const knowledgeId = computed(
() => props.knowledgeIdProp || (route.query.id as string) || '', () => props.knowledgeIdProp || (route.query.id as string) || '',
); );
const resetDialogState = () => { const resetDialogState = () => {
batchReady.value = false;
duplicatePolicy.value = 'SKIP';
fileUploadRef.value?.reset?.(); fileUploadRef.value?.reset?.();
}; };
const closeDialog = () => { const handleBatchStateChange = (state: { ready: boolean }) => {
batchReady.value = state.ready;
};
const closeDialog = async () => {
if (submitting.value) { if (submitting.value) {
return false; return false;
} }
dialogVisible.value = false; try {
resetDialogState(); if (props.enableBulkAuto) {
return true; await fileUploadRef.value?.cancelCurrentBatch?.();
} else {
resetDialogState();
}
dialogVisible.value = false;
batchReady.value = false;
duplicatePolicy.value = 'SKIP';
return true;
} catch (error: any) {
ElMessage.error(
error?.message || $t('documentCollection.importDoc.cancelBatchFailed'),
);
return false;
}
}; };
const openDialog = () => { const openDialog = () => {
@@ -108,6 +141,39 @@ const createTasks = async () => {
} }
}; };
const startBatchImport = async (importMode: 'AUTO' | 'MANUAL') => {
const selectedBatchId = fileUploadRef.value?.getBatchId?.();
if (!selectedBatchId || !fileUploadRef.value?.isBatchReady?.()) {
ElMessage.error($t('documentCollection.importDoc.uploadCompleteFirst'));
return;
}
submitting.value = true;
try {
const res = await props.requestClient.post(
'/api/v1/document/import/batch/start',
{
batchId: selectedBatchId,
duplicatePolicy: duplicatePolicy.value,
importMode,
knowledgeId: knowledgeId.value,
},
);
if (res.errorCode !== 0) {
return;
}
ElMessage.success(
importMode === 'AUTO'
? $t('documentCollection.importDoc.autoImportStarted')
: $t('documentCollection.importDoc.manualImportStarted'),
);
dialogVisible.value = false;
resetDialogState();
emits('imported');
} finally {
submitting.value = false;
}
};
defineExpose({ defineExpose({
closeDialog, closeDialog,
openDialog, openDialog,
@@ -124,15 +190,69 @@ defineExpose({
:confirm-loading="submitting" :confirm-loading="submitting"
:confirm-text="$t('button.importFile')" :confirm-text="$t('button.importFile')"
:submitting="submitting" :submitting="submitting"
:show-footer="!enableBulkAuto"
width="xl" width="xl"
@confirm="createTasks" @confirm="createTasks"
> >
<div class="import-dialog"> <div class="import-dialog">
<p class="import-dialog__tip"> <p class="import-dialog__tip">
{{ $t('documentCollection.importDoc.uploadCreateTip') }} {{
$t(
enableBulkAuto
? 'documentCollection.importDoc.batchUploadTip'
: 'documentCollection.importDoc.uploadCreateTip',
)
}}
</p> </p>
<ImportKnowledgeFileContainer ref="fileUploadRef" /> <ImportKnowledgeFileContainer
ref="fileUploadRef"
:batch-mode="enableBulkAuto"
:knowledge-id="knowledgeId"
@batch-state-change="handleBatchStateChange"
/>
<div v-if="enableBulkAuto && batchReady" class="duplicate-policy">
<span class="duplicate-policy__label">
{{ $t('documentCollection.importDoc.duplicatePolicy') }}
</span>
<ElRadioGroup v-model="duplicatePolicy" size="small">
<ElRadioButton value="SKIP">
{{ $t('documentCollection.importDoc.skipDuplicates') }}
</ElRadioButton>
<ElRadioButton value="OVERWRITE">
{{ $t('documentCollection.importDoc.overwriteDuplicates') }}
</ElRadioButton>
<ElRadioButton value="REIMPORT">
{{ $t('documentCollection.importDoc.reimportDuplicates') }}
</ElRadioButton>
</ElRadioGroup>
</div>
<div v-if="enableBulkAuto" class="import-dialog__footer">
<ElButton :disabled="submitting" @click="closeDialog">
{{ $t('button.cancel') }}
</ElButton>
<ElButton
:disabled="submitting || !batchReady"
:loading="submitting"
@click="startBatchImport('MANUAL')"
>
{{ $t('documentCollection.importDoc.manualImport') }}
</ElButton>
<ElTooltip
:content="$t('documentCollection.importDoc.autoImportTip')"
placement="top"
>
<ElButton
type="primary"
:disabled="submitting || !batchReady"
:loading="submitting"
@click="startBatchImport('AUTO')"
>
{{ $t('documentCollection.importDoc.autoImport') }}
<ElIcon class="import-dialog__info"><InfoFilled /></ElIcon>
</ElButton>
</ElTooltip>
</div>
</div> </div>
</EasyFlowFormModal> </EasyFlowFormModal>
</template> </template>
@@ -151,6 +271,36 @@ defineExpose({
color: var(--el-text-color-secondary); color: var(--el-text-color-secondary);
} }
.import-dialog__footer {
display: flex;
flex-wrap: wrap;
gap: 8px;
justify-content: flex-end;
padding-top: 16px;
border-top: 1px solid var(--el-border-color-lighter);
}
.duplicate-policy {
display: flex;
flex-wrap: wrap;
gap: 8px 12px;
align-items: center;
min-height: 32px;
}
.duplicate-policy__label {
font-size: 13px;
color: var(--el-text-color-secondary);
}
.import-dialog__footer :deep(.el-button + .el-button) {
margin-left: 0;
}
.import-dialog__info {
margin-left: 6px;
}
:deep(.upload-demo) { :deep(.upload-demo) {
width: 100%; width: 100%;
} }

View File

@@ -1,14 +1,33 @@
<script setup lang="ts"> <script setup lang="ts">
import { ref } from 'vue'; import { computed, h, ref, watch } from 'vue';
import { useAppConfig } from '@easyflow/hooks';
import { $t } from '@easyflow/locales'; import { $t } from '@easyflow/locales';
import { useAccessStore } from '@easyflow/stores';
import { ElButton, ElProgress, ElTable, ElTableColumn } from 'element-plus'; import { FolderOpened, UploadFilled } from '@element-plus/icons-vue';
import {
ElButton,
ElIcon,
ElMessage,
ElProgress,
ElTable,
ElTableColumn,
} from 'element-plus';
import {
ElTableV2,
ElAutoResizer as TableV2AutoResizer,
} from 'element-plus/es/components/table-v2/index.mjs';
import { formatFileSize } from '#/api/common/file'; import { formatFileSize } from '#/api/common/file';
import { api } from '#/api/request';
import DragFileUpload from '#/components/upload/DragFileUpload.vue'; import DragFileUpload from '#/components/upload/DragFileUpload.vue';
interface FileInfo { import { resolveDocumentUploadResponse } from './document-import-upload-response';
import 'element-plus/es/components/table-v2/style/css.mjs';
interface LegacyFileInfo {
uid: string; uid: string;
fileName: string; fileName: string;
progressUpload: number; progressUpload: number;
@@ -16,99 +35,607 @@ interface FileInfo {
status: string; status: string;
filePath: string; filePath: string;
} }
const fileData = ref<FileInfo[]>([]);
const filesPath = ref([]); interface BatchFileInfo {
clientFileKey: string;
error?: string;
file: File;
fileName: string;
fileSize: number;
itemId?: string;
progressUpload: number;
relativePath: string;
status: 'error' | 'queued' | 'success' | 'uploading';
uid: string;
}
interface BatchCreateResponse {
batchId: string;
items: Array<{
clientFileKey: string;
itemId: string;
}>;
uploadConcurrency: number;
}
const props = defineProps({
batchMode: {
type: Boolean,
default: false,
},
knowledgeId: {
type: String,
default: '',
},
});
const emit = defineEmits<{
batchStateChange: [
state: {
ready: boolean;
uploading: boolean;
},
];
}>();
const MAX_FILE_COUNT = 2000;
const MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024;
const MAX_TOTAL_SIZE_BYTES = 1024 * 1024 * 1024;
const SUPPORTED_EXTENSIONS = new Set([
'docx',
'md',
'pdf',
'pptx',
'txt',
'xlsx',
]);
const fileData = ref<LegacyFileInfo[]>([]);
const filesPath = ref<any[]>([]);
const dragUploadRef = ref<InstanceType<typeof DragFileUpload>>(); const dragUploadRef = ref<InstanceType<typeof DragFileUpload>>();
const batchFiles = ref<BatchFileInfo[]>([]);
const batchId = ref('');
const folderInputRef = ref<HTMLInputElement>();
const fileInputRef = ref<HTMLInputElement>();
const activeRequests = new Set<XMLHttpRequest>();
const accessStore = useAccessStore();
const { apiURL } = useAppConfig(import.meta.env, import.meta.env.PROD);
let batchGeneration = 0;
const batchUploading = computed(() =>
batchFiles.value.some(
(item) => item.status === 'queued' || item.status === 'uploading',
),
);
const batchReady = computed(
() =>
Boolean(batchId.value) &&
batchFiles.value.length > 0 &&
batchFiles.value.every((item) => item.status === 'success'),
);
watch(
[batchReady, batchUploading],
([ready, uploading]) => {
emit('batchStateChange', { ready, uploading });
},
{ immediate: true },
);
function resolveUploadProgressStatus(
status: BatchFileInfo['status'],
): 'exception' | 'success' | undefined {
if (status === 'success') return 'success';
if (status === 'error') return 'exception';
return undefined;
}
const batchColumns = computed(() => [
{
cellRenderer: ({ rowData }: any) =>
h(
'span',
{
class: 'batch-file-name',
title: rowData.relativePath,
},
rowData.relativePath,
),
dataKey: 'relativePath',
flexGrow: 1,
key: 'relativePath',
title: $t('documentCollection.importDoc.fileName'),
width: 420,
},
{
cellRenderer: ({ rowData }: any) =>
h(ElProgress, {
percentage: Math.round(rowData.progressUpload || 0),
status: resolveUploadProgressStatus(rowData.status),
strokeWidth: 8,
}),
dataKey: 'progressUpload',
key: 'progressUpload',
title: $t('documentCollection.importDoc.progressUpload'),
width: 220,
},
{
cellRenderer: ({ rowData }: any) =>
h('span', formatFileSize(rowData.fileSize)),
dataKey: 'fileSize',
key: 'fileSize',
title: $t('documentCollection.importDoc.fileSize'),
width: 132,
},
{
cellRenderer: ({ rowData }: any) => {
if (rowData.status === 'error') {
return h(
ElButton,
{
link: true,
title:
rowData.error || $t('documentCollection.importDoc.uploadFailed'),
type: 'primary',
onClick: () => retryBatchUpload(rowData),
},
() => $t('documentCollection.importDoc.retryUpload'),
);
}
return h(
'span',
{
class: rowData.status === 'success' ? 'upload-state--success' : '',
title: rowData.error || '',
},
rowData.status === 'success'
? $t('documentCollection.importDoc.uploaded')
: $t('documentCollection.importDoc.uploading'),
);
},
dataKey: 'status',
key: 'status',
title: $t('common.handle'),
width: 112,
},
]);
function resetState() {
batchGeneration += 1;
for (const request of activeRequests) {
request.abort();
}
activeRequests.clear();
fileData.value = [];
filesPath.value = [];
batchFiles.value = [];
batchId.value = '';
dragUploadRef.value?.clearFiles?.();
if (folderInputRef.value) folderInputRef.value.value = '';
if (fileInputRef.value) fileInputRef.value.value = '';
}
async function cancelCurrentBatch() {
for (const request of activeRequests) {
request.abort();
}
activeRequests.clear();
const currentBatchId = batchId.value;
if (currentBatchId) {
const response = await api.post('/api/v1/document/import/batch/cancel', {
batchId: currentBatchId,
knowledgeId: props.knowledgeId,
});
if (response.errorCode !== 0) {
throw new Error(
response.message ||
$t('documentCollection.importDoc.cancelBatchFailed'),
);
}
}
resetState();
}
defineExpose({ defineExpose({
cancelCurrentBatch,
getBatchId() {
return batchId.value;
},
getFilesData() { getFilesData() {
return fileData.value.filter((item) => item.filePath); return fileData.value.filter((item) => item.filePath);
}, },
reset() { isBatchReady() {
fileData.value = []; return batchReady.value;
filesPath.value = [];
dragUploadRef.value?.clearFiles?.();
}, },
isUploading() {
return batchUploading.value;
},
reset: resetState,
}); });
function handleSuccess(response: any) { function handleSuccess(response: any) {
filesPath.value = response.data; filesPath.value = response.data;
} }
function handleChange(file: any) { function handleChange(file: any) {
const existingFile = fileData.value.find((item) => item.uid === file.uid); const existingFile = fileData.value.find((item) => item.uid === file.uid);
if (existingFile) { if (existingFile) {
fileData.value = fileData.value.map((item) => { fileData.value = fileData.value.map((item) =>
if (item.uid === file.uid) { item.uid === file.uid
return { ? {
...item, ...item,
fileSize: file.size, filePath: file?.response?.data?.path,
progressUpload: file.percentage, fileSize: file.size,
status: file.status, progressUpload: file.percentage,
filePath: file?.response?.data?.path, status: file.status,
}; }
} : item,
return item; );
}); return;
} else { }
fileData.value.push({ fileData.value.push({
uid: file.uid, fileName: file.name,
fileName: file.name, filePath: file?.response?.data?.path,
progressUpload: file.percentage, fileSize: file.size,
fileSize: file.size, progressUpload: file.percentage,
status: file.status, status: file.status,
filePath: file?.response?.data?.path, uid: file.uid,
}); });
}
function handleRemove(row: LegacyFileInfo) {
fileData.value = fileData.value.filter((item) => item.uid !== row.uid);
}
function triggerFileSelect() {
if (!batchUploading.value) {
fileInputRef.value?.click();
} }
} }
function handleRemove(row: any) { function triggerFolderSelect(event: Event) {
fileData.value = fileData.value.filter((item) => item.uid !== row.uid); event.stopPropagation();
if (!batchUploading.value) {
folderInputRef.value?.click();
}
}
async function handleNativeSelection(event: Event) {
const input = event.target as HTMLInputElement;
const files = [...(input.files || [])];
input.value = '';
if (files.length > 0) {
await prepareBatch(files);
}
}
async function handleDrop(event: DragEvent) {
if (batchUploading.value) return;
const files = [...(event.dataTransfer?.files || [])];
if (files.length > 0) {
await prepareBatch(files);
}
}
async function prepareBatch(files: File[]) {
const generation = ++batchGeneration;
const accepted: File[] = [];
let totalBytes = 0;
let ignoredCount = 0;
for (const file of files) {
const relativePath = getRelativePath(file);
if (
relativePath.includes('/__MACOSX/') ||
relativePath.split('/').at(-1) === '.DS_Store'
) {
ignoredCount++;
continue;
}
const extension = file.name.split('.').pop()?.toLowerCase() || '';
if (!SUPPORTED_EXTENSIONS.has(extension)) {
ignoredCount++;
continue;
}
if (file.size > MAX_FILE_SIZE_BYTES) {
ElMessage.warning($t('documentCollection.importDoc.singleFileLimit'));
return;
}
totalBytes += file.size;
accepted.push(file);
}
if (accepted.length === 0) {
ElMessage.warning($t('documentCollection.importDoc.noSupportedFiles'));
return;
}
if (accepted.length > MAX_FILE_COUNT) {
ElMessage.warning($t('documentCollection.importDoc.fileCountLimit'));
return;
}
if (totalBytes > MAX_TOTAL_SIZE_BYTES) {
ElMessage.warning($t('documentCollection.importDoc.folderSizeLimit'));
return;
}
if (ignoredCount > 0) {
ElMessage.info($t('documentCollection.importDoc.unsupportedSkipped'));
}
batchFiles.value = await Promise.all(
accepted.map(async (file, index) => {
const relativePath = getRelativePath(file);
return {
clientFileKey: await createClientFileKey(relativePath),
file,
fileName: file.name,
fileSize: file.size,
progressUpload: 0,
relativePath,
status: 'queued',
uid: `${Date.now()}-${index}`,
} satisfies BatchFileInfo;
}),
);
batchId.value = '';
try {
const response = await api.post('/api/v1/document/import/batch/create', {
files: batchFiles.value.map((item) => ({
clientFileKey: item.clientFileKey,
fileName: item.fileName,
fileSize: item.fileSize,
relativePath: item.relativePath,
})),
knowledgeId: props.knowledgeId,
});
if (response.errorCode !== 0 || !response.data) {
throw new Error(
response.message ||
$t('documentCollection.importDoc.createBatchFailed'),
);
}
const data = response.data as BatchCreateResponse;
if (generation !== batchGeneration) {
await api.post('/api/v1/document/import/batch/cancel', {
batchId: data.batchId,
knowledgeId: props.knowledgeId,
});
return;
}
batchId.value = String(data.batchId);
const itemMap = new Map(
data.items.map((item) => [item.clientFileKey, String(item.itemId)]),
);
for (const item of batchFiles.value) {
item.itemId = itemMap.get(item.clientFileKey);
}
await uploadWithWorkers(Math.max(1, Number(data.uploadConcurrency || 3)));
} catch (error: any) {
for (const item of batchFiles.value) {
item.status = 'error';
item.error =
error?.message || $t('documentCollection.importDoc.createBatchFailed');
}
}
}
async function uploadWithWorkers(concurrency: number) {
let nextIndex = 0;
const worker = async () => {
while (nextIndex < batchFiles.value.length) {
const index = nextIndex++;
const item = batchFiles.value[index];
if (item) await uploadBatchItem(item);
}
};
await Promise.all(
Array.from({ length: Math.min(concurrency, batchFiles.value.length) }, () =>
worker(),
),
);
}
async function retryBatchUpload(item: BatchFileInfo) {
if (!item.itemId || !batchId.value) return;
await uploadBatchItem(item);
}
function uploadBatchItem(item: BatchFileInfo) {
return new Promise<void>((resolve) => {
if (!item.itemId || !batchId.value) {
item.status = 'error';
item.error = $t('documentCollection.importDoc.createBatchFailed');
resolve();
return;
}
const request = new XMLHttpRequest();
activeRequests.add(request);
item.status = 'uploading';
item.error = undefined;
request.open(
'POST',
`${apiURL}/api/v1/document/import/batch/${batchId.value}/item/${item.itemId}/upload?knowledgeId=${encodeURIComponent(props.knowledgeId)}`,
);
if (accessStore.accessToken) {
request.setRequestHeader('easyflow-token', accessStore.accessToken);
}
request.setRequestHeader('Accept', 'application/json');
request.upload.addEventListener('progress', (progressEvent) => {
if (progressEvent.lengthComputable) {
item.progressUpload = Math.round(
(progressEvent.loaded * 100) / progressEvent.total,
);
}
});
request.addEventListener('load', () => {
activeRequests.delete(request);
try {
const response = resolveDocumentUploadResponse({
fallbackMessage: $t('documentCollection.importDoc.uploadFailed'),
responseText: request.responseText || '',
status: request.status,
statusText: request.statusText,
});
if (!response.success) {
throw new Error(response.message);
}
item.progressUpload = 100;
item.status = 'success';
} catch (error: any) {
item.status = 'error';
item.error =
error?.message || $t('documentCollection.importDoc.uploadFailed');
}
resolve();
});
request.addEventListener('error', () => {
activeRequests.delete(request);
item.status = 'error';
item.error = $t('documentCollection.importDoc.uploadFailed');
resolve();
});
request.addEventListener('abort', () => {
activeRequests.delete(request);
resolve();
});
const formData = new FormData();
formData.append('file', item.file, item.fileName);
request.send(formData);
});
}
function getRelativePath(file: File) {
return (
(file as File & { webkitRelativePath?: string }).webkitRelativePath ||
file.name
).replaceAll('\\', '/');
}
async function createClientFileKey(relativePath: string) {
const source = relativePath;
if (globalThis.crypto?.subtle) {
const digest = await globalThis.crypto.subtle.digest(
'SHA-256',
new TextEncoder().encode(source),
);
return [...new Uint8Array(digest)]
.map((value) => value.toString(16).padStart(2, '0'))
.join('');
}
return [...source]
.reduce(
(hash, char) => (hash * 31 + (char.codePointAt(0) || 0)) >>> 0,
2_166_136_261,
)
.toString(16)
.padStart(64, '0');
} }
</script> </script>
<template> <template>
<div class="import-file-container"> <div class="import-file-container">
<DragFileUpload <template v-if="batchMode">
ref="dragUploadRef" <input
@success="handleSuccess" ref="fileInputRef"
@on-change="handleChange" class="native-file-input"
/> type="file"
<div class="import-file-container__table"> multiple
<ElTable :data="fileData" style="width: 100%" size="large"> accept=".txt,.pdf,.docx,.md,.pptx,.xlsx"
<ElTableColumn @change="handleNativeSelection"
prop="fileName" />
:label="$t('documentCollection.importDoc.fileName')" <input
width="250" ref="folderInputRef"
/> class="native-file-input"
<ElTableColumn type="file"
prop="progressUpload" multiple
:label="$t('documentCollection.importDoc.progressUpload')" webkitdirectory
width="180" @change="handleNativeSelection"
/>
<div
class="batch-drop-zone"
:class="{ 'batch-drop-zone--disabled': batchUploading }"
role="button"
tabindex="0"
@click="triggerFileSelect"
@keydown.enter="triggerFileSelect"
@dragover.prevent
@drop.prevent="handleDrop"
>
<ElIcon class="batch-drop-zone__icon"><UploadFilled /></ElIcon>
<div class="batch-drop-zone__title">
{{ $t('documentCollection.importDoc.batchUploadTitle') }}
</div>
<div class="batch-drop-zone__description">
{{ $t('documentCollection.importDoc.batchUploadDescription') }}
</div>
<ElButton
:icon="FolderOpened"
:disabled="batchUploading"
@click="triggerFolderSelect"
> >
<template #default="{ row }"> {{ $t('documentCollection.importDoc.selectFolder') }}
<ElProgress </ElButton>
:percentage="row.progressUpload" </div>
v-if="row.status === 'success'"
status="success" <div class="batch-table">
<TableV2AutoResizer>
<template #default="{ height, width }">
<ElTableV2
:columns="batchColumns"
:data="batchFiles"
:height="height"
:width="width"
:row-height="52"
fixed
row-key="uid"
/> />
<ElProgress v-else :percentage="row.progressUpload" />
</template> </template>
</ElTableColumn> </TableV2AutoResizer>
<ElTableColumn </div>
prop="fileSize" </template>
:label="$t('documentCollection.importDoc.fileSize')"
> <template v-else>
<template #default="{ row }"> <DragFileUpload
<span>{{ formatFileSize(row.fileSize) }}</span> ref="dragUploadRef"
</template> @success="handleSuccess"
</ElTableColumn> @on-change="handleChange"
<ElTableColumn :label="$t('common.handle')"> />
<template #default="{ row }"> <div class="import-file-container__table">
<ElButton type="danger" size="small" @click="handleRemove(row)"> <ElTable :data="fileData" style="width: 100%" size="large">
{{ $t('button.delete') }} <ElTableColumn
</ElButton> prop="fileName"
</template> :label="$t('documentCollection.importDoc.fileName')"
</ElTableColumn> width="250"
</ElTable> />
</div> <ElTableColumn
prop="progressUpload"
:label="$t('documentCollection.importDoc.progressUpload')"
width="180"
>
<template #default="{ row }">
<ElProgress
v-if="row.status === 'success'"
:percentage="row.progressUpload"
status="success"
/>
<ElProgress v-else :percentage="row.progressUpload" />
</template>
</ElTableColumn>
<ElTableColumn
prop="fileSize"
:label="$t('documentCollection.importDoc.fileSize')"
>
<template #default="{ row }">
<span>{{ formatFileSize(row.fileSize) }}</span>
</template>
</ElTableColumn>
<ElTableColumn :label="$t('common.handle')">
<template #default="{ row }">
<ElButton type="danger" size="small" @click="handleRemove(row)">
{{ $t('button.delete') }}
</ElButton>
</template>
</ElTableColumn>
</ElTable>
</div>
</template>
</div> </div>
</template> </template>
@@ -116,10 +643,80 @@ function handleRemove(row: any) {
.import-file-container { .import-file-container {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
gap: 12px; gap: 16px;
} }
.import-file-container__table { .import-file-container__table {
margin-top: 2px; margin-top: 2px;
} }
.native-file-input {
display: none;
}
.batch-drop-zone {
display: flex;
flex-direction: column;
gap: 8px;
align-items: center;
justify-content: center;
min-height: 208px;
padding: 24px;
cursor: pointer;
background: hsl(var(--surface-contrast-soft) / 54%);
border: 1px dashed var(--el-color-primary-light-5);
border-radius: 16px;
transition:
background-color 0.2s,
border-color 0.2s;
}
.batch-drop-zone:hover,
.batch-drop-zone:focus-visible {
background: var(--el-color-primary-light-9);
border-color: var(--el-color-primary);
outline: none;
}
.batch-drop-zone--disabled {
cursor: wait;
opacity: 0.72;
}
.batch-drop-zone__icon {
font-size: 48px;
color: var(--el-color-primary);
}
.batch-drop-zone__title {
font-size: 16px;
font-weight: 600;
color: var(--el-text-color-primary);
}
.batch-drop-zone__description {
margin-bottom: 4px;
font-size: 13px;
color: var(--el-text-color-secondary);
text-align: center;
}
.batch-table {
width: 100%;
height: 312px;
overflow: hidden;
border: 1px solid var(--el-border-color-lighter);
border-radius: 12px;
}
:deep(.batch-file-name) {
display: block;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
:deep(.upload-state--success) {
color: var(--el-color-success);
}
</style> </style>

View File

@@ -0,0 +1,67 @@
import { describe, expect, it } from 'vitest';
import { resolveDocumentTaskErrorText } from './document-import-error';
const translate = (key: string) => `translated:${key}`;
describe('resolveDocumentTaskErrorText', () => {
it('uses the stable backend error code', () => {
expect(
resolveDocumentTaskErrorText(
{ lastTaskErrorCode: 'parse_service_unavailable' },
translate,
),
).toBe('translated:documentCollection.importDoc.parseServiceUnavailable');
});
it('supports the error code stored in document options', () => {
expect(
resolveDocumentTaskErrorText(
{ options: { 'task.errorCode': 'pending_timeout' } },
translate,
),
).toBe('translated:documentCollection.importDoc.taskPendingTimeout');
});
it('normalizes legacy MinerU 503 errors', () => {
expect(
resolveDocumentTaskErrorText(
{
lastTaskError:
'MinerU request failed: path=/tasks, status=503, body=',
},
translate,
),
).toBe('translated:documentCollection.importDoc.parseServiceUnavailable');
});
it('distinguishes legacy internal document access errors from MinerU errors', () => {
expect(
resolveDocumentTaskErrorText(
{
lastTaskError:
'下载文档 URL 失败: http://127.0.0.1/file.docx; 远端文档地址不允许访问非公网目标',
},
translate,
),
).toBe('translated:documentCollection.importDoc.documentSourceUnavailable');
});
it('uses the stable document source error code', () => {
expect(
resolveDocumentTaskErrorText(
{ lastTaskErrorCode: 'document_source_unavailable' },
translate,
),
).toBe('translated:documentCollection.importDoc.documentSourceUnavailable');
});
it('keeps unknown business errors intact', () => {
expect(
resolveDocumentTaskErrorText(
{ lastTaskError: '文档内容为空' },
translate,
),
).toBe('文档内容为空');
});
});

View File

@@ -0,0 +1,67 @@
interface DocumentTaskErrorSource {
lastTaskError?: string;
lastTaskErrorCode?: string;
options?: Record<string, unknown>;
}
const ERROR_MESSAGE_KEYS: Record<string, string> = {
document_source_unavailable:
'documentCollection.importDoc.documentSourceUnavailable',
execution_interrupted:
'documentCollection.importDoc.taskExecutionInterrupted',
parse_service_timeout: 'documentCollection.importDoc.parseServiceTimeout',
parse_service_unavailable:
'documentCollection.importDoc.parseServiceUnavailable',
pending_timeout: 'documentCollection.importDoc.taskPendingTimeout',
};
const resolveLegacyErrorCode = (message: string) => {
const normalized = message.toLowerCase();
if (
normalized.includes('下载文档 url 失败') ||
normalized.includes('读取批量导入文件失败') ||
normalized.includes('远端文档地址不允许访问非公网目标')
) {
return 'document_source_unavailable';
}
if (
normalized.includes('status=502') ||
normalized.includes('status=503') ||
normalized.includes('status=504') ||
normalized.includes('connection refused') ||
normalized.includes('failed to connect') ||
normalized.includes('failed to call mineru endpoint') ||
normalized.includes('no route to host') ||
normalized.includes('unknown host')
) {
return 'parse_service_unavailable';
}
if (
normalized.includes('timed out') ||
normalized.includes('timeout') ||
normalized.includes('超时')
) {
return normalized.includes('排队')
? 'pending_timeout'
: 'parse_service_timeout';
}
if (normalized.includes('任务执行中断')) {
return 'execution_interrupted';
}
return '';
};
/**
* 将后端稳定错误码及历史原始错误转换为面向用户的本地化文案。
*/
export const resolveDocumentTaskErrorText = (
row: DocumentTaskErrorSource,
translate: (key: string) => string,
) => {
const storedCode =
row.lastTaskErrorCode ||
String(row.options?.['task.errorCode'] || '') ||
resolveLegacyErrorCode(row.lastTaskError || '');
const messageKey = ERROR_MESSAGE_KEYS[storedCode];
return messageKey ? translate(messageKey) : row.lastTaskError || '';
};

View File

@@ -0,0 +1,71 @@
import { describe, expect, it } from 'vitest';
import { resolveDocumentUploadResponse } from './document-import-upload-response';
const fallbackMessage = '文件上传失败';
describe('resolveDocumentUploadResponse', () => {
it.each([
['数字错误码', '{"errorCode":0}'],
['字符串错误码', '{"errorCode":"0"}'],
['兼容 code 字段', '{"code":0}'],
['带 BOM 的响应', '\uFEFF{"errorCode":0}'],
])('接受%s', (_name, responseText) => {
expect(
resolveDocumentUploadResponse({
fallbackMessage,
responseText,
status: 200,
}),
).toEqual({ success: true });
});
it('接受无响应体的 204 响应', () => {
expect(
resolveDocumentUploadResponse({
fallbackMessage,
responseText: '',
status: 204,
}),
).toEqual({ success: true });
});
it('返回业务错误消息', () => {
expect(
resolveDocumentUploadResponse({
fallbackMessage,
responseText: '{"errorCode":4091,"message":"文件已存在"}',
status: 409,
}),
).toEqual({
message: '文件已存在',
success: false,
});
});
it('拒绝成功状态下的未知响应格式', () => {
expect(
resolveDocumentUploadResponse({
fallbackMessage,
responseText: 'binary-response',
status: 200,
}),
).toEqual({
message: fallbackMessage,
success: false,
});
});
it('拒绝缺少成功错误码的空响应', () => {
expect(
resolveDocumentUploadResponse({
fallbackMessage,
responseText: '',
status: 200,
}),
).toEqual({
message: fallbackMessage,
success: false,
});
});
});

View File

@@ -0,0 +1,75 @@
interface UploadResponsePayload {
code?: unknown;
error?: unknown;
errorCode?: unknown;
message?: unknown;
msg?: unknown;
}
interface ResolveUploadResponseOptions {
fallbackMessage: string;
responseText: string;
status: number;
statusText?: string;
}
interface UploadResponseResult {
message?: string;
success: boolean;
}
function resolveMessage(
payload: undefined | UploadResponsePayload,
statusText: string | undefined,
fallbackMessage: string,
) {
const message =
payload?.message ?? payload?.error ?? payload?.msg ?? statusText;
return typeof message === 'string' && message.trim()
? message.trim()
: fallbackMessage;
}
export function resolveDocumentUploadResponse({
fallbackMessage,
responseText,
status,
statusText,
}: ResolveUploadResponseOptions): UploadResponseResult {
const isHttpSuccess = status >= 200 && status < 300;
const normalizedBody = responseText.replace(/^\uFEFF/, '').trim();
let payload: undefined | UploadResponsePayload;
if (normalizedBody) {
try {
payload = JSON.parse(normalizedBody) as UploadResponsePayload;
} catch {
return {
message: fallbackMessage,
success: false,
};
}
}
if (!isHttpSuccess) {
return {
message: resolveMessage(payload, statusText, fallbackMessage),
success: false,
};
}
if (status === 204) {
return { success: true };
}
const rawErrorCode = payload?.errorCode ?? payload?.code;
const errorCode = Number(rawErrorCode);
if (!Number.isFinite(errorCode) || errorCode !== 0) {
return {
message: resolveMessage(payload, statusText, fallbackMessage),
success: false,
};
}
return { success: true };
}

View File

@@ -32,7 +32,10 @@ interface Entity {
deptId: number | string; deptId: number | string;
expiredAt: Date | null | string; expiredAt: Date | null | string;
permissionIds: (number | string)[]; // 绑定值:权限 ID 数组 permissionIds: (number | string)[]; // 绑定值:权限 ID 数组
knowledgeShareEnabled: boolean; knowledgeReadEnabled: boolean;
knowledgeImportEnabled: boolean;
knowledgeMaintenanceEnabled: boolean;
knowledgeShareEnabled?: boolean;
workflowApiEnabled: boolean; workflowApiEnabled: boolean;
id?: number; // 编辑时的主键 id?: number; // 编辑时的主键
} }
@@ -51,7 +54,9 @@ const entity = ref<Entity>({
deptId: '', deptId: '',
expiredAt: null, expiredAt: null,
permissionIds: [], permissionIds: [],
knowledgeShareEnabled: false, knowledgeReadEnabled: false,
knowledgeImportEnabled: false,
knowledgeMaintenanceEnabled: false,
workflowApiEnabled: false, workflowApiEnabled: false,
}); });
// 加载状态 // 加载状态
@@ -121,7 +126,14 @@ function getResourcePermissionList() {
function createDefaultEntity(row: Partial<Entity> = {}): Entity { function createDefaultEntity(row: Partial<Entity> = {}): Entity {
const permissionIds = row.permissionIds || []; const permissionIds = row.permissionIds || [];
const knowledgeShareEnabled = Boolean(row.knowledgeShareEnabled); const legacyKnowledgeEnabled = Boolean(row.knowledgeShareEnabled);
const knowledgeReadEnabled = Boolean(
row.knowledgeReadEnabled ?? legacyKnowledgeEnabled,
);
const knowledgeImportEnabled = Boolean(
row.knowledgeImportEnabled ?? legacyKnowledgeEnabled,
);
const knowledgeMaintenanceEnabled = Boolean(row.knowledgeMaintenanceEnabled);
const workflowApiEnabled = Boolean(row.workflowApiEnabled); const workflowApiEnabled = Boolean(row.workflowApiEnabled);
return { return {
apiKey: '', apiKey: '',
@@ -130,7 +142,9 @@ function createDefaultEntity(row: Partial<Entity> = {}): Entity {
expiredAt: null, expiredAt: null,
...row, ...row,
permissionIds, permissionIds,
knowledgeShareEnabled, knowledgeReadEnabled,
knowledgeImportEnabled,
knowledgeMaintenanceEnabled,
workflowApiEnabled, workflowApiEnabled,
}; };
} }
@@ -189,7 +203,9 @@ function closeDialog() {
deptId: '', deptId: '',
expiredAt: null, expiredAt: null,
permissionIds: [], permissionIds: [],
knowledgeShareEnabled: false, knowledgeReadEnabled: false,
knowledgeImportEnabled: false,
knowledgeMaintenanceEnabled: false,
workflowApiEnabled: false, workflowApiEnabled: false,
}; };
isAdd.value = true; isAdd.value = true;
@@ -264,10 +280,22 @@ defineExpose({
</ElCheckbox> </ElCheckbox>
</ElCheckboxGroup> </ElCheckboxGroup>
<ElCheckbox <ElCheckbox
v-model="entity.knowledgeShareEnabled" v-model="entity.knowledgeReadEnabled"
class="permission-checkbox" class="permission-checkbox"
> >
{{ $t('sysApiKey.knowledgeSharePermission') }} {{ $t('sysApiKey.knowledgeReadPermission') }}
</ElCheckbox>
<ElCheckbox
v-model="entity.knowledgeImportEnabled"
class="permission-checkbox"
>
{{ $t('sysApiKey.knowledgeImportPermission') }}
</ElCheckbox>
<ElCheckbox
v-model="entity.knowledgeMaintenanceEnabled"
class="permission-checkbox"
>
{{ $t('sysApiKey.knowledgeMaintenancePermission') }}
</ElCheckbox> </ElCheckbox>
<ElCheckbox <ElCheckbox
v-model="entity.workflowApiEnabled" v-model="entity.workflowApiEnabled"