EasyFlow-Tech-Dev/EasyFlow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完善知识库批量导入与公共 API

Browse Source 
- 新增批量异步导入、状态查询、失败重试与中断恢复链路

- 拆分知识库读取、导入、维护权限并完善 Public API 契约

- 补充数据库迁移、管理端交互、接口说明与相关测试
This commit is contained in:
陈子默
2026-08-03 11:13:48 +08:00
parent 6df3dd9981
commit 51dbfd41b6
99 changed files with 16274 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentController.java
View File

@@ -11,8 +11,11 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.http.MediaType;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.task.DocumentImportBatchAppService;
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
import tech.easyflow.ai.entity.Document;
import tech.easyflow.ai.entity.DocumentCollection;
@@ -83,6 +86,9 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
@Autowired
private DocumentImportTaskStatusStreamService documentImportTaskStatusStreamService;
@Autowired
private DocumentImportBatchAppService documentImportBatchAppService;
@Value("${easyflow.storage.local.root:}")
private String fileUploadPath;
@@ -312,6 +318,142 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
return documentService.retryIndexTask(request);
}
/**
* 创建文档批量上传清单。
*
* @param request 文件清单
* @return 批次与服务端文件项
*/
@PostMapping("import/batch/create")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportBatchDtos.CreateResponse> createImportBatch(
@JsonBody DocumentImportBatchDtos.CreateRequest request) {
if (request == null || request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return Result.ok(documentImportBatchAppService.createBatch(request));
}
/**
* 上传一个批次文件。
*
* @param batchId 批次 ID
* @param itemId 文件项 ID
* @param knowledgeId 知识库 ID
* @param file 上传文件
* @return 文件项状态
*/
@PostMapping(value = "import/batch/{batchId}/item/{itemId}/upload",
consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportBatchDtos.ItemResponse> uploadImportBatchItem(
@PathVariable BigInteger batchId,
@PathVariable BigInteger itemId,
@RequestParam BigInteger knowledgeId,
@RequestPart("file") MultipartFile file) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库");
return Result.ok(documentImportBatchAppService.uploadItem(
knowledgeId, batchId, itemId, file));
}
/**
* 启动手动或自动批量导入。
*
* @param request 启动请求
* @return 批次状态
*/
@PostMapping("import/batch/start")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportBatchDtos.StatusResponse> startImportBatch(
@JsonBody DocumentImportBatchDtos.StartRequest request) {
if (request == null || request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return Result.ok(documentImportBatchAppService.startBatch(request));
}
/**
* 取消一个尚未启动的上传批次。
*
* @param knowledgeId 知识库 ID
* @param batchId 批次 ID
* @return 空结果
*/
@PostMapping("import/batch/cancel")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<Void> cancelImportBatch(
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
@JsonBody(value = "batchId", required = true) BigInteger batchId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库");
documentImportBatchAppService.cancelBatch(knowledgeId, batchId);
return Result.ok();
}
/**
* 查询批次状态。
*
* @param knowledgeId 知识库 ID
* @param batchId 批次 ID
* @return 批次状态
*/
@GetMapping("import/batch/status")
@SaCheckPermission("/api/v1/documentCollection/query")
public Result<DocumentImportBatchDtos.StatusResponse> getImportBatchStatus(
@RequestParam BigInteger knowledgeId,
@RequestParam BigInteger batchId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.READ, "无权限访问知识库");
return Result.ok(documentImportBatchAppService.getBatchStatus(knowledgeId, batchId));
}
/**
* 查询知识库最近一个自动导入批次。
*
* @param knowledgeId 知识库 ID
* @return 最近批次状态
*/
@GetMapping("import/batch/current")
@SaCheckPermission("/api/v1/documentCollection/query")
public Result<DocumentImportBatchDtos.StatusResponse> getCurrentImportBatch(
@RequestParam BigInteger knowledgeId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.READ, "无权限访问知识库");
return Result.ok(documentImportBatchAppService.getLatestAutoBatch(knowledgeId));
}
/**
* 继续中断或部分失败的自动导入批次。
*
* @param knowledgeId 知识库 ID
* @param batchId 批次 ID
* @return 继续后的批次状态
*/
@PostMapping("import/batch/continue")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportBatchDtos.StatusResponse> continueImportBatch(
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
@JsonBody(value = "batchId", required = true) BigInteger batchId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.MANAGE, "无权限管理知识库");
return Result.ok(documentImportBatchAppService.continueBatch(knowledgeId, batchId));
}
/**
* 根据解析、分块或向量化失败阶段统一重试。
*
* @param request 重试请求
* @return 重试任务状态
*/
@PostMapping("import/task/retry")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportDtos.TaskStartIndexResponse> retryImportTask(
@JsonBody DocumentImportDtos.TaskRetryRequest request) {
if (request == null || request.getKnowledgeId() == null || request.getDocumentId() == null) {
throw new BusinessException("重试信息不完整");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return documentService.retryFailedTask(request);
}
/**
* 更新 entity
*

121
easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/system/SysApiKeyController.java
View File

@@ -6,18 +6,21 @@ import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.table.TableInfo;
import com.mybatisflex.core.table.TableInfoFactory;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import tech.easyflow.ai.service.KnowledgeSharePermissionService;
import tech.easyflow.ai.service.WorkflowApiPermissionService;
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.satoken.util.SaTokenUtil;
import tech.easyflow.common.util.IdUtil;
import tech.easyflow.common.vo.PkVo;
import tech.easyflow.common.web.controller.BaseCurdController;
import tech.easyflow.common.web.jsonbody.JsonBody;
import tech.easyflow.system.entity.SysApiKey;
import tech.easyflow.system.entity.SysApiKeyResourceMapping;
import tech.easyflow.system.service.SysApiKeyResourceMappingService;
@@ -29,6 +32,7 @@ import java.time.LocalDate;
import java.time.ZoneId;
import java.util.Date;
import java.util.List;
import java.util.Set;
/**
* 控制层。
@@ -83,12 +87,57 @@ public class SysApiKeyController extends BaseCurdController<SysApiKeyService, Sy
return Result.ok(new PkVo(pkArgs));
}
/**
* 更新访问令牌基础信息与授权。
*
* <p>权限开关不映射数据库列，权限更新请求可能只包含主键与权限字段。
* 此时跳过主表更新，避免 MyBatis-Flex 生成空的 {@code SET} 子句。</p>
*
* @param entity 待更新的访问令牌
* @return 更新结果
*/
@Override
@PostMapping("/update")
@Transactional(rollbackFor = Exception.class)
public Result<?> update(@JsonBody SysApiKey entity) {
if (entity == null || entity.getId() == null) {
return Result.fail("访问令牌 ID 不能为空");
}
if (!hasPersistentUpdateFields(entity) && !hasPermissionUpdateFields(entity)) {
return Result.fail("没有可更新的访问令牌字段");
}
if (hasNewKnowledgePermissionFields(entity)
&& !hasCompleteKnowledgePermissionFields(entity)) {
return Result.fail("知识库读取、导入、维护权限必须同时提交");
}
if (service.getById(entity.getId()) == null) {
return Result.fail("访问令牌不存在");
}
Result<?> beforeResult = onSaveOrUpdateBefore(entity, false);
if (beforeResult != null) {
return beforeResult;
}
if (hasPersistentUpdateFields(entity)) {
service.updateById(entity);
}
onSaveOrUpdateAfter(entity, false);
return Result.ok();
}
@Override
protected void onSaveOrUpdateAfter(SysApiKey entity, boolean isSave) {
if (entity.getPermissionIds() != null) {
sysApiKeyResourceMappingService.authInterface(entity);
}
if (entity.getKnowledgeShareEnabled() != null) {
if (hasNewKnowledgePermissionFields(entity)) {
knowledgeSharePermissionService.replaceApiPermissions(
entity.getId(),
Boolean.TRUE.equals(entity.getKnowledgeReadEnabled()),
Boolean.TRUE.equals(entity.getKnowledgeImportEnabled()),
Boolean.TRUE.equals(entity.getKnowledgeMaintenanceEnabled())
);
} else if (entity.getKnowledgeShareEnabled() != null) {
// 兼容旧客户端：开启旧总开关只授予读取和导入，维护权限保持关闭。
knowledgeSharePermissionService.replaceApiShareEnabled(entity.getId(), entity.getKnowledgeShareEnabled());
}
if (entity.getWorkflowApiEnabled() != null) {
@@ -130,11 +179,18 @@ public class SysApiKeyController extends BaseCurdController<SysApiKeyService, Sy
List<BigInteger> resourceIds = sysApiKeyResourceMappingService.listAs(interfaceWrapper, BigInteger.class);
entity.setPermissionIds(resourceIds);
QueryWrapper knowledgeWrapper = QueryWrapper.create()
.select(SysApiKeyResourceMapping::getId)
.eq(SysApiKeyResourceMapping::getApiKeyId, entity.getId())
.eq(SysApiKeyResourceMapping::getResourceType, "KNOWLEDGE");
entity.setKnowledgeShareEnabled(sysApiKeyResourceMappingService.count(knowledgeWrapper) > 0);
Set<String> knowledgeScopes =
knowledgeSharePermissionService.getApiPermissionScopes(entity.getId());
boolean readEnabled =
knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
boolean importEnabled =
knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_IMPORT.name());
boolean maintenanceEnabled =
knowledgeScopes.contains(KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
entity.setKnowledgeReadEnabled(readEnabled);
entity.setKnowledgeImportEnabled(importEnabled);
entity.setKnowledgeMaintenanceEnabled(maintenanceEnabled);
entity.setKnowledgeShareEnabled(readEnabled || importEnabled || maintenanceEnabled);
QueryWrapper workflowWrapper = QueryWrapper.create()
.select(SysApiKeyResourceMapping::getId)
@@ -142,4 +198,57 @@ public class SysApiKeyController extends BaseCurdController<SysApiKeyService, Sy
.eq(SysApiKeyResourceMapping::getResourceType, WorkflowApiPermissionService.RESOURCE_TYPE_WORKFLOW);
entity.setWorkflowApiEnabled(sysApiKeyResourceMappingService.count(workflowWrapper) > 0);
}
/**
* 判断请求是否提交了任一新版知识库权限字段。
*
* @param entity 访问令牌
* @return 是否提交新版字段
*/
private boolean hasNewKnowledgePermissionFields(SysApiKey entity) {
return entity.getKnowledgeReadEnabled() != null
|| entity.getKnowledgeImportEnabled() != null
|| entity.getKnowledgeMaintenanceEnabled() != null;
}
/**
* 判断请求是否完整提交三个新版知识库权限字段。
*
* @param entity 访问令牌
* @return 三个字段是否均已提交
*/
private boolean hasCompleteKnowledgePermissionFields(SysApiKey entity) {
return entity.getKnowledgeReadEnabled() != null
&& entity.getKnowledgeImportEnabled() != null
&& entity.getKnowledgeMaintenanceEnabled() != null;
}
/**
* 判断请求是否包含主表可持久化字段。
*
* @param entity 访问令牌
* @return 是否需要更新访问令牌主表
*/
private boolean hasPersistentUpdateFields(SysApiKey entity) {
return entity.getApiKey() != null
|| entity.getCreated() != null
|| entity.getStatus() != null
|| entity.getDeptId() != null
|| entity.getTenantId() != null
|| entity.getExpiredAt() != null
|| entity.getCreatedBy() != null;
}
/**
* 判断请求是否包含任一非主表权限字段。
*
* @param entity 访问令牌
* @return 是否需要更新权限映射
*/
private boolean hasPermissionUpdateFields(SysApiKey entity) {
return entity.getPermissionIds() != null
|| entity.getKnowledgeShareEnabled() != null
|| hasNewKnowledgePermissionFields(entity)
|| entity.getWorkflowApiEnabled() != null;
}
}