EasyFlow-Tech-Dev/EasyFlow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 重构知识库文档导入任务化流程

Browse Source 
- 新增上传建单、异步解析、分块处理与异步向量化闭环

- 收口分享页权限、完成态检索过滤与 SSE 局部状态刷新
This commit is contained in:
陈子默
2026-04-15 19:27:22 +08:00
parent a41b50959e
commit 2689adfa40
56 changed files with 6376 additions and 1060 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentCollectionController.java
View File

@@ -146,10 +146,14 @@ public class DocumentCollectionController extends BaseCurdController<DocumentCol
)
public Result<List<KnowledgeSearchResultItem>> search(@RequestParam BigInteger knowledgeId,
@RequestParam String keyword,
@RequestParam(required = false) String retrievalMode) {
@RequestParam(required = false) String retrievalMode,
@RequestParam(required = false) Integer docRecallMaxNum,
@RequestParam(required = false) Double simThreshold) {
KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
request.setKnowledgeId(knowledgeId);
request.setQuery(keyword);
request.setLimit(docRecallMaxNum);
request.setMinSimilarity(simThreshold);
request.setRetrievalMode(KnowledgeRetrievalModes.parse(retrievalMode));
request.setCallerType("API");
request.setCallerId(String.valueOf(knowledgeId));

79
easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/DocumentController.java
View File

@@ -8,9 +8,12 @@ import jakarta.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ClassPathResource;
import org.springframework.http.MediaType;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
import tech.easyflow.ai.entity.Document;
import tech.easyflow.ai.entity.DocumentCollection;
import tech.easyflow.ai.entity.DocumentCollectionSplitParams;
@@ -77,6 +80,9 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
@Autowired
private ResourceAccessService resourceAccessService;
@Autowired
private DocumentImportTaskStatusStreamService documentImportTaskStatusStreamService;
@Value("${easyflow.storage.local.root:}")
private String fileUploadPath;
@@ -233,6 +239,79 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
return documentService.commitImport(request);
}
@PostMapping("import/task/create")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportDtos.TaskCreateResponse> createImportTask(@JsonBody DocumentImportDtos.TaskCreateRequest request) {
if (request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return documentService.createImportTask(request);
}
@GetMapping("import/task/detail")
@SaCheckPermission("/api/v1/documentCollection/query")
public Result<DocumentImportDtos.TaskDetailResponse> getImportTaskDetail(@RequestParam BigInteger taskId) {
Result<DocumentImportDtos.TaskDetailResponse> result = documentService.getImportTaskDetail(taskId);
if (result.getData() != null && result.getData().getKnowledgeId() != null) {
getDocumentCollection(result.getData().getKnowledgeId().toString(), ResourceAction.READ, "无权限访问知识库");
}
return result;
}
/**
* 订阅知识库文档任务状态流。
*
* @param knowledgeId 知识库 ID
* @return SSE 推送连接
*/
@PostMapping(value = "import/task/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
@SaCheckPermission("/api/v1/documentCollection/query")
public SseEmitter streamImportTask(@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId) {
getDocumentCollection(knowledgeId.toString(), ResourceAction.READ, "无权限访问知识库");
return documentImportTaskStatusStreamService.subscribe(knowledgeId);
}
@PostMapping("import/task/preview")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportDtos.PreviewResponse> previewImportTask(@JsonBody DocumentImportDtos.PreviewRequest request) {
if (request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return documentService.previewImportTask(request);
}
@PostMapping("import/task/startIndex")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportDtos.TaskStartIndexResponse> startIndexTask(@JsonBody DocumentImportDtos.TaskStartIndexRequest request) {
if (request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return documentService.startIndexTask(request);
}
@PostMapping("import/task/retryParse")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportDtos.TaskStartIndexResponse> retryParseTask(@JsonBody DocumentImportDtos.TaskRetryRequest request) {
if (request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return documentService.retryParseTask(request);
}
@PostMapping("import/task/retryIndex")
@SaCheckPermission("/api/v1/documentCollection/save")
public Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(@JsonBody DocumentImportDtos.TaskRetryRequest request) {
if (request.getKnowledgeId() == null) {
throw new BusinessException("知识库id不能为空");
}
getDocumentCollection(request.getKnowledgeId().toString(), ResourceAction.MANAGE, "无权限管理知识库");
return documentService.retryIndexTask(request);
}
/**
* 更新 entity
*

130
easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/ai/ShareKnowledgeController.java
View File

@@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.ai.documentimport.DocumentImportDtos;
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
import tech.easyflow.ai.dto.KnowledgeShareLimitedConfigRequest;
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
import tech.easyflow.ai.entity.Document;
@@ -42,6 +44,7 @@ import tech.easyflow.ai.service.KnowledgeShareService;
import tech.easyflow.ai.service.ModelService;
import tech.easyflow.ai.vo.FaqImportResultVo;
import tech.easyflow.ai.vo.KnowledgeShareAuthContext;
import tech.easyflow.ai.vo.KnowledgeShareViewDetail;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.filestorage.FileStorageService;
import tech.easyflow.common.vo.UploadResVo;
@@ -99,6 +102,8 @@ public class ShareKnowledgeController {
private KnowledgeEmbeddingService knowledgeEmbeddingService;
@Resource(name = "default")
private FileStorageService fileStorageService;
@Resource
private DocumentImportTaskStatusStreamService documentImportTaskStatusStreamService;
/**
* 获取知识库详情。
@@ -107,14 +112,17 @@ public class ShareKnowledgeController {
* @return 知识库详情
*/
@GetMapping("/documentCollection/detail")
public Result<DocumentCollection> detail(@RequestParam String shareKey) {
public Result<KnowledgeShareViewDetail> detail(@RequestParam String shareKey) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
null,
KnowledgeShareActionScope.VIEW.name()
);
audit(context, "访问知识库分享页", "KNOWLEDGE_SHARE_URL_ACCESS", false, auditDetail("knowledgeId", context.getKnowledge().getId()));
return Result.ok(context.getKnowledge());
KnowledgeShareViewDetail detail = new KnowledgeShareViewDetail();
detail.setKnowledge(context.getKnowledge());
detail.setPermissionScopes(new java.util.ArrayList<String>(context.getShare().getPermissionScopeSet()));
return Result.ok(detail);
}
/**
@@ -234,6 +242,26 @@ public class ShareKnowledgeController {
return Result.ok(documentService.getDocumentList(context.getKnowledge().getId().toString(), pageSize, pageNumber, title));
}
/**
* 订阅分享知识库的文档任务状态流。
*
* @param shareKey 分享访问密钥
* @param knowledgeId 知识库 ID
* @return SSE 推送连接
*/
@PostMapping(value = "/document/import/task/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public SseEmitter streamDocumentTask(
@RequestParam String shareKey,
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
knowledgeId,
KnowledgeShareActionScope.VIEW.name()
);
return documentImportTaskStatusStreamService.subscribe(context.getKnowledge().getId());
}
/**
* 下载文档。
*/
@@ -344,6 +372,104 @@ public class ShareKnowledgeController {
return documentService.commitImport(request);
}
@PostMapping("/document/import/task/create")
public Result<DocumentImportDtos.TaskCreateResponse> createImportTask(
@RequestParam String shareKey,
@JsonBody DocumentImportDtos.TaskCreateRequest request
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
request == null ? null : request.getKnowledgeId(),
KnowledgeShareActionScope.CONTENT_CREATE.name()
);
BigInteger knowledgeId = resolveKnowledgeId(context, request == null ? null : request.getKnowledgeId());
request.setKnowledgeId(knowledgeId);
audit(context, "创建分享文档导入任务", "KNOWLEDGE_SHARE_URL_WRITE", true, auditDetail("knowledgeId", knowledgeId));
return documentService.createImportTask(request);
}
@GetMapping("/document/import/task/detail")
public Result<DocumentImportDtos.TaskDetailResponse> getImportTaskDetail(
@RequestParam String shareKey,
@RequestParam BigInteger taskId
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
null,
KnowledgeShareActionScope.VIEW.name()
);
Result<DocumentImportDtos.TaskDetailResponse> result = documentService.getImportTaskDetail(taskId);
BigInteger knowledgeId = result.getData() == null ? null : result.getData().getKnowledgeId();
if (knowledgeId == null || knowledgeId.compareTo(context.getKnowledge().getId()) != 0) {
throw new BusinessException("任务不存在");
}
return result;
}
@PostMapping("/document/import/task/preview")
public Result<DocumentImportDtos.PreviewResponse> previewImportTask(
@RequestParam String shareKey,
@JsonBody DocumentImportDtos.PreviewRequest request
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
request == null ? null : request.getKnowledgeId(),
KnowledgeShareActionScope.CONTENT_CREATE.name()
);
BigInteger knowledgeId = resolveKnowledgeId(context, request == null ? null : request.getKnowledgeId());
request.setKnowledgeId(knowledgeId);
audit(context, "预览分享文档分块", "KNOWLEDGE_SHARE_URL_WRITE", true, auditDetail("knowledgeId", knowledgeId));
return documentService.previewImportTask(request);
}
@PostMapping("/document/import/task/startIndex")
public Result<DocumentImportDtos.TaskStartIndexResponse> startIndexTask(
@RequestParam String shareKey,
@JsonBody DocumentImportDtos.TaskStartIndexRequest request
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
request == null ? null : request.getKnowledgeId(),
KnowledgeShareActionScope.CONTENT_CREATE.name()
);
BigInteger knowledgeId = resolveKnowledgeId(context, request == null ? null : request.getKnowledgeId());
request.setKnowledgeId(knowledgeId);
audit(context, "启动分享文档向量化", "KNOWLEDGE_SHARE_URL_WRITE", true, auditDetail("knowledgeId", knowledgeId));
return documentService.startIndexTask(request);
}
@PostMapping("/document/import/task/retryParse")
public Result<DocumentImportDtos.TaskStartIndexResponse> retryParseTask(
@RequestParam String shareKey,
@JsonBody DocumentImportDtos.TaskRetryRequest request
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
request == null ? null : request.getKnowledgeId(),
KnowledgeShareActionScope.CONTENT_CREATE.name()
);
BigInteger knowledgeId = resolveKnowledgeId(context, request == null ? null : request.getKnowledgeId());
request.setKnowledgeId(knowledgeId);
audit(context, "重试分享文档解析", "KNOWLEDGE_SHARE_URL_WRITE", true, auditDetail("knowledgeId", knowledgeId));
return documentService.retryParseTask(request);
}
@PostMapping("/document/import/task/retryIndex")
public Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(
@RequestParam String shareKey,
@JsonBody DocumentImportDtos.TaskRetryRequest request
) {
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
shareKey,
request == null ? null : request.getKnowledgeId(),
KnowledgeShareActionScope.CONTENT_CREATE.name()
);
BigInteger knowledgeId = resolveKnowledgeId(context, request == null ? null : request.getKnowledgeId());
request.setKnowledgeId(knowledgeId);
audit(context, "重试分享文档向量化", "KNOWLEDGE_SHARE_URL_WRITE", true, auditDetail("knowledgeId", knowledgeId));
return documentService.retryIndexTask(request);
}
/**
* Chunk 分页。
*/

77
easyflow-api/easyflow-api-public/src/main/java/tech/easyflow/publicapi/controller/PublicKnowledgeShareController.java
View File

@@ -224,6 +224,83 @@ public class PublicKnowledgeShareController {
return documentService.commitImport(request);
}
@PostMapping("/document/import/task/create")
public Result<DocumentImportDtos.TaskCreateResponse> createImportTask(
@RequestHeader("ApiKey") String apiKey,
@JsonBody DocumentImportDtos.TaskCreateRequest request,
HttpServletRequest servletRequest
) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name());
requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API创建文档导入任务", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.createImportTask(request);
}
@GetMapping("/document/import/task/detail")
public Result<DocumentImportDtos.TaskDetailResponse> getImportTaskDetail(
@RequestHeader("ApiKey") String apiKey,
@RequestParam BigInteger knowledgeId,
@RequestParam BigInteger taskId,
HttpServletRequest request
) {
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeShareActionScope.VIEW.name());
requireDocumentKnowledge(knowledgeId);
Result<DocumentImportDtos.TaskDetailResponse> result = documentService.getImportTaskDetail(taskId);
if (result.getData() == null || result.getData().getKnowledgeId() == null
|| result.getData().getKnowledgeId().compareTo(knowledgeId) != 0) {
throw new BusinessException("任务不存在");
}
return result;
}
@PostMapping("/document/import/task/preview")
public Result<DocumentImportDtos.PreviewResponse> previewImportTask(
@RequestHeader("ApiKey") String apiKey,
@JsonBody DocumentImportDtos.PreviewRequest request,
HttpServletRequest servletRequest
) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name());
requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API预览文档分块", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.previewImportTask(request);
}
@PostMapping("/document/import/task/startIndex")
public Result<DocumentImportDtos.TaskStartIndexResponse> startIndexTask(
@RequestHeader("ApiKey") String apiKey,
@JsonBody DocumentImportDtos.TaskStartIndexRequest request,
HttpServletRequest servletRequest
) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name());
requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API启动文档向量化", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.startIndexTask(request);
}
@PostMapping("/document/import/task/retryParse")
public Result<DocumentImportDtos.TaskStartIndexResponse> retryParseTask(
@RequestHeader("ApiKey") String apiKey,
@JsonBody DocumentImportDtos.TaskRetryRequest request,
HttpServletRequest servletRequest
) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name());
requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API重试文档解析", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.retryParseTask(request);
}
@PostMapping("/document/import/task/retryIndex")
public Result<DocumentImportDtos.TaskStartIndexResponse> retryIndexTask(
@RequestHeader("ApiKey") String apiKey,
@JsonBody DocumentImportDtos.TaskRetryRequest request,
HttpServletRequest servletRequest
) {
assertApiShare(apiKey, servletRequest.getRequestURI(), request.getKnowledgeId(), KnowledgeShareActionScope.CONTENT_CREATE.name());
requireDocumentKnowledge(request.getKnowledgeId());
audit(apiKey, "API重试文档向量化", "KNOWLEDGE_API_SHARE_WRITE", servletRequest.getRequestURI(), Map.of("knowledgeId", request.getKnowledgeId()));
return documentService.retryIndexTask(request);
}
/**
* Chunk 分页。
*/