Compare commits
13 Commits
1bd9810518
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 9722bea701 | |||
| 386cebf342 | |||
| c7cac61ce8 | |||
| b99e275371 | |||
| ebade41e40 | |||
| cc7f0c1a43 | |||
| c4799760cf | |||
| 6cbc330f55 | |||
| d7b0d442eb | |||
| 198d592dd6 | |||
| 7257f41eb8 | |||
| e9fc0bd810 | |||
| 9ef20119a9 |
@@ -1,23 +1,18 @@
|
|||||||
package tech.easyflow.admin.controller.ai;
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncRetryRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatusRequest;
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
|
||||||
import tech.easyflow.ai.entity.Model;
|
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
|
||||||
import tech.easyflow.ai.service.ModelService;
|
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
|
||||||
import tech.easyflow.common.annotation.UsePermission;
|
import tech.easyflow.common.annotation.UsePermission;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
import tech.easyflow.common.web.controller.BaseController;
|
||||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||||
import com.easyagents.core.document.Document;
|
|
||||||
import com.easyagents.core.store.DocumentStore;
|
|
||||||
import com.easyagents.core.store.StoreOptions;
|
|
||||||
import com.easyagents.core.store.StoreResult;
|
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
@@ -28,12 +23,8 @@ import tech.easyflow.system.enums.ResourceAction;
|
|||||||
import tech.easyflow.system.enums.ResourceLookup;
|
import tech.easyflow.system.enums.ResourceLookup;
|
||||||
import tech.easyflow.system.permission.resource.RequireResourceAccess;
|
import tech.easyflow.system.permission.resource.RequireResourceAccess;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 控制层。
|
* 控制层。
|
||||||
@@ -44,19 +35,12 @@ import java.util.Map;
|
|||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/api/v1/documentChunk")
|
@RequestMapping("/api/v1/documentChunk")
|
||||||
@UsePermission(moduleName = "/api/v1/documentCollection")
|
@UsePermission(moduleName = "/api/v1/documentCollection")
|
||||||
public class DocumentChunkController extends BaseCurdController<DocumentChunkService, DocumentChunk> {
|
public class DocumentChunkController extends BaseController {
|
||||||
|
|
||||||
@Resource
|
private final DocumentChunkService documentChunkService;
|
||||||
DocumentCollectionService documentCollectionService;
|
|
||||||
|
|
||||||
@Resource
|
|
||||||
ModelService modelService;
|
|
||||||
|
|
||||||
@Resource
|
|
||||||
DocumentChunkService documentChunkService;
|
|
||||||
|
|
||||||
public DocumentChunkController(DocumentChunkService service) {
|
public DocumentChunkController(DocumentChunkService service) {
|
||||||
super(service);
|
this.documentChunkService = service;
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("page")
|
@GetMapping("page")
|
||||||
@@ -68,9 +52,30 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
|
|||||||
idExpr = "#request.getParameter('documentId')",
|
idExpr = "#request.getParameter('documentId')",
|
||||||
denyMessage = "无权限访问知识库"
|
denyMessage = "无权限访问知识库"
|
||||||
)
|
)
|
||||||
@Override
|
public Result<Page<DocumentChunk>> page(
|
||||||
public Result<Page<DocumentChunk>> page(HttpServletRequest request, String sortKey, String sortType, Long pageNumber, Long pageSize) {
|
HttpServletRequest request,
|
||||||
return super.page(request, sortKey, sortType, pageNumber, pageSize);
|
Long pageNumber,
|
||||||
|
Long pageSize
|
||||||
|
) {
|
||||||
|
String documentIdValue = request.getParameter("documentId");
|
||||||
|
if (documentIdValue == null || documentIdValue.isBlank()) {
|
||||||
|
return Result.<Page<DocumentChunk>>fail("documentId不能为空", null);
|
||||||
|
}
|
||||||
|
BigInteger documentId;
|
||||||
|
try {
|
||||||
|
documentId = new BigInteger(documentIdValue);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
return Result.<Page<DocumentChunk>>fail("documentId格式不正确", null);
|
||||||
|
}
|
||||||
|
long normalizedPageNumber = pageNumber == null || pageNumber < 1 ? 1 : pageNumber;
|
||||||
|
long normalizedPageSize = pageSize == null || pageSize < 1 ? 10 : pageSize;
|
||||||
|
QueryWrapper query = QueryWrapper.create()
|
||||||
|
.eq(DocumentChunk::getDocumentId, documentId)
|
||||||
|
.orderBy("sorting asc");
|
||||||
|
return Result.ok(documentChunkService.page(
|
||||||
|
new Page<>(normalizedPageNumber, normalizedPageSize),
|
||||||
|
query
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("update")
|
@PostMapping("update")
|
||||||
@@ -79,43 +84,23 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
|
|||||||
resource = CategoryResourceType.KNOWLEDGE,
|
resource = CategoryResourceType.KNOWLEDGE,
|
||||||
action = ResourceAction.MANAGE,
|
action = ResourceAction.MANAGE,
|
||||||
lookup = ResourceLookup.DOCUMENT_CHUNK_ID,
|
lookup = ResourceLookup.DOCUMENT_CHUNK_ID,
|
||||||
idExpr = "#documentChunk.id",
|
idExpr = "#request.id",
|
||||||
denyMessage = "无权限管理知识库"
|
denyMessage = "无权限管理知识库"
|
||||||
)
|
)
|
||||||
public Result<?> update(@JsonBody DocumentChunk documentChunk) {
|
public Result<?> update(
|
||||||
boolean success = service.updateById(documentChunk);
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
if (success){
|
DocumentChunkContentUpdateRequest request
|
||||||
DocumentChunk record = documentChunkService.getById(documentChunk.getId());
|
) {
|
||||||
DocumentCollection knowledge = documentCollectionService.getById(record.getDocumentCollectionId());
|
DocumentChunk current = documentChunkService.getById(request.getId());
|
||||||
if (knowledge == null) {
|
if (current == null) {
|
||||||
return Result.fail(1, "知识库不存在");
|
return Result.fail(1, "记录不存在");
|
||||||
}
|
}
|
||||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
DocumentChunk updated = documentChunkService.updateContent(
|
||||||
if (documentStore == null) {
|
current.getDocumentCollectionId(),
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
current.getId(),
|
||||||
}
|
request.getContent()
|
||||||
try {
|
);
|
||||||
// 设置向量模型
|
return Result.ok(updated);
|
||||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
|
||||||
documentStore.setEmbeddingModel(embeddingModel);
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
|
||||||
Document document = Document.of(documentChunk.getContent());
|
|
||||||
document.setId(documentChunk.getId());
|
|
||||||
Map<String, Object> metadata = new HashMap<>();
|
|
||||||
metadata.put("keywords", documentChunk.getMetadataKeyWords());
|
|
||||||
metadata.put("questions", documentChunk.getMetadataQuestions());
|
|
||||||
document.setMetadataMap(metadata);
|
|
||||||
StoreResult result = documentStore.update(document, options); // 更新已有记录
|
|
||||||
return Result.ok(result);
|
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Result.ok(false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("removeChunk")
|
@PostMapping("removeChunk")
|
||||||
@@ -127,36 +112,58 @@ public class DocumentChunkController extends BaseCurdController<DocumentChunkSer
|
|||||||
idExpr = "#chunkId",
|
idExpr = "#chunkId",
|
||||||
denyMessage = "无权限管理知识库"
|
denyMessage = "无权限管理知识库"
|
||||||
)
|
)
|
||||||
public Result<?> remove(@JsonBody(value = "id", required = true) BigInteger chunkId) {
|
public Result<?> removeChunk(@JsonBody(value = "id", required = true) BigInteger chunkId) {
|
||||||
DocumentChunk docChunk = documentChunkService.getById(chunkId);
|
DocumentChunk docChunk = documentChunkService.getById(chunkId);
|
||||||
if (docChunk == null) {
|
if (docChunk == null) {
|
||||||
return Result.fail(1, "记录不存在");
|
return Result.fail(1, "记录不存在");
|
||||||
}
|
}
|
||||||
DocumentCollection knowledge = documentCollectionService.getById(docChunk.getDocumentCollectionId());
|
return Result.ok(documentChunkService.deleteChunk(
|
||||||
if (knowledge == null) {
|
docChunk.getDocumentCollectionId(),
|
||||||
return Result.fail(2, "知识库不存在");
|
chunkId
|
||||||
|
));
|
||||||
}
|
}
|
||||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
|
||||||
if (documentStore == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量库");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
// 设置向量模型
|
|
||||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(4, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
|
||||||
documentStore.setEmbeddingModel(embeddingModel);
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
|
||||||
List<BigInteger> deleteList = new ArrayList<>();
|
|
||||||
deleteList.add(chunkId);
|
|
||||||
documentStore.delete(deleteList, options);
|
|
||||||
documentChunkService.removeChunk(knowledge, chunkId);
|
|
||||||
|
|
||||||
return super.remove(chunkId);
|
@PostMapping("syncStatus")
|
||||||
} finally {
|
@SaCheckPermission("/api/v1/documentCollection/query")
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
@RequireResourceAccess(
|
||||||
|
resource = CategoryResourceType.KNOWLEDGE,
|
||||||
|
action = ResourceAction.READ,
|
||||||
|
lookup = ResourceLookup.DOCUMENT_ID,
|
||||||
|
idExpr = "#request.documentId",
|
||||||
|
denyMessage = "无权限访问知识库"
|
||||||
|
)
|
||||||
|
public Result<List<DocumentChunkSyncStatus>> syncStatus(
|
||||||
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkSyncStatusRequest request
|
||||||
|
) {
|
||||||
|
return Result.ok(documentChunkService.listIndexSyncStatus(
|
||||||
|
null,
|
||||||
|
request.getDocumentId(),
|
||||||
|
request.getIds()
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("retrySync")
|
||||||
|
@SaCheckPermission("/api/v1/documentCollection/save")
|
||||||
|
@RequireResourceAccess(
|
||||||
|
resource = CategoryResourceType.KNOWLEDGE,
|
||||||
|
action = ResourceAction.MANAGE,
|
||||||
|
lookup = ResourceLookup.DOCUMENT_CHUNK_ID,
|
||||||
|
idExpr = "#request.id",
|
||||||
|
denyMessage = "无权限管理知识库"
|
||||||
|
)
|
||||||
|
public Result<?> retrySync(
|
||||||
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkSyncRetryRequest request
|
||||||
|
) {
|
||||||
|
DocumentChunk current = documentChunkService.getById(request.getId());
|
||||||
|
if (current == null || request.getIndexSyncVersion() == null) {
|
||||||
|
return Result.fail(1, "记录不存在或同步版本缺失");
|
||||||
|
}
|
||||||
|
return Result.ok(documentChunkService.retryIndexSync(
|
||||||
|
current.getDocumentCollectionId(),
|
||||||
|
current.getId(),
|
||||||
|
request.getIndexSyncVersion()
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -118,12 +118,10 @@ public class DocumentController extends BaseCurdController<DocumentService, Docu
|
|||||||
List<Serializable> ids = Collections.singletonList(id);
|
List<Serializable> ids = Collections.singletonList(id);
|
||||||
Result<?> result = onRemoveBefore(ids);
|
Result<?> result = onRemoveBefore(ids);
|
||||||
if (result != null) return result;
|
if (result != null) return result;
|
||||||
boolean isSuccess = documentService.removeDoc(id);
|
boolean success = documentService.removeDoc(id);
|
||||||
if (!isSuccess){
|
if (success) {
|
||||||
return Result.ok(false);
|
|
||||||
}
|
|
||||||
boolean success = service.removeById(id);
|
|
||||||
onRemoveAfter(ids);
|
onRemoveAfter(ids);
|
||||||
|
}
|
||||||
return Result.ok(success);
|
return Result.ok(success);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,10 +18,12 @@ import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
|||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||||
import tech.easyflow.ai.entity.Plugin;
|
import tech.easyflow.ai.entity.Plugin;
|
||||||
import tech.easyflow.ai.entity.PluginItem;
|
import tech.easyflow.ai.entity.PluginItem;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
|
import tech.easyflow.ai.entity.WorkflowExecResult;
|
||||||
import tech.easyflow.ai.enums.PluginType;
|
import tech.easyflow.ai.enums.PluginType;
|
||||||
import tech.easyflow.ai.plugin.workflow.snapshot.WorkflowPluginSnapshotResolver;
|
import tech.easyflow.ai.plugin.workflow.snapshot.WorkflowPluginSnapshotResolver;
|
||||||
import tech.easyflow.ai.service.PluginService;
|
import tech.easyflow.ai.service.PluginService;
|
||||||
@@ -29,6 +31,7 @@ import tech.easyflow.ai.service.PluginItemService;
|
|||||||
import tech.easyflow.ai.service.AgentResourceReferenceService;
|
import tech.easyflow.ai.service.AgentResourceReferenceService;
|
||||||
import tech.easyflow.ai.service.PluginVisibilityService;
|
import tech.easyflow.ai.service.PluginVisibilityService;
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
|
import tech.easyflow.ai.service.WorkflowExecResultService;
|
||||||
import tech.easyflow.common.constant.Constants;
|
import tech.easyflow.common.constant.Constants;
|
||||||
import tech.easyflow.common.annotation.UsePermission;
|
import tech.easyflow.common.annotation.UsePermission;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
@@ -91,11 +94,15 @@ public class PluginItemController extends BaseCurdController<PluginItemService,
|
|||||||
@Resource
|
@Resource
|
||||||
private WorkflowService workflowService;
|
private WorkflowService workflowService;
|
||||||
@Resource
|
@Resource
|
||||||
|
private WorkflowExecResultService workflowExecResultService;
|
||||||
|
@Resource
|
||||||
private ChainExecutor chainExecutor;
|
private ChainExecutor chainExecutor;
|
||||||
@Resource
|
@Resource
|
||||||
private TinyFlowService tinyFlowService;
|
private TinyFlowService tinyFlowService;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowCheckService workflowCheckService;
|
private WorkflowCheckService workflowCheckService;
|
||||||
|
@Resource
|
||||||
|
private WorkflowResumeService workflowResumeService;
|
||||||
|
|
||||||
@PostMapping("/tool/save")
|
@PostMapping("/tool/save")
|
||||||
@SaCheckPermission("/api/v1/plugin/save")
|
@SaCheckPermission("/api/v1/plugin/save")
|
||||||
@@ -215,6 +222,7 @@ public class PluginItemController extends BaseCurdController<PluginItemService,
|
|||||||
@SaCheckPermission("/api/v1/plugin/query")
|
@SaCheckPermission("/api/v1/plugin/query")
|
||||||
public Result<ChainInfo> pluginToolTestChainStatus(@JsonBody(value = "executeId", required = true) String executeId,
|
public Result<ChainInfo> pluginToolTestChainStatus(@JsonBody(value = "executeId", required = true) String executeId,
|
||||||
@JsonBody("nodes") List<NodeInfo> nodes) {
|
@JsonBody("nodes") List<NodeInfo> nodes) {
|
||||||
|
assertPluginTestExecutionOwnership(executeId);
|
||||||
return Result.ok(tinyFlowService.getChainStatus(executeId, nodes));
|
return Result.ok(tinyFlowService.getChainStatus(executeId, nodes));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,10 +237,33 @@ public class PluginItemController extends BaseCurdController<PluginItemService,
|
|||||||
@SaCheckPermission("/api/v1/plugin/query")
|
@SaCheckPermission("/api/v1/plugin/query")
|
||||||
public Result<Void> pluginToolTestResume(@JsonBody(value = "executeId", required = true) String executeId,
|
public Result<Void> pluginToolTestResume(@JsonBody(value = "executeId", required = true) String executeId,
|
||||||
@JsonBody("confirmParams") Map<String, Object> confirmParams) {
|
@JsonBody("confirmParams") Map<String, Object> confirmParams) {
|
||||||
chainExecutor.resumeAsync(executeId, confirmParams);
|
assertPluginTestExecutionOwnership(executeId);
|
||||||
|
workflowResumeService.resume(executeId, confirmParams);
|
||||||
return Result.ok();
|
return Result.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验插件试运行实例由当前登录用户发起。
|
||||||
|
*
|
||||||
|
* @param executeId 执行实例 ID
|
||||||
|
*/
|
||||||
|
private void assertPluginTestExecutionOwnership(String executeId) {
|
||||||
|
if (StrUtil.isBlank(executeId)) {
|
||||||
|
throw new BusinessException("执行ID不能为空");
|
||||||
|
}
|
||||||
|
WorkflowExecResult record = workflowExecResultService.getByExecKey(executeId);
|
||||||
|
if (record == null) {
|
||||||
|
throw new BusinessException(404, 404, "工作流执行记录不存在或已过期");
|
||||||
|
}
|
||||||
|
LoginAccount currentAccount = SaTokenUtil.getLoginAccount();
|
||||||
|
if (currentAccount == null
|
||||||
|
|| currentAccount.getId() == null
|
||||||
|
|| record.getCreatedBy() == null
|
||||||
|
|| !currentAccount.getId().toString().equals(record.getCreatedBy())) {
|
||||||
|
throw new BusinessException(403, 403, "无权限访问当前插件试运行实例");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void handleArray(JSONArray array) {
|
private void handleArray(JSONArray array) {
|
||||||
for (Object o : array) {
|
for (Object o : array) {
|
||||||
JSONObject obj = (JSONObject) o;
|
JSONObject obj = (JSONObject) o;
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
package tech.easyflow.admin.controller.ai;
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
import cn.hutool.core.io.IoUtil;
|
import cn.hutool.core.io.IoUtil;
|
||||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
|
||||||
import com.easyagents.core.store.DocumentStore;
|
|
||||||
import com.easyagents.core.store.StoreOptions;
|
|
||||||
import com.easyagents.core.store.StoreResult;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.query.QueryColumn;
|
import com.mybatisflex.core.query.QueryColumn;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
@@ -22,6 +18,11 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
|||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||||
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
|
import tech.easyflow.ai.documentimport.task.DocumentImportTaskStatusStreamService;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncRetryRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatusRequest;
|
||||||
import tech.easyflow.ai.dto.KnowledgeShareLimitedConfigRequest;
|
import tech.easyflow.ai.dto.KnowledgeShareLimitedConfigRequest;
|
||||||
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
|
import tech.easyflow.ai.dto.KnowledgeSearchResultItem;
|
||||||
import tech.easyflow.ai.entity.Document;
|
import tech.easyflow.ai.entity.Document;
|
||||||
@@ -43,7 +44,6 @@ import tech.easyflow.ai.service.KnowledgeEmbeddingService;
|
|||||||
import tech.easyflow.ai.service.KnowledgeShareAuditService;
|
import tech.easyflow.ai.service.KnowledgeShareAuditService;
|
||||||
import tech.easyflow.ai.service.KnowledgeShareService;
|
import tech.easyflow.ai.service.KnowledgeShareService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
|
||||||
import tech.easyflow.ai.vo.FaqImportResultVo;
|
import tech.easyflow.ai.vo.FaqImportResultVo;
|
||||||
import tech.easyflow.ai.vo.KnowledgeShareAuthContext;
|
import tech.easyflow.ai.vo.KnowledgeShareAuthContext;
|
||||||
import tech.easyflow.ai.vo.KnowledgeShareViewDetail;
|
import tech.easyflow.ai.vo.KnowledgeShareViewDetail;
|
||||||
@@ -62,7 +62,6 @@ import java.net.URLEncoder;
|
|||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
@@ -505,43 +504,27 @@ public class ShareKnowledgeController {
|
|||||||
@PostMapping("/documentChunk/update")
|
@PostMapping("/documentChunk/update")
|
||||||
public Result<?> updateDocumentChunk(
|
public Result<?> updateDocumentChunk(
|
||||||
@RequestParam String shareKey,
|
@RequestParam String shareKey,
|
||||||
@JsonBody DocumentChunk documentChunk
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkContentUpdateRequest request
|
||||||
) {
|
) {
|
||||||
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
|
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
|
||||||
shareKey,
|
shareKey,
|
||||||
null,
|
null,
|
||||||
KnowledgeShareActionScope.CONTENT_UPDATE.name()
|
KnowledgeShareActionScope.CONTENT_UPDATE.name()
|
||||||
);
|
);
|
||||||
DocumentChunk current = documentChunkService.getById(documentChunk.getId());
|
DocumentChunk current = documentChunkService.getById(request.getId());
|
||||||
if (current == null || current.getDocumentCollectionId() == null
|
if (current == null || current.getDocumentCollectionId() == null
|
||||||
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
||||||
throw new BusinessException("记录不存在");
|
throw new BusinessException("记录不存在");
|
||||||
}
|
}
|
||||||
boolean success = documentChunkService.updateById(documentChunk);
|
DocumentChunk updated = documentChunkService.updateContent(
|
||||||
if (success) {
|
context.getKnowledge().getId(),
|
||||||
DocumentStore documentStore = context.getKnowledge().toDocumentStore();
|
current.getId(),
|
||||||
if (documentStore == null) {
|
request.getContent()
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
);
|
||||||
}
|
|
||||||
try {
|
|
||||||
Model model = modelService.getModelInstance(context.getKnowledge().getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
|
||||||
documentStore.setEmbeddingModel(embeddingModel);
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(context.getKnowledge().getVectorStoreCollection());
|
|
||||||
com.easyagents.core.document.Document doc = com.easyagents.core.document.Document.of(documentChunk.getContent());
|
|
||||||
doc.setId(documentChunk.getId());
|
|
||||||
StoreResult result = documentStore.update(doc, options);
|
|
||||||
audit(context, "更新分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
audit(context, "更新分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
||||||
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", documentChunk.getId()));
|
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", request.getId()));
|
||||||
return Result.ok(result);
|
return Result.ok(updated);
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Result.ok(false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -562,25 +545,50 @@ public class ShareKnowledgeController {
|
|||||||
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
|| current.getDocumentCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
||||||
return Result.fail(1, "记录不存在");
|
return Result.fail(1, "记录不存在");
|
||||||
}
|
}
|
||||||
DocumentStore documentStore = context.getKnowledge().toDocumentStore();
|
DocumentChunkDeleteResult removed = documentChunkService.deleteChunk(
|
||||||
if (documentStore == null) {
|
context.getKnowledge().getId(),
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
chunkId
|
||||||
}
|
);
|
||||||
try {
|
|
||||||
Model model = modelService.getModelInstance(context.getKnowledge().getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
documentStore.setEmbeddingModel(model.toEmbeddingModel());
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(context.getKnowledge().getVectorStoreCollection());
|
|
||||||
documentStore.delete(Collections.singletonList(chunkId), options);
|
|
||||||
documentChunkService.removeById(chunkId);
|
|
||||||
audit(context, "删除分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
audit(context, "删除分享文档 Chunk", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
||||||
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", chunkId));
|
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", chunkId));
|
||||||
return Result.ok(true);
|
return Result.ok(removed);
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/documentChunk/syncStatus")
|
||||||
|
public Result<List<DocumentChunkSyncStatus>> documentChunkSyncStatus(
|
||||||
|
@RequestParam String shareKey,
|
||||||
|
@JsonBody DocumentChunkSyncStatusRequest request
|
||||||
|
) {
|
||||||
|
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
|
||||||
|
shareKey, null, KnowledgeShareActionScope.VIEW.name()
|
||||||
|
);
|
||||||
|
Document document = documentService.getById(request.getDocumentId());
|
||||||
|
if (document == null || document.getCollectionId() == null
|
||||||
|
|| document.getCollectionId().compareTo(context.getKnowledge().getId()) != 0) {
|
||||||
|
throw new BusinessException("文档不存在");
|
||||||
|
}
|
||||||
|
return Result.ok(documentChunkService.listIndexSyncStatus(
|
||||||
|
context.getKnowledge().getId(), request.getDocumentId(), request.getIds()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/documentChunk/retrySync")
|
||||||
|
public Result<DocumentChunk> retryDocumentChunkSync(
|
||||||
|
@RequestParam String shareKey,
|
||||||
|
@JsonBody DocumentChunkSyncRetryRequest request
|
||||||
|
) {
|
||||||
|
KnowledgeShareAuthContext context = knowledgeShareService.assertUrlShareAccess(
|
||||||
|
shareKey, null, KnowledgeShareActionScope.CONTENT_UPDATE.name()
|
||||||
|
);
|
||||||
|
if (request.getIndexSyncVersion() == null) {
|
||||||
|
throw new BusinessException("同步版本不能为空");
|
||||||
|
}
|
||||||
|
DocumentChunk chunk = documentChunkService.retryIndexSync(
|
||||||
|
context.getKnowledge().getId(), request.getId(), request.getIndexSyncVersion()
|
||||||
|
);
|
||||||
|
audit(context, "重试分享文档 Chunk 索引同步", "KNOWLEDGE_SHARE_URL_WRITE", true,
|
||||||
|
auditDetail("knowledgeId", context.getKnowledge().getId(), "chunkId", request.getId()));
|
||||||
|
return Result.ok(chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
package tech.easyflow.admin.controller.ai;
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
import com.easyagents.flow.core.chain.ChainStatus;
|
|
||||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
@@ -14,6 +13,7 @@ import tech.easyflow.admin.service.ai.WorkflowChatEventStream;
|
|||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.entity.WorkflowExecResult;
|
import tech.easyflow.ai.entity.WorkflowExecResult;
|
||||||
@@ -64,6 +64,8 @@ public class WorkflowChatController {
|
|||||||
@Resource
|
@Resource
|
||||||
private ChainExecutor chainExecutor;
|
private ChainExecutor chainExecutor;
|
||||||
@Resource
|
@Resource
|
||||||
|
private WorkflowResumeService workflowResumeService;
|
||||||
|
@Resource
|
||||||
private WorkflowExecResultService execResultService;
|
private WorkflowExecResultService execResultService;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowExecStepService execStepService;
|
private WorkflowExecStepService execStepService;
|
||||||
@@ -171,19 +173,8 @@ public class WorkflowChatController {
|
|||||||
@JsonBody("confirmParams")
|
@JsonBody("confirmParams")
|
||||||
Map<String, Object> confirmParams
|
Map<String, Object> confirmParams
|
||||||
) {
|
) {
|
||||||
WorkflowExecResult record = assertExecutionOwnership(executeId);
|
assertExecutionOwnership(executeId);
|
||||||
if (record.getStatus() != null
|
workflowResumeService.resume(executeId, confirmParams);
|
||||||
&& (record.getStatus() == ChainStatus.SUCCEEDED.getValue()
|
|
||||||
|| record.getStatus() == ChainStatus.FAILED.getValue()
|
|
||||||
|| record.getStatus() == ChainStatus.CANCELLED.getValue())) {
|
|
||||||
throw new BusinessException("当前工作流执行已结束");
|
|
||||||
}
|
|
||||||
chainExecutor.resumeAsync(
|
|
||||||
executeId,
|
|
||||||
confirmParams == null
|
|
||||||
? new LinkedHashMap<>()
|
|
||||||
: new LinkedHashMap<>(confirmParams)
|
|
||||||
);
|
|
||||||
return Result.ok();
|
return Result.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
|||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.enums.PublishStatus;
|
import tech.easyflow.ai.enums.PublishStatus;
|
||||||
import tech.easyflow.ai.publish.WorkflowPublishAppService;
|
import tech.easyflow.ai.publish.WorkflowPublishAppService;
|
||||||
@@ -94,6 +95,8 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
|
|||||||
@Resource
|
@Resource
|
||||||
private WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
private WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
||||||
@Resource
|
@Resource
|
||||||
|
private WorkflowResumeService workflowResumeService;
|
||||||
|
@Resource
|
||||||
private ResourceAccessService resourceAccessService;
|
private ResourceAccessService resourceAccessService;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
||||||
@@ -324,12 +327,7 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
|
|||||||
)
|
)
|
||||||
public Result<Void> resume(@JsonBody(value = "executeId", required = true) String executeId,
|
public Result<Void> resume(@JsonBody(value = "executeId", required = true) String executeId,
|
||||||
@JsonBody("confirmParams") Map<String, Object> confirmParams) {
|
@JsonBody("confirmParams") Map<String, Object> confirmParams) {
|
||||||
if (!chainExecutor.resumeAsyncIfSuspended(executeId, confirmParams)) {
|
workflowResumeService.resume(executeId, confirmParams);
|
||||||
throw new BusinessException(
|
|
||||||
409,
|
|
||||||
40901,
|
|
||||||
"当前执行状态不可恢复,仅暂停中的工作流允许恢复");
|
|
||||||
}
|
|
||||||
return Result.ok();
|
return Result.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -52,11 +52,17 @@ public record WorkflowDesignerOptionsView(
|
|||||||
* @param id 知识库 ID
|
* @param id 知识库 ID
|
||||||
* @param title 知识库标题
|
* @param title 知识库标题
|
||||||
* @param description 知识库描述
|
* @param description 知识库描述
|
||||||
|
* @param vectorEmbedModelId Embedding 模型 ID
|
||||||
|
* @param dimensionOfVectorModel 向量维度
|
||||||
|
* @param vectorStoreEnabled 是否可用于向量检索
|
||||||
*/
|
*/
|
||||||
public record KnowledgeOption(
|
public record KnowledgeOption(
|
||||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger id,
|
@JsonSerialize(using = ToStringSerializer.class) BigInteger id,
|
||||||
String title,
|
String title,
|
||||||
String description
|
String description,
|
||||||
|
@JsonSerialize(using = ToStringSerializer.class) BigInteger vectorEmbedModelId,
|
||||||
|
Integer dimensionOfVectorModel,
|
||||||
|
Boolean vectorStoreEnabled
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import com.mybatisflex.core.query.QueryWrapper;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import tech.easyflow.admin.model.ai.WorkflowDesignerOptionsView;
|
import tech.easyflow.admin.model.ai.WorkflowDesignerOptionsView;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.knowledge.WorkflowKnowledgeContractService;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.entity.Model;
|
import tech.easyflow.ai.entity.Model;
|
||||||
import tech.easyflow.ai.entity.ModelProvider;
|
import tech.easyflow.ai.entity.ModelProvider;
|
||||||
@@ -75,6 +76,7 @@ public class WorkflowDesignerOptionService {
|
|||||||
private final DatacenterSourceService datacenterSourceService;
|
private final DatacenterSourceService datacenterSourceService;
|
||||||
private final DatacenterDatasetRegistryService datacenterDatasetRegistryService;
|
private final DatacenterDatasetRegistryService datacenterDatasetRegistryService;
|
||||||
private final DatacenterDatasetQueryService datacenterDatasetQueryService;
|
private final DatacenterDatasetQueryService datacenterDatasetQueryService;
|
||||||
|
private final WorkflowKnowledgeContractService workflowKnowledgeContractService;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建工作流设计器选项服务。
|
* 创建工作流设计器选项服务。
|
||||||
@@ -93,6 +95,7 @@ public class WorkflowDesignerOptionService {
|
|||||||
* @param datacenterSourceService 数据源服务
|
* @param datacenterSourceService 数据源服务
|
||||||
* @param datacenterDatasetRegistryService 数据集注册服务
|
* @param datacenterDatasetRegistryService 数据集注册服务
|
||||||
* @param datacenterDatasetQueryService 数据集查询服务
|
* @param datacenterDatasetQueryService 数据集查询服务
|
||||||
|
* @param workflowKnowledgeContractService 工作流知识库契约服务
|
||||||
*/
|
*/
|
||||||
public WorkflowDesignerOptionService(
|
public WorkflowDesignerOptionService(
|
||||||
ModelService modelService,
|
ModelService modelService,
|
||||||
@@ -108,7 +111,8 @@ public class WorkflowDesignerOptionService {
|
|||||||
ResourceAccessService resourceAccessService,
|
ResourceAccessService resourceAccessService,
|
||||||
DatacenterSourceService datacenterSourceService,
|
DatacenterSourceService datacenterSourceService,
|
||||||
DatacenterDatasetRegistryService datacenterDatasetRegistryService,
|
DatacenterDatasetRegistryService datacenterDatasetRegistryService,
|
||||||
DatacenterDatasetQueryService datacenterDatasetQueryService) {
|
DatacenterDatasetQueryService datacenterDatasetQueryService,
|
||||||
|
WorkflowKnowledgeContractService workflowKnowledgeContractService) {
|
||||||
this.modelService = modelService;
|
this.modelService = modelService;
|
||||||
this.documentCollectionService = documentCollectionService;
|
this.documentCollectionService = documentCollectionService;
|
||||||
this.pluginService = pluginService;
|
this.pluginService = pluginService;
|
||||||
@@ -123,6 +127,7 @@ public class WorkflowDesignerOptionService {
|
|||||||
this.datacenterSourceService = datacenterSourceService;
|
this.datacenterSourceService = datacenterSourceService;
|
||||||
this.datacenterDatasetRegistryService = datacenterDatasetRegistryService;
|
this.datacenterDatasetRegistryService = datacenterDatasetRegistryService;
|
||||||
this.datacenterDatasetQueryService = datacenterDatasetQueryService;
|
this.datacenterDatasetQueryService = datacenterDatasetQueryService;
|
||||||
|
this.workflowKnowledgeContractService = workflowKnowledgeContractService;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -163,6 +168,7 @@ public class WorkflowDesignerOptionService {
|
|||||||
LoginAccount account = requireAccount();
|
LoginAccount account = requireAccount();
|
||||||
Set<BigInteger> modelIds = new HashSet<>();
|
Set<BigInteger> modelIds = new HashSet<>();
|
||||||
Set<BigInteger> knowledgeIds = new HashSet<>();
|
Set<BigInteger> knowledgeIds = new HashSet<>();
|
||||||
|
List<List<BigInteger>> knowledgeGroups = new ArrayList<>();
|
||||||
Set<BigInteger> checkedPluginItemIds = new HashSet<>();
|
Set<BigInteger> checkedPluginItemIds = new HashSet<>();
|
||||||
Set<BigInteger> checkedWorkflowIds = new HashSet<>();
|
Set<BigInteger> checkedWorkflowIds = new HashSet<>();
|
||||||
Set<BigInteger> checkedSourceIds = new HashSet<>();
|
Set<BigInteger> checkedSourceIds = new HashSet<>();
|
||||||
@@ -177,14 +183,22 @@ public class WorkflowDesignerOptionService {
|
|||||||
if (data == null) {
|
if (data == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
String nodeType = data.getString("type");
|
String nodeType = node.getString("type");
|
||||||
|
String dataType = data.getString("type");
|
||||||
|
if (nodeType != null && !nodeType.isBlank()
|
||||||
|
&& dataType != null && !dataType.isBlank()
|
||||||
|
&& !Objects.equals(nodeType, dataType)) {
|
||||||
|
throw new BusinessException("工作流节点类型与节点数据类型不一致");
|
||||||
|
}
|
||||||
if (nodeType == null || nodeType.isBlank()) {
|
if (nodeType == null || nodeType.isBlank()) {
|
||||||
nodeType = node.getString("type");
|
nodeType = dataType;
|
||||||
}
|
}
|
||||||
if ("llmNode".equals(nodeType)) {
|
if ("llmNode".equals(nodeType)) {
|
||||||
addReferenceId(modelIds, readReferenceId(data, "llmId", "模型"));
|
addReferenceId(modelIds, readReferenceId(data, "llmId", "模型"));
|
||||||
} else if ("knowledgeNode".equals(nodeType)) {
|
} else if ("knowledgeNode".equals(nodeType)) {
|
||||||
addReferenceId(knowledgeIds, readReferenceId(data, "knowledgeId", "知识库"));
|
List<BigInteger> nodeKnowledgeIds = readKnowledgeReferenceIds(data);
|
||||||
|
knowledgeIds.addAll(nodeKnowledgeIds);
|
||||||
|
knowledgeGroups.add(nodeKnowledgeIds);
|
||||||
} else if ("plugin-node".equals(nodeType)) {
|
} else if ("plugin-node".equals(nodeType)) {
|
||||||
BigInteger pluginItemId = readReferenceId(data, "pluginId", "插件工具");
|
BigInteger pluginItemId = readReferenceId(data, "pluginId", "插件工具");
|
||||||
if (pluginItemId != null && checkedPluginItemIds.add(pluginItemId)) {
|
if (pluginItemId != null && checkedPluginItemIds.add(pluginItemId)) {
|
||||||
@@ -198,6 +212,8 @@ public class WorkflowDesignerOptionService {
|
|||||||
}
|
}
|
||||||
assertModelReferences(modelIds, account);
|
assertModelReferences(modelIds, account);
|
||||||
assertKnowledgeReferences(knowledgeIds, account);
|
assertKnowledgeReferences(knowledgeIds, account);
|
||||||
|
workflowKnowledgeContractService.assertMultiKnowledgeContracts(
|
||||||
|
knowledgeGroups, account.getTenantId());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -460,17 +476,55 @@ public class WorkflowDesignerOptionService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private List<WorkflowDesignerOptionsView.KnowledgeOption> listKnowledgeOptions(LoginAccount account) {
|
private List<WorkflowDesignerOptionsView.KnowledgeOption> listKnowledgeOptions(LoginAccount account) {
|
||||||
return documentCollectionService.list(QueryWrapper.create()
|
List<DocumentCollection> collections = documentCollectionService.list(QueryWrapper.create()
|
||||||
.eq(DocumentCollection::getTenantId, account.getTenantId())
|
.eq(DocumentCollection::getTenantId, account.getTenantId())
|
||||||
.orderBy(DocumentCollection::getModified, false))
|
.orderBy(DocumentCollection::getModified, false));
|
||||||
.stream()
|
Set<BigInteger> vectorReadyIds = workflowKnowledgeContractService
|
||||||
|
.findVectorReadyKnowledgeIds(collections, account.getTenantId());
|
||||||
|
return collections.stream()
|
||||||
.filter(item -> resourceAccessService.canAccess(
|
.filter(item -> resourceAccessService.canAccess(
|
||||||
account, CategoryResourceType.KNOWLEDGE, item, ResourceAction.USE))
|
account, CategoryResourceType.KNOWLEDGE, item, ResourceAction.USE))
|
||||||
.map(item -> new WorkflowDesignerOptionsView.KnowledgeOption(
|
.map(item -> new WorkflowDesignerOptionsView.KnowledgeOption(
|
||||||
item.getId(), item.getTitle(), item.getDescription()))
|
item.getId(),
|
||||||
|
item.getTitle(),
|
||||||
|
item.getDescription(),
|
||||||
|
item.getVectorEmbedModelId(),
|
||||||
|
item.getDimensionOfVectorModel(),
|
||||||
|
vectorReadyIds.contains(item.getId())))
|
||||||
.toList();
|
.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private List<BigInteger> readKnowledgeReferenceIds(JSONObject data) {
|
||||||
|
if (data.containsKey("knowledgeIds")) {
|
||||||
|
Object rawIds = data.get("knowledgeIds");
|
||||||
|
if (!(rawIds instanceof JSONArray ids) || ids.isEmpty()) {
|
||||||
|
throw new BusinessException("知识库节点至少需要选择一个知识库");
|
||||||
|
}
|
||||||
|
List<BigInteger> result = new ArrayList<>();
|
||||||
|
for (Object id : ids) {
|
||||||
|
BigInteger parsed = parseReferenceId(id, "知识库");
|
||||||
|
if (result.contains(parsed)) {
|
||||||
|
throw new BusinessException("知识库节点不能重复选择同一知识库");
|
||||||
|
}
|
||||||
|
result.add(parsed);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
BigInteger legacyId = readReferenceId(data, "knowledgeId", "知识库");
|
||||||
|
return legacyId == null ? List.of() : List.of(legacyId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigInteger parseReferenceId(Object value, String resourceName) {
|
||||||
|
if (value == null || String.valueOf(value).isBlank()) {
|
||||||
|
throw new BusinessException(resourceName + "ID不能为空");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return new BigInteger(String.valueOf(value));
|
||||||
|
} catch (NumberFormatException exception) {
|
||||||
|
throw new BusinessException(resourceName + "ID无效");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void addReferenceId(Set<BigInteger> resourceIds, BigInteger resourceId) {
|
private void addReferenceId(Set<BigInteger> resourceIds, BigInteger resourceId) {
|
||||||
if (resourceId != null) {
|
if (resourceId != null) {
|
||||||
resourceIds.add(resourceId);
|
resourceIds.add(resourceId);
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
|||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||||
import tech.easyflow.ai.entity.WorkflowExecResult;
|
import tech.easyflow.ai.entity.WorkflowExecResult;
|
||||||
import tech.easyflow.ai.entity.WorkflowExecStep;
|
import tech.easyflow.ai.entity.WorkflowExecStep;
|
||||||
@@ -46,6 +47,7 @@ public class WorkflowPublicChatService {
|
|||||||
private final WorkflowPublicChatAccessGuard accessGuard;
|
private final WorkflowPublicChatAccessGuard accessGuard;
|
||||||
private final WorkflowChatEventStream eventStream;
|
private final WorkflowChatEventStream eventStream;
|
||||||
private final ChainExecutor chainExecutor;
|
private final ChainExecutor chainExecutor;
|
||||||
|
private final WorkflowResumeService workflowResumeService;
|
||||||
private final WorkflowExecResultService execResultService;
|
private final WorkflowExecResultService execResultService;
|
||||||
private final WorkflowExecStepService execStepService;
|
private final WorkflowExecStepService execStepService;
|
||||||
|
|
||||||
@@ -57,6 +59,7 @@ public class WorkflowPublicChatService {
|
|||||||
WorkflowPublicChatAccessGuard accessGuard,
|
WorkflowPublicChatAccessGuard accessGuard,
|
||||||
WorkflowChatEventStream eventStream,
|
WorkflowChatEventStream eventStream,
|
||||||
ChainExecutor chainExecutor,
|
ChainExecutor chainExecutor,
|
||||||
|
WorkflowResumeService workflowResumeService,
|
||||||
WorkflowExecResultService execResultService,
|
WorkflowExecResultService execResultService,
|
||||||
WorkflowExecStepService execStepService
|
WorkflowExecStepService execStepService
|
||||||
) {
|
) {
|
||||||
@@ -67,6 +70,7 @@ public class WorkflowPublicChatService {
|
|||||||
this.accessGuard = accessGuard;
|
this.accessGuard = accessGuard;
|
||||||
this.eventStream = eventStream;
|
this.eventStream = eventStream;
|
||||||
this.chainExecutor = chainExecutor;
|
this.chainExecutor = chainExecutor;
|
||||||
|
this.workflowResumeService = workflowResumeService;
|
||||||
this.execResultService = execResultService;
|
this.execResultService = execResultService;
|
||||||
this.execStepService = execStepService;
|
this.execStepService = execStepService;
|
||||||
}
|
}
|
||||||
@@ -185,17 +189,8 @@ public class WorkflowPublicChatService {
|
|||||||
) {
|
) {
|
||||||
WorkflowPublicChatContext context = contextResolver.resolveActive(
|
WorkflowPublicChatContext context = contextResolver.resolveActive(
|
||||||
shareKey, visitorId);
|
shareKey, visitorId);
|
||||||
WorkflowExecResult record = assertExecutionOwnership(
|
assertExecutionOwnership(context, executeId);
|
||||||
context, executeId);
|
workflowResumeService.resume(executeId, confirmParams);
|
||||||
if (isTerminal(record.getStatus())) {
|
|
||||||
throw new BusinessException("当前工作流执行已结束");
|
|
||||||
}
|
|
||||||
chainExecutor.resumeAsync(
|
|
||||||
executeId,
|
|
||||||
confirmParams == null
|
|
||||||
? new LinkedHashMap<>()
|
|
||||||
: new LinkedHashMap<>(confirmParams)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -253,13 +248,6 @@ public class WorkflowPublicChatService {
|
|||||||
return record;
|
return record;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isTerminal(Integer status) {
|
|
||||||
return status != null
|
|
||||||
&& (status == ChainStatus.SUCCEEDED.getValue()
|
|
||||||
|| status == ChainStatus.FAILED.getValue()
|
|
||||||
|| status == ChainStatus.CANCELLED.getValue());
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, Object> buildExecutionDetail(
|
private Map<String, Object> buildExecutionDetail(
|
||||||
WorkflowExecResult record,
|
WorkflowExecResult record,
|
||||||
List<WorkflowExecStep> steps,
|
List<WorkflowExecStep> steps,
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
|
import org.testng.Assert;
|
||||||
|
import org.testng.annotations.Test;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkContentUpdateRequest;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
|
import tech.easyflow.common.web.controller.BaseController;
|
||||||
|
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||||
|
import tech.easyflow.system.permission.resource.RequireResourceAccess;
|
||||||
|
|
||||||
|
import java.lang.reflect.Field;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.lang.reflect.Parameter;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文档分块维护接口契约测试。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkControllerContractTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void updateEndpointsShouldOnlyAcceptChunkIdAndContent() throws Exception {
|
||||||
|
Method adminUpdate = DocumentChunkController.class.getDeclaredMethod(
|
||||||
|
"update",
|
||||||
|
DocumentChunkContentUpdateRequest.class
|
||||||
|
);
|
||||||
|
Method shareUpdate = ShareKnowledgeController.class.getDeclaredMethod(
|
||||||
|
"updateDocumentChunk",
|
||||||
|
String.class,
|
||||||
|
DocumentChunkContentUpdateRequest.class
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertNotNull(adminUpdate);
|
||||||
|
Assert.assertNotNull(shareUpdate);
|
||||||
|
assertStrictJsonBody(adminUpdate.getParameters()[0]);
|
||||||
|
assertStrictJsonBody(shareUpdate.getParameters()[1]);
|
||||||
|
Assert.assertEquals(
|
||||||
|
adminUpdate.getAnnotation(RequireResourceAccess.class).idExpr(),
|
||||||
|
"#request.id"
|
||||||
|
);
|
||||||
|
Set<String> fields = Arrays.stream(
|
||||||
|
DocumentChunkContentUpdateRequest.class.getDeclaredFields()
|
||||||
|
)
|
||||||
|
.map(Field::getName)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
Assert.assertEquals(Set.of("id", "content"), fields);
|
||||||
|
Assert.assertEquals(
|
||||||
|
DocumentChunkController.class.getMethod(
|
||||||
|
"update",
|
||||||
|
DocumentChunkContentUpdateRequest.class
|
||||||
|
).getDeclaringClass(),
|
||||||
|
DocumentChunkController.class
|
||||||
|
);
|
||||||
|
Assert.assertEquals(
|
||||||
|
Arrays.stream(DocumentChunkController.class.getDeclaredMethods())
|
||||||
|
.filter(method -> method.getName().equals("update"))
|
||||||
|
.filter(method -> !method.isBridge() && !method.isSynthetic())
|
||||||
|
.count(),
|
||||||
|
1L
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void controllerShouldNotExposeGenericWriteEndpoints() {
|
||||||
|
Assert.assertEquals(DocumentChunkController.class.getSuperclass(), BaseController.class);
|
||||||
|
Set<String> postMappings = Arrays.stream(DocumentChunkController.class.getMethods())
|
||||||
|
.map(method -> method.getAnnotation(PostMapping.class))
|
||||||
|
.filter(annotation -> annotation != null)
|
||||||
|
.flatMap(annotation -> Arrays.stream(annotation.value()))
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
|
Assert.assertEquals(
|
||||||
|
postMappings,
|
||||||
|
Set.of("update", "removeChunk", "syncStatus", "retrySync")
|
||||||
|
);
|
||||||
|
Assert.assertFalse(postMappings.contains("save"));
|
||||||
|
Assert.assertFalse(postMappings.contains("remove"));
|
||||||
|
Assert.assertFalse(postMappings.contains("removeBatch"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void adminUpdateAndDeleteShouldUseUnifiedMaintenanceService() {
|
||||||
|
DocumentChunkService service = mock(DocumentChunkService.class);
|
||||||
|
DocumentChunkController controller = new DocumentChunkController(service);
|
||||||
|
|
||||||
|
DocumentChunk current = new DocumentChunk();
|
||||||
|
current.setId(BigInteger.ONE);
|
||||||
|
current.setDocumentCollectionId(BigInteger.TWO);
|
||||||
|
when(service.getById(BigInteger.ONE)).thenReturn(current);
|
||||||
|
|
||||||
|
DocumentChunkContentUpdateRequest request = new DocumentChunkContentUpdateRequest();
|
||||||
|
request.setId(BigInteger.ONE);
|
||||||
|
request.setContent("updated");
|
||||||
|
controller.update(request);
|
||||||
|
controller.removeChunk(BigInteger.ONE);
|
||||||
|
|
||||||
|
verify(service).updateContent(BigInteger.TWO, BigInteger.ONE, "updated");
|
||||||
|
verify(service).deleteChunk(BigInteger.TWO, BigInteger.ONE);
|
||||||
|
verify(service, never()).updateById(any(DocumentChunk.class));
|
||||||
|
verify(service, never()).removeById(any());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void assertStrictJsonBody(Parameter parameter) {
|
||||||
|
JsonBody jsonBody = parameter.getAnnotation(JsonBody.class);
|
||||||
|
Assert.assertNotNull(jsonBody);
|
||||||
|
Assert.assertTrue(jsonBody.required());
|
||||||
|
Assert.assertFalse(jsonBody.skipConvertError());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,21 +7,27 @@ import org.testng.Assert;
|
|||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
import tech.easyflow.ai.entity.Plugin;
|
import tech.easyflow.ai.entity.Plugin;
|
||||||
import tech.easyflow.ai.entity.PluginItem;
|
import tech.easyflow.ai.entity.PluginItem;
|
||||||
|
import tech.easyflow.ai.entity.WorkflowExecResult;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.service.AgentResourceReferenceService;
|
import tech.easyflow.ai.service.AgentResourceReferenceService;
|
||||||
import tech.easyflow.ai.service.PluginItemService;
|
import tech.easyflow.ai.service.PluginItemService;
|
||||||
import tech.easyflow.ai.service.PluginService;
|
import tech.easyflow.ai.service.PluginService;
|
||||||
import tech.easyflow.ai.service.PluginVisibilityService;
|
import tech.easyflow.ai.service.PluginVisibilityService;
|
||||||
|
import tech.easyflow.ai.service.WorkflowExecResultService;
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.mockStatic;
|
import static org.mockito.Mockito.mockStatic;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.verifyNoInteractions;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -68,6 +74,61 @@ public class PluginItemControllerTest {
|
|||||||
verify(visibilityService).assertPluginVisible(1L, BigInteger.TEN, "无权限删除该插件工具");
|
verify(visibilityService).assertPluginVisible(1L, BigInteger.TEN, "无权限删除该插件工具");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证当前用户不能恢复其他用户发起的插件试运行实例。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testResumeShouldRejectAnotherUsersExecution() {
|
||||||
|
PluginItemService pluginItemService = mock(PluginItemService.class);
|
||||||
|
WorkflowExecResultService execResultService = mock(WorkflowExecResultService.class);
|
||||||
|
WorkflowResumeService resumeService = mock(WorkflowResumeService.class);
|
||||||
|
WorkflowExecResult record = new WorkflowExecResult();
|
||||||
|
record.setCreatedBy(BigInteger.ONE.toString());
|
||||||
|
when(execResultService.getByExecKey("execution-1")).thenReturn(record);
|
||||||
|
|
||||||
|
PluginItemController controller = new PluginItemController(pluginItemService);
|
||||||
|
setField(controller, "workflowExecResultService", execResultService);
|
||||||
|
setField(controller, "workflowResumeService", resumeService);
|
||||||
|
LoginAccount currentAccount = new LoginAccount();
|
||||||
|
currentAccount.setId(BigInteger.TWO);
|
||||||
|
|
||||||
|
try (MockedStatic<SaTokenUtil> login = mockStatic(SaTokenUtil.class)) {
|
||||||
|
login.when(SaTokenUtil::getLoginAccount).thenReturn(currentAccount);
|
||||||
|
BusinessException error = Assert.expectThrows(
|
||||||
|
BusinessException.class,
|
||||||
|
() -> controller.pluginToolTestResume("execution-1", Map.of())
|
||||||
|
);
|
||||||
|
Assert.assertEquals(error.getHttpStatus(), 403);
|
||||||
|
Assert.assertEquals(error.getErrorCode(), 403);
|
||||||
|
}
|
||||||
|
verifyNoInteractions(resumeService);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证当前用户可以恢复自己发起的插件试运行实例。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testResumeShouldAllowExecutionOwner() {
|
||||||
|
PluginItemService pluginItemService = mock(PluginItemService.class);
|
||||||
|
WorkflowExecResultService execResultService = mock(WorkflowExecResultService.class);
|
||||||
|
WorkflowResumeService resumeService = mock(WorkflowResumeService.class);
|
||||||
|
WorkflowExecResult record = new WorkflowExecResult();
|
||||||
|
record.setCreatedBy(BigInteger.ONE.toString());
|
||||||
|
when(execResultService.getByExecKey("execution-1")).thenReturn(record);
|
||||||
|
|
||||||
|
PluginItemController controller = new PluginItemController(pluginItemService);
|
||||||
|
setField(controller, "workflowExecResultService", execResultService);
|
||||||
|
setField(controller, "workflowResumeService", resumeService);
|
||||||
|
LoginAccount currentAccount = new LoginAccount();
|
||||||
|
currentAccount.setId(BigInteger.ONE);
|
||||||
|
|
||||||
|
try (MockedStatic<SaTokenUtil> login = mockStatic(SaTokenUtil.class)) {
|
||||||
|
login.when(SaTokenUtil::getLoginAccount).thenReturn(currentAccount);
|
||||||
|
controller.pluginToolTestResume("execution-1", Map.of("choice", "A"));
|
||||||
|
}
|
||||||
|
verify(resumeService).resume("execution-1", Map.of("choice", "A"));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建插件工具。
|
* 创建插件工具。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -6,7 +6,10 @@ import org.mockito.MockedStatic;
|
|||||||
import org.testng.Assert;
|
import org.testng.Assert;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.knowledge.WorkflowKnowledgeContractService;
|
||||||
import tech.easyflow.ai.entity.Model;
|
import tech.easyflow.ai.entity.Model;
|
||||||
|
import tech.easyflow.ai.entity.ModelProvider;
|
||||||
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.plugin.workflow.snapshot.WorkflowPluginSnapshotResolver;
|
import tech.easyflow.ai.plugin.workflow.snapshot.WorkflowPluginSnapshotResolver;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
@@ -129,6 +132,96 @@ public class WorkflowDesignerOptionServiceTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldAcceptCompatibleMultiKnowledgeReferences() {
|
||||||
|
ModelService modelService = mock(ModelService.class);
|
||||||
|
DocumentCollectionService knowledgeService =
|
||||||
|
mock(DocumentCollectionService.class);
|
||||||
|
ResourceAccessService accessService = mock(ResourceAccessService.class);
|
||||||
|
when(accessService.canAccess(any(), any(), any(), any()))
|
||||||
|
.thenReturn(true);
|
||||||
|
when(knowledgeService.listByIds(any()))
|
||||||
|
.thenReturn(List.of(
|
||||||
|
knowledge(1, 7, 3),
|
||||||
|
knowledge(2, 7, 3)));
|
||||||
|
when(modelService.listModelInstances(any()))
|
||||||
|
.thenReturn(List.of(embeddingModel(7)));
|
||||||
|
WorkflowDesignerOptionService service = createService(
|
||||||
|
modelService,
|
||||||
|
knowledgeService,
|
||||||
|
mock(DatacenterSourceService.class),
|
||||||
|
mock(WorkflowService.class),
|
||||||
|
accessService);
|
||||||
|
|
||||||
|
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
||||||
|
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(loginAccount());
|
||||||
|
|
||||||
|
service.assertContentReferences("""
|
||||||
|
{"nodes":[{"type":"knowledgeNode","data":{
|
||||||
|
"knowledgeIds":["1","2"],"retrievalMode":"VECTOR"
|
||||||
|
}}]}
|
||||||
|
""");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldRejectIncompatibleMultiKnowledgeReferences() {
|
||||||
|
ModelService modelService = mock(ModelService.class);
|
||||||
|
DocumentCollectionService knowledgeService =
|
||||||
|
mock(DocumentCollectionService.class);
|
||||||
|
ResourceAccessService accessService = mock(ResourceAccessService.class);
|
||||||
|
when(accessService.canAccess(any(), any(), any(), any()))
|
||||||
|
.thenReturn(true);
|
||||||
|
when(knowledgeService.listByIds(any()))
|
||||||
|
.thenReturn(List.of(
|
||||||
|
knowledge(1, 7, 3),
|
||||||
|
knowledge(2, 8, 3)));
|
||||||
|
when(modelService.listModelInstances(any()))
|
||||||
|
.thenReturn(List.of(embeddingModel(7), embeddingModel(8)));
|
||||||
|
WorkflowDesignerOptionService service = createService(
|
||||||
|
modelService,
|
||||||
|
knowledgeService,
|
||||||
|
mock(DatacenterSourceService.class),
|
||||||
|
mock(WorkflowService.class),
|
||||||
|
accessService);
|
||||||
|
|
||||||
|
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
||||||
|
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(loginAccount());
|
||||||
|
|
||||||
|
BusinessException exception = Assert.expectThrows(
|
||||||
|
BusinessException.class,
|
||||||
|
() -> service.assertContentReferences("""
|
||||||
|
{"nodes":[{"type":"knowledgeNode","data":{
|
||||||
|
"knowledgeIds":["1","2"],"retrievalMode":"VECTOR"
|
||||||
|
}}]}
|
||||||
|
"""));
|
||||||
|
|
||||||
|
Assert.assertTrue(exception.getMessage().contains("Embedding"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void shouldRejectConflictingRootAndDataNodeTypes() {
|
||||||
|
WorkflowDesignerOptionService service = createService(
|
||||||
|
mock(ModelService.class),
|
||||||
|
mock(DocumentCollectionService.class),
|
||||||
|
mock(DatacenterSourceService.class));
|
||||||
|
|
||||||
|
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
||||||
|
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(loginAccount());
|
||||||
|
|
||||||
|
BusinessException exception = Assert.expectThrows(
|
||||||
|
BusinessException.class,
|
||||||
|
() -> service.assertContentReferences("""
|
||||||
|
{"nodes":[{"type":"knowledgeNode","data":{
|
||||||
|
"type":"llmNode","knowledgeId":"1"
|
||||||
|
}}]}
|
||||||
|
"""));
|
||||||
|
|
||||||
|
Assert.assertTrue(exception.getMessage().contains("类型"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private WorkflowDesignerOptionService createService(
|
private WorkflowDesignerOptionService createService(
|
||||||
ModelService modelService,
|
ModelService modelService,
|
||||||
DocumentCollectionService knowledgeService,
|
DocumentCollectionService knowledgeService,
|
||||||
@@ -142,6 +235,20 @@ public class WorkflowDesignerOptionServiceTest {
|
|||||||
DatacenterSourceService sourceService,
|
DatacenterSourceService sourceService,
|
||||||
WorkflowService workflowService) {
|
WorkflowService workflowService) {
|
||||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||||
|
return createService(
|
||||||
|
modelService,
|
||||||
|
knowledgeService,
|
||||||
|
sourceService,
|
||||||
|
workflowService,
|
||||||
|
resourceAccessService);
|
||||||
|
}
|
||||||
|
|
||||||
|
private WorkflowDesignerOptionService createService(
|
||||||
|
ModelService modelService,
|
||||||
|
DocumentCollectionService knowledgeService,
|
||||||
|
DatacenterSourceService sourceService,
|
||||||
|
WorkflowService workflowService,
|
||||||
|
ResourceAccessService resourceAccessService) {
|
||||||
return new WorkflowDesignerOptionService(
|
return new WorkflowDesignerOptionService(
|
||||||
modelService,
|
modelService,
|
||||||
knowledgeService,
|
knowledgeService,
|
||||||
@@ -156,10 +263,40 @@ public class WorkflowDesignerOptionServiceTest {
|
|||||||
resourceAccessService,
|
resourceAccessService,
|
||||||
sourceService,
|
sourceService,
|
||||||
mock(DatacenterDatasetRegistryService.class),
|
mock(DatacenterDatasetRegistryService.class),
|
||||||
mock(DatacenterDatasetQueryService.class)
|
mock(DatacenterDatasetQueryService.class),
|
||||||
|
new WorkflowKnowledgeContractService(
|
||||||
|
knowledgeService, modelService)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private DocumentCollection knowledge(
|
||||||
|
long id, long embeddingModelId, int dimension) {
|
||||||
|
DocumentCollection collection = new DocumentCollection();
|
||||||
|
collection.setId(BigInteger.valueOf(id));
|
||||||
|
collection.setTenantId(BigInteger.valueOf(100));
|
||||||
|
collection.setVectorEmbedModelId(BigInteger.valueOf(embeddingModelId));
|
||||||
|
collection.setDimensionOfVectorModel(dimension);
|
||||||
|
collection.setVectorStoreEnable(true);
|
||||||
|
collection.setVectorStoreCollection("collection_" + id);
|
||||||
|
return collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Model embeddingModel(long id) {
|
||||||
|
Model model = new Model();
|
||||||
|
model.setId(BigInteger.valueOf(id));
|
||||||
|
model.setTenantId(BigInteger.valueOf(100));
|
||||||
|
model.setModelType(Model.MODEL_TYPES[1]);
|
||||||
|
model.setProviderId(BigInteger.ONE);
|
||||||
|
model.setModelName("embedding-" + id);
|
||||||
|
model.setEndpoint("https://embedding.example");
|
||||||
|
model.setRequestPath("/v1/embeddings");
|
||||||
|
ModelProvider provider = new ModelProvider();
|
||||||
|
provider.setId(BigInteger.ONE);
|
||||||
|
provider.setProviderType("openai");
|
||||||
|
model.setModelProvider(provider);
|
||||||
|
return model;
|
||||||
|
}
|
||||||
|
|
||||||
private LoginAccount loginAccount() {
|
private LoginAccount loginAccount() {
|
||||||
LoginAccount account = new LoginAccount();
|
LoginAccount account = new LoginAccount();
|
||||||
account.setId(BigInteger.ONE);
|
account.setId(BigInteger.ONE);
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import org.testng.Assert;
|
|||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.entity.WorkflowExecResult;
|
import tech.easyflow.ai.entity.WorkflowExecResult;
|
||||||
@@ -162,6 +163,8 @@ public class WorkflowPublicChatServiceTest {
|
|||||||
WorkflowChatEventStream eventStream = mock(
|
WorkflowChatEventStream eventStream = mock(
|
||||||
WorkflowChatEventStream.class);
|
WorkflowChatEventStream.class);
|
||||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||||
|
WorkflowResumeService workflowResumeService =
|
||||||
|
mock(WorkflowResumeService.class);
|
||||||
WorkflowExecResultService execResultService = mock(
|
WorkflowExecResultService execResultService = mock(
|
||||||
WorkflowExecResultService.class);
|
WorkflowExecResultService.class);
|
||||||
WorkflowExecStepService execStepService = mock(
|
WorkflowExecStepService execStepService = mock(
|
||||||
@@ -196,6 +199,7 @@ public class WorkflowPublicChatServiceTest {
|
|||||||
accessGuard,
|
accessGuard,
|
||||||
eventStream,
|
eventStream,
|
||||||
chainExecutor,
|
chainExecutor,
|
||||||
|
workflowResumeService,
|
||||||
execResultService,
|
execResultService,
|
||||||
execStepService
|
execStepService
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
package tech.easyflow.publicapi.controller;
|
package tech.easyflow.publicapi.controller;
|
||||||
|
|
||||||
import cn.hutool.core.io.IoUtil;
|
import cn.hutool.core.io.IoUtil;
|
||||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
|
||||||
import com.easyagents.core.store.DocumentStore;
|
|
||||||
import com.easyagents.core.store.StoreOptions;
|
|
||||||
import com.easyagents.core.store.StoreResult;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
@@ -18,11 +14,15 @@ import org.springframework.web.bind.annotation.RequestParam;
|
|||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkAsyncUpdateResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncRetryRequest;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatusRequest;
|
||||||
import tech.easyflow.ai.entity.Document;
|
import tech.easyflow.ai.entity.Document;
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.entity.FaqItem;
|
import tech.easyflow.ai.entity.FaqItem;
|
||||||
import tech.easyflow.ai.entity.Model;
|
|
||||||
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
|
import tech.easyflow.ai.enums.KnowledgeApiPermissionScope;
|
||||||
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalModes;
|
||||||
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
@@ -33,9 +33,7 @@ import tech.easyflow.ai.service.FaqCategoryService;
|
|||||||
import tech.easyflow.ai.service.FaqItemService;
|
import tech.easyflow.ai.service.FaqItemService;
|
||||||
import tech.easyflow.ai.service.KnowledgeShareAuditService;
|
import tech.easyflow.ai.service.KnowledgeShareAuditService;
|
||||||
import tech.easyflow.ai.service.KnowledgeSharePermissionService;
|
import tech.easyflow.ai.service.KnowledgeSharePermissionService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
|
||||||
import tech.easyflow.ai.service.impl.KnowledgeSharePermissionServiceImpl;
|
import tech.easyflow.ai.service.impl.KnowledgeSharePermissionServiceImpl;
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
|
||||||
import tech.easyflow.ai.vo.FaqImportResultVo;
|
import tech.easyflow.ai.vo.FaqImportResultVo;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.filestorage.FileStorageService;
|
import tech.easyflow.common.filestorage.FileStorageService;
|
||||||
@@ -81,8 +79,6 @@ public class PublicKnowledgeShareController {
|
|||||||
private FaqItemService faqItemService;
|
private FaqItemService faqItemService;
|
||||||
@Resource
|
@Resource
|
||||||
private FaqCategoryService faqCategoryService;
|
private FaqCategoryService faqCategoryService;
|
||||||
@Resource
|
|
||||||
private ModelService modelService;
|
|
||||||
@Resource(name = "default")
|
@Resource(name = "default")
|
||||||
private FileStorageService fileStorageService;
|
private FileStorageService fileStorageService;
|
||||||
|
|
||||||
@@ -351,37 +347,20 @@ public class PublicKnowledgeShareController {
|
|||||||
public Result<?> updateDocumentChunk(
|
public Result<?> updateDocumentChunk(
|
||||||
@RequestHeader("ApiKey") String apiKey,
|
@RequestHeader("ApiKey") String apiKey,
|
||||||
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
|
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
|
||||||
@JsonBody DocumentChunk documentChunk,
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunk documentChunk,
|
||||||
HttpServletRequest request
|
HttpServletRequest request
|
||||||
) {
|
) {
|
||||||
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
||||||
requireDocumentKnowledge(knowledgeId);
|
requireDocumentKnowledge(knowledgeId);
|
||||||
DocumentChunk current = requireDocumentChunk(documentChunk.getId(), knowledgeId);
|
DocumentChunk current = requireDocumentChunk(documentChunk.getId(), knowledgeId);
|
||||||
boolean success = documentChunkService.updateById(documentChunk);
|
DocumentChunk updated = documentChunkService.updateContent(
|
||||||
if (success) {
|
knowledgeId,
|
||||||
DocumentCollection knowledge = documentCollectionService.getById(knowledgeId);
|
current.getId(),
|
||||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
documentChunk.getContent()
|
||||||
if (documentStore == null) {
|
);
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
|
||||||
documentStore.setEmbeddingModel(embeddingModel);
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
|
||||||
com.easyagents.core.document.Document doc = com.easyagents.core.document.Document.of(documentChunk.getContent());
|
|
||||||
doc.setId(current.getId());
|
|
||||||
StoreResult result = documentStore.update(doc, options);
|
|
||||||
audit(apiKey, "API更新文档 Chunk", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", documentChunk.getId()));
|
audit(apiKey, "API更新文档 Chunk", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", documentChunk.getId()));
|
||||||
return Result.ok(result);
|
return Result.ok(DocumentChunkAsyncUpdateResult.from(updated));
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Result.ok(false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -397,25 +376,51 @@ public class PublicKnowledgeShareController {
|
|||||||
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
assertApiShare(apiKey, request.getRequestURI(), knowledgeId, KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
||||||
requireDocumentKnowledge(knowledgeId);
|
requireDocumentKnowledge(knowledgeId);
|
||||||
requireDocumentChunk(chunkId, knowledgeId);
|
requireDocumentChunk(chunkId, knowledgeId);
|
||||||
DocumentCollection knowledge = documentCollectionService.getById(knowledgeId);
|
DocumentChunkDeleteResult removed = documentChunkService.deleteChunk(
|
||||||
DocumentStore documentStore = knowledge.toDocumentStore();
|
knowledgeId,
|
||||||
if (documentStore == null) {
|
chunkId
|
||||||
return Result.fail(2, "知识库没有配置向量库");
|
);
|
||||||
}
|
|
||||||
try {
|
|
||||||
Model model = modelService.getModelInstance(knowledge.getVectorEmbedModelId());
|
|
||||||
if (model == null) {
|
|
||||||
return Result.fail(3, "知识库没有配置向量模型");
|
|
||||||
}
|
|
||||||
documentStore.setEmbeddingModel(model.toEmbeddingModel());
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(knowledge.getVectorStoreCollection());
|
|
||||||
documentStore.delete(Collections.singletonList(chunkId), options);
|
|
||||||
documentChunkService.removeById(chunkId);
|
|
||||||
audit(apiKey, "API删除文档 Chunk", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", chunkId));
|
audit(apiKey, "API删除文档 Chunk", "KNOWLEDGE_API_SHARE_WRITE", request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", chunkId));
|
||||||
return Result.ok(true);
|
return Result.ok(removed != null);
|
||||||
} finally {
|
|
||||||
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/documentChunk/syncStatus")
|
||||||
|
public Result<List<DocumentChunkSyncStatus>> documentChunkSyncStatus(
|
||||||
|
@RequestHeader("ApiKey") String apiKey,
|
||||||
|
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
|
||||||
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkSyncStatusRequest statusRequest,
|
||||||
|
HttpServletRequest request
|
||||||
|
) {
|
||||||
|
assertApiShare(apiKey, request.getRequestURI(), knowledgeId,
|
||||||
|
KnowledgeApiPermissionScope.KNOWLEDGE_READ.name());
|
||||||
|
requireDocumentKnowledge(knowledgeId);
|
||||||
|
requireDocument(statusRequest.getDocumentId(), knowledgeId);
|
||||||
|
return Result.ok(documentChunkService.listIndexSyncStatus(
|
||||||
|
knowledgeId, statusRequest.getDocumentId(), statusRequest.getIds()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/documentChunk/retrySync")
|
||||||
|
public Result<DocumentChunk> retryDocumentChunkSync(
|
||||||
|
@RequestHeader("ApiKey") String apiKey,
|
||||||
|
@JsonBody(value = "knowledgeId", required = true) BigInteger knowledgeId,
|
||||||
|
@JsonBody(required = true, skipConvertError = false)
|
||||||
|
DocumentChunkSyncRetryRequest retryRequest,
|
||||||
|
HttpServletRequest request
|
||||||
|
) {
|
||||||
|
assertApiShare(apiKey, request.getRequestURI(), knowledgeId,
|
||||||
|
KnowledgeApiPermissionScope.KNOWLEDGE_MAINTENANCE.name());
|
||||||
|
requireDocumentKnowledge(knowledgeId);
|
||||||
|
if (retryRequest.getIndexSyncVersion() == null) {
|
||||||
|
throw new BusinessException("同步版本不能为空");
|
||||||
|
}
|
||||||
|
DocumentChunk chunk = documentChunkService.retryIndexSync(
|
||||||
|
knowledgeId, retryRequest.getId(), retryRequest.getIndexSyncVersion()
|
||||||
|
);
|
||||||
|
audit(apiKey, "API重试文档 Chunk 索引同步", "KNOWLEDGE_API_SHARE_WRITE",
|
||||||
|
request.getRequestURI(), Map.of("knowledgeId", knowledgeId, "chunkId", retryRequest.getId()));
|
||||||
|
return Result.ok(chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
|||||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||||
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiPreparedUpload;
|
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiPreparedUpload;
|
||||||
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadLifecycleService;
|
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadLifecycleService;
|
||||||
@@ -70,6 +71,8 @@ public class PublicWorkflowController {
|
|||||||
@Resource
|
@Resource
|
||||||
private WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
private WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
||||||
@Resource
|
@Resource
|
||||||
|
private WorkflowResumeService workflowResumeService;
|
||||||
|
@Resource
|
||||||
private WorkflowApiPermissionService workflowApiPermissionService;
|
private WorkflowApiPermissionService workflowApiPermissionService;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowExecResultService workflowExecResultService;
|
private WorkflowExecResultService workflowExecResultService;
|
||||||
@@ -250,14 +253,7 @@ public class PublicWorkflowController {
|
|||||||
SysApiKey apiKey = workflowApiPermissionService.assertWorkflowApi(request.getHeader("ApiKey"), request.getRequestURI());
|
SysApiKey apiKey = workflowApiPermissionService.assertWorkflowApi(request.getHeader("ApiKey"), request.getRequestURI());
|
||||||
WorkflowExecResult execResult = assertApiKeyExecutionOwnership(apiKey, executeId);
|
WorkflowExecResult execResult = assertApiKeyExecutionOwnership(apiKey, executeId);
|
||||||
assertWorkflowExecutionResumable(execResult);
|
assertWorkflowExecutionResumable(execResult);
|
||||||
if (!chainExecutor.resumeAsyncIfSuspended(
|
workflowResumeService.resume(executeId, confirmParams);
|
||||||
executeId,
|
|
||||||
confirmParams)) {
|
|
||||||
throw new BusinessException(
|
|
||||||
409,
|
|
||||||
40901,
|
|
||||||
"当前执行状态不可恢复,仅暂停中的工作流允许恢复");
|
|
||||||
}
|
|
||||||
return Result.ok();
|
return Result.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -394,6 +394,7 @@ public final class WorkflowRunAsyncErrorProfile
|
|||||||
*/
|
*/
|
||||||
private boolean isStableBusinessCode(int code) {
|
private boolean isStableBusinessCode(int code) {
|
||||||
return (code >= 40011 && code <= 40017)
|
return (code >= 40011 && code <= 40017)
|
||||||
|
|| code == 40031
|
||||||
|| (code >= 40101 && code <= 40103)
|
|| (code >= 40101 && code <= 40103)
|
||||||
|| (code >= 40301 && code <= 40302)
|
|| (code >= 40301 && code <= 40302)
|
||||||
|| (code >= 40401 && code <= 40402)
|
|| (code >= 40401 && code <= 40402)
|
||||||
@@ -412,7 +413,8 @@ public final class WorkflowRunAsyncErrorProfile
|
|||||||
* @return 对外 HTTP 状态
|
* @return 对外 HTTP 状态
|
||||||
*/
|
*/
|
||||||
private int normalizeHttpStatus(int code, int fallback) {
|
private int normalizeHttpStatus(int code, int fallback) {
|
||||||
if (code >= 40011 && code <= 40017) {
|
if ((code >= 40011 && code <= 40017)
|
||||||
|
|| code == 40031) {
|
||||||
return 400;
|
return 400;
|
||||||
}
|
}
|
||||||
if (code >= 40101 && code <= 40103) {
|
if (code >= 40101 && code <= 40103) {
|
||||||
|
|||||||
@@ -66,6 +66,8 @@ public class PublicKnowledgeShareControllerContractTest {
|
|||||||
JsonBody chunkBody = chunk.getAnnotation(JsonBody.class);
|
JsonBody chunkBody = chunk.getAnnotation(JsonBody.class);
|
||||||
Assert.assertNotNull(chunkBody);
|
Assert.assertNotNull(chunkBody);
|
||||||
Assert.assertEquals("", chunkBody.value());
|
Assert.assertEquals("", chunkBody.value());
|
||||||
|
Assert.assertTrue(chunkBody.required());
|
||||||
|
Assert.assertFalse(chunkBody.skipConvertError());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import org.springframework.test.util.ReflectionTestUtils;
|
|||||||
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.entity.WorkflowExecResult;
|
import tech.easyflow.ai.entity.WorkflowExecResult;
|
||||||
import tech.easyflow.ai.enums.PublishStatus;
|
import tech.easyflow.ai.enums.PublishStatus;
|
||||||
@@ -44,6 +45,7 @@ public class PublicWorkflowControllerBehaviorTest {
|
|||||||
|
|
||||||
private PublicWorkflowController controller;
|
private PublicWorkflowController controller;
|
||||||
private ChainExecutor chainExecutor;
|
private ChainExecutor chainExecutor;
|
||||||
|
private WorkflowResumeService workflowResumeService;
|
||||||
private TinyFlowService tinyFlowService;
|
private TinyFlowService tinyFlowService;
|
||||||
private HttpServletRequest request;
|
private HttpServletRequest request;
|
||||||
|
|
||||||
@@ -54,6 +56,7 @@ public class PublicWorkflowControllerBehaviorTest {
|
|||||||
public void setUp() {
|
public void setUp() {
|
||||||
controller = new PublicWorkflowController();
|
controller = new PublicWorkflowController();
|
||||||
chainExecutor = Mockito.mock(ChainExecutor.class);
|
chainExecutor = Mockito.mock(ChainExecutor.class);
|
||||||
|
workflowResumeService = Mockito.mock(WorkflowResumeService.class);
|
||||||
tinyFlowService = Mockito.mock(TinyFlowService.class);
|
tinyFlowService = Mockito.mock(TinyFlowService.class);
|
||||||
WorkflowApiPermissionService permissionService =
|
WorkflowApiPermissionService permissionService =
|
||||||
Mockito.mock(WorkflowApiPermissionService.class);
|
Mockito.mock(WorkflowApiPermissionService.class);
|
||||||
@@ -79,6 +82,10 @@ public class PublicWorkflowControllerBehaviorTest {
|
|||||||
controller,
|
controller,
|
||||||
"chainExecutor",
|
"chainExecutor",
|
||||||
chainExecutor);
|
chainExecutor);
|
||||||
|
ReflectionTestUtils.setField(
|
||||||
|
controller,
|
||||||
|
"workflowResumeService",
|
||||||
|
workflowResumeService);
|
||||||
ReflectionTestUtils.setField(
|
ReflectionTestUtils.setField(
|
||||||
controller,
|
controller,
|
||||||
"tinyFlowService",
|
"tinyFlowService",
|
||||||
@@ -108,10 +115,12 @@ public class PublicWorkflowControllerBehaviorTest {
|
|||||||
public void resumeShouldRejectNonSuspendedExecution() {
|
public void resumeShouldRejectNonSuspendedExecution() {
|
||||||
when(request.getRequestURI()).thenReturn(
|
when(request.getRequestURI()).thenReturn(
|
||||||
"/public-api/workflow/resume");
|
"/public-api/workflow/resume");
|
||||||
when(chainExecutor.resumeAsyncIfSuspended(
|
Mockito.doThrow(new BusinessException(
|
||||||
EXECUTE_ID,
|
409,
|
||||||
Map.of("approved", true)))
|
40901,
|
||||||
.thenReturn(false);
|
"当前执行状态不可恢复,仅暂停中的工作流允许恢复"))
|
||||||
|
.when(workflowResumeService)
|
||||||
|
.resume(EXECUTE_ID, Map.of("approved", true));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
controller.resume(
|
controller.resume(
|
||||||
@@ -124,7 +133,7 @@ public class PublicWorkflowControllerBehaviorTest {
|
|||||||
Assert.assertEquals(40901, exception.getErrorCode());
|
Assert.assertEquals(40901, exception.getErrorCode());
|
||||||
}
|
}
|
||||||
|
|
||||||
verify(chainExecutor).resumeAsyncIfSuspended(
|
verify(workflowResumeService).resume(
|
||||||
EXECUTE_ID,
|
EXECUTE_ID,
|
||||||
Map.of("approved", true));
|
Map.of("approved", true));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -167,6 +167,28 @@ public class WorkflowRunAsyncErrorProfileTest {
|
|||||||
resolution.modelAndView.getModel().get("message"));
|
resolution.modelAndView.getModel().get("message"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证确认节点恢复校验保留专用错误码,不回退为运行参数错误。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldKeepResumeValidationCode() {
|
||||||
|
Resolution resolution = resolve(
|
||||||
|
"/public-api/workflow/resume",
|
||||||
|
MediaType.APPLICATION_JSON_VALUE,
|
||||||
|
new BusinessException(
|
||||||
|
400,
|
||||||
|
40031,
|
||||||
|
"确认参数[模板类型]包含未配置选项"));
|
||||||
|
|
||||||
|
Assert.assertEquals(400, resolution.response.getStatus());
|
||||||
|
Assert.assertEquals(
|
||||||
|
40031,
|
||||||
|
resolution.modelAndView.getModel().get("errorCode"));
|
||||||
|
Assert.assertEquals(
|
||||||
|
"确认参数[模板类型]包含未配置选项",
|
||||||
|
resolution.modelAndView.getModel().get("message"));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证 API Key 无效和两层权限错误保持可区分。
|
* 验证 API Key 无效和两层权限错误保持可区分。
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
|||||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowResumeService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
import tech.easyflow.common.annotation.UsePermission;
|
import tech.easyflow.common.annotation.UsePermission;
|
||||||
@@ -54,6 +55,8 @@ public class UcWorkflowController extends BaseCurdController<WorkflowService, Wo
|
|||||||
@Resource
|
@Resource
|
||||||
private WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
private WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
||||||
@Resource
|
@Resource
|
||||||
|
private WorkflowResumeService workflowResumeService;
|
||||||
|
@Resource
|
||||||
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
||||||
|
|
||||||
public UcWorkflowController(WorkflowService service) {
|
public UcWorkflowController(WorkflowService service) {
|
||||||
@@ -163,12 +166,7 @@ public class UcWorkflowController extends BaseCurdController<WorkflowService, Wo
|
|||||||
)
|
)
|
||||||
public Result<Void> resume(@JsonBody(value = "executeId", required = true) String executeId,
|
public Result<Void> resume(@JsonBody(value = "executeId", required = true) String executeId,
|
||||||
@JsonBody("confirmParams") Map<String, Object> confirmParams) {
|
@JsonBody("confirmParams") Map<String, Object> confirmParams) {
|
||||||
if (!chainExecutor.resumeAsyncIfSuspended(executeId, confirmParams)) {
|
workflowResumeService.resume(executeId, confirmParams);
|
||||||
throw new BusinessException(
|
|
||||||
409,
|
|
||||||
40901,
|
|
||||||
"当前执行状态不可恢复,仅暂停中的工作流允许恢复");
|
|
||||||
}
|
|
||||||
return Result.ok();
|
return Result.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package tech.easyflow.common.cache;
|
package tech.easyflow.common.cache;
|
||||||
|
|
||||||
|
import jakarta.annotation.PreDestroy;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
@@ -10,6 +11,11 @@ import org.springframework.stereotype.Component;
|
|||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
|
import java.util.concurrent.ScheduledFuture;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.function.Supplier;
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -27,6 +33,13 @@ public class RedisLockExecutor {
|
|||||||
private static final DefaultRedisScript<Long> NEXT_FENCING_TOKEN_SCRIPT;
|
private static final DefaultRedisScript<Long> NEXT_FENCING_TOKEN_SCRIPT;
|
||||||
private static final DefaultRedisScript<Long> ACQUIRE_FENCED_LOCK_SCRIPT;
|
private static final DefaultRedisScript<Long> ACQUIRE_FENCED_LOCK_SCRIPT;
|
||||||
|
|
||||||
|
private final ScheduledExecutorService lockRenewalExecutor =
|
||||||
|
Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||||
|
Thread thread = new Thread(runnable, "easyflow-redis-lock-renewal");
|
||||||
|
thread.setDaemon(true);
|
||||||
|
return thread;
|
||||||
|
});
|
||||||
|
|
||||||
static {
|
static {
|
||||||
RELEASE_LOCK_SCRIPT = new DefaultRedisScript<>();
|
RELEASE_LOCK_SCRIPT = new DefaultRedisScript<>();
|
||||||
RELEASE_LOCK_SCRIPT.setScriptText(
|
RELEASE_LOCK_SCRIPT.setScriptText(
|
||||||
@@ -94,6 +107,66 @@ public class RedisLockExecutor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在自动续租的分布式锁保护下执行任务。
|
||||||
|
*
|
||||||
|
* <p>适用于包含数据库锁等待或外部持久化操作、无法由固定租约严格覆盖的管理命令。
|
||||||
|
* 若执行期间确认锁已丢失,则不向调用方返回成功。</p>
|
||||||
|
*/
|
||||||
|
public void executeWithRenewingLock(
|
||||||
|
String lockKey,
|
||||||
|
Duration waitTimeout,
|
||||||
|
Duration leaseTimeout,
|
||||||
|
Runnable task) {
|
||||||
|
executeWithRenewingLock(lockKey, waitTimeout, leaseTimeout, () -> {
|
||||||
|
task.run();
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在自动续租的分布式锁保护下执行有返回值任务。
|
||||||
|
*/
|
||||||
|
public <T> T executeWithRenewingLock(
|
||||||
|
String lockKey,
|
||||||
|
Duration waitTimeout,
|
||||||
|
Duration leaseTimeout,
|
||||||
|
Supplier<T> task) {
|
||||||
|
LockHandle handle = acquire(lockKey, waitTimeout, leaseTimeout);
|
||||||
|
AtomicBoolean lost = new AtomicBoolean();
|
||||||
|
long renewalIntervalMillis = Math.max(1L, leaseTimeout.toMillis() / 3L);
|
||||||
|
ScheduledFuture<?> renewal = lockRenewalExecutor.scheduleWithFixedDelay(
|
||||||
|
() -> {
|
||||||
|
try {
|
||||||
|
if (!handle.renew()) {
|
||||||
|
lost.set(true);
|
||||||
|
}
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
lost.set(true);
|
||||||
|
log.warn("分布式锁续租失败,当前命令不得返回成功: lockKey={}",
|
||||||
|
lockKey, exception);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
renewalIntervalMillis,
|
||||||
|
renewalIntervalMillis,
|
||||||
|
TimeUnit.MILLISECONDS);
|
||||||
|
try {
|
||||||
|
T result = task.get();
|
||||||
|
if (lost.get()) {
|
||||||
|
throw new IllegalStateException("执行期间分布式锁已丢失,lockKey=" + lockKey);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} finally {
|
||||||
|
renewal.cancel(false);
|
||||||
|
handle.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@PreDestroy
|
||||||
|
public void shutdownLockRenewalExecutor() {
|
||||||
|
lockRenewalExecutor.shutdownNow();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取显式释放的分布式锁句柄。
|
* 获取显式释放的分布式锁句柄。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ import org.springframework.data.redis.core.script.RedisScript;
|
|||||||
import java.lang.reflect.Field;
|
import java.lang.reflect.Field;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link RedisLockExecutor} 回归测试。
|
* {@link RedisLockExecutor} 回归测试。
|
||||||
@@ -147,6 +150,96 @@ public class RedisLockExecutorTest {
|
|||||||
String.valueOf(Duration.ofDays(4).toMillis())));
|
String.valueOf(Duration.ofDays(4).toMillis())));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void renewingLockShouldRenewBeforeLongRunningCommandCompletes() throws Exception {
|
||||||
|
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||||
|
ValueOperations<String, String> valueOperations = mockValueOperations(true);
|
||||||
|
CountDownLatch renewed = new CountDownLatch(1);
|
||||||
|
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||||
|
Mockito.when(redisTemplate.execute(
|
||||||
|
ArgumentMatchers.<RedisScript<Long>>any(),
|
||||||
|
ArgumentMatchers.<List<String>>any(),
|
||||||
|
ArgumentMatchers.anyString(),
|
||||||
|
ArgumentMatchers.anyString()
|
||||||
|
)).thenAnswer(invocation -> {
|
||||||
|
renewed.countDown();
|
||||||
|
return 1L;
|
||||||
|
});
|
||||||
|
|
||||||
|
RedisLockExecutor executor = new RedisLockExecutor();
|
||||||
|
setRedisTemplate(executor, redisTemplate);
|
||||||
|
try {
|
||||||
|
executor.executeWithRenewingLock(
|
||||||
|
"easyflow:test:renewing-lock",
|
||||||
|
Duration.ZERO,
|
||||||
|
Duration.ofMillis(60),
|
||||||
|
() -> {
|
||||||
|
try {
|
||||||
|
Assert.assertTrue(renewed.await(1, TimeUnit.SECONDS));
|
||||||
|
} catch (InterruptedException exception) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new AssertionError("等待锁续租时被中断", exception);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
executor.shutdownLockRenewalExecutor();
|
||||||
|
}
|
||||||
|
|
||||||
|
Mockito.verify(redisTemplate, Mockito.atLeastOnce()).execute(
|
||||||
|
ArgumentMatchers.<RedisScript<Long>>any(),
|
||||||
|
ArgumentMatchers.eq(List.of("easyflow:test:renewing-lock")),
|
||||||
|
ArgumentMatchers.anyString(),
|
||||||
|
ArgumentMatchers.eq("60"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void renewingLockMustNotReturnSuccessAfterRenewalThrows() throws Exception {
|
||||||
|
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||||
|
ValueOperations<String, String> valueOperations = mockValueOperations(true);
|
||||||
|
CountDownLatch renewalAttempted = new CountDownLatch(1);
|
||||||
|
AtomicInteger scriptCalls = new AtomicInteger();
|
||||||
|
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||||
|
Mockito.when(redisTemplate.execute(
|
||||||
|
ArgumentMatchers.<RedisScript<Long>>any(),
|
||||||
|
ArgumentMatchers.<List<String>>any(),
|
||||||
|
ArgumentMatchers.anyString(),
|
||||||
|
ArgumentMatchers.anyString()
|
||||||
|
)).thenAnswer(invocation -> {
|
||||||
|
if (scriptCalls.incrementAndGet() == 1) {
|
||||||
|
renewalAttempted.countDown();
|
||||||
|
throw new IllegalStateException("redis unavailable");
|
||||||
|
}
|
||||||
|
return 1L;
|
||||||
|
});
|
||||||
|
|
||||||
|
RedisLockExecutor executor = new RedisLockExecutor();
|
||||||
|
setRedisTemplate(executor, redisTemplate);
|
||||||
|
try {
|
||||||
|
try {
|
||||||
|
executor.executeWithRenewingLock(
|
||||||
|
"easyflow:test:renewal-failure",
|
||||||
|
Duration.ZERO,
|
||||||
|
Duration.ofMillis(60),
|
||||||
|
() -> {
|
||||||
|
try {
|
||||||
|
Assert.assertTrue(renewalAttempted.await(1, TimeUnit.SECONDS));
|
||||||
|
// 等待续租线程把失败结果发布到调用线程;业务任务与续租
|
||||||
|
// 同时完成时,锁仍处于原租约内且 callback 已结束。
|
||||||
|
Thread.sleep(50L);
|
||||||
|
} catch (InterruptedException exception) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new AssertionError(exception);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Assert.fail("续租异常后不应返回成功");
|
||||||
|
} catch (IllegalStateException exception) {
|
||||||
|
Assert.assertTrue(exception.getMessage().contains("分布式锁已丢失"));
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
executor.shutdownLockRenewalExecutor();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
private ValueOperations<String, String> mockValueOperations(boolean acquired) {
|
private ValueOperations<String, String> mockValueOperations(boolean acquired) {
|
||||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||||
|
|||||||
@@ -67,7 +67,9 @@ public abstract class AbstractAgentAsyncSubTools implements AsyncSubTools {
|
|||||||
* @param arguments 调用参数
|
* @param arguments 调用参数
|
||||||
* @return 执行结果
|
* @return 执行结果
|
||||||
*/
|
*/
|
||||||
protected abstract AgentToolExecutionResult executeBusiness(Map<String, Object> arguments);
|
protected abstract AgentToolExecutionResult executeBusiness(
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
@@ -92,7 +94,7 @@ public abstract class AbstractAgentAsyncSubTools implements AsyncSubTools {
|
|||||||
record.getMetadata().put("toolDisplayName", displayName());
|
record.getMetadata().put("toolDisplayName", displayName());
|
||||||
appendEvent(record, "SUBMITTED", displayName() + "任务已提交");
|
appendEvent(record, "SUBMITTED", displayName() + "任务已提交");
|
||||||
taskStore.create(record);
|
taskStore.create(record);
|
||||||
dispatch(sessionId, record.getTaskId(), record.getArguments());
|
dispatch(sessionId, record.getTaskId(), record.getArguments(), context);
|
||||||
|
|
||||||
AsyncToolSubmitResult result = new AsyncToolSubmitResult();
|
AsyncToolSubmitResult result = new AsyncToolSubmitResult();
|
||||||
result.setTaskId(taskId);
|
result.setTaskId(taskId);
|
||||||
@@ -157,16 +159,23 @@ public abstract class AbstractAgentAsyncSubTools implements AsyncSubTools {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void dispatch(String sessionId, String taskId, Map<String, Object> arguments) {
|
private void dispatch(String sessionId,
|
||||||
|
String taskId,
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
try {
|
try {
|
||||||
taskExecutor.execute(() -> executeTask(sessionId, taskId, arguments));
|
taskExecutor.execute(() -> executeTask(
|
||||||
|
sessionId, taskId, arguments, context));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
taskStore.update(sessionId, taskId, record -> fail(record, e));
|
taskStore.update(sessionId, taskId, record -> fail(record, e));
|
||||||
throw new BusinessException("提交异步工具任务失败:" + safeMessage(e));
|
throw new BusinessException("提交异步工具任务失败:" + safeMessage(e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void executeTask(String sessionId, String taskId, Map<String, Object> arguments) {
|
private void executeTask(String sessionId,
|
||||||
|
String taskId,
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
try {
|
try {
|
||||||
taskStore.update(sessionId, taskId, record -> {
|
taskStore.update(sessionId, taskId, record -> {
|
||||||
record.setStatus(AsyncToolTaskStatus.RUNNING);
|
record.setStatus(AsyncToolTaskStatus.RUNNING);
|
||||||
@@ -174,7 +183,8 @@ public abstract class AbstractAgentAsyncSubTools implements AsyncSubTools {
|
|||||||
appendEvent(record, "RUNNING", displayName() + "任务执行中");
|
appendEvent(record, "RUNNING", displayName() + "任务执行中");
|
||||||
return record;
|
return record;
|
||||||
});
|
});
|
||||||
AgentToolExecutionResult executionResult = executeBusiness(arguments);
|
AgentToolExecutionResult executionResult = executeBusiness(
|
||||||
|
arguments, context);
|
||||||
taskStore.update(sessionId, taskId, record -> {
|
taskStore.update(sessionId, taskId, record -> {
|
||||||
record.setStatus(AsyncToolTaskStatus.SUCCEEDED);
|
record.setStatus(AsyncToolTaskStatus.SUCCEEDED);
|
||||||
record.setSummary(displayName() + "任务已完成");
|
record.setSummary(displayName() + "任务已完成");
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package tech.easyflow.agent.runtime.asynctool;
|
package tech.easyflow.agent.runtime.asynctool;
|
||||||
|
|
||||||
|
import com.easyagents.agent.runtime.tool.AgentToolContext;
|
||||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||||
import tech.easyflow.agent.enums.AgentToolType;
|
import tech.easyflow.agent.enums.AgentToolType;
|
||||||
import tech.easyflow.agent.runtime.tool.AgentToolExecutionResult;
|
import tech.easyflow.agent.runtime.tool.AgentToolExecutionResult;
|
||||||
@@ -82,7 +83,9 @@ public class PluginAsyncSubTools extends AbstractAgentAsyncSubTools {
|
|||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
protected AgentToolExecutionResult executeBusiness(Map<String, Object> arguments) {
|
protected AgentToolExecutionResult executeBusiness(
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
return pluginToolExecutor.execute(pluginItem, plugin, arguments);
|
return pluginToolExecutor.execute(pluginItem, plugin, arguments);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package tech.easyflow.agent.runtime.asynctool;
|
package tech.easyflow.agent.runtime.asynctool;
|
||||||
|
|
||||||
|
import com.easyagents.agent.runtime.tool.AgentToolContext;
|
||||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||||
import tech.easyflow.agent.enums.AgentToolType;
|
import tech.easyflow.agent.enums.AgentToolType;
|
||||||
import tech.easyflow.agent.runtime.tool.AgentToolExecutionResult;
|
import tech.easyflow.agent.runtime.tool.AgentToolExecutionResult;
|
||||||
@@ -77,7 +78,9 @@ public class WorkflowAsyncSubTools extends AbstractAgentAsyncSubTools {
|
|||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
protected AgentToolExecutionResult executeBusiness(Map<String, Object> arguments) {
|
protected AgentToolExecutionResult executeBusiness(
|
||||||
return workflowToolExecutor.execute(workflow, arguments);
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
|
return workflowToolExecutor.execute(workflow, arguments, context);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ public class AgentToolRuntimeCompiler {
|
|||||||
Tool tool = workflowToolExecutor.buildTool(workflow);
|
Tool tool = workflowToolExecutor.buildTool(workflow);
|
||||||
AgentToolSpec spec = toToolSpec(tool, binding);
|
AgentToolSpec spec = toToolSpec(tool, binding);
|
||||||
AgentToolInvoker invoker = (arguments, context) -> invokeSafely(spec.getName(), binding, context,
|
AgentToolInvoker invoker = (arguments, context) -> invokeSafely(spec.getName(), binding, context,
|
||||||
() -> workflowToolExecutor.execute(workflow, arguments).getResult());
|
() -> workflowToolExecutor.execute(workflow, arguments, context).getResult());
|
||||||
return new CompiledSyncTool(spec, invoker);
|
return new CompiledSyncTool(spec, invoker);
|
||||||
}
|
}
|
||||||
if (type == AgentToolType.PLUGIN) {
|
if (type == AgentToolType.PLUGIN) {
|
||||||
|
|||||||
@@ -2,13 +2,19 @@ package tech.easyflow.agent.runtime.tool;
|
|||||||
|
|
||||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||||
import com.easyagents.core.model.chat.tool.Tool;
|
import com.easyagents.core.model.chat.tool.Tool;
|
||||||
|
import com.easyagents.agent.runtime.AgentRuntimeContext;
|
||||||
|
import com.easyagents.agent.runtime.tool.AgentToolContext;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import tech.easyflow.ai.easyagents.tool.WorkflowTool;
|
import tech.easyflow.ai.easyagents.tool.WorkflowTool;
|
||||||
import tech.easyflow.ai.easyagentsflow.repository.FrozenWorkflowDefinitionRegistry;
|
import tech.easyflow.ai.easyagentsflow.repository.FrozenWorkflowDefinitionRegistry;
|
||||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
|
import tech.easyflow.common.constant.Constants;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -59,11 +65,63 @@ public class WorkflowToolExecutor {
|
|||||||
* @return 执行结果
|
* @return 执行结果
|
||||||
*/
|
*/
|
||||||
public AgentToolExecutionResult execute(Workflow workflow, Map<String, Object> arguments) {
|
public AgentToolExecutionResult execute(Workflow workflow, Map<String, Object> arguments) {
|
||||||
|
return execute(workflow, arguments, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用 Agent 调用身份执行 Workflow 工具。
|
||||||
|
*
|
||||||
|
* @param workflow 工作流
|
||||||
|
* @param arguments 执行参数
|
||||||
|
* @param context Agent 工具上下文
|
||||||
|
* @return 执行结果
|
||||||
|
*/
|
||||||
|
public AgentToolExecutionResult execute(Workflow workflow,
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
|
Map<String, Object> variables = arguments == null
|
||||||
|
? new LinkedHashMap<>()
|
||||||
|
: new LinkedHashMap<>(arguments);
|
||||||
|
variables.remove(Constants.LOGIN_USER_KEY);
|
||||||
|
LoginAccount account = toLoginAccount(context);
|
||||||
|
if (account != null) {
|
||||||
|
variables.put(Constants.LOGIN_USER_KEY, account);
|
||||||
|
}
|
||||||
Object result = chainExecutor.executeWithoutSuspension(
|
Object result = chainExecutor.executeWithoutSuspension(
|
||||||
definitionId(workflow), arguments == null ? Map.of() : arguments);
|
definitionId(workflow), variables);
|
||||||
return new AgentToolExecutionResult(result, resolveBusinessExecutionId(result));
|
return new AgentToolExecutionResult(result, resolveBusinessExecutionId(result));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private LoginAccount toLoginAccount(AgentToolContext context) {
|
||||||
|
AgentRuntimeContext runtimeContext = context == null
|
||||||
|
? null
|
||||||
|
: context.getRuntimeContext();
|
||||||
|
BigInteger userId = positiveId(runtimeContext == null
|
||||||
|
? null
|
||||||
|
: runtimeContext.getUserId());
|
||||||
|
BigInteger tenantId = positiveId(runtimeContext == null
|
||||||
|
? null
|
||||||
|
: runtimeContext.getTenantId());
|
||||||
|
if (userId == null || tenantId == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
LoginAccount account = new LoginAccount();
|
||||||
|
account.setId(userId);
|
||||||
|
account.setTenantId(tenantId);
|
||||||
|
account.setLoginName(runtimeContext.getUserName());
|
||||||
|
account.setNickname(runtimeContext.getUserName());
|
||||||
|
return account;
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigInteger positiveId(String value) {
|
||||||
|
try {
|
||||||
|
BigInteger id = new BigInteger(value);
|
||||||
|
return id.signum() > 0 ? id : null;
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private String definitionId(Workflow workflow) {
|
private String definitionId(Workflow workflow) {
|
||||||
if (frozenDefinitionRegistry != null && workflow != null
|
if (frozenDefinitionRegistry != null && workflow != null
|
||||||
&& workflow.getContent() != null && !workflow.getContent().isBlank()) {
|
&& workflow.getContent() != null && !workflow.getContent().isBlank()) {
|
||||||
|
|||||||
@@ -146,7 +146,9 @@ public class AbstractAgentAsyncSubToolsTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected AgentToolExecutionResult executeBusiness(Map<String, Object> arguments) {
|
protected AgentToolExecutionResult executeBusiness(
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
return new AgentToolExecutionResult(Map.of("echo", arguments.get("keyword")), "business-run-1");
|
return new AgentToolExecutionResult(Map.of("echo", arguments.get("keyword")), "business-run-1");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -153,7 +153,10 @@ public class WorkflowPluginAsyncSubToolsTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public AgentToolExecutionResult execute(Workflow workflow, Map<String, Object> arguments) {
|
public AgentToolExecutionResult execute(
|
||||||
|
Workflow workflow,
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
return new AgentToolExecutionResult(businessResult, "workflow-run-1");
|
return new AgentToolExecutionResult(businessResult, "workflow-run-1");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -145,7 +145,10 @@ public class AgentToolRuntimeCompilerTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public AgentToolExecutionResult execute(Workflow workflow, Map<String, Object> arguments) {
|
public AgentToolExecutionResult execute(
|
||||||
|
Workflow workflow,
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
throw new IllegalStateException("jdbc:mysql://internal:3306 secret-token");
|
throw new IllegalStateException("jdbc:mysql://internal:3306 secret-token");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -262,7 +265,10 @@ public class AgentToolRuntimeCompilerTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public AgentToolExecutionResult execute(Workflow workflow, Map<String, Object> arguments) {
|
public AgentToolExecutionResult execute(
|
||||||
|
Workflow workflow,
|
||||||
|
Map<String, Object> arguments,
|
||||||
|
AgentToolContext context) {
|
||||||
return new AgentToolExecutionResult(Map.of("ok", true), "wf-run-1");
|
return new AgentToolExecutionResult(Map.of("ok", true), "wf-run-1");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
package tech.easyflow.agent.runtime.tool;
|
||||||
|
|
||||||
|
import com.easyagents.agent.runtime.AgentRuntimeContext;
|
||||||
|
import com.easyagents.agent.runtime.tool.AgentToolContext;
|
||||||
|
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.repository.FrozenWorkflowDefinitionRegistry;
|
||||||
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
|
import tech.easyflow.common.constant.Constants;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.mockito.ArgumentMatchers.anyMap;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Agent Workflow 工具调用上下文测试。
|
||||||
|
*/
|
||||||
|
public class WorkflowToolExecutorTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public void shouldForwardAgentIdentityToFrozenWorkflowVariables() {
|
||||||
|
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||||
|
FrozenWorkflowDefinitionRegistry registry =
|
||||||
|
mock(FrozenWorkflowDefinitionRegistry.class);
|
||||||
|
Workflow workflow = new Workflow();
|
||||||
|
workflow.setId(BigInteger.valueOf(101));
|
||||||
|
workflow.setContent("{\"nodes\":[]}");
|
||||||
|
when(registry.register(workflow)).thenReturn("agent-frozen:101:hash");
|
||||||
|
when(chainExecutor.executeWithoutSuspension(anyString(), anyMap()))
|
||||||
|
.thenReturn(Map.of("ok", true));
|
||||||
|
WorkflowToolExecutor executor = new WorkflowToolExecutor(
|
||||||
|
chainExecutor, registry);
|
||||||
|
|
||||||
|
AgentRuntimeContext runtimeContext = new AgentRuntimeContext();
|
||||||
|
runtimeContext.setUserId("7");
|
||||||
|
runtimeContext.setTenantId("9");
|
||||||
|
runtimeContext.setUserName("测试用户");
|
||||||
|
AgentToolContext context = new AgentToolContext();
|
||||||
|
context.setRuntimeContext(runtimeContext);
|
||||||
|
Map<String, Object> arguments = new LinkedHashMap<>();
|
||||||
|
arguments.put("question", "问题");
|
||||||
|
LoginAccount forgedAccount = new LoginAccount();
|
||||||
|
forgedAccount.setId(BigInteger.valueOf(999));
|
||||||
|
forgedAccount.setTenantId(BigInteger.valueOf(999));
|
||||||
|
arguments.put(Constants.LOGIN_USER_KEY, forgedAccount);
|
||||||
|
|
||||||
|
executor.execute(workflow, arguments, context);
|
||||||
|
|
||||||
|
ArgumentCaptor<Map<String, Object>> variables =
|
||||||
|
ArgumentCaptor.forClass(Map.class);
|
||||||
|
verify(chainExecutor).executeWithoutSuspension(
|
||||||
|
org.mockito.ArgumentMatchers.eq("agent-frozen:101:hash"),
|
||||||
|
variables.capture());
|
||||||
|
LoginAccount account = (LoginAccount) variables.getValue()
|
||||||
|
.get(Constants.LOGIN_USER_KEY);
|
||||||
|
Assert.assertEquals(BigInteger.valueOf(7), account.getId());
|
||||||
|
Assert.assertEquals(BigInteger.valueOf(9), account.getTenantId());
|
||||||
|
Assert.assertEquals("测试用户", account.getLoginName());
|
||||||
|
Assert.assertSame(forgedAccount,
|
||||||
|
arguments.get(Constants.LOGIN_USER_KEY));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public void shouldDropReservedIdentityWithoutAgentContext() {
|
||||||
|
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||||
|
FrozenWorkflowDefinitionRegistry registry =
|
||||||
|
mock(FrozenWorkflowDefinitionRegistry.class);
|
||||||
|
Workflow workflow = new Workflow();
|
||||||
|
workflow.setId(BigInteger.valueOf(101));
|
||||||
|
workflow.setContent("{\"nodes\":[]}");
|
||||||
|
when(registry.register(workflow)).thenReturn("agent-frozen:101:hash");
|
||||||
|
when(chainExecutor.executeWithoutSuspension(anyString(), anyMap()))
|
||||||
|
.thenReturn(Map.of("ok", true));
|
||||||
|
WorkflowToolExecutor executor = new WorkflowToolExecutor(
|
||||||
|
chainExecutor, registry);
|
||||||
|
Map<String, Object> arguments = new LinkedHashMap<>();
|
||||||
|
arguments.put(Constants.LOGIN_USER_KEY, new LoginAccount());
|
||||||
|
|
||||||
|
executor.execute(workflow, arguments);
|
||||||
|
|
||||||
|
ArgumentCaptor<Map<String, Object>> variables =
|
||||||
|
ArgumentCaptor.forClass(Map.class);
|
||||||
|
verify(chainExecutor).executeWithoutSuspension(
|
||||||
|
anyString(), variables.capture());
|
||||||
|
Assert.assertFalse(variables.getValue()
|
||||||
|
.containsKey(Constants.LOGIN_USER_KEY));
|
||||||
|
Assert.assertTrue(arguments.containsKey(Constants.LOGIN_USER_KEY));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -53,6 +53,10 @@
|
|||||||
<groupId>com.easyagents</groupId>
|
<groupId>com.easyagents</groupId>
|
||||||
<artifactId>easy-agents-spring-boot-starter</artifactId>
|
<artifactId>easy-agents-spring-boot-starter</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.milvus</groupId>
|
||||||
|
<artifactId>milvus-sdk-java</artifactId>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.google.re2j</groupId>
|
<groupId>com.google.re2j</groupId>
|
||||||
<artifactId>re2j</artifactId>
|
<artifactId>re2j</artifactId>
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package tech.easyflow.ai.config;
|
||||||
|
|
||||||
|
import com.easyagents.store.milvus.MilvusClientManager;
|
||||||
|
import jakarta.annotation.PreDestroy;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.context.annotation.Lazy;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 应用级 Milvus 客户端池。
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
@Lazy
|
||||||
|
public class AiMilvusClientManager extends MilvusClientManager {
|
||||||
|
|
||||||
|
public AiMilvusClientManager(AiMilvusConfig config) {
|
||||||
|
super(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
@PreDestroy
|
||||||
|
public void destroy() {
|
||||||
|
close();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,6 +17,14 @@ public class AiMilvusConfig extends MilvusVectorStoreConfig {
|
|||||||
config.setPassword(getPassword());
|
config.setPassword(getPassword());
|
||||||
config.setAutoCreateCollection(isAutoCreateCollection());
|
config.setAutoCreateCollection(isAutoCreateCollection());
|
||||||
config.setDefaultCollectionName(collectionName);
|
config.setDefaultCollectionName(collectionName);
|
||||||
|
config.setPoolMaxTotal(getPoolMaxTotal());
|
||||||
|
config.setPoolMaxTotalPerKey(getPoolMaxTotalPerKey());
|
||||||
|
config.setPoolMaxIdlePerKey(getPoolMaxIdlePerKey());
|
||||||
|
config.setPoolMinIdlePerKey(getPoolMinIdlePerKey());
|
||||||
|
config.setPoolMaxWaitMillis(getPoolMaxWaitMillis());
|
||||||
|
config.setPoolEvictionIntervalMillis(getPoolEvictionIntervalMillis());
|
||||||
|
config.setPoolMinEvictableIdleMillis(getPoolMinEvictableIdleMillis());
|
||||||
|
config.setSearchTimeoutMillis(getSearchTimeoutMillis());
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,8 @@ import tech.easyflow.ai.documentimport.task.DocumentImportStatusBroadcastPropert
|
|||||||
DocumentImportBulkProperties.class,
|
DocumentImportBulkProperties.class,
|
||||||
DocumentImportParseMonitorProperties.class,
|
DocumentImportParseMonitorProperties.class,
|
||||||
DocumentImportStatusBroadcastProperties.class,
|
DocumentImportStatusBroadcastProperties.class,
|
||||||
RagHealthProperties.class
|
RagHealthProperties.class,
|
||||||
|
MultiKnowledgeRetrievalProperties.class
|
||||||
})
|
})
|
||||||
@AutoConfiguration
|
@AutoConfiguration
|
||||||
public class AiModuleConfig {
|
public class AiModuleConfig {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ public class EasyFlowThreadPoolProperties {
|
|||||||
private Pool sse = new Pool(4, 16, 2000, 30, true);
|
private Pool sse = new Pool(4, 16, 2000, 30, true);
|
||||||
private Pool documentImport = new Pool(2, 4, 200, 60, true);
|
private Pool documentImport = new Pool(2, 4, 200, 60, true);
|
||||||
private Pool agentAsyncTool = new Pool(2, 8, 200, 60, true);
|
private Pool agentAsyncTool = new Pool(2, 8, 200, 60, true);
|
||||||
|
private Pool knowledgeRetrieval = new Pool(4, 8, 64, 30, true);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取 SSE 线程池配置。
|
* 获取 SSE 线程池配置。
|
||||||
@@ -66,6 +67,14 @@ public class EasyFlowThreadPoolProperties {
|
|||||||
this.agentAsyncTool = agentAsyncTool;
|
this.agentAsyncTool = agentAsyncTool;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Pool getKnowledgeRetrieval() {
|
||||||
|
return knowledgeRetrieval;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setKnowledgeRetrieval(Pool knowledgeRetrieval) {
|
||||||
|
this.knowledgeRetrieval = knowledgeRetrieval;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 线程池配置项。
|
* 线程池配置项。
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package tech.easyflow.ai.config;
|
||||||
|
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 工作流多知识库向量检索的资源与时限配置。
|
||||||
|
*/
|
||||||
|
@ConfigurationProperties(prefix = "easyflow.ai.knowledge.multi-retrieval")
|
||||||
|
public class MultiKnowledgeRetrievalProperties {
|
||||||
|
|
||||||
|
private int maxSources = 8;
|
||||||
|
private int candidateMultiplier = 5;
|
||||||
|
private int perSourceCandidateLimit = 50;
|
||||||
|
private int totalCandidateLimit = 400;
|
||||||
|
private double minVectorScore = 0.6D;
|
||||||
|
private Duration perSourceTimeout = Duration.ofSeconds(10);
|
||||||
|
private Duration totalTimeout = Duration.ofSeconds(20);
|
||||||
|
|
||||||
|
public int getMaxSources() {
|
||||||
|
return maxSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMaxSources(int maxSources) {
|
||||||
|
this.maxSources = maxSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getCandidateMultiplier() {
|
||||||
|
return candidateMultiplier;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCandidateMultiplier(int candidateMultiplier) {
|
||||||
|
this.candidateMultiplier = candidateMultiplier;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getPerSourceCandidateLimit() {
|
||||||
|
return perSourceCandidateLimit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPerSourceCandidateLimit(int perSourceCandidateLimit) {
|
||||||
|
this.perSourceCandidateLimit = perSourceCandidateLimit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getTotalCandidateLimit() {
|
||||||
|
return totalCandidateLimit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTotalCandidateLimit(int totalCandidateLimit) {
|
||||||
|
this.totalCandidateLimit = totalCandidateLimit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public double getMinVectorScore() {
|
||||||
|
return minVectorScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMinVectorScore(double minVectorScore) {
|
||||||
|
this.minVectorScore = minVectorScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Duration getPerSourceTimeout() {
|
||||||
|
return perSourceTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPerSourceTimeout(Duration perSourceTimeout) {
|
||||||
|
this.perSourceTimeout = perSourceTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Duration getTotalTimeout() {
|
||||||
|
return totalTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTotalTimeout(Duration totalTimeout) {
|
||||||
|
this.totalTimeout = totalTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 启动期校验全部有界配置。
|
||||||
|
*/
|
||||||
|
public void validate() {
|
||||||
|
if (maxSources < 2 || maxSources > 64) {
|
||||||
|
throw new IllegalArgumentException("多知识库最大来源数必须在 2 到 64 之间");
|
||||||
|
}
|
||||||
|
if (candidateMultiplier < 1 || candidateMultiplier > 20) {
|
||||||
|
throw new IllegalArgumentException("多知识库候选倍率必须在 1 到 20 之间");
|
||||||
|
}
|
||||||
|
if (perSourceCandidateLimit < 1 || perSourceCandidateLimit > 1000) {
|
||||||
|
throw new IllegalArgumentException("单知识库候选上限必须在 1 到 1000 之间");
|
||||||
|
}
|
||||||
|
long derivedCandidateLimit = (long) maxSources
|
||||||
|
* perSourceCandidateLimit;
|
||||||
|
if (totalCandidateLimit < derivedCandidateLimit
|
||||||
|
|| totalCandidateLimit > 10000) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"多知识库总候选上限不能小于最大来源数与单库候选上限的乘积");
|
||||||
|
}
|
||||||
|
if (!Double.isFinite(minVectorScore) || minVectorScore < 0D || minVectorScore > 1D) {
|
||||||
|
throw new IllegalArgumentException("多知识库向量阈值必须在 0 到 1 之间");
|
||||||
|
}
|
||||||
|
if (perSourceTimeout == null || perSourceTimeout.isZero() || perSourceTimeout.isNegative()) {
|
||||||
|
throw new IllegalArgumentException("单知识库超时时间必须大于 0");
|
||||||
|
}
|
||||||
|
if (totalTimeout == null || totalTimeout.isZero() || totalTimeout.isNegative()
|
||||||
|
|| totalTimeout.compareTo(perSourceTimeout) < 0) {
|
||||||
|
throw new IllegalArgumentException("节点总超时时间不能小于单知识库超时时间");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,16 +25,23 @@ public class RagHealthIndicator {
|
|||||||
public static class RagMilvusHealthIndicator extends CachedHealthIndicatorSupport implements HealthIndicator {
|
public static class RagMilvusHealthIndicator extends CachedHealthIndicatorSupport implements HealthIndicator {
|
||||||
|
|
||||||
private final AiMilvusConfig aiMilvusConfig;
|
private final AiMilvusConfig aiMilvusConfig;
|
||||||
|
private final AiMilvusClientManager milvusClientManager;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建 Milvus 健康检查器。
|
* 创建 Milvus 健康检查器。
|
||||||
*
|
*
|
||||||
* @param aiMilvusConfig Milvus 配置
|
* @param aiMilvusConfig Milvus 配置
|
||||||
|
* @param milvusClientManager 应用级 Milvus 客户端池
|
||||||
* @param healthProperties RAG 健康检查配置
|
* @param healthProperties RAG 健康检查配置
|
||||||
*/
|
*/
|
||||||
public RagMilvusHealthIndicator(AiMilvusConfig aiMilvusConfig, RagHealthProperties healthProperties) {
|
public RagMilvusHealthIndicator(
|
||||||
|
AiMilvusConfig aiMilvusConfig,
|
||||||
|
AiMilvusClientManager milvusClientManager,
|
||||||
|
RagHealthProperties healthProperties
|
||||||
|
) {
|
||||||
super(healthProperties);
|
super(healthProperties);
|
||||||
this.aiMilvusConfig = aiMilvusConfig;
|
this.aiMilvusConfig = aiMilvusConfig;
|
||||||
|
this.milvusClientManager = milvusClientManager;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -51,8 +58,10 @@ public class RagHealthIndicator {
|
|||||||
protected Health doHealthCheck() {
|
protected Health doHealthCheck() {
|
||||||
MilvusVectorStore vectorStore = null;
|
MilvusVectorStore vectorStore = null;
|
||||||
try {
|
try {
|
||||||
|
milvusClientManager.reconfigureIfNeeded(aiMilvusConfig);
|
||||||
vectorStore = new MilvusVectorStore(
|
vectorStore = new MilvusVectorStore(
|
||||||
aiMilvusConfig.copyForCollection("__rag_health_probe__")
|
aiMilvusConfig.copyForCollection("__rag_health_probe__"),
|
||||||
|
milvusClientManager
|
||||||
);
|
);
|
||||||
if (vectorStore.checkAvailable()) {
|
if (vectorStore.checkAvailable()) {
|
||||||
return Health.up().withDetail("uri", aiMilvusConfig.getUri()).build();
|
return Health.up().withDetail("uri", aiMilvusConfig.getUri()).build();
|
||||||
|
|||||||
@@ -104,4 +104,28 @@ public class ThreadPoolConfig {
|
|||||||
executor.initialize();
|
executor.initialize();
|
||||||
return executor;
|
return executor;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建工作流多知识库检索线程池。
|
||||||
|
*
|
||||||
|
* @return 多知识库检索线程池
|
||||||
|
*/
|
||||||
|
@Bean(name = "knowledgeRetrievalExecutor")
|
||||||
|
public ThreadPoolTaskExecutor knowledgeRetrievalExecutor() {
|
||||||
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||||
|
EasyFlowThreadPoolProperties.Pool pool = properties.getKnowledgeRetrieval();
|
||||||
|
executor.setCorePoolSize(pool.getCoreSize());
|
||||||
|
executor.setMaxPoolSize(pool.getMaxSize());
|
||||||
|
executor.setQueueCapacity(pool.getQueueCapacity());
|
||||||
|
executor.setKeepAliveSeconds(pool.getKeepAliveSeconds());
|
||||||
|
executor.setAllowCoreThreadTimeOut(pool.isAllowCoreThreadTimeout());
|
||||||
|
executor.setThreadNamePrefix("knowledge-retrieval-");
|
||||||
|
executor.setRejectedExecutionHandler((runnable, executorService) -> {
|
||||||
|
log.error("多知识库检索线程池过载,active={}, queue={}",
|
||||||
|
executorService.getActiveCount(), executorService.getQueue().size());
|
||||||
|
throw new BusinessException("知识库检索繁忙,请稍后重试");
|
||||||
|
});
|
||||||
|
executor.initialize();
|
||||||
|
return executor;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -75,6 +75,10 @@ public class DocumentParseBridgeException extends RuntimeException {
|
|||||||
return new DocumentParseBridgeException("task_failed", message);
|
return new DocumentParseBridgeException("task_failed", message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static DocumentParseBridgeException taskNotFound(String message, Throwable cause) {
|
||||||
|
return new DocumentParseBridgeException("task_not_found", message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
public static DocumentParseBridgeException resultFetchFailed(String message, Throwable cause) {
|
public static DocumentParseBridgeException resultFetchFailed(String message, Throwable cause) {
|
||||||
return new DocumentParseBridgeException("result_fetch_failed", message, cause);
|
return new DocumentParseBridgeException("result_fetch_failed", message, cause);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import com.easyagents.document.core.entity.ParseResponse;
|
|||||||
import com.easyagents.document.core.entity.ParseResult;
|
import com.easyagents.document.core.entity.ParseResult;
|
||||||
import com.easyagents.document.core.entity.ParseTaskInfo;
|
import com.easyagents.document.core.entity.ParseTaskInfo;
|
||||||
import com.easyagents.document.core.entity.ParseTaskStatus;
|
import com.easyagents.document.core.entity.ParseTaskStatus;
|
||||||
|
import com.easyagents.document.core.exception.DocumentAsyncTaskNotFoundException;
|
||||||
import com.easyagents.document.pdf.PdfDocumentParseService;
|
import com.easyagents.document.pdf.PdfDocumentParseService;
|
||||||
import com.easyagents.document.pptx.PptxDocumentParseService;
|
import com.easyagents.document.pptx.PptxDocumentParseService;
|
||||||
import com.easyagents.document.xlsx.XlsxDocumentParseService;
|
import com.easyagents.document.xlsx.XlsxDocumentParseService;
|
||||||
@@ -140,6 +141,8 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
|||||||
return parseResultMapper.map(taskStatus);
|
return parseResultMapper.map(taskStatus);
|
||||||
} catch (DocumentParseBridgeException e) {
|
} catch (DocumentParseBridgeException e) {
|
||||||
throw e;
|
throw e;
|
||||||
|
} catch (DocumentAsyncTaskNotFoundException e) {
|
||||||
|
throw DocumentParseBridgeException.taskNotFound("异步文档解析任务不存在", e);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw DocumentParseBridgeException.taskFailed("查询异步文档解析任务状态失败", e);
|
throw DocumentParseBridgeException.taskFailed("查询异步文档解析任务状态失败", e);
|
||||||
}
|
}
|
||||||
@@ -175,6 +178,9 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
|||||||
} catch (DocumentParseBridgeException e) {
|
} catch (DocumentParseBridgeException e) {
|
||||||
LOG.error("桥接服务获取异步解析结果失败: providerTaskId={}", taskId, e);
|
LOG.error("桥接服务获取异步解析结果失败: providerTaskId={}", taskId, e);
|
||||||
throw e;
|
throw e;
|
||||||
|
} catch (DocumentAsyncTaskNotFoundException e) {
|
||||||
|
LOG.warn("桥接服务异步解析执行已丢失: providerTaskId={}", taskId);
|
||||||
|
throw DocumentParseBridgeException.taskNotFound("异步文档解析任务不存在", e);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
LOG.error("桥接服务获取异步解析结果异常: providerTaskId={}", taskId, e);
|
LOG.error("桥接服务获取异步解析结果异常: providerTaskId={}", taskId, e);
|
||||||
throw DocumentParseBridgeException.resultFetchFailed("获取异步文档解析结果失败", e);
|
throw DocumentParseBridgeException.resultFetchFailed("获取异步文档解析结果失败", e);
|
||||||
@@ -212,6 +218,9 @@ public class DocumentParseBridgeServiceImpl implements DocumentParseBridgeServic
|
|||||||
} catch (DocumentParseBridgeException e) {
|
} catch (DocumentParseBridgeException e) {
|
||||||
LOG.error("桥接服务查询异步解析任务状态失败: providerTaskId={}", taskId, e);
|
LOG.error("桥接服务查询异步解析任务状态失败: providerTaskId={}", taskId, e);
|
||||||
throw e;
|
throw e;
|
||||||
|
} catch (DocumentAsyncTaskNotFoundException e) {
|
||||||
|
LOG.warn("桥接服务异步解析执行已丢失: providerTaskId={}", taskId);
|
||||||
|
throw DocumentParseBridgeException.taskNotFound("异步文档解析任务不存在", e);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
LOG.error("桥接服务查询异步解析任务状态异常: providerTaskId={}", taskId, e);
|
LOG.error("桥接服务查询异步解析任务状态异常: providerTaskId={}", taskId, e);
|
||||||
throw DocumentParseBridgeException.taskFailed("聚合查询异步文档解析任务信息失败", e);
|
throw DocumentParseBridgeException.taskFailed("聚合查询异步文档解析任务信息失败", e);
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package tech.easyflow.ai.document.support;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识库导入与工作流文档解析共用的文件格式策略。
|
||||||
|
*
|
||||||
|
* @author Codex
|
||||||
|
* @since 2026-09-02
|
||||||
|
*/
|
||||||
|
public final class DocumentParseFilePolicy {
|
||||||
|
|
||||||
|
private static final List<String> SUPPORTED_EXTENSION_ORDER =
|
||||||
|
List.of("txt", "pdf", "docx", "md", "pptx", "xlsx", "csv");
|
||||||
|
private static final Set<String> SUPPORTED_EXTENSIONS =
|
||||||
|
Set.copyOf(SUPPORTED_EXTENSION_ORDER);
|
||||||
|
private static final String SUPPORTED_TYPE_LABEL =
|
||||||
|
SUPPORTED_EXTENSION_ORDER.stream()
|
||||||
|
.map(extension -> extension.toUpperCase(Locale.ROOT))
|
||||||
|
.collect(Collectors.joining("、"));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 禁止实例化格式策略工具类。
|
||||||
|
*/
|
||||||
|
private DocumentParseFilePolicy() {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 返回只读的支持格式集合。
|
||||||
|
*
|
||||||
|
* @return 支持的文件扩展名
|
||||||
|
*/
|
||||||
|
public static Set<String> supportedExtensions() {
|
||||||
|
return SUPPORTED_EXTENSIONS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 返回面向用户展示的支持格式列表。
|
||||||
|
*
|
||||||
|
* @return 大写扩展名列表
|
||||||
|
*/
|
||||||
|
public static String supportedTypeLabel() {
|
||||||
|
return SUPPORTED_TYPE_LABEL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断扩展名是否属于文档解析支持范围。
|
||||||
|
*
|
||||||
|
* @param extension 文件扩展名
|
||||||
|
* @return 支持时返回 {@code true}
|
||||||
|
*/
|
||||||
|
public static boolean isSupportedExtension(String extension) {
|
||||||
|
return SUPPORTED_EXTENSIONS.contains(normalizeExtension(extension));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断文件名是否属于文档解析支持范围。
|
||||||
|
*
|
||||||
|
* @param fileName 文件名
|
||||||
|
* @return 支持时返回 {@code true}
|
||||||
|
*/
|
||||||
|
public static boolean isSupportedFileName(String fileName) {
|
||||||
|
return isSupportedExtension(extensionOf(fileName));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 提取文件名中的小写扩展名。
|
||||||
|
*
|
||||||
|
* @param fileName 文件名
|
||||||
|
* @return 小写扩展名;无扩展名时返回空字符串
|
||||||
|
*/
|
||||||
|
public static String extensionOf(String fileName) {
|
||||||
|
String normalizedName = fileName == null ? "" : fileName.trim();
|
||||||
|
int dotIndex = normalizedName.lastIndexOf('.');
|
||||||
|
return dotIndex < 0 || dotIndex == normalizedName.length() - 1
|
||||||
|
? ""
|
||||||
|
: normalizeExtension(normalizedName.substring(dotIndex + 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String normalizeExtension(String extension) {
|
||||||
|
return extension == null ? "" : extension.trim().toLowerCase(Locale.ROOT);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块检索索引同步状态常量。
|
||||||
|
*/
|
||||||
|
public final class DocumentChunkSyncState {
|
||||||
|
|
||||||
|
private static final String PARENT_LOCK_PREFIX =
|
||||||
|
"easyflow:lock:document-chunk-parent:";
|
||||||
|
private static final String SYNC_LOCK_PREFIX =
|
||||||
|
"easyflow:lock:document-chunk-sync:";
|
||||||
|
|
||||||
|
public static final String SYNCED = "SYNCED";
|
||||||
|
public static final String PENDING = "PENDING";
|
||||||
|
public static final String FAILED = "FAILED";
|
||||||
|
|
||||||
|
public static final String TASK_RUNNING = "RUNNING";
|
||||||
|
public static final String TASK_SUCCEEDED = "SUCCEEDED";
|
||||||
|
public static final String TASK_SUPERSEDED = "SUPERSEDED";
|
||||||
|
|
||||||
|
public static final String OPERATION_UPSERT = "UPSERT";
|
||||||
|
public static final String OPERATION_DELETE = "DELETE";
|
||||||
|
|
||||||
|
public static String parentLockKey(Object documentId) {
|
||||||
|
return PARENT_LOCK_PREFIX + documentId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String syncLockKey(Object chunkId) {
|
||||||
|
return SYNC_LOCK_PREFIX + chunkId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DocumentChunkSyncState() {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,414 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import com.easyagents.core.model.embedding.EmbeddingModel;
|
||||||
|
import com.easyagents.core.model.embedding.EmbeddingOptions;
|
||||||
|
import com.easyagents.core.store.DocumentStore;
|
||||||
|
import com.easyagents.core.store.StoreOptions;
|
||||||
|
import com.easyagents.core.store.StoreResult;
|
||||||
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
|
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
|
||||||
|
import com.easyagents.store.milvus.MilvusVectorStore;
|
||||||
|
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusClientManager;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusConfig;
|
||||||
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
import tech.easyflow.ai.entity.Model;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.ai.service.ModelService;
|
||||||
|
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 持久化分块索引同步任务的投递、执行和恢复。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
public class DocumentChunkSyncTaskAppService {
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentChunkSyncTaskAppService.class);
|
||||||
|
private static final int MAX_ATTEMPTS = 5;
|
||||||
|
private static final int DISPATCH_LIMIT = 100;
|
||||||
|
private static final long REDISPATCH_MILLIS = 5_000L;
|
||||||
|
private static final long LEASE_MILLIS = 300_000L;
|
||||||
|
|
||||||
|
private final DocumentChunkSyncTaskMapper taskMapper;
|
||||||
|
private final DocumentChunkMapper chunkMapper;
|
||||||
|
private final DocumentCollectionService collectionService;
|
||||||
|
private final ModelService modelService;
|
||||||
|
private final SearcherFactory searcherFactory;
|
||||||
|
private final DocumentChunkSyncTaskProducer producer;
|
||||||
|
private final PlatformTransactionManager transactionManager;
|
||||||
|
private final RedisLockExecutor redisLockExecutor;
|
||||||
|
private final AiMilvusConfig milvusConfig;
|
||||||
|
private final ObjectProvider<AiMilvusClientManager> milvusClientManagerProvider;
|
||||||
|
|
||||||
|
public DocumentChunkSyncTaskAppService(
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper,
|
||||||
|
DocumentChunkMapper chunkMapper,
|
||||||
|
DocumentCollectionService collectionService,
|
||||||
|
ModelService modelService,
|
||||||
|
SearcherFactory searcherFactory,
|
||||||
|
DocumentChunkSyncTaskProducer producer,
|
||||||
|
PlatformTransactionManager transactionManager,
|
||||||
|
RedisLockExecutor redisLockExecutor,
|
||||||
|
AiMilvusConfig milvusConfig,
|
||||||
|
ObjectProvider<AiMilvusClientManager> milvusClientManagerProvider
|
||||||
|
) {
|
||||||
|
this.taskMapper = taskMapper;
|
||||||
|
this.chunkMapper = chunkMapper;
|
||||||
|
this.collectionService = collectionService;
|
||||||
|
this.modelService = modelService;
|
||||||
|
this.searcherFactory = searcherFactory;
|
||||||
|
this.producer = producer;
|
||||||
|
this.transactionManager = transactionManager;
|
||||||
|
this.redisLockExecutor = redisLockExecutor;
|
||||||
|
this.milvusConfig = milvusConfig;
|
||||||
|
this.milvusClientManagerProvider = milvusClientManagerProvider;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DocumentChunkSyncTask createTask(
|
||||||
|
DocumentChunk chunk,
|
||||||
|
DocumentCollection collection,
|
||||||
|
String operation,
|
||||||
|
long version,
|
||||||
|
Date now
|
||||||
|
) {
|
||||||
|
taskMapper.supersedeOlder(chunk.getId(), version, now);
|
||||||
|
DocumentChunkSyncTask task = new DocumentChunkSyncTask();
|
||||||
|
task.setChunkId(chunk.getId());
|
||||||
|
task.setDocumentId(chunk.getDocumentId());
|
||||||
|
task.setDocumentCollectionId(chunk.getDocumentCollectionId());
|
||||||
|
task.setVectorCollection(collection.getVectorStoreCollection());
|
||||||
|
task.setOperation(operation);
|
||||||
|
task.setSyncVersion(version);
|
||||||
|
task.setStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
task.setAttemptCount(0);
|
||||||
|
task.setNextRetryAt(now);
|
||||||
|
task.setCreated(now);
|
||||||
|
task.setModified(now);
|
||||||
|
if (taskMapper.insert(task) <= 0) {
|
||||||
|
throw new IllegalStateException("创建分块索引同步任务失败");
|
||||||
|
}
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void dispatchBestEffort(BigInteger taskId) {
|
||||||
|
try {
|
||||||
|
producer.send(taskId);
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.warn("分块索引同步消息投递失败,等待数据库补投: taskId={}", taskId, exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void dispatchPendingTasks() {
|
||||||
|
Date now = new Date();
|
||||||
|
taskMapper.recoverExpired(now);
|
||||||
|
Date redispatchBefore = new Date(now.getTime() - REDISPATCH_MILLIS);
|
||||||
|
for (DocumentChunkSyncTask task : taskMapper.selectPendingDue(
|
||||||
|
now,
|
||||||
|
redispatchBefore,
|
||||||
|
DISPATCH_LIMIT
|
||||||
|
)) {
|
||||||
|
if (taskMapper.markDispatched(task.getId(), now, redispatchBefore) <= 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
producer.send(task.getId());
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.warn("补投分块索引同步消息失败: taskId={}", task.getId(), exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void handleTask(BigInteger taskId) {
|
||||||
|
DocumentChunkSyncTask snapshot = taskMapper.selectOneById(taskId);
|
||||||
|
if (snapshot == null || !DocumentChunkSyncState.PENDING.equals(snapshot.getStatus())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
redisLockExecutor.executeWithRenewingLock(
|
||||||
|
DocumentChunkSyncState.syncLockKey(snapshot.getChunkId()),
|
||||||
|
Duration.ofSeconds(3),
|
||||||
|
Duration.ofMinutes(5),
|
||||||
|
() -> {
|
||||||
|
claimAndExecute(taskId, snapshot.getChunkId());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void claimAndExecute(BigInteger taskId, BigInteger chunkId) {
|
||||||
|
Date now = new Date();
|
||||||
|
String token = UUID.randomUUID().toString();
|
||||||
|
if (taskMapper.claim(
|
||||||
|
taskId,
|
||||||
|
token,
|
||||||
|
new Date(now.getTime() + LEASE_MILLIS),
|
||||||
|
now
|
||||||
|
) <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
new TransactionTemplate(transactionManager).executeWithoutResult(status ->
|
||||||
|
executeOwnedTask(taskId, chunkId, token)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void executeOwnedTask(
|
||||||
|
BigInteger taskId,
|
||||||
|
BigInteger chunkId,
|
||||||
|
String token
|
||||||
|
) {
|
||||||
|
// FOR UPDATE 必须是本事务的第一次读取。MySQL REPEATABLE READ 下,
|
||||||
|
// 若先做普通查询会建立旧快照,使删除或新版本建单已经提交后仍读取旧分块。
|
||||||
|
taskMapper.lockChunkTasks(chunkId);
|
||||||
|
DocumentChunkSyncTask task = taskMapper.selectOneById(taskId);
|
||||||
|
if (task == null
|
||||||
|
|| !DocumentChunkSyncState.TASK_RUNNING.equals(task.getStatus())
|
||||||
|
|| !token.equals(task.getExecutionToken())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isSuperseded(task)) {
|
||||||
|
finishTask(task, token, DocumentChunkSyncState.TASK_SUPERSEDED, null, null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
synchronizeIndexes(task);
|
||||||
|
finishSuccess(task, token);
|
||||||
|
} catch (IndexSyncException exception) {
|
||||||
|
LOG.warn("分块索引同步失败: taskId={}, chunkId={}, code={}",
|
||||||
|
task.getId(), task.getChunkId(), exception.code, exception.getCause());
|
||||||
|
finishFailure(task, token, exception.code, exception.getMessage());
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.error("分块索引同步发生未分类异常: taskId={}, chunkId={}",
|
||||||
|
task.getId(), task.getChunkId(), exception);
|
||||||
|
finishFailure(task, token, "INDEX_SYNC_FAILED", "检索索引同步失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isSuperseded(DocumentChunkSyncTask task) {
|
||||||
|
DocumentChunk chunk = chunkMapper.selectOneById(task.getChunkId());
|
||||||
|
if (DocumentChunkSyncState.OPERATION_DELETE.equals(task.getOperation())) {
|
||||||
|
return chunk != null;
|
||||||
|
}
|
||||||
|
return chunk == null
|
||||||
|
|| chunk.getIndexSyncVersion() == null
|
||||||
|
|| chunk.getIndexSyncVersion().longValue() != task.getSyncVersion().longValue()
|
||||||
|
|| !DocumentChunkSyncState.PENDING.equals(chunk.getIndexSyncStatus());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void synchronizeIndexes(DocumentChunkSyncTask task) {
|
||||||
|
if (DocumentChunkSyncState.OPERATION_DELETE.equals(task.getOperation())) {
|
||||||
|
synchronizeDelete(task);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
synchronizeUpsert(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void synchronizeUpsert(DocumentChunkSyncTask task) {
|
||||||
|
DocumentChunk chunk = chunkMapper.selectOneById(task.getChunkId());
|
||||||
|
DocumentCollection collection = collectionService.getById(task.getDocumentCollectionId());
|
||||||
|
if (chunk == null || collection == null) {
|
||||||
|
throw new IndexSyncException("INDEX_SOURCE_MISSING", "分块或知识库不存在", null);
|
||||||
|
}
|
||||||
|
StoreContext context = prepareUpsertContext(collection, task.getVectorCollection());
|
||||||
|
try {
|
||||||
|
com.easyagents.core.document.Document document = toSearchDocument(chunk, task.getDocumentCollectionId());
|
||||||
|
StoreResult vectorResult = context.documentStore.update(
|
||||||
|
Collections.singletonList(document),
|
||||||
|
context.storeOptions
|
||||||
|
);
|
||||||
|
if (vectorResult == null || !vectorResult.isSuccess()) {
|
||||||
|
throw new IndexSyncException("VECTOR_UPSERT_FAILED", "向量索引更新失败", null);
|
||||||
|
}
|
||||||
|
if (context.searcher != null
|
||||||
|
&& !context.searcher.addDocuments(Collections.singletonList(document))) {
|
||||||
|
throw new IndexSyncException("KEYWORD_UPSERT_FAILED", "关键词索引更新失败", null);
|
||||||
|
}
|
||||||
|
} catch (IndexSyncException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new IndexSyncException("INDEX_UPSERT_FAILED", "检索索引更新失败", exception);
|
||||||
|
} finally {
|
||||||
|
DocumentStoreLifecycleSupport.closeQuietly(context.documentStore);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void synchronizeDelete(DocumentChunkSyncTask task) {
|
||||||
|
MilvusVectorStoreConfig storeConfig = milvusConfig.copyForCollection(task.getVectorCollection());
|
||||||
|
AiMilvusClientManager clientManager = milvusClientManagerProvider.getObject();
|
||||||
|
clientManager.reconfigureIfNeeded(milvusConfig);
|
||||||
|
DocumentStore documentStore = new MilvusVectorStore(
|
||||||
|
storeConfig,
|
||||||
|
clientManager
|
||||||
|
);
|
||||||
|
StoreOptions storeOptions = StoreOptions.ofCollectionName(task.getVectorCollection());
|
||||||
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
|
try {
|
||||||
|
StoreResult result = documentStore.delete(
|
||||||
|
Collections.singletonList(task.getChunkId().toString()),
|
||||||
|
storeOptions
|
||||||
|
);
|
||||||
|
if (result == null || !result.isSuccess()) {
|
||||||
|
throw new IndexSyncException("VECTOR_DELETE_FAILED", "向量索引删除失败", null);
|
||||||
|
}
|
||||||
|
if (searcher != null && !searcher.deleteDocument(task.getChunkId())) {
|
||||||
|
throw new IndexSyncException("KEYWORD_DELETE_FAILED", "关键词索引删除失败", null);
|
||||||
|
}
|
||||||
|
} catch (IndexSyncException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new IndexSyncException("INDEX_DELETE_FAILED", "检索索引删除失败", exception);
|
||||||
|
} finally {
|
||||||
|
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private StoreContext prepareUpsertContext(
|
||||||
|
DocumentCollection collection,
|
||||||
|
String vectorCollection
|
||||||
|
) {
|
||||||
|
DocumentStore documentStore = collection.toDocumentStore();
|
||||||
|
if (documentStore == null) {
|
||||||
|
throw new IndexSyncException("VECTOR_STORE_MISSING", "知识库没有配置向量库", null);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Model model = modelService.getModelInstance(collection.getVectorEmbedModelId());
|
||||||
|
if (model == null) {
|
||||||
|
throw new IndexSyncException("EMBEDDING_MODEL_MISSING", "知识库没有配置向量模型", null);
|
||||||
|
}
|
||||||
|
EmbeddingModel embeddingModel = model.toEmbeddingModel();
|
||||||
|
documentStore.setEmbeddingModel(embeddingModel);
|
||||||
|
StoreOptions options = StoreOptions.ofCollectionName(vectorCollection);
|
||||||
|
EmbeddingOptions embeddingOptions = new EmbeddingOptions();
|
||||||
|
embeddingOptions.setModel(model.getModelName());
|
||||||
|
embeddingOptions.setDimensions(collection.getDimensionOfVectorModel());
|
||||||
|
options.setEmbeddingOptions(embeddingOptions);
|
||||||
|
options.setIndexName(vectorCollection);
|
||||||
|
return new StoreContext(documentStore, options, searcherFactory.getSearcher());
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
DocumentStoreLifecycleSupport.closeQuietly(documentStore);
|
||||||
|
throw exception;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private com.easyagents.core.document.Document toSearchDocument(
|
||||||
|
DocumentChunk chunk,
|
||||||
|
BigInteger knowledgeId
|
||||||
|
) {
|
||||||
|
com.easyagents.core.document.Document document =
|
||||||
|
com.easyagents.core.document.Document.of(chunk.getContent());
|
||||||
|
document.setId(chunk.getId());
|
||||||
|
document.addMetadata(KeywordSearchMetadataKeys.KNOWLEDGE_ID, knowledgeId.toString());
|
||||||
|
return document;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void finishSuccess(DocumentChunkSyncTask task, String token) {
|
||||||
|
new TransactionTemplate(transactionManager).executeWithoutResult(status -> {
|
||||||
|
if (taskMapper.finishOwned(
|
||||||
|
task.getId(), token, DocumentChunkSyncState.TASK_SUCCEEDED, null, null, new Date()
|
||||||
|
) > 0 && DocumentChunkSyncState.OPERATION_UPSERT.equals(task.getOperation())) {
|
||||||
|
chunkMapper.updateSyncState(
|
||||||
|
task.getChunkId(), task.getSyncVersion(), DocumentChunkSyncState.SYNCED, null, null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private void finishTask(
|
||||||
|
DocumentChunkSyncTask task,
|
||||||
|
String token,
|
||||||
|
String status,
|
||||||
|
String errorCode,
|
||||||
|
String errorMessage
|
||||||
|
) {
|
||||||
|
taskMapper.finishOwned(task.getId(), token, status, errorCode, errorMessage, new Date());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void finishFailure(
|
||||||
|
DocumentChunkSyncTask task,
|
||||||
|
String token,
|
||||||
|
String errorCode,
|
||||||
|
String errorMessage
|
||||||
|
) {
|
||||||
|
int attempts = task.getAttemptCount() == null ? 1 : task.getAttemptCount();
|
||||||
|
boolean deleteOperation = DocumentChunkSyncState.OPERATION_DELETE.equals(
|
||||||
|
task.getOperation()
|
||||||
|
);
|
||||||
|
// 删除后已没有页面实体承载手动重试入口,因此清理任务必须保留为持久化 tombstone。
|
||||||
|
boolean exhausted = !deleteOperation && attempts >= MAX_ATTEMPTS;
|
||||||
|
String nextStatus = exhausted ? DocumentChunkSyncState.FAILED : DocumentChunkSyncState.PENDING;
|
||||||
|
long delaySeconds = Math.min(1L << Math.min(attempts, 6), 60L);
|
||||||
|
Date now = new Date();
|
||||||
|
Date nextRetryAt = exhausted ? now : new Date(now.getTime() + delaySeconds * 1_000L);
|
||||||
|
new TransactionTemplate(transactionManager).executeWithoutResult(status -> {
|
||||||
|
if (taskMapper.failOrRetryOwned(
|
||||||
|
task.getId(), token, nextStatus, nextRetryAt, errorCode, errorMessage, now
|
||||||
|
) > 0 && exhausted && DocumentChunkSyncState.OPERATION_UPSERT.equals(task.getOperation())) {
|
||||||
|
chunkMapper.updateSyncState(
|
||||||
|
task.getChunkId(), task.getSyncVersion(), DocumentChunkSyncState.FAILED,
|
||||||
|
errorCode, errorMessage
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public DocumentChunk retry(BigInteger chunkId, long syncVersion) {
|
||||||
|
Date now = new Date();
|
||||||
|
DocumentChunk chunk = chunkMapper.selectOneById(chunkId);
|
||||||
|
if (chunk == null || chunk.getIndexSyncVersion() == null
|
||||||
|
|| chunk.getIndexSyncVersion().longValue() != syncVersion
|
||||||
|
|| !DocumentChunkSyncState.FAILED.equals(chunk.getIndexSyncStatus())) {
|
||||||
|
throw new IllegalStateException("分块同步状态已变化,请刷新后重试");
|
||||||
|
}
|
||||||
|
DocumentChunkSyncTask task = taskMapper.selectFailed(chunkId, syncVersion);
|
||||||
|
if (task == null) {
|
||||||
|
throw new IllegalStateException("未找到可重试的索引同步任务");
|
||||||
|
}
|
||||||
|
new TransactionTemplate(transactionManager).executeWithoutResult(status -> {
|
||||||
|
if (taskMapper.retryFailed(task.getId(), chunkId, syncVersion, now) <= 0) {
|
||||||
|
throw new IllegalStateException("索引同步任务状态已变化");
|
||||||
|
}
|
||||||
|
if (chunkMapper.updateSyncState(
|
||||||
|
chunkId, syncVersion, DocumentChunkSyncState.PENDING, null, null
|
||||||
|
) <= 0) {
|
||||||
|
throw new IllegalStateException("分块同步状态已变化");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
dispatchBestEffort(task.getId());
|
||||||
|
chunk.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
chunk.setIndexSyncErrorCode(null);
|
||||||
|
chunk.setIndexSyncErrorMessage(null);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
private record StoreContext(
|
||||||
|
DocumentStore documentStore,
|
||||||
|
StoreOptions storeOptions,
|
||||||
|
DocumentSearcher searcher
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class IndexSyncException extends RuntimeException {
|
||||||
|
private final String code;
|
||||||
|
|
||||||
|
private IndexSyncException(String code, String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
this.code = code;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.common.mq.config.MQProperties;
|
||||||
|
import tech.easyflow.common.mq.core.MQConsumerHandler;
|
||||||
|
import tech.easyflow.common.mq.core.MQDeferException;
|
||||||
|
import tech.easyflow.common.mq.core.MQMessage;
|
||||||
|
import tech.easyflow.common.mq.core.MQSubscription;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步消息消费者。
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class DocumentChunkSyncTaskConsumer implements MQConsumerHandler {
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentChunkSyncTaskConsumer.class);
|
||||||
|
|
||||||
|
private final DocumentChunkSyncTaskAppService appService;
|
||||||
|
private final MQProperties mqProperties;
|
||||||
|
|
||||||
|
public DocumentChunkSyncTaskConsumer(
|
||||||
|
DocumentChunkSyncTaskAppService appService,
|
||||||
|
MQProperties mqProperties
|
||||||
|
) {
|
||||||
|
this.appService = appService;
|
||||||
|
this.mqProperties = mqProperties;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MQSubscription subscription() {
|
||||||
|
MQSubscription subscription = new MQSubscription();
|
||||||
|
subscription.setTopic(DocumentChunkSyncTaskProducer.TOPIC);
|
||||||
|
subscription.setConsumerGroup(DocumentChunkSyncTaskProducer.GROUP);
|
||||||
|
subscription.setShardCount(Math.max(
|
||||||
|
mqProperties.getRedis().getChatPersistShardCount(),
|
||||||
|
1
|
||||||
|
));
|
||||||
|
subscription.setBatchEnabled(false);
|
||||||
|
return subscription;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void handle(List<MQMessage> messages) {
|
||||||
|
for (MQMessage message : messages) {
|
||||||
|
DocumentChunkSyncTaskMessage event = JSON.parseObject(
|
||||||
|
message.getBody(),
|
||||||
|
DocumentChunkSyncTaskMessage.class
|
||||||
|
);
|
||||||
|
if (event == null || event.getTaskId() == null) {
|
||||||
|
LOG.warn("忽略非法分块索引同步消息: messageId={}", message.getMessageId());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
appService.handleTask(event.getTaskId());
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.error("分块索引同步任务状态处理失败: taskId={}", event.getTaskId(), exception);
|
||||||
|
throw new MQDeferException("分块索引同步任务暂时无法处理", exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步消息。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkSyncTaskMessage implements Serializable {
|
||||||
|
private BigInteger taskId;
|
||||||
|
private Date occurredAt;
|
||||||
|
|
||||||
|
public BigInteger getTaskId() { return taskId; }
|
||||||
|
public void setTaskId(BigInteger taskId) { this.taskId = taskId; }
|
||||||
|
public Date getOccurredAt() { return occurredAt; }
|
||||||
|
public void setOccurredAt(Date occurredAt) { this.occurredAt = occurredAt; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.common.cache.DistributedScheduledLock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 补投待同步任务并回收过期租约。
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class DocumentChunkSyncTaskMonitor {
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentChunkSyncTaskMonitor.class);
|
||||||
|
|
||||||
|
private final DocumentChunkSyncTaskAppService appService;
|
||||||
|
|
||||||
|
public DocumentChunkSyncTaskMonitor(DocumentChunkSyncTaskAppService appService) {
|
||||||
|
this.appService = appService;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Scheduled(fixedDelayString = "${easyflow.ai.document-chunk-sync.dispatch-interval:2s}",
|
||||||
|
initialDelayString = "${easyflow.ai.document-chunk-sync.dispatch-interval:2s}")
|
||||||
|
@DistributedScheduledLock(key = "easyflow:schedule:document-chunk-index-sync", leaseSeconds = 2L)
|
||||||
|
public void dispatchPendingTasks() {
|
||||||
|
try {
|
||||||
|
appService.dispatchPendingTasks();
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
LOG.error("分块索引同步补投失败", exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import tech.easyflow.common.mq.core.MQMessage;
|
||||||
|
import tech.easyflow.common.mq.core.MQProducer;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步消息生产者。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
public class DocumentChunkSyncTaskProducer {
|
||||||
|
|
||||||
|
static final String TOPIC = "document-chunk-index-sync";
|
||||||
|
static final String GROUP = "document-chunk-index-sync-group";
|
||||||
|
|
||||||
|
private final MQProducer mqProducer;
|
||||||
|
|
||||||
|
public DocumentChunkSyncTaskProducer(MQProducer mqProducer) {
|
||||||
|
this.mqProducer = mqProducer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void send(BigInteger taskId) {
|
||||||
|
Date now = new Date();
|
||||||
|
DocumentChunkSyncTaskMessage event = new DocumentChunkSyncTaskMessage();
|
||||||
|
event.setTaskId(taskId);
|
||||||
|
event.setOccurredAt(now);
|
||||||
|
MQMessage message = new MQMessage();
|
||||||
|
message.setMessageId("chunk-sync-" + taskId);
|
||||||
|
message.setTopic(TOPIC);
|
||||||
|
message.setKey(String.valueOf(taskId));
|
||||||
|
message.setCreatedAt(now);
|
||||||
|
message.setBody(JSON.toJSONString(event));
|
||||||
|
mqProducer.send(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import org.springframework.transaction.support.TransactionSynchronization;
|
|||||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
import tech.easyflow.ai.document.support.DocumentParseFilePolicy;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext;
|
import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult;
|
import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult;
|
||||||
@@ -63,7 +64,7 @@ public class DocumentImportBatchAppService {
|
|||||||
|
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(DocumentImportBatchAppService.class);
|
private static final Logger LOG = LoggerFactory.getLogger(DocumentImportBatchAppService.class);
|
||||||
private static final Set<String> SUPPORTED_EXTENSIONS =
|
private static final Set<String> SUPPORTED_EXTENSIONS =
|
||||||
DocumentImportFormatPolicy.supportedExtensions();
|
DocumentParseFilePolicy.supportedExtensions();
|
||||||
private static final Duration BATCH_MUTATION_LOCK_LEASE = Duration.ofMinutes(30);
|
private static final Duration BATCH_MUTATION_LOCK_LEASE = Duration.ofMinutes(30);
|
||||||
private static final Duration RECOVERY_DISPATCH_LEASE = Duration.ofMinutes(2);
|
private static final Duration RECOVERY_DISPATCH_LEASE = Duration.ofMinutes(2);
|
||||||
private static final Duration RECOVERY_LEASE_RENEW_INTERVAL =
|
private static final Duration RECOVERY_LEASE_RENEW_INTERVAL =
|
||||||
@@ -79,6 +80,7 @@ public class DocumentImportBatchAppService {
|
|||||||
private final DocumentMapper documentMapper;
|
private final DocumentMapper documentMapper;
|
||||||
private final RedisLockExecutor redisLockExecutor;
|
private final RedisLockExecutor redisLockExecutor;
|
||||||
private final DocumentImportBatchCircuitBreaker circuitBreaker;
|
private final DocumentImportBatchCircuitBreaker circuitBreaker;
|
||||||
|
private final DocumentImportRecoveryCoordinator recoveryCoordinator;
|
||||||
|
|
||||||
@Resource(name = "default")
|
@Resource(name = "default")
|
||||||
private FileStorageService storageService;
|
private FileStorageService storageService;
|
||||||
@@ -96,6 +98,7 @@ public class DocumentImportBatchAppService {
|
|||||||
* @param documentMapper 文档 Mapper
|
* @param documentMapper 文档 Mapper
|
||||||
* @param redisLockExecutor 分布式锁执行器
|
* @param redisLockExecutor 分布式锁执行器
|
||||||
* @param circuitBreaker 自动导入批次熔断器
|
* @param circuitBreaker 自动导入批次熔断器
|
||||||
|
* @param recoveryCoordinator 恢复令牌事务协调器
|
||||||
*/
|
*/
|
||||||
public DocumentImportBatchAppService(DocumentImportBatchService batchService,
|
public DocumentImportBatchAppService(DocumentImportBatchService batchService,
|
||||||
DocumentImportBatchItemService itemService,
|
DocumentImportBatchItemService itemService,
|
||||||
@@ -106,7 +109,8 @@ public class DocumentImportBatchAppService {
|
|||||||
DocumentImportBatchItemMapper itemMapper,
|
DocumentImportBatchItemMapper itemMapper,
|
||||||
DocumentMapper documentMapper,
|
DocumentMapper documentMapper,
|
||||||
RedisLockExecutor redisLockExecutor,
|
RedisLockExecutor redisLockExecutor,
|
||||||
DocumentImportBatchCircuitBreaker circuitBreaker) {
|
DocumentImportBatchCircuitBreaker circuitBreaker,
|
||||||
|
DocumentImportRecoveryCoordinator recoveryCoordinator) {
|
||||||
this.batchService = batchService;
|
this.batchService = batchService;
|
||||||
this.itemService = itemService;
|
this.itemService = itemService;
|
||||||
this.batchTracker = batchTracker;
|
this.batchTracker = batchTracker;
|
||||||
@@ -117,6 +121,7 @@ public class DocumentImportBatchAppService {
|
|||||||
this.documentMapper = documentMapper;
|
this.documentMapper = documentMapper;
|
||||||
this.redisLockExecutor = redisLockExecutor;
|
this.redisLockExecutor = redisLockExecutor;
|
||||||
this.circuitBreaker = circuitBreaker;
|
this.circuitBreaker = circuitBreaker;
|
||||||
|
this.recoveryCoordinator = recoveryCoordinator;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1245,13 +1250,10 @@ public class DocumentImportBatchAppService {
|
|||||||
Date claimedAt = new Date();
|
Date claimedAt = new Date();
|
||||||
Date leaseUntil = new Date(
|
Date leaseUntil = new Date(
|
||||||
claimedAt.getTime() + RECOVERY_DISPATCH_LEASE.toMillis());
|
claimedAt.getTime() + RECOVERY_DISPATCH_LEASE.toMillis());
|
||||||
if (batchMapper.claimRecoveryPending(
|
|
||||||
batchId, recoveryToken, leaseUntil, claimedAt) <= 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
DocumentImportBatch claimedBatch =
|
DocumentImportBatch claimedBatch =
|
||||||
batchMapper.selectClaimedRecovery(batchId, recoveryToken);
|
recoveryCoordinator.claim(
|
||||||
|
batchId, recoveryToken, leaseUntil, claimedAt);
|
||||||
if (claimedBatch == null) {
|
if (claimedBatch == null) {
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"批次恢复调度令牌已失效,旧持有者停止恢复: "
|
"批次恢复调度令牌已失效,旧持有者停止恢复: "
|
||||||
@@ -1283,9 +1285,8 @@ public class DocumentImportBatchAppService {
|
|||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
int finalized = batchMapper.finalizeRecoveryPending(
|
if (!recoveryCoordinator.finalizeRecovery(
|
||||||
batchId, recoveryToken, new Date());
|
batchId, recoveryToken, new Date())) {
|
||||||
if (finalized <= 0) {
|
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"批次恢复待办已变更,当前实例跳过收尾: "
|
"批次恢复待办已变更,当前实例跳过收尾: "
|
||||||
+ "batchId={}, recoveryToken={}",
|
+ "batchId={}, recoveryToken={}",
|
||||||
@@ -1298,7 +1299,8 @@ public class DocumentImportBatchAppService {
|
|||||||
if (!circuitBreaker.interruptRecoveryBatch(
|
if (!circuitBreaker.interruptRecoveryBatch(
|
||||||
batchId, recoveryToken, error)) {
|
batchId, recoveryToken, error)) {
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"批次恢复异常发生时令牌已失效,跳过旧持有者熔断: "
|
"批次恢复异常发生时令牌已失效或已有并发推进,"
|
||||||
|
+ "跳过当前持有者熔断: "
|
||||||
+ "batchId={}, recoveryToken={}",
|
+ "batchId={}, recoveryToken={}",
|
||||||
batchId,
|
batchId,
|
||||||
recoveryToken
|
recoveryToken
|
||||||
@@ -1349,13 +1351,13 @@ public class DocumentImportBatchAppService {
|
|||||||
Date renewedLeaseUntil = new Date(
|
Date renewedLeaseUntil = new Date(
|
||||||
nowMillis + RECOVERY_DISPATCH_LEASE.toMillis()
|
nowMillis + RECOVERY_DISPATCH_LEASE.toMillis()
|
||||||
);
|
);
|
||||||
int renewed = batchMapper.renewRecoveryPendingLease(
|
boolean renewed = recoveryCoordinator.renew(
|
||||||
batchId,
|
batchId,
|
||||||
recoveryToken,
|
recoveryToken,
|
||||||
renewedLeaseUntil,
|
renewedLeaseUntil,
|
||||||
now
|
now
|
||||||
);
|
);
|
||||||
if (renewed <= 0) {
|
if (!renewed) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
renewAfter.set(nowMillis + renewIntervalMillis);
|
renewAfter.set(nowMillis + renewIntervalMillis);
|
||||||
@@ -1456,7 +1458,11 @@ public class DocumentImportBatchAppService {
|
|||||||
int dotIndex = fileName == null ? -1 : fileName.lastIndexOf('.');
|
int dotIndex = fileName == null ? -1 : fileName.lastIndexOf('.');
|
||||||
String extension = dotIndex < 0 ? "" : fileName.substring(dotIndex + 1).toLowerCase(Locale.ROOT);
|
String extension = dotIndex < 0 ? "" : fileName.substring(dotIndex + 1).toLowerCase(Locale.ROOT);
|
||||||
if (!SUPPORTED_EXTENSIONS.contains(extension)) {
|
if (!SUPPORTED_EXTENSIONS.contains(extension)) {
|
||||||
throw new BusinessException("暂不支持该文件格式");
|
throw new BusinessException(
|
||||||
|
"暂不支持该文件格式,仅支持 "
|
||||||
|
+ DocumentParseFilePolicy.supportedTypeLabel()
|
||||||
|
+ " 文件"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,9 @@ import org.springframework.stereotype.Service;
|
|||||||
import org.springframework.transaction.annotation.Propagation;
|
import org.springframework.transaction.annotation.Propagation;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||||
|
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||||
import tech.easyflow.ai.entity.DocumentImportTask;
|
import tech.easyflow.ai.entity.DocumentImportTask;
|
||||||
|
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||||
import tech.easyflow.ai.enums.DocumentImportMode;
|
import tech.easyflow.ai.enums.DocumentImportMode;
|
||||||
import tech.easyflow.ai.enums.DocumentImportTaskStatus;
|
import tech.easyflow.ai.enums.DocumentImportTaskStatus;
|
||||||
@@ -23,6 +25,8 @@ import java.sql.SQLRecoverableException;
|
|||||||
import java.sql.SQLTimeoutException;
|
import java.sql.SQLTimeoutException;
|
||||||
import java.sql.SQLTransientConnectionException;
|
import java.sql.SQLTransientConnectionException;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
import java.util.concurrent.RejectedExecutionException;
|
import java.util.concurrent.RejectedExecutionException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -135,13 +139,18 @@ public class DocumentImportBatchCircuitBreaker {
|
|||||||
if (recoveryToken == null || recoveryToken.isBlank()) {
|
if (recoveryToken == null || recoveryToken.isBlank()) {
|
||||||
throw new IllegalArgumentException("恢复调度令牌不能为空");
|
throw new IllegalArgumentException("恢复调度令牌不能为空");
|
||||||
}
|
}
|
||||||
|
DocumentImportRecoveryException recoveryError =
|
||||||
|
findCause(error, DocumentImportRecoveryException.class);
|
||||||
return interruptBatch(
|
return interruptBatch(
|
||||||
batchId,
|
batchId,
|
||||||
resolveReason(error),
|
resolveReason(error),
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
error,
|
error,
|
||||||
recoveryToken
|
recoveryToken,
|
||||||
|
recoveryError == null
|
||||||
|
? List.of()
|
||||||
|
: recoveryError.getFailureFenceItemIds()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -211,7 +220,7 @@ public class DocumentImportBatchCircuitBreaker {
|
|||||||
String phase,
|
String phase,
|
||||||
Throwable error) {
|
Throwable error) {
|
||||||
return interruptBatch(
|
return interruptBatch(
|
||||||
batchId, reason, taskId, phase, error, null);
|
batchId, reason, taskId, phase, error, null, List.of());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -223,6 +232,7 @@ public class DocumentImportBatchCircuitBreaker {
|
|||||||
* @param phase 触发任务阶段,可为空
|
* @param phase 触发任务阶段,可为空
|
||||||
* @param error 原始异常
|
* @param error 原始异常
|
||||||
* @param recoveryToken 恢复调度令牌,可为空
|
* @param recoveryToken 恢复调度令牌,可为空
|
||||||
|
* @param recoveryFailureItemIds 零重排时的初始失败项围栏
|
||||||
* @return 批次是否已经停止运行;围栏失效时返回 {@code false}
|
* @return 批次是否已经停止运行;围栏失效时返回 {@code false}
|
||||||
*/
|
*/
|
||||||
private boolean interruptBatch(BigInteger batchId,
|
private boolean interruptBatch(BigInteger batchId,
|
||||||
@@ -230,11 +240,17 @@ public class DocumentImportBatchCircuitBreaker {
|
|||||||
BigInteger taskId,
|
BigInteger taskId,
|
||||||
String phase,
|
String phase,
|
||||||
Throwable error,
|
Throwable error,
|
||||||
String recoveryToken) {
|
String recoveryToken,
|
||||||
|
List<BigInteger> recoveryFailureItemIds) {
|
||||||
if (batchId == null) {
|
if (batchId == null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
DocumentImportBatch batch = batchMapper.selectOneById(batchId);
|
boolean fencedRecoveryFailure = recoveryToken != null
|
||||||
|
&& recoveryFailureItemIds != null
|
||||||
|
&& !recoveryFailureItemIds.isEmpty();
|
||||||
|
// 批次行是后续 task/item 状态收口的根锁;所有熔断路径先锁定
|
||||||
|
// batch,避免多表围栏更新与任务完成事务形成 task -> batch 逆序。
|
||||||
|
DocumentImportBatch batch = batchMapper.selectForUpdate(batchId);
|
||||||
if (batch == null) {
|
if (batch == null) {
|
||||||
LOG.warn(
|
LOG.warn(
|
||||||
"忽略无法关联批次的自动导入熔断请求: taskId={}, batchId={}",
|
"忽略无法关联批次的自动导入熔断请求: taskId={}, batchId={}",
|
||||||
@@ -249,6 +265,39 @@ public class DocumentImportBatchCircuitBreaker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Date now = new Date();
|
Date now = new Date();
|
||||||
|
if (fencedRecoveryFailure) {
|
||||||
|
if (!Boolean.TRUE.equals(batch.getRecoveryPending())
|
||||||
|
|| !Objects.equals(recoveryToken, batch.getRecoveryToken())
|
||||||
|
|| batch.getRecoveryLeaseUntil() == null
|
||||||
|
|| !batch.getRecoveryLeaseUntil().after(now)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (itemMapper.countActiveItems(batchId) > 0) {
|
||||||
|
LOG.info(
|
||||||
|
"恢复失败熔断发现批次已有活跃文件项,保留运行批次: "
|
||||||
|
+ "batchId={}",
|
||||||
|
batchId
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (BigInteger itemId : recoveryFailureItemIds) {
|
||||||
|
DocumentImportBatchItem item =
|
||||||
|
itemMapper.selectForUpdate(itemId);
|
||||||
|
if (item == null
|
||||||
|
|| !batchId.equals(item.getBatchId())
|
||||||
|
|| !DocumentImportBatchItemStatus.FAILED
|
||||||
|
.name().equals(item.getStatus())) {
|
||||||
|
LOG.info(
|
||||||
|
"恢复失败熔断发现等价并发推进,保留运行批次: "
|
||||||
|
+ "batchId={}, itemId={}, status={}",
|
||||||
|
batchId,
|
||||||
|
itemId,
|
||||||
|
item == null ? null : item.getStatus()
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
int interrupted;
|
int interrupted;
|
||||||
if (recoveryToken != null) {
|
if (recoveryToken != null) {
|
||||||
interrupted = batchMapper.interruptOwnedRecoveryBatch(
|
interrupted = batchMapper.interruptOwnedRecoveryBatch(
|
||||||
@@ -305,6 +354,14 @@ public class DocumentImportBatchCircuitBreaker {
|
|||||||
* @return 稳定错误码与用户可见摘要
|
* @return 稳定错误码与用户可见摘要
|
||||||
*/
|
*/
|
||||||
private InterruptionReason resolveReason(Throwable error) {
|
private InterruptionReason resolveReason(Throwable error) {
|
||||||
|
DocumentImportRecoveryException recoveryError =
|
||||||
|
findCause(error, DocumentImportRecoveryException.class);
|
||||||
|
if (recoveryError != null) {
|
||||||
|
return new InterruptionReason(
|
||||||
|
recoveryError.getCode(),
|
||||||
|
recoveryError.getUserMessage()
|
||||||
|
);
|
||||||
|
}
|
||||||
if (containsRedisFailure(error)) {
|
if (containsRedisFailure(error)) {
|
||||||
return new InterruptionReason(
|
return new InterruptionReason(
|
||||||
REDIS_UNAVAILABLE,
|
REDIS_UNAVAILABLE,
|
||||||
@@ -329,6 +386,25 @@ public class DocumentImportBatchCircuitBreaker {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从异常链中查找指定类型。
|
||||||
|
*
|
||||||
|
* @param error 原始异常
|
||||||
|
* @param type 目标异常类型
|
||||||
|
* @param <T> 异常类型
|
||||||
|
* @return 匹配异常;不存在时返回 {@code null}
|
||||||
|
*/
|
||||||
|
private <T extends Throwable> T findCause(Throwable error, Class<T> type) {
|
||||||
|
Throwable current = error;
|
||||||
|
while (current != null) {
|
||||||
|
if (type.isInstance(current)) {
|
||||||
|
return type.cast(current);
|
||||||
|
}
|
||||||
|
current = current.getCause();
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 判断异常链是否来自 Redis/Lettuce。
|
* 判断异常链是否来自 Redis/Lettuce。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -27,6 +27,18 @@ import java.util.Date;
|
|||||||
@Service
|
@Service
|
||||||
public class DocumentImportBatchTracker {
|
public class DocumentImportBatchTracker {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 单文件恢复失败原因的事务内处理结果。
|
||||||
|
*/
|
||||||
|
public enum FailedRetryErrorOutcome {
|
||||||
|
/** 失败原因已写回,文件项仍等待后续人工恢复。 */
|
||||||
|
ERROR_RECORDED,
|
||||||
|
/** 等价并发操作已把文件项推进出失败状态。 */
|
||||||
|
ALREADY_ADVANCED,
|
||||||
|
/** 批次或文件项已不再允许当前恢复请求写回。 */
|
||||||
|
REJECTED
|
||||||
|
}
|
||||||
|
|
||||||
private final DocumentImportBatchService batchService;
|
private final DocumentImportBatchService batchService;
|
||||||
private final DocumentImportBatchItemService itemService;
|
private final DocumentImportBatchItemService itemService;
|
||||||
private final DocumentImportBatchMapper batchMapper;
|
private final DocumentImportBatchMapper batchMapper;
|
||||||
@@ -64,6 +76,31 @@ public class DocumentImportBatchTracker {
|
|||||||
return batch;
|
return batch;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在任务行变更前锁定所属批次,统一 batch -> task 锁序。
|
||||||
|
*
|
||||||
|
* <p>自动导入的任务生命周期由运行批次驱动,批次结束后不允许
|
||||||
|
* 旧任务继续推进。手动导入的上传批次可以先结束,用户之后再手动
|
||||||
|
* 执行分块和索引,因此只参与锁序,不以批次终态阻断后续任务。</p>
|
||||||
|
*
|
||||||
|
* @param batchId 批次 ID;无批次任务传入 {@code null}
|
||||||
|
* @return 批次存在且允许当前任务推进时为 {@code true}
|
||||||
|
*/
|
||||||
|
public boolean lockBatchForTaskMutation(BigInteger batchId) {
|
||||||
|
if (batchId == null) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
DocumentImportBatch batch = batchMapper.selectForUpdate(batchId);
|
||||||
|
if (batch == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (DocumentImportMode.AUTO.name().equals(batch.getImportMode())) {
|
||||||
|
return DocumentImportBatchStatus.RUNNING.name().equals(
|
||||||
|
batch.getStatus());
|
||||||
|
}
|
||||||
|
return DocumentImportMode.MANUAL.name().equals(batch.getImportMode());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 查询批次文件项。
|
* 查询批次文件项。
|
||||||
*
|
*
|
||||||
@@ -107,10 +144,8 @@ public class DocumentImportBatchTracker {
|
|||||||
if (itemId == null) {
|
if (itemId == null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
DocumentImportBatchItem current = requireItem(itemId);
|
|
||||||
int attemptDelta = (status == DocumentImportBatchItemStatus.PENDING
|
int attemptDelta = (status == DocumentImportBatchItemStatus.PENDING
|
||||||
|| status == DocumentImportBatchItemStatus.RUNNING)
|
|| status == DocumentImportBatchItemStatus.RUNNING)
|
||||||
&& DocumentImportBatchItemStatus.FAILED.name().equals(current.getStatus())
|
|
||||||
? 1
|
? 1
|
||||||
: 0;
|
: 0;
|
||||||
return transitionItem(itemId, stage, status, errorSummary,
|
return transitionItem(itemId, stage, status, errorSummary,
|
||||||
@@ -162,28 +197,34 @@ public class DocumentImportBatchTracker {
|
|||||||
if (itemId == null) {
|
if (itemId == null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
for (int attempt = 0; attempt < 3; attempt++) {
|
LockedBatchItem locked = lockBatchThenItem(itemId);
|
||||||
DocumentImportBatchItem current = requireItem(itemId);
|
if (DocumentImportBatchStatus.INTERRUPTED.name()
|
||||||
|
.equals(locked.batch.getStatus())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
DocumentImportBatchItem current = locked.item;
|
||||||
DocumentImportBatchItemStatus currentStatus =
|
DocumentImportBatchItemStatus currentStatus =
|
||||||
DocumentImportBatchItemStatus.valueOf(current.getStatus());
|
DocumentImportBatchItemStatus.valueOf(current.getStatus());
|
||||||
if (!isAllowedTransition(currentStatus, status)) {
|
if (!isAllowedTransition(currentStatus, status)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
String expectedStatus = current.getStatus();
|
|
||||||
Date now = new Date();
|
Date now = new Date();
|
||||||
|
int effectiveAttemptDelta = currentStatus == DocumentImportBatchItemStatus.FAILED
|
||||||
|
? Math.max(0, attemptDelta)
|
||||||
|
: 0;
|
||||||
int updated = itemMapper.transitionStatus(
|
int updated = itemMapper.transitionStatus(
|
||||||
itemId,
|
itemId,
|
||||||
expectedStatus,
|
current.getStatus(),
|
||||||
stage.name(),
|
stage.name(),
|
||||||
status.name(),
|
status.name(),
|
||||||
errorSummary,
|
errorSummary,
|
||||||
failureCode,
|
failureCode,
|
||||||
retryable,
|
retryable,
|
||||||
Math.max(0, attemptDelta),
|
effectiveAttemptDelta,
|
||||||
now
|
now
|
||||||
);
|
);
|
||||||
if (updated <= 0) {
|
if (updated <= 0) {
|
||||||
continue;
|
return false;
|
||||||
}
|
}
|
||||||
CounterDelta delta = CounterDelta.between(current, status, retryable);
|
CounterDelta delta = CounterDelta.between(current, status, retryable);
|
||||||
if (!delta.isZero()) {
|
if (!delta.isZero()) {
|
||||||
@@ -203,8 +244,6 @@ public class DocumentImportBatchTracker {
|
|||||||
refreshBatch(current.getBatchId());
|
refreshBatch(current.getBatchId());
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 校验文件项状态机,拒绝迟到任务覆盖终态。
|
* 校验文件项状态机,拒绝迟到任务覆盖终态。
|
||||||
@@ -243,7 +282,12 @@ public class DocumentImportBatchTracker {
|
|||||||
*/
|
*/
|
||||||
@org.springframework.transaction.annotation.Transactional
|
@org.springframework.transaction.annotation.Transactional
|
||||||
public void bindDocument(BigInteger itemId, BigInteger documentId) {
|
public void bindDocument(BigInteger itemId, BigInteger documentId) {
|
||||||
DocumentImportBatchItem item = requireItem(itemId);
|
LockedBatchItem locked = lockBatchThenItem(itemId);
|
||||||
|
if (DocumentImportBatchStatus.INTERRUPTED.name()
|
||||||
|
.equals(locked.batch.getStatus())) {
|
||||||
|
throw new BusinessException("导入批次已中断,请继续批次后重试");
|
||||||
|
}
|
||||||
|
DocumentImportBatchItem item = locked.item;
|
||||||
if (documentId.equals(item.getDocumentId())
|
if (documentId.equals(item.getDocumentId())
|
||||||
&& DocumentImportBatchItemStatus.PENDING.name().equals(item.getStatus())) {
|
&& DocumentImportBatchItemStatus.PENDING.name().equals(item.getStatus())) {
|
||||||
return;
|
return;
|
||||||
@@ -252,6 +296,11 @@ public class DocumentImportBatchTracker {
|
|||||||
boolean recoveringFailedItem =
|
boolean recoveringFailedItem =
|
||||||
DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
|
DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
|
||||||
&& item.getDocumentId() == null;
|
&& item.getDocumentId() == null;
|
||||||
|
if (recoveringFailedItem
|
||||||
|
&& !DocumentImportBatchStatus.RUNNING.name()
|
||||||
|
.equals(locked.batch.getStatus())) {
|
||||||
|
throw new BusinessException("导入批次状态已变化,请刷新后重试");
|
||||||
|
}
|
||||||
int updated = recoveringFailedItem
|
int updated = recoveringFailedItem
|
||||||
? itemMapper.bindFailedDocument(itemId, documentId, now)
|
? itemMapper.bindFailedDocument(itemId, documentId, now)
|
||||||
: itemMapper.bindDocument(itemId, documentId, now);
|
: itemMapper.bindDocument(itemId, documentId, now);
|
||||||
@@ -268,6 +317,91 @@ public class DocumentImportBatchTracker {
|
|||||||
refreshBatch(item.getBatchId());
|
refreshBatch(item.getBatchId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将引用已丢失文档的失败项重新绑定到恢复创建的新文档。
|
||||||
|
*
|
||||||
|
* @param itemId 文件项 ID
|
||||||
|
* @param missingDocumentId 已不存在的旧文档 ID
|
||||||
|
* @param documentId 恢复创建的新文档 ID
|
||||||
|
*/
|
||||||
|
@org.springframework.transaction.annotation.Transactional
|
||||||
|
public void replaceMissingDocument(BigInteger itemId,
|
||||||
|
BigInteger missingDocumentId,
|
||||||
|
BigInteger documentId) {
|
||||||
|
LockedBatchItem locked = lockBatchThenItem(itemId);
|
||||||
|
if (!DocumentImportBatchStatus.RUNNING.name()
|
||||||
|
.equals(locked.batch.getStatus())) {
|
||||||
|
throw new BusinessException("导入批次状态已变化,请刷新后重试");
|
||||||
|
}
|
||||||
|
DocumentImportBatchItem item = locked.item;
|
||||||
|
if (!DocumentImportBatchItemStatus.FAILED.name().equals(item.getStatus())
|
||||||
|
|| !java.util.Objects.equals(missingDocumentId, item.getDocumentId())) {
|
||||||
|
throw new BusinessException("导入文件状态已变化,请刷新后重试");
|
||||||
|
}
|
||||||
|
Date now = new Date();
|
||||||
|
if (itemMapper.replaceMissingDocument(
|
||||||
|
itemId, missingDocumentId, documentId, now) <= 0) {
|
||||||
|
throw new BusinessException("原文档状态已变化,请刷新后重试");
|
||||||
|
}
|
||||||
|
batchMapper.refreshCountersFromItems(item.getBatchId(), now);
|
||||||
|
refreshBatch(item.getBatchId());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按批次到文件项的固定顺序取得失败项恢复权。
|
||||||
|
*
|
||||||
|
* @param itemId 文件项 ID
|
||||||
|
* @return 可恢复文件项;状态已变化时返回 null
|
||||||
|
*/
|
||||||
|
@org.springframework.transaction.annotation.Transactional
|
||||||
|
public DocumentImportBatchItem lockFailedItemForRetry(BigInteger itemId) {
|
||||||
|
if (itemId == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
LockedBatchItem locked = lockBatchThenItem(itemId);
|
||||||
|
if (!DocumentImportBatchStatus.RUNNING.name()
|
||||||
|
.equals(locked.batch.getStatus())
|
||||||
|
|| !DocumentImportBatchItemStatus.FAILED.name()
|
||||||
|
.equals(locked.item.getStatus())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return locked.item;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在运行批次内保留单文件恢复失败原因。
|
||||||
|
*
|
||||||
|
* @param itemId 文件项 ID
|
||||||
|
* @param batchId 批次 ID
|
||||||
|
* @param errorSummary 错误摘要
|
||||||
|
* @return 失败原因写回、并发推进或拒绝结果
|
||||||
|
*/
|
||||||
|
@org.springframework.transaction.annotation.Transactional(
|
||||||
|
propagation = org.springframework.transaction.annotation.Propagation.REQUIRES_NEW)
|
||||||
|
public FailedRetryErrorOutcome updateFailedRetryError(
|
||||||
|
BigInteger itemId,
|
||||||
|
BigInteger batchId,
|
||||||
|
String errorSummary
|
||||||
|
) {
|
||||||
|
if (itemId == null || batchId == null) {
|
||||||
|
return FailedRetryErrorOutcome.REJECTED;
|
||||||
|
}
|
||||||
|
LockedBatchItem locked = lockBatchThenItem(itemId);
|
||||||
|
if (!batchId.equals(locked.batch.getId())
|
||||||
|
|| !DocumentImportBatchStatus.RUNNING.name()
|
||||||
|
.equals(locked.batch.getStatus())) {
|
||||||
|
return FailedRetryErrorOutcome.REJECTED;
|
||||||
|
}
|
||||||
|
if (!DocumentImportBatchItemStatus.FAILED.name()
|
||||||
|
.equals(locked.item.getStatus())) {
|
||||||
|
return FailedRetryErrorOutcome.ALREADY_ADVANCED;
|
||||||
|
}
|
||||||
|
return itemMapper.updateFailedRetryError(
|
||||||
|
itemId, batchId, errorSummary, new Date()) > 0
|
||||||
|
? FailedRetryErrorOutcome.ERROR_RECORDED
|
||||||
|
: FailedRetryErrorOutcome.REJECTED;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 原子完成文件上传并增量更新批次上传数。
|
* 原子完成文件上传并增量更新批次上传数。
|
||||||
*
|
*
|
||||||
@@ -280,7 +414,12 @@ public class DocumentImportBatchTracker {
|
|||||||
public boolean completeUpload(BigInteger itemId,
|
public boolean completeUpload(BigInteger itemId,
|
||||||
String filePath,
|
String filePath,
|
||||||
String storageLocator) {
|
String storageLocator) {
|
||||||
DocumentImportBatchItem item = requireItem(itemId);
|
LockedBatchItem locked = lockBatchThenItem(itemId);
|
||||||
|
if (DocumentImportBatchStatus.INTERRUPTED.name()
|
||||||
|
.equals(locked.batch.getStatus())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
DocumentImportBatchItem item = locked.item;
|
||||||
if (DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus())) {
|
if (DocumentImportBatchItemStatus.UPLOADED.name().equals(item.getStatus())) {
|
||||||
return filePath.equals(item.getFilePath())
|
return filePath.equals(item.getFilePath())
|
||||||
&& storageLocator.equals(item.getStorageLocator());
|
&& storageLocator.equals(item.getStorageLocator());
|
||||||
@@ -304,6 +443,34 @@ public class DocumentImportBatchTracker {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 先锁批次再锁文件项,统一所有计数迁移路径的加锁顺序。
|
||||||
|
*
|
||||||
|
* @param itemId 文件项 ID
|
||||||
|
* @return 已锁定的批次与文件项
|
||||||
|
*/
|
||||||
|
private LockedBatchItem lockBatchThenItem(BigInteger itemId) {
|
||||||
|
DocumentImportBatchItem candidate = requireItem(itemId);
|
||||||
|
DocumentImportBatch batch = batchMapper.selectForUpdate(candidate.getBatchId());
|
||||||
|
if (batch == null) {
|
||||||
|
throw new BusinessException("导入批次不存在");
|
||||||
|
}
|
||||||
|
DocumentImportBatchItem item = itemMapper.selectForUpdate(itemId);
|
||||||
|
if (item == null) {
|
||||||
|
throw new BusinessException("导入文件不存在");
|
||||||
|
}
|
||||||
|
if (!java.util.Objects.equals(batch.getId(), item.getBatchId())) {
|
||||||
|
throw new IllegalStateException("导入文件批次归属已变化");
|
||||||
|
}
|
||||||
|
return new LockedBatchItem(batch, item);
|
||||||
|
}
|
||||||
|
|
||||||
|
private record LockedBatchItem(
|
||||||
|
DocumentImportBatch batch,
|
||||||
|
DocumentImportBatchItem item
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 记录文件项成功后需要清理的历史文档。
|
* 记录文件项成功后需要清理的历史文档。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
package tech.easyflow.ai.documentimport.task;
|
|
||||||
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 知识库文档导入格式统一策略。
|
|
||||||
*
|
|
||||||
* @author Codex
|
|
||||||
* @since 2026-08-04
|
|
||||||
*/
|
|
||||||
public final class DocumentImportFormatPolicy {
|
|
||||||
|
|
||||||
private static final Set<String> SUPPORTED_EXTENSIONS =
|
|
||||||
Set.of("txt", "pdf", "docx", "md", "pptx", "xlsx", "csv");
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 禁止实例化格式策略工具类。
|
|
||||||
*/
|
|
||||||
private DocumentImportFormatPolicy() {
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回只读的支持格式集合。
|
|
||||||
*
|
|
||||||
* @return 支持的文件扩展名
|
|
||||||
*/
|
|
||||||
public static Set<String> supportedExtensions() {
|
|
||||||
return SUPPORTED_EXTENSIONS;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断扩展名是否属于知识库导入支持范围。
|
|
||||||
*
|
|
||||||
* @param extension 已转换为小写的文件扩展名
|
|
||||||
* @return 支持时返回 {@code true}
|
|
||||||
*/
|
|
||||||
public static boolean isSupported(String extension) {
|
|
||||||
return SUPPORTED_EXTENSIONS.contains(extension);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package tech.easyflow.ai.documentimport.task;
|
||||||
|
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Propagation;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在独立短事务中维护批次恢复令牌。
|
||||||
|
*
|
||||||
|
* <p>继续操作通常从原事务的 {@code afterCommit} 回调启动。将领取、续租和
|
||||||
|
* 收尾放入独立事务,可保证恢复令牌在单文件重试事务开始前已经提交,并避免
|
||||||
|
* 复用刚完成提交但尚未解绑的事务资源。</p>
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
public class DocumentImportRecoveryCoordinator {
|
||||||
|
|
||||||
|
private final DocumentImportBatchMapper batchMapper;
|
||||||
|
|
||||||
|
public DocumentImportRecoveryCoordinator(DocumentImportBatchMapper batchMapper) {
|
||||||
|
this.batchMapper = batchMapper;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 领取恢复待办并读取本次令牌对应的恢复参数。
|
||||||
|
*
|
||||||
|
* @param batchId 批次 ID
|
||||||
|
* @param recoveryToken 恢复令牌
|
||||||
|
* @param leaseUntil 租约到期时间
|
||||||
|
* @param claimedAt 领取时间
|
||||||
|
* @return 已领取批次;领取失败时返回 null
|
||||||
|
*/
|
||||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
|
public DocumentImportBatch claim(BigInteger batchId,
|
||||||
|
String recoveryToken,
|
||||||
|
Date leaseUntil,
|
||||||
|
Date claimedAt) {
|
||||||
|
if (batchMapper.claimRecoveryPending(
|
||||||
|
batchId, recoveryToken, leaseUntil, claimedAt) <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return batchMapper.selectClaimedRecovery(batchId, recoveryToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 续期当前恢复令牌。
|
||||||
|
*
|
||||||
|
* @return 当前令牌仍有效且续期成功时返回 true
|
||||||
|
*/
|
||||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
|
public boolean renew(BigInteger batchId,
|
||||||
|
String recoveryToken,
|
||||||
|
Date leaseUntil,
|
||||||
|
Date renewedAt) {
|
||||||
|
return batchMapper.renewRecoveryPendingLease(
|
||||||
|
batchId, recoveryToken, leaseUntil, renewedAt) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按真实计数收尾并清除恢复待办。
|
||||||
|
*
|
||||||
|
* @return 当前令牌仍有效且收尾成功时返回 true
|
||||||
|
*/
|
||||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
|
public boolean finalizeRecovery(BigInteger batchId,
|
||||||
|
String recoveryToken,
|
||||||
|
Date finalizedAt) {
|
||||||
|
return batchMapper.finalizeRecoveryPending(
|
||||||
|
batchId, recoveryToken, finalizedAt) > 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package tech.easyflow.ai.documentimport.task;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批次恢复未能重新排队任何失败项。
|
||||||
|
*
|
||||||
|
* <p>该异常只携带稳定错误码和用户可见摘要,供恢复令牌持有者
|
||||||
|
* 中断批次,避免“继续”请求在没有启动任何新任务时表现为成功。</p>
|
||||||
|
*/
|
||||||
|
public class DocumentImportRecoveryException extends RuntimeException {
|
||||||
|
|
||||||
|
private static final String ERROR_CODE = "document_import_recovery_failed";
|
||||||
|
private static final String USER_MESSAGE =
|
||||||
|
"未能重新排队任何失败文件,请查看文件错误后继续";
|
||||||
|
private final List<BigInteger> failureFenceItemIds;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建批次恢复失败异常。
|
||||||
|
*/
|
||||||
|
public DocumentImportRecoveryException() {
|
||||||
|
this(List.of());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建带初始失败项快照的批次恢复失败异常。
|
||||||
|
*
|
||||||
|
* @param failureFenceItemIds 领取恢复时仍为失败的文件项 ID
|
||||||
|
*/
|
||||||
|
public DocumentImportRecoveryException(
|
||||||
|
Collection<BigInteger> failureFenceItemIds
|
||||||
|
) {
|
||||||
|
super(USER_MESSAGE);
|
||||||
|
this.failureFenceItemIds = failureFenceItemIds == null
|
||||||
|
? List.of()
|
||||||
|
: failureFenceItemIds.stream()
|
||||||
|
.filter(java.util.Objects::nonNull)
|
||||||
|
.distinct()
|
||||||
|
.sorted()
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取稳定错误码。
|
||||||
|
*
|
||||||
|
* @return 稳定错误码
|
||||||
|
*/
|
||||||
|
public String getCode() {
|
||||||
|
return ERROR_CODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取用户可见摘要。
|
||||||
|
*
|
||||||
|
* @return 用户可见摘要
|
||||||
|
*/
|
||||||
|
public String getUserMessage() {
|
||||||
|
return USER_MESSAGE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取用于原子熔断围栏的初始失败项。
|
||||||
|
*
|
||||||
|
* @return 按 ID 排序的不可变列表
|
||||||
|
*/
|
||||||
|
public List<BigInteger> getFailureFenceItemIds() {
|
||||||
|
return failureFenceItemIds;
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory;
|
|||||||
import org.springframework.dao.DuplicateKeyException;
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
import tech.easyflow.ai.document.support.DocumentParseFilePolicy;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext;
|
import tech.easyflow.ai.documentimport.DocumentImportBatchCreateContext;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult;
|
import tech.easyflow.ai.documentimport.DocumentImportBatchRetryResult;
|
||||||
@@ -60,7 +61,7 @@ public class KnowledgeImportBatchFacade {
|
|||||||
private static final Duration INCOMPLETE_SUBMISSION_TIMEOUT =
|
private static final Duration INCOMPLETE_SUBMISSION_TIMEOUT =
|
||||||
Duration.ofMinutes(30);
|
Duration.ofMinutes(30);
|
||||||
private static final Set<String> SUPPORTED_EXTENSIONS =
|
private static final Set<String> SUPPORTED_EXTENSIONS =
|
||||||
DocumentImportFormatPolicy.supportedExtensions();
|
DocumentParseFilePolicy.supportedExtensions();
|
||||||
|
|
||||||
private final DocumentImportBatchAppService batchAppService;
|
private final DocumentImportBatchAppService batchAppService;
|
||||||
private final DocumentImportBatchTracker batchTracker;
|
private final DocumentImportBatchTracker batchTracker;
|
||||||
@@ -889,7 +890,13 @@ public class KnowledgeImportBatchFacade {
|
|||||||
*/
|
*/
|
||||||
private void assertSupportedExtension(String fileName) {
|
private void assertSupportedExtension(String fileName) {
|
||||||
if (!SUPPORTED_EXTENSIONS.contains(extension(fileName))) {
|
if (!SUPPORTED_EXTENSIONS.contains(extension(fileName))) {
|
||||||
throw new BusinessException(415, 41502, "暂不支持该文件格式");
|
throw new BusinessException(
|
||||||
|
415,
|
||||||
|
41502,
|
||||||
|
"暂不支持该文件格式,仅支持 "
|
||||||
|
+ DocumentParseFilePolicy.supportedTypeLabel()
|
||||||
|
+ " 文件"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 兼容 success 字段的异步分块更新结果。
|
||||||
|
*/
|
||||||
|
public record DocumentChunkAsyncUpdateResult(
|
||||||
|
boolean success,
|
||||||
|
BigInteger id,
|
||||||
|
String indexSyncStatus,
|
||||||
|
Long indexSyncVersion
|
||||||
|
) {
|
||||||
|
public static DocumentChunkAsyncUpdateResult from(DocumentChunk chunk) {
|
||||||
|
return new DocumentChunkAsyncUpdateResult(
|
||||||
|
true,
|
||||||
|
chunk.getId(),
|
||||||
|
chunk.getIndexSyncStatus(),
|
||||||
|
chunk.getIndexSyncVersion()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块正文更新请求,只允许修改 Markdown 内容。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkContentUpdateRequest implements Serializable {
|
||||||
|
|
||||||
|
private BigInteger id;
|
||||||
|
private String content;
|
||||||
|
|
||||||
|
public BigInteger getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setId(BigInteger id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getContent() {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setContent(String content) {
|
||||||
|
this.content = content;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块删除结果。
|
||||||
|
*/
|
||||||
|
public record DocumentChunkDeleteResult(
|
||||||
|
BigInteger id,
|
||||||
|
BigInteger documentId,
|
||||||
|
long remainingChunkCount
|
||||||
|
) implements Serializable {
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 重试分块索引同步请求。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkSyncRetryRequest {
|
||||||
|
private BigInteger id;
|
||||||
|
private Long indexSyncVersion;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public Long getIndexSyncVersion() { return indexSyncVersion; }
|
||||||
|
public void setIndexSyncVersion(Long indexSyncVersion) { this.indexSyncVersion = indexSyncVersion; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步状态。
|
||||||
|
*/
|
||||||
|
public record DocumentChunkSyncStatus(
|
||||||
|
BigInteger id,
|
||||||
|
String indexSyncStatus,
|
||||||
|
Long indexSyncVersion,
|
||||||
|
String indexSyncErrorCode,
|
||||||
|
String indexSyncErrorMessage
|
||||||
|
) {
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package tech.easyflow.ai.dto;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量查询分块索引同步状态。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkSyncStatusRequest {
|
||||||
|
private BigInteger documentId;
|
||||||
|
private List<BigInteger> ids;
|
||||||
|
|
||||||
|
public BigInteger getDocumentId() { return documentId; }
|
||||||
|
public void setDocumentId(BigInteger documentId) { this.documentId = documentId; }
|
||||||
|
public List<BigInteger> getIds() { return ids; }
|
||||||
|
public void setIds(List<BigInteger> ids) { this.ids = ids; }
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import com.alibaba.fastjson2.JSONObject;
|
|||||||
import com.easyagents.flow.core.chain.Chain;
|
import com.easyagents.flow.core.chain.Chain;
|
||||||
import com.easyagents.flow.core.knowledge.Knowledge;
|
import com.easyagents.flow.core.knowledge.Knowledge;
|
||||||
import com.easyagents.flow.core.knowledge.KnowledgeProvider;
|
import com.easyagents.flow.core.knowledge.KnowledgeProvider;
|
||||||
|
import com.easyagents.flow.core.knowledge.KnowledgeSearchRequest;
|
||||||
import com.easyagents.flow.core.node.KnowledgeNode;
|
import com.easyagents.flow.core.node.KnowledgeNode;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
@@ -14,6 +15,7 @@ import tech.easyflow.ai.service.DocumentCollectionService;
|
|||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@@ -26,6 +28,9 @@ public class KnowledgeProviderImpl implements KnowledgeProvider {
|
|||||||
@Resource
|
@Resource
|
||||||
private DocumentCollectionService documentCollectionService;
|
private DocumentCollectionService documentCollectionService;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private WorkflowMultiKnowledgeRetrievalService multiKnowledgeRetrievalService;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取知识库检索器。
|
* 获取知识库检索器。
|
||||||
*
|
*
|
||||||
@@ -44,24 +49,86 @@ public class KnowledgeProviderImpl implements KnowledgeProvider {
|
|||||||
int limit,
|
int limit,
|
||||||
KnowledgeNode knowledgeNode,
|
KnowledgeNode knowledgeNode,
|
||||||
Chain chain) {
|
Chain chain) {
|
||||||
|
return searchSingle(
|
||||||
|
new BigInteger(id.toString()),
|
||||||
|
keyword,
|
||||||
|
limit,
|
||||||
|
knowledgeNode);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> search(KnowledgeSearchRequest request) {
|
||||||
|
if (request == null || request.getKnowledgeIds().isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
List<BigInteger> knowledgeIds = new ArrayList<>();
|
||||||
|
for (Object id : request.getKnowledgeIds()) {
|
||||||
|
try {
|
||||||
|
knowledgeIds.add(new BigInteger(String.valueOf(id)));
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new IllegalArgumentException("知识库 ID 无效: " + id, exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (knowledgeIds.size() == 1) {
|
||||||
|
List<Map<String, Object>> documents = searchSingle(
|
||||||
|
knowledgeIds.get(0),
|
||||||
|
request.getKeyword(),
|
||||||
|
request.getLimit(),
|
||||||
|
request.getKnowledgeNode());
|
||||||
|
return buildOutputs(documents);
|
||||||
|
}
|
||||||
|
if (!"VECTOR".equalsIgnoreCase(request.getRetrievalMode())) {
|
||||||
|
throw new IllegalArgumentException("多知识库检索仅支持 VECTOR 模式");
|
||||||
|
}
|
||||||
|
MultiKnowledgeRetrievalResult result = multiKnowledgeRetrievalService.search(
|
||||||
|
knowledgeIds,
|
||||||
|
request.getKeyword(),
|
||||||
|
request.getLimit(),
|
||||||
|
request.getKnowledgeNode() == null
|
||||||
|
? null
|
||||||
|
: request.getKnowledgeNode().getId(),
|
||||||
|
request.getChain());
|
||||||
|
List<Map<String, Object>> documents = new ArrayList<>();
|
||||||
|
for (Document document : result.getDocuments()) {
|
||||||
|
documents.add(toWorkflowDocument(
|
||||||
|
document,
|
||||||
|
document.getMetadata("knowledgeId", null)));
|
||||||
|
}
|
||||||
|
return buildOutputs(documents);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Map<String, Object>> searchSingle(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
String keyword,
|
||||||
|
int limit,
|
||||||
|
KnowledgeNode knowledgeNode) {
|
||||||
KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
|
KnowledgeRetrievalRequest request = new KnowledgeRetrievalRequest();
|
||||||
request.setKnowledgeId(new BigInteger(id.toString()));
|
request.setKnowledgeId(knowledgeId);
|
||||||
request.setQuery(keyword);
|
request.setQuery(keyword);
|
||||||
request.setLimit(limit);
|
request.setLimit(limit);
|
||||||
request.setRetrievalMode(KnowledgeRetrievalModes.parse(knowledgeNode.getRetrievalMode()));
|
request.setRetrievalMode(KnowledgeRetrievalModes.parse(
|
||||||
|
knowledgeNode == null
|
||||||
|
? null
|
||||||
|
: knowledgeNode.getRetrievalMode()));
|
||||||
request.setCallerType("WORKFLOW");
|
request.setCallerType("WORKFLOW");
|
||||||
request.setCallerId(knowledgeNode.getId());
|
request.setCallerId(knowledgeNode == null ? null : knowledgeNode.getId());
|
||||||
List<Document> documents = documentCollectionService.search(request);
|
List<Document> documents = documentCollectionService.search(request);
|
||||||
if (limit > 0 && documents.size() > limit) {
|
if (limit > 0 && documents.size() > limit) {
|
||||||
documents = new ArrayList<>(documents.subList(0, limit));
|
documents = new ArrayList<>(documents.subList(0, limit));
|
||||||
}
|
}
|
||||||
List<Map<String, Object>> res = new ArrayList<>();
|
List<Map<String, Object>> result = new ArrayList<>();
|
||||||
for (Document document : documents) {
|
for (Document document : documents) {
|
||||||
res.add(toWorkflowDocument(document, id));
|
result.add(toWorkflowDocument(document, knowledgeId));
|
||||||
}
|
}
|
||||||
return res;
|
return result;
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
private Map<String, Object> buildOutputs(List<Map<String, Object>> documents) {
|
||||||
|
Map<String, Object> outputs = new LinkedHashMap<>();
|
||||||
|
outputs.put("documents", documents);
|
||||||
|
return outputs;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -74,7 +141,7 @@ public class KnowledgeProviderImpl implements KnowledgeProvider {
|
|||||||
private Map<String, Object> toWorkflowDocument(
|
private Map<String, Object> toWorkflowDocument(
|
||||||
Document document, Object knowledgeId) {
|
Document document, Object knowledgeId) {
|
||||||
JSONObject result = JSONObject.from(document);
|
JSONObject result = JSONObject.from(document);
|
||||||
result.put("title", document.getTitle());
|
result.put("title", resolveWorkflowTitle(document));
|
||||||
result.put("content", document.getContent());
|
result.put("content", document.getContent());
|
||||||
result.put(
|
result.put(
|
||||||
"documentId",
|
"documentId",
|
||||||
@@ -82,4 +149,30 @@ public class KnowledgeProviderImpl implements KnowledgeProvider {
|
|||||||
result.put("knowledgeId", knowledgeId);
|
result.put("knowledgeId", knowledgeId);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取工作流文档标题,兼容历史检索结果只在元数据中保存来源标题的情况。
|
||||||
|
*
|
||||||
|
* @param document 检索文档
|
||||||
|
* @return 工作流文档标题
|
||||||
|
*/
|
||||||
|
private String resolveWorkflowTitle(Document document) {
|
||||||
|
String title = trimToNull(document.getTitle());
|
||||||
|
if (title != null) {
|
||||||
|
return title;
|
||||||
|
}
|
||||||
|
title = trimToNull(document.getMetadata("sourceFileName"));
|
||||||
|
if (title != null) {
|
||||||
|
return title;
|
||||||
|
}
|
||||||
|
return trimToNull(document.getMetadata("question"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String trimToNull(Object value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String text = String.valueOf(value).trim();
|
||||||
|
return text.isEmpty() ? null : text;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package tech.easyflow.ai.easyagentsflow.knowledge;
|
||||||
|
|
||||||
|
import com.easyagents.core.document.Document;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 多知识库检索结果及可观察状态。
|
||||||
|
*/
|
||||||
|
public class MultiKnowledgeRetrievalResult {
|
||||||
|
|
||||||
|
private final List<Document> documents;
|
||||||
|
private final Map<String, Object> summary;
|
||||||
|
private final List<Map<String, Object>> sourceStatuses;
|
||||||
|
|
||||||
|
public MultiKnowledgeRetrievalResult(
|
||||||
|
List<Document> documents,
|
||||||
|
Map<String, Object> summary,
|
||||||
|
List<Map<String, Object>> sourceStatuses) {
|
||||||
|
this.documents = documents == null
|
||||||
|
? Collections.emptyList()
|
||||||
|
: Collections.unmodifiableList(new ArrayList<>(documents));
|
||||||
|
this.summary = summary == null
|
||||||
|
? Collections.emptyMap()
|
||||||
|
: Collections.unmodifiableMap(new LinkedHashMap<>(summary));
|
||||||
|
this.sourceStatuses = sourceStatuses == null
|
||||||
|
? Collections.emptyList()
|
||||||
|
: Collections.unmodifiableList(new ArrayList<>(sourceStatuses));
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Document> getDocuments() {
|
||||||
|
return documents;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, Object> getSummary() {
|
||||||
|
return summary;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Map<String, Object>> getSourceStatuses() {
|
||||||
|
return sourceStatuses;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,579 @@
|
|||||||
|
package tech.easyflow.ai.easyagentsflow.knowledge;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
|
import com.alibaba.fastjson2.JSONArray;
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
import tech.easyflow.ai.entity.Model;
|
||||||
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.ai.service.ModelService;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.security.NoSuchAlgorithmException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HexFormat;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 工作流多知识库 Embedding 契约校验与发布快照服务。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
public class WorkflowKnowledgeContractService {
|
||||||
|
|
||||||
|
public static final String SNAPSHOT_KEY = "knowledgeContracts";
|
||||||
|
|
||||||
|
private final DocumentCollectionService documentCollectionService;
|
||||||
|
private final ModelService modelService;
|
||||||
|
|
||||||
|
public WorkflowKnowledgeContractService(
|
||||||
|
DocumentCollectionService documentCollectionService,
|
||||||
|
ModelService modelService) {
|
||||||
|
this.documentCollectionService = documentCollectionService;
|
||||||
|
this.modelService = modelService;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量计算设计器中真正具备向量检索条件的知识库。
|
||||||
|
*
|
||||||
|
* @param collections 候选知识库
|
||||||
|
* @param tenantId 当前租户
|
||||||
|
* @return 可用知识库 ID
|
||||||
|
*/
|
||||||
|
public Set<BigInteger> findVectorReadyKnowledgeIds(
|
||||||
|
List<DocumentCollection> collections,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
if (collections == null || collections.isEmpty()) {
|
||||||
|
return Collections.emptySet();
|
||||||
|
}
|
||||||
|
Map<BigInteger, Model> models = loadModels(collections);
|
||||||
|
Set<BigInteger> result = new LinkedHashSet<>();
|
||||||
|
for (DocumentCollection collection : collections) {
|
||||||
|
if (isVectorReady(collection, models, tenantId)) {
|
||||||
|
result.add(collection.getId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验保存阶段全部多知识库节点的 Embedding 契约。
|
||||||
|
*
|
||||||
|
* @param knowledgeGroups 各知识库节点的有序引用
|
||||||
|
* @param tenantId 当前租户
|
||||||
|
*/
|
||||||
|
public void assertMultiKnowledgeContracts(
|
||||||
|
List<List<BigInteger>> knowledgeGroups,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
resolveContext(onlyMultiGroups(knowledgeGroups), tenantId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解析工作流引用的全部知识库,并校验存在性与租户归属。
|
||||||
|
*
|
||||||
|
* @param content 工作流内容
|
||||||
|
* @param tenantId 工作流租户
|
||||||
|
* @return 按工作流首次引用顺序排列的知识库
|
||||||
|
*/
|
||||||
|
public List<DocumentCollection> resolveReferencedCollections(
|
||||||
|
String content,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
List<List<BigInteger>> groups = readKnowledgeGroups(content);
|
||||||
|
if (groups.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
Set<BigInteger> ids = new LinkedHashSet<>();
|
||||||
|
groups.forEach(ids::addAll);
|
||||||
|
Map<BigInteger, DocumentCollection> collections = loadCollections(
|
||||||
|
ids, tenantId);
|
||||||
|
return ids.stream().map(collections::get).toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验工作流内容并生成不包含凭据的多知识库发布契约。
|
||||||
|
*
|
||||||
|
* @param content 工作流内容
|
||||||
|
* @param tenantId 工作流租户
|
||||||
|
* @return 稳定的发布契约列表
|
||||||
|
*/
|
||||||
|
public List<Map<String, Object>> buildSnapshotContracts(
|
||||||
|
String content,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
List<List<BigInteger>> groups = readKnowledgeGroups(content).stream()
|
||||||
|
.filter(group -> group.size() > 1)
|
||||||
|
.toList();
|
||||||
|
ContractContext context = resolveContext(groups, tenantId);
|
||||||
|
if (groups.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
Set<BigInteger> orderedIds = new LinkedHashSet<>();
|
||||||
|
groups.forEach(orderedIds::addAll);
|
||||||
|
List<Map<String, Object>> result = new ArrayList<>();
|
||||||
|
for (BigInteger id : orderedIds) {
|
||||||
|
result.add(toContract(
|
||||||
|
context.collections.get(id),
|
||||||
|
context.models,
|
||||||
|
tenantId));
|
||||||
|
}
|
||||||
|
return List.copyOf(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 审批真正发布前重新校验提交快照与当前知识库契约是否一致。
|
||||||
|
*
|
||||||
|
* @param resourceSnapshot 待发布工作流快照
|
||||||
|
*/
|
||||||
|
public void assertSnapshotCurrent(Map<String, Object> resourceSnapshot) {
|
||||||
|
if (resourceSnapshot == null) {
|
||||||
|
throw new BusinessException("工作流发布快照不能为空");
|
||||||
|
}
|
||||||
|
String content = text(resourceSnapshot.get("content"));
|
||||||
|
BigInteger tenantId = bigInteger(resourceSnapshot.get("tenantId"));
|
||||||
|
List<Map<String, Object>> current = buildSnapshotContracts(
|
||||||
|
content, tenantId);
|
||||||
|
List<Map<String, Object>> frozen = readSnapshotContracts(
|
||||||
|
resourceSnapshot.get(SNAPSHOT_KEY));
|
||||||
|
if (!current.equals(frozen)) {
|
||||||
|
throw new BusinessException("工作流引用的知识库 Embedding 配置已变化,请重新提交发布");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 已发布工作流运行时校验本节点引用仍与发布契约一致。
|
||||||
|
*
|
||||||
|
* @param publishedSnapshot 已发布工作流快照
|
||||||
|
* @param collections 当前节点知识库
|
||||||
|
*/
|
||||||
|
public Model assertPublishedContracts(
|
||||||
|
Map<String, Object> publishedSnapshot,
|
||||||
|
List<DocumentCollection> collections) {
|
||||||
|
if (collections == null || collections.size() < 2) {
|
||||||
|
throw new BusinessException("多知识库检索至少需要两个知识库");
|
||||||
|
}
|
||||||
|
if (publishedSnapshot == null || publishedSnapshot.isEmpty()) {
|
||||||
|
throw new BusinessException("已发布工作流缺少知识库契约");
|
||||||
|
}
|
||||||
|
BigInteger tenantId = bigInteger(publishedSnapshot.get("tenantId"));
|
||||||
|
ContractContext context = resolveLoadedContext(collections, tenantId);
|
||||||
|
Model embeddingModel = requireCompatibleEmbeddingModel(
|
||||||
|
collections, context, tenantId);
|
||||||
|
Map<String, Map<String, Object>> frozenById = new LinkedHashMap<>();
|
||||||
|
for (Map<String, Object> contract : readSnapshotContracts(
|
||||||
|
publishedSnapshot.get(SNAPSHOT_KEY))) {
|
||||||
|
frozenById.put(text(contract.get("knowledgeId")), contract);
|
||||||
|
}
|
||||||
|
for (DocumentCollection collection : collections) {
|
||||||
|
Map<String, Object> frozen = frozenById.get(
|
||||||
|
String.valueOf(collection.getId()));
|
||||||
|
Map<String, Object> current = toContract(
|
||||||
|
collection, context.models, tenantId);
|
||||||
|
if (!current.equals(frozen)) {
|
||||||
|
throw new BusinessException("已发布工作流的知识库 Embedding 配置已变化,请重新发布");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return embeddingModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验 Agent 冻结定义中的契约指纹与当前有效配置一致。
|
||||||
|
*
|
||||||
|
* @param expectedFingerprint 冻结定义中的契约指纹
|
||||||
|
* @param collections 定义引用的全部多知识库
|
||||||
|
* @param tenantId 冻结定义租户
|
||||||
|
*/
|
||||||
|
public Model assertFrozenContractFingerprint(
|
||||||
|
String expectedFingerprint,
|
||||||
|
List<DocumentCollection> allCollections,
|
||||||
|
List<DocumentCollection> currentCollections,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
ContractContext context = resolveLoadedContext(
|
||||||
|
allCollections, tenantId);
|
||||||
|
Model embeddingModel = requireCompatibleEmbeddingModel(
|
||||||
|
currentCollections, context, tenantId);
|
||||||
|
List<Map<String, Object>> current = new ArrayList<>();
|
||||||
|
for (DocumentCollection collection : allCollections) {
|
||||||
|
current.add(toContract(collection, context.models, tenantId));
|
||||||
|
}
|
||||||
|
if (!Objects.equals(expectedFingerprint, fingerprint(current))) {
|
||||||
|
throw new BusinessException("Agent 冻结工作流的知识库 Embedding 配置已变化,请重新发布 Agent");
|
||||||
|
}
|
||||||
|
return embeddingModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用已加载的知识库快照校验当前多库契约,并返回同一次批量读取的模型快照。
|
||||||
|
*
|
||||||
|
* @param collections 当前节点知识库快照
|
||||||
|
* @param tenantId 执行租户
|
||||||
|
* @return 已校验的 Embedding 模型快照
|
||||||
|
*/
|
||||||
|
public Model requireCompatibleEmbeddingModel(
|
||||||
|
List<DocumentCollection> collections,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
ContractContext context = resolveLoadedContext(collections, tenantId);
|
||||||
|
return requireCompatibleEmbeddingModel(
|
||||||
|
collections, context, tenantId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 计算发布快照中规范知识库契约的稳定指纹。
|
||||||
|
*
|
||||||
|
* @param contracts 发布快照契约
|
||||||
|
* @return SHA-256 十六进制指纹
|
||||||
|
*/
|
||||||
|
public String fingerprintSnapshotContracts(Object contracts) {
|
||||||
|
return fingerprint(readSnapshotContracts(contracts));
|
||||||
|
}
|
||||||
|
|
||||||
|
private ContractContext resolveContext(
|
||||||
|
List<List<BigInteger>> groups,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
if (groups == null || groups.isEmpty()) {
|
||||||
|
return ContractContext.empty();
|
||||||
|
}
|
||||||
|
if (tenantId == null) {
|
||||||
|
throw new BusinessException("工作流租户不能为空");
|
||||||
|
}
|
||||||
|
Set<BigInteger> ids = new LinkedHashSet<>();
|
||||||
|
groups.forEach(ids::addAll);
|
||||||
|
Map<BigInteger, DocumentCollection> collections = loadCollections(
|
||||||
|
ids, tenantId);
|
||||||
|
Map<BigInteger, Model> models = loadModels(collections.values());
|
||||||
|
for (List<BigInteger> group : groups) {
|
||||||
|
assertCompatibleGroup(group, collections, models, tenantId);
|
||||||
|
}
|
||||||
|
return new ContractContext(collections, models);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ContractContext resolveLoadedContext(
|
||||||
|
List<DocumentCollection> collections,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
if (collections == null || collections.isEmpty()) {
|
||||||
|
throw new BusinessException("工作流引用的知识库不存在或已失效");
|
||||||
|
}
|
||||||
|
if (tenantId == null) {
|
||||||
|
throw new BusinessException("工作流租户不能为空");
|
||||||
|
}
|
||||||
|
Map<BigInteger, DocumentCollection> byId = new LinkedHashMap<>();
|
||||||
|
for (DocumentCollection collection : collections) {
|
||||||
|
if (collection == null || collection.getId() == null) {
|
||||||
|
throw new BusinessException("工作流引用的知识库不存在或已失效");
|
||||||
|
}
|
||||||
|
if (!Objects.equals(tenantId, collection.getTenantId())) {
|
||||||
|
throw new BusinessException("工作流引用了其他租户的知识库");
|
||||||
|
}
|
||||||
|
if (byId.put(collection.getId(), collection) != null) {
|
||||||
|
throw new BusinessException("知识库节点不能重复选择同一知识库");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new ContractContext(byId, loadModels(collections));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Model requireCompatibleEmbeddingModel(
|
||||||
|
List<DocumentCollection> collections,
|
||||||
|
ContractContext context,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
if (collections == null || collections.size() < 2) {
|
||||||
|
throw new BusinessException("多知识库检索至少需要两个知识库");
|
||||||
|
}
|
||||||
|
List<BigInteger> ids = collections.stream()
|
||||||
|
.map(DocumentCollection::getId)
|
||||||
|
.toList();
|
||||||
|
assertCompatibleGroup(
|
||||||
|
ids, context.collections, context.models, tenantId);
|
||||||
|
Model embeddingModel = context.models.get(
|
||||||
|
collections.get(0).getVectorEmbedModelId());
|
||||||
|
if (embeddingModel == null) {
|
||||||
|
throw new BusinessException("知识库 Embedding 模型不存在或已失效");
|
||||||
|
}
|
||||||
|
return embeddingModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertCompatibleGroup(
|
||||||
|
List<BigInteger> group,
|
||||||
|
Map<BigInteger, DocumentCollection> collections,
|
||||||
|
Map<BigInteger, Model> models,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
DocumentCollection first = collections.get(group.get(0));
|
||||||
|
if (!isVectorReady(first, models, tenantId)) {
|
||||||
|
throw new BusinessException("知识库未完成有效的向量检索配置: " + first.getTitle());
|
||||||
|
}
|
||||||
|
for (BigInteger id : group) {
|
||||||
|
DocumentCollection current = collections.get(id);
|
||||||
|
if (!isVectorReady(current, models, tenantId)) {
|
||||||
|
throw new BusinessException("知识库未完成有效的向量检索配置: " + current.getTitle());
|
||||||
|
}
|
||||||
|
if (!Objects.equals(
|
||||||
|
first.getVectorEmbedModelId(),
|
||||||
|
current.getVectorEmbedModelId())
|
||||||
|
|| !Objects.equals(
|
||||||
|
first.getDimensionOfVectorModel(),
|
||||||
|
current.getDimensionOfVectorModel())
|
||||||
|
|| !Objects.equals(
|
||||||
|
first.getTenantId(), current.getTenantId())) {
|
||||||
|
throw new BusinessException("多知识库检索要求使用相同的 Embedding 模型和向量维度");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isVectorReady(
|
||||||
|
DocumentCollection collection,
|
||||||
|
Map<BigInteger, Model> models,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
if (collection == null
|
||||||
|
|| collection.getId() == null
|
||||||
|
|| !Objects.equals(collection.getTenantId(), tenantId)
|
||||||
|
|| !Boolean.TRUE.equals(collection.getVectorStoreEnable())
|
||||||
|
|| collection.getVectorEmbedModelId() == null
|
||||||
|
|| collection.getDimensionOfVectorModel() == null
|
||||||
|
|| collection.getDimensionOfVectorModel() <= 0
|
||||||
|
|| collection.getVectorStoreCollection() == null
|
||||||
|
|| collection.getVectorStoreCollection().isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
Model model = models.get(collection.getVectorEmbedModelId());
|
||||||
|
return model != null
|
||||||
|
&& Model.MODEL_TYPES[1].equals(model.getModelType())
|
||||||
|
&& model.getModelProvider() != null
|
||||||
|
&& text(model.getModelProvider().getProviderType()) != null
|
||||||
|
&& Objects.equals(model.getTenantId(), tenantId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<BigInteger, Model> loadModels(
|
||||||
|
Collection<DocumentCollection> collections) {
|
||||||
|
Set<BigInteger> modelIds = new LinkedHashSet<>();
|
||||||
|
for (DocumentCollection collection : collections) {
|
||||||
|
if (collection != null && collection.getVectorEmbedModelId() != null) {
|
||||||
|
modelIds.add(collection.getVectorEmbedModelId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (modelIds.isEmpty()) {
|
||||||
|
return Collections.emptyMap();
|
||||||
|
}
|
||||||
|
List<Model> loaded = modelService.listModelInstances(modelIds);
|
||||||
|
Map<BigInteger, Model> result = new LinkedHashMap<>();
|
||||||
|
if (loaded != null) {
|
||||||
|
for (Model model : loaded) {
|
||||||
|
if (model != null && model.getId() != null) {
|
||||||
|
result.put(model.getId(), model);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> toContract(
|
||||||
|
DocumentCollection collection,
|
||||||
|
Map<BigInteger, Model> models,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
if (!isVectorReady(collection, models, tenantId)) {
|
||||||
|
throw new BusinessException("知识库未完成有效的向量检索配置: "
|
||||||
|
+ (collection == null ? "" : collection.getTitle()));
|
||||||
|
}
|
||||||
|
Model model = models.get(collection.getVectorEmbedModelId());
|
||||||
|
Map<String, Object> contract = new LinkedHashMap<>();
|
||||||
|
contract.put("knowledgeId", String.valueOf(collection.getId()));
|
||||||
|
contract.put("tenantId", String.valueOf(collection.getTenantId()));
|
||||||
|
contract.put("embeddingModelId", String.valueOf(model.getId()));
|
||||||
|
contract.put("embeddingDimension", collection.getDimensionOfVectorModel());
|
||||||
|
contract.put("vectorStoreCollection", collection.getVectorStoreCollection());
|
||||||
|
contract.put("vectorStoreType", nullableText(collection.getVectorStoreType()));
|
||||||
|
contract.put("modelProviderId", nullableString(model.getProviderId()));
|
||||||
|
contract.put(
|
||||||
|
"modelProviderType",
|
||||||
|
nullableText(model.getModelProvider().getProviderType()));
|
||||||
|
contract.put("modelType", model.getModelType());
|
||||||
|
contract.put("modelName", nullableText(model.getModelName()));
|
||||||
|
contract.put("modelEndpoint", nullableText(model.getEndpoint()));
|
||||||
|
contract.put("modelRequestPath", nullableText(model.getRequestPath()));
|
||||||
|
return contract;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<BigInteger, DocumentCollection> loadCollections(
|
||||||
|
Set<BigInteger> ids,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
if (tenantId == null) {
|
||||||
|
throw new BusinessException("工作流租户不能为空");
|
||||||
|
}
|
||||||
|
List<DocumentCollection> loaded = documentCollectionService.listByIds(ids);
|
||||||
|
Map<BigInteger, DocumentCollection> collections = new LinkedHashMap<>();
|
||||||
|
if (loaded != null) {
|
||||||
|
for (DocumentCollection collection : loaded) {
|
||||||
|
if (collection != null && collection.getId() != null) {
|
||||||
|
collections.put(collection.getId(), collection);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (collections.size() != ids.size()) {
|
||||||
|
throw new BusinessException("工作流引用的知识库不存在或已失效");
|
||||||
|
}
|
||||||
|
for (DocumentCollection collection : collections.values()) {
|
||||||
|
if (!Objects.equals(tenantId, collection.getTenantId())) {
|
||||||
|
throw new BusinessException("工作流引用了其他租户的知识库");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return collections;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<List<BigInteger>> readKnowledgeGroups(String content) {
|
||||||
|
if (content == null || content.isBlank()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
JSONObject root;
|
||||||
|
try {
|
||||||
|
root = JSON.parseObject(content);
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new BusinessException("工作流内容不是合法JSON");
|
||||||
|
}
|
||||||
|
JSONArray nodes = root.getJSONArray("nodes");
|
||||||
|
if (nodes == null || nodes.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<List<BigInteger>> groups = new ArrayList<>();
|
||||||
|
for (int index = 0; index < nodes.size(); index++) {
|
||||||
|
JSONObject node = nodes.getJSONObject(index);
|
||||||
|
JSONObject data = node == null ? null : node.getJSONObject("data");
|
||||||
|
if (data == null || !"knowledgeNode".equals(nodeType(node, data))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
List<BigInteger> group = new ArrayList<>();
|
||||||
|
Set<BigInteger> unique = new LinkedHashSet<>();
|
||||||
|
Object rawIds = data.get("knowledgeIds");
|
||||||
|
if (rawIds instanceof JSONArray ids && !ids.isEmpty()) {
|
||||||
|
for (Object rawId : ids) {
|
||||||
|
BigInteger id = bigInteger(rawId);
|
||||||
|
if (!unique.add(id)) {
|
||||||
|
throw new BusinessException("知识库节点不能重复选择同一知识库");
|
||||||
|
}
|
||||||
|
group.add(id);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
group.add(bigInteger(data.get("knowledgeId")));
|
||||||
|
}
|
||||||
|
groups.add(List.copyOf(group));
|
||||||
|
}
|
||||||
|
return List.copyOf(groups);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<List<BigInteger>> onlyMultiGroups(
|
||||||
|
List<List<BigInteger>> knowledgeGroups) {
|
||||||
|
if (knowledgeGroups == null || knowledgeGroups.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
return knowledgeGroups.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(group -> group.size() > 1)
|
||||||
|
.map(List::copyOf)
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Map<String, Object>> readSnapshotContracts(Object value) {
|
||||||
|
if (value == null) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
JSONArray array;
|
||||||
|
try {
|
||||||
|
array = value instanceof JSONArray jsonArray
|
||||||
|
? jsonArray
|
||||||
|
: JSON.parseArray(JSON.toJSONString(value));
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new BusinessException("工作流知识库发布契约无效");
|
||||||
|
}
|
||||||
|
List<Map<String, Object>> result = new ArrayList<>();
|
||||||
|
for (int index = 0; index < array.size(); index++) {
|
||||||
|
JSONObject object = array.getJSONObject(index);
|
||||||
|
if (object == null) {
|
||||||
|
throw new BusinessException("工作流知识库发布契约无效");
|
||||||
|
}
|
||||||
|
Map<String, Object> contract = new LinkedHashMap<>();
|
||||||
|
contract.put("knowledgeId", text(object.get("knowledgeId")));
|
||||||
|
contract.put("tenantId", text(object.get("tenantId")));
|
||||||
|
contract.put("embeddingModelId", text(object.get("embeddingModelId")));
|
||||||
|
contract.put("embeddingDimension", object.getInteger("embeddingDimension"));
|
||||||
|
contract.put("vectorStoreCollection", nullableText(object.getString("vectorStoreCollection")));
|
||||||
|
contract.put("vectorStoreType", nullableText(object.getString("vectorStoreType")));
|
||||||
|
contract.put("modelProviderId", nullableText(object.getString("modelProviderId")));
|
||||||
|
contract.put("modelProviderType", nullableText(object.getString("modelProviderType")));
|
||||||
|
contract.put("modelType", nullableText(object.getString("modelType")));
|
||||||
|
contract.put("modelName", nullableText(object.getString("modelName")));
|
||||||
|
contract.put("modelEndpoint", nullableText(object.getString("modelEndpoint")));
|
||||||
|
contract.put("modelRequestPath", nullableText(object.getString("modelRequestPath")));
|
||||||
|
result.add(contract);
|
||||||
|
}
|
||||||
|
return List.copyOf(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nodeType(JSONObject node, JSONObject data) {
|
||||||
|
String rootType = text(node.getString("type"));
|
||||||
|
String dataType = text(data.getString("type"));
|
||||||
|
if (rootType != null && dataType != null
|
||||||
|
&& !Objects.equals(rootType, dataType)) {
|
||||||
|
throw new BusinessException("工作流节点类型与节点数据类型不一致");
|
||||||
|
}
|
||||||
|
return rootType == null ? dataType : rootType;
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigInteger bigInteger(Object value) {
|
||||||
|
String normalized = text(value);
|
||||||
|
if (normalized == null) {
|
||||||
|
throw new BusinessException("知识库或租户 ID 无效");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
BigInteger result = new BigInteger(normalized);
|
||||||
|
if (result.signum() <= 0) {
|
||||||
|
throw new NumberFormatException("non-positive");
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} catch (NumberFormatException exception) {
|
||||||
|
throw new BusinessException("知识库或租户 ID 无效");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String text(Object value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String result = String.valueOf(value).trim();
|
||||||
|
return result.isEmpty() ? null : result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nullableText(String value) {
|
||||||
|
return text(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nullableString(Object value) {
|
||||||
|
return value == null ? null : String.valueOf(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String fingerprint(Object value) {
|
||||||
|
try {
|
||||||
|
byte[] bytes = JSON.toJSONString(value)
|
||||||
|
.getBytes(StandardCharsets.UTF_8);
|
||||||
|
return HexFormat.of().formatHex(
|
||||||
|
MessageDigest.getInstance("SHA-256").digest(bytes));
|
||||||
|
} catch (NoSuchAlgorithmException exception) {
|
||||||
|
throw new IllegalStateException("SHA-256 is unavailable", exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private record ContractContext(
|
||||||
|
Map<BigInteger, DocumentCollection> collections,
|
||||||
|
Map<BigInteger, Model> models) {
|
||||||
|
|
||||||
|
private static ContractContext empty() {
|
||||||
|
return new ContractContext(
|
||||||
|
Collections.emptyMap(),
|
||||||
|
Collections.emptyMap());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,6 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.repository;
|
package tech.easyflow.ai.easyagentsflow.repository;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
import com.easyagents.flow.core.chain.ChainDefinition;
|
import com.easyagents.flow.core.chain.ChainDefinition;
|
||||||
import com.easyagents.flow.core.node.ConfirmNode;
|
import com.easyagents.flow.core.node.ConfirmNode;
|
||||||
import com.easyagents.flow.core.parser.ChainParser;
|
import com.easyagents.flow.core.parser.ChainParser;
|
||||||
@@ -84,9 +85,35 @@ public class AgentWorkflowSnapshotFactory {
|
|||||||
snapshot.put("englishName", workflow.getEnglishName());
|
snapshot.put("englishName", workflow.getEnglishName());
|
||||||
snapshot.put("revision", workflow.getRevision());
|
snapshot.put("revision", workflow.getRevision());
|
||||||
snapshot.put("content", prepared.content());
|
snapshot.put("content", prepared.content());
|
||||||
|
snapshot.put("tenantId", workflow.getTenantId());
|
||||||
|
snapshot.put(
|
||||||
|
"publishedSnapshotJson",
|
||||||
|
knowledgeRuntimeSnapshot(workflow));
|
||||||
return snapshot;
|
return snapshot;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 提取冻结执行所需的知识库契约,不携带工作流快照中的其他字段。
|
||||||
|
*
|
||||||
|
* @param workflow 已发布工作流
|
||||||
|
* @return 租户与知识库契约白名单
|
||||||
|
*/
|
||||||
|
Map<String, Object> knowledgeRuntimeSnapshot(Workflow workflow) {
|
||||||
|
Map<String, Object> runtimeSnapshot = new LinkedHashMap<>();
|
||||||
|
runtimeSnapshot.put("tenantId", workflow.getTenantId());
|
||||||
|
Map<String, Object> publishedSnapshot =
|
||||||
|
workflow.getPublishedSnapshotJson();
|
||||||
|
Object contracts = publishedSnapshot == null
|
||||||
|
? null
|
||||||
|
: publishedSnapshot.get("knowledgeContracts");
|
||||||
|
runtimeSnapshot.put(
|
||||||
|
"knowledgeContracts",
|
||||||
|
contracts == null
|
||||||
|
? java.util.List.of()
|
||||||
|
: JSON.parse(JSON.toJSONString(contracts)));
|
||||||
|
return runtimeSnapshot;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Agent 可执行 Workflow 的准备结果。
|
* Agent 可执行 Workflow 的准备结果。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -2,8 +2,11 @@ package tech.easyflow.ai.easyagentsflow.repository;
|
|||||||
|
|
||||||
import com.easyagents.flow.core.chain.ChainDefinition;
|
import com.easyagents.flow.core.chain.ChainDefinition;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.knowledge.WorkflowKnowledgeContractService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.security.MessageDigest;
|
import java.security.MessageDigest;
|
||||||
import java.security.NoSuchAlgorithmException;
|
import java.security.NoSuchAlgorithmException;
|
||||||
@@ -24,6 +27,7 @@ public class FrozenWorkflowDefinitionRegistry {
|
|||||||
private static final int MAX_ENTRIES = 512;
|
private static final int MAX_ENTRIES = 512;
|
||||||
|
|
||||||
private final AgentWorkflowSnapshotFactory snapshotFactory;
|
private final AgentWorkflowSnapshotFactory snapshotFactory;
|
||||||
|
private final WorkflowKnowledgeContractService workflowKnowledgeContractService;
|
||||||
private final Map<String, ChainDefinition> definitions =
|
private final Map<String, ChainDefinition> definitions =
|
||||||
new LinkedHashMap<>(32, 0.75F, true);
|
new LinkedHashMap<>(32, 0.75F, true);
|
||||||
private final Map<String, Workflow> workflows =
|
private final Map<String, Workflow> workflows =
|
||||||
@@ -33,9 +37,13 @@ public class FrozenWorkflowDefinitionRegistry {
|
|||||||
* 创建冻结定义注册表。
|
* 创建冻结定义注册表。
|
||||||
*
|
*
|
||||||
* @param snapshotFactory Agent Workflow 冻结快照工厂
|
* @param snapshotFactory Agent Workflow 冻结快照工厂
|
||||||
|
* @param workflowKnowledgeContractService 知识库契约服务
|
||||||
*/
|
*/
|
||||||
public FrozenWorkflowDefinitionRegistry(AgentWorkflowSnapshotFactory snapshotFactory) {
|
public FrozenWorkflowDefinitionRegistry(
|
||||||
|
AgentWorkflowSnapshotFactory snapshotFactory,
|
||||||
|
WorkflowKnowledgeContractService workflowKnowledgeContractService) {
|
||||||
this.snapshotFactory = snapshotFactory;
|
this.snapshotFactory = snapshotFactory;
|
||||||
|
this.workflowKnowledgeContractService = workflowKnowledgeContractService;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -48,7 +56,19 @@ public class FrozenWorkflowDefinitionRegistry {
|
|||||||
public String register(Workflow workflow) {
|
public String register(Workflow workflow) {
|
||||||
AgentWorkflowSnapshotFactory.PreparedWorkflow prepared = snapshotFactory.prepare(workflow);
|
AgentWorkflowSnapshotFactory.PreparedWorkflow prepared = snapshotFactory.prepare(workflow);
|
||||||
String preparedContent = prepared.content();
|
String preparedContent = prepared.content();
|
||||||
String id = PREFIX + workflow.getId() + ":" + sha256(preparedContent);
|
Map<String, Object> runtimeSnapshot =
|
||||||
|
snapshotFactory.knowledgeRuntimeSnapshot(workflow);
|
||||||
|
if (workflow.getTenantId() == null
|
||||||
|
|| workflow.getTenantId().signum() <= 0) {
|
||||||
|
throw new BusinessException("绑定工作流租户快照不完整,请重新发布工作流");
|
||||||
|
}
|
||||||
|
String contractFingerprint = workflowKnowledgeContractService
|
||||||
|
.fingerprintSnapshotContracts(
|
||||||
|
runtimeSnapshot.get("knowledgeContracts"));
|
||||||
|
String id = PREFIX + workflow.getId()
|
||||||
|
+ ":" + workflow.getTenantId()
|
||||||
|
+ ":" + contractFingerprint
|
||||||
|
+ ":" + sha256(preparedContent);
|
||||||
synchronized (definitions) {
|
synchronized (definitions) {
|
||||||
if (definitions.containsKey(id)) {
|
if (definitions.containsKey(id)) {
|
||||||
definitions.get(id);
|
definitions.get(id);
|
||||||
@@ -59,7 +79,7 @@ public class FrozenWorkflowDefinitionRegistry {
|
|||||||
definition.setName(workflow.getEnglishName());
|
definition.setName(workflow.getEnglishName());
|
||||||
definition.setDescription(workflow.getDescription());
|
definition.setDescription(workflow.getDescription());
|
||||||
definitions.put(id, definition);
|
definitions.put(id, definition);
|
||||||
workflows.put(id, workflow);
|
workflows.put(id, frozenWorkflow(workflow, runtimeSnapshot));
|
||||||
while (definitions.size() > MAX_ENTRIES) {
|
while (definitions.size() > MAX_ENTRIES) {
|
||||||
String eldest = definitions.keySet().iterator().next();
|
String eldest = definitions.keySet().iterator().next();
|
||||||
definitions.remove(eldest);
|
definitions.remove(eldest);
|
||||||
@@ -100,9 +120,66 @@ public class FrozenWorkflowDefinitionRegistry {
|
|||||||
* @return 是否冻结定义
|
* @return 是否冻结定义
|
||||||
*/
|
*/
|
||||||
public boolean isFrozen(String definitionId) {
|
public boolean isFrozen(String definitionId) {
|
||||||
|
return isFrozenDefinitionId(definitionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断定义 ID 是否属于冻结 Agent 工作流,不触发注册表实例创建。
|
||||||
|
*
|
||||||
|
* @param definitionId 定义 ID
|
||||||
|
* @return 是否冻结定义
|
||||||
|
*/
|
||||||
|
public static boolean isFrozenDefinitionId(String definitionId) {
|
||||||
return definitionId != null && definitionId.startsWith(PREFIX);
|
return definitionId != null && definitionId.startsWith(PREFIX);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解析冻结定义中可独立校验的租户和知识库契约指纹。
|
||||||
|
*
|
||||||
|
* @param definitionId 冻结定义 ID
|
||||||
|
* @return 冻结定义身份
|
||||||
|
*/
|
||||||
|
public static FrozenDefinitionIdentity parseIdentity(String definitionId) {
|
||||||
|
if (!isFrozenDefinitionId(definitionId)) {
|
||||||
|
throw new IllegalArgumentException("Not a frozen workflow definition");
|
||||||
|
}
|
||||||
|
String[] parts = definitionId.substring(PREFIX.length())
|
||||||
|
.split(":", 4);
|
||||||
|
if (parts.length != 4
|
||||||
|
|| !parts[2].matches("[0-9a-f]{64}")
|
||||||
|
|| !parts[3].matches("[0-9a-f]{64}")) {
|
||||||
|
throw new IllegalArgumentException("Invalid frozen workflow definition");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
BigInteger workflowId = new BigInteger(parts[0]);
|
||||||
|
BigInteger tenantId = new BigInteger(parts[1]);
|
||||||
|
if (workflowId.signum() <= 0 || tenantId.signum() <= 0) {
|
||||||
|
throw new NumberFormatException("non-positive");
|
||||||
|
}
|
||||||
|
return new FrozenDefinitionIdentity(
|
||||||
|
workflowId, tenantId, parts[2]);
|
||||||
|
} catch (NumberFormatException exception) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Invalid frozen workflow definition", exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Workflow frozenWorkflow(
|
||||||
|
Workflow source,
|
||||||
|
Map<String, Object> runtimeSnapshot) {
|
||||||
|
Workflow frozen = new Workflow();
|
||||||
|
frozen.setId(source.getId());
|
||||||
|
frozen.setTenantId(source.getTenantId());
|
||||||
|
frozen.setTitle(source.getTitle());
|
||||||
|
frozen.setDescription(source.getDescription());
|
||||||
|
frozen.setEnglishName(source.getEnglishName());
|
||||||
|
frozen.setRevision(source.getRevision());
|
||||||
|
frozen.setContent(source.getContent());
|
||||||
|
frozen.setPublishedSnapshotJson(
|
||||||
|
new LinkedHashMap<>(runtimeSnapshot));
|
||||||
|
return frozen;
|
||||||
|
}
|
||||||
|
|
||||||
private String sha256(String content) {
|
private String sha256(String content) {
|
||||||
try {
|
try {
|
||||||
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256")
|
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256")
|
||||||
@@ -111,4 +188,17 @@ public class FrozenWorkflowDefinitionRegistry {
|
|||||||
throw new IllegalStateException("SHA-256 is unavailable", exception);
|
throw new IllegalStateException("SHA-256 is unavailable", exception);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 冻结定义中不依赖进程内 LRU 状态的执行身份。
|
||||||
|
*
|
||||||
|
* @param workflowId 工作流 ID
|
||||||
|
* @param tenantId 工作流租户
|
||||||
|
* @param knowledgeContractFingerprint 知识库契约指纹
|
||||||
|
*/
|
||||||
|
public record FrozenDefinitionIdentity(
|
||||||
|
BigInteger workflowId,
|
||||||
|
BigInteger tenantId,
|
||||||
|
String knowledgeContractFingerprint) {
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.service;
|
package tech.easyflow.ai.easyagentsflow.service;
|
||||||
|
|
||||||
|
import com.easyagents.document.core.exception.DocumentParseException;
|
||||||
import com.easyagents.flow.core.chain.ChainState;
|
import com.easyagents.flow.core.chain.ChainState;
|
||||||
import com.easyagents.flow.core.chain.ExceptionSummary;
|
import com.easyagents.flow.core.chain.ExceptionSummary;
|
||||||
import com.easyagents.flow.core.chain.NodeState;
|
import com.easyagents.flow.core.chain.NodeState;
|
||||||
@@ -141,9 +142,12 @@ public class TinyFlowService {
|
|||||||
node.setResult(resolved);
|
node.setResult(resolved);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 只有当参数不为空时才覆盖
|
if (nodeState != null
|
||||||
if (chainState.getSuspendForParameters() != null) {
|
&& nodeState.getStatus() == NodeStatus.SUSPEND
|
||||||
|
&& chainState.getSuspendForParameters() != null) {
|
||||||
node.setSuspendForParameters(chainState.getSuspendForParameters());
|
node.setSuspendForParameters(chainState.getSuspendForParameters());
|
||||||
|
} else {
|
||||||
|
node.setSuspendForParameters(null);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -165,6 +169,10 @@ public class TinyFlowService {
|
|||||||
String rootMessage = StringUtil.hasText(error.getRootCauseMessage())
|
String rootMessage = StringUtil.hasText(error.getRootCauseMessage())
|
||||||
? error.getRootCauseMessage()
|
? error.getRootCauseMessage()
|
||||||
: error.getMessage();
|
: error.getMessage();
|
||||||
|
if (DocumentParseException.class.getName().equals(rootClass)
|
||||||
|
&& StringUtil.hasText(rootMessage)) {
|
||||||
|
return rootMessage;
|
||||||
|
}
|
||||||
if (StringUtil.noText(rootClass)) {
|
if (StringUtil.noText(rootClass)) {
|
||||||
return rootMessage;
|
return rootMessage;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,9 +3,14 @@ package tech.easyflow.ai.easyagentsflow.service;
|
|||||||
import com.alibaba.fastjson2.JSON;
|
import com.alibaba.fastjson2.JSON;
|
||||||
import com.alibaba.fastjson2.JSONArray;
|
import com.alibaba.fastjson2.JSONArray;
|
||||||
import com.alibaba.fastjson2.JSONObject;
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import com.easyagents.flow.core.chain.DataType;
|
||||||
|
import com.easyagents.flow.core.node.ConfirmNode;
|
||||||
import com.easyagents.flow.core.parser.ChainParser;
|
import com.easyagents.flow.core.parser.ChainParser;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
|
import tech.easyflow.ai.config.MultiKnowledgeRetrievalProperties;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckIssue;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckIssue;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckResult;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckResult;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||||
@@ -40,6 +45,7 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -47,18 +53,29 @@ import java.util.stream.Collectors;
|
|||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
public class WorkflowCheckService {
|
public class WorkflowCheckService {
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(
|
||||||
|
WorkflowCheckService.class);
|
||||||
|
private static final long SLOW_CHECK_THRESHOLD_MS = 500L;
|
||||||
private static final String LEVEL_ERROR = "ERROR";
|
private static final String LEVEL_ERROR = "ERROR";
|
||||||
private static final String LEVEL_WARNING = "WARNING";
|
private static final String LEVEL_WARNING = "WARNING";
|
||||||
private static final String TYPE_START = "startNode";
|
private static final String TYPE_START = "startNode";
|
||||||
private static final String TYPE_END = "endNode";
|
private static final String TYPE_END = "endNode";
|
||||||
private static final String TYPE_LOOP = "loopNode";
|
private static final String TYPE_LOOP = "loopNode";
|
||||||
private static final String TYPE_CONDITION = "conditionNode";
|
private static final String TYPE_CONDITION = "conditionNode";
|
||||||
|
private static final String TYPE_CONFIRM = "confirmNode";
|
||||||
|
private static final String TYPE_KNOWLEDGE = "knowledgeNode";
|
||||||
|
private static final Set<String> CONFIRM_ARRAY_LEFT_OPERATORS = Set.of(
|
||||||
|
"contains", "notContains", "isEmpty", "isNotEmpty");
|
||||||
private static final String TYPE_WORKFLOW = "workflow-node";
|
private static final String TYPE_WORKFLOW = "workflow-node";
|
||||||
private static final String TYPE_PLUGIN = "plugin-node";
|
private static final String TYPE_PLUGIN = "plugin-node";
|
||||||
private static final String TYPE_MAKE_FILE = "make-file";
|
private static final String TYPE_MAKE_FILE = "make-file";
|
||||||
private static final String SYSTEM_START_PARAM_NAME = "user_input";
|
private static final String SYSTEM_START_PARAM_NAME = "user_input";
|
||||||
private static final int MIN_LOOP_COUNT = 1;
|
private static final int MIN_LOOP_COUNT = 1;
|
||||||
private static final int MAX_LOOP_COUNT = 300;
|
private static final int MAX_LOOP_COUNT = 300;
|
||||||
|
private static final int DEFAULT_MAX_KNOWLEDGE_SOURCES = 8;
|
||||||
|
private static final int DEFAULT_MAX_MULTI_KNOWLEDGE_LIMIT = 200;
|
||||||
|
private static final Pattern COMPLETE_VARIABLE_REFERENCE =
|
||||||
|
Pattern.compile("^\\{\\{\\s*[^\\s{}][^{}]*?\\s*}}$");
|
||||||
private static final String JOIN_MODE_ANY = "any";
|
private static final String JOIN_MODE_ANY = "any";
|
||||||
private static final String JOIN_MODE_ALL = "all";
|
private static final String JOIN_MODE_ALL = "all";
|
||||||
|
|
||||||
@@ -74,6 +91,8 @@ public class WorkflowCheckService {
|
|||||||
private PluginItemService pluginItemService;
|
private PluginItemService pluginItemService;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowPluginSnapshotResolver workflowPluginSnapshotResolver;
|
private WorkflowPluginSnapshotResolver workflowPluginSnapshotResolver;
|
||||||
|
@Resource
|
||||||
|
private MultiKnowledgeRetrievalProperties multiKnowledgeRetrievalProperties;
|
||||||
|
|
||||||
public WorkflowCheckResult checkWorkflow(BigInteger workflowId, WorkflowCheckStage stage) {
|
public WorkflowCheckResult checkWorkflow(BigInteger workflowId, WorkflowCheckStage stage) {
|
||||||
if (workflowId == null) {
|
if (workflowId == null) {
|
||||||
@@ -90,9 +109,11 @@ public class WorkflowCheckService {
|
|||||||
if (stage == null) {
|
if (stage == null) {
|
||||||
throw new BusinessException("校验阶段不能为空");
|
throw new BusinessException("校验阶段不能为空");
|
||||||
}
|
}
|
||||||
|
long checkStartedAt = System.nanoTime();
|
||||||
List<WorkflowCheckIssue> issues = new ArrayList<>();
|
List<WorkflowCheckIssue> issues = new ArrayList<>();
|
||||||
Set<String> issueKeys = new LinkedHashSet<>();
|
Set<String> issueKeys = new LinkedHashSet<>();
|
||||||
ParsedWorkflow parsedWorkflow = parseAndCheckBase(content, issues, issueKeys);
|
ParsedWorkflow parsedWorkflow = parseAndCheckBase(content, issues, issueKeys);
|
||||||
|
long baseFinishedAt = System.nanoTime();
|
||||||
if (parsedWorkflow != null) {
|
if (parsedWorkflow != null) {
|
||||||
List<NodeView> startNodes = parsedWorkflow.nodes.stream()
|
List<NodeView> startNodes = parsedWorkflow.nodes.stream()
|
||||||
.filter(node -> TYPE_START.equals(node.type))
|
.filter(node -> TYPE_START.equals(node.type))
|
||||||
@@ -100,11 +121,22 @@ public class WorkflowCheckService {
|
|||||||
checkStartFormSchema(startNodes, issues, issueKeys);
|
checkStartFormSchema(startNodes, issues, issueKeys);
|
||||||
checkPluginSchemaHashes(parsedWorkflow, issues, issueKeys);
|
checkPluginSchemaHashes(parsedWorkflow, issues, issueKeys);
|
||||||
}
|
}
|
||||||
|
long schemaFinishedAt = System.nanoTime();
|
||||||
|
|
||||||
if (stage == WorkflowCheckStage.PRE_EXECUTE && parsedWorkflow != null) {
|
if (stage == WorkflowCheckStage.PRE_EXECUTE && parsedWorkflow != null) {
|
||||||
runStrictChecks(content, parsedWorkflow, currentWorkflowId, issues, issueKeys);
|
runStrictChecks(content, parsedWorkflow, currentWorkflowId, issues, issueKeys);
|
||||||
}
|
}
|
||||||
return buildResult(stage, issues);
|
WorkflowCheckResult result = buildResult(stage, issues);
|
||||||
|
logSlowCheck(
|
||||||
|
stage,
|
||||||
|
currentWorkflowId,
|
||||||
|
parsedWorkflow,
|
||||||
|
issues.size(),
|
||||||
|
checkStartedAt,
|
||||||
|
baseFinishedAt,
|
||||||
|
schemaFinishedAt,
|
||||||
|
System.nanoTime());
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void checkOrThrow(String content, WorkflowCheckStage stage, BigInteger currentWorkflowId) {
|
public void checkOrThrow(String content, WorkflowCheckStage stage, BigInteger currentWorkflowId) {
|
||||||
@@ -169,6 +201,21 @@ public class WorkflowCheckService {
|
|||||||
if (!StringUtils.hasText(node.type) || !parserMap.containsKey(node.type)) {
|
if (!StringUtils.hasText(node.type) || !parserMap.containsKey(node.type)) {
|
||||||
addIssue(issues, issueKeys, "NODE_TYPE_UNKNOWN", "节点类型无法识别: " + safe(node.type), node.id, null, node.name);
|
addIssue(issues, issueKeys, "NODE_TYPE_UNKNOWN", "节点类型无法识别: " + safe(node.type), node.id, null, node.name);
|
||||||
}
|
}
|
||||||
|
String dataType = node.data == null
|
||||||
|
? null
|
||||||
|
: trimToNull(node.data.getString("type"));
|
||||||
|
if (StringUtils.hasText(node.type)
|
||||||
|
&& StringUtils.hasText(dataType)
|
||||||
|
&& !Objects.equals(node.type, dataType)) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"NODE_TYPE_MISMATCH",
|
||||||
|
"节点类型与节点数据类型不一致",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
}
|
||||||
if (StringUtils.hasText(node.parentId) && node.parentId.equals(node.id)) {
|
if (StringUtils.hasText(node.parentId) && node.parentId.equals(node.id)) {
|
||||||
addIssue(issues, issueKeys, "NODE_PARENT_SELF", "节点不能引用自己作为父节点", node.id, null, node.name);
|
addIssue(issues, issueKeys, "NODE_PARENT_SELF", "节点不能引用自己作为父节点", node.id, null, node.name);
|
||||||
}
|
}
|
||||||
@@ -184,6 +231,9 @@ public class WorkflowCheckService {
|
|||||||
}
|
}
|
||||||
checkLoopConfigurations(nodes, nodeMap, issues, issueKeys);
|
checkLoopConfigurations(nodes, nodeMap, issues, issueKeys);
|
||||||
checkConditionConfigurations(nodes, issues, issueKeys);
|
checkConditionConfigurations(nodes, issues, issueKeys);
|
||||||
|
checkKnowledgeConfigurations(nodes, issues, issueKeys);
|
||||||
|
checkConfirmConfigurations(nodes, issues, issueKeys);
|
||||||
|
checkConfirmOutputReferences(nodes, issues, issueKeys);
|
||||||
|
|
||||||
List<EdgeView> edges = new ArrayList<>();
|
List<EdgeView> edges = new ArrayList<>();
|
||||||
Set<String> edgeIds = new HashSet<>();
|
Set<String> edgeIds = new HashSet<>();
|
||||||
@@ -239,6 +289,195 @@ public class WorkflowCheckService {
|
|||||||
return parsedWorkflow;
|
return parsedWorkflow;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验知识库节点的新旧引用字段和多库向量模式约束。
|
||||||
|
*/
|
||||||
|
private void checkKnowledgeConfigurations(
|
||||||
|
List<NodeView> nodes,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
for (NodeView node : nodes) {
|
||||||
|
if (!TYPE_KNOWLEDGE.equals(node.type) || node.data == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
checkKnowledgeOutputContract(node, issues, issueKeys);
|
||||||
|
List<String> knowledgeIds = new ArrayList<>();
|
||||||
|
if (node.data.containsKey("knowledgeIds")) {
|
||||||
|
Object rawIds = node.data.get("knowledgeIds");
|
||||||
|
if (!(rawIds instanceof JSONArray ids) || ids.isEmpty()) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"KNOWLEDGE_IDS_INVALID",
|
||||||
|
"知识库节点至少需要选择一个知识库",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Set<String> unique = new LinkedHashSet<>();
|
||||||
|
for (Object rawId : ids) {
|
||||||
|
String id = trimToNull(rawId == null
|
||||||
|
? null
|
||||||
|
: String.valueOf(rawId));
|
||||||
|
if (id == null || !id.matches("[0-9]+")) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"KNOWLEDGE_IDS_INVALID",
|
||||||
|
"知识库节点包含无效的知识库ID",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!unique.add(id)) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"KNOWLEDGE_IDS_DUPLICATE",
|
||||||
|
"知识库节点不能重复选择同一知识库",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
knowledgeIds.addAll(unique);
|
||||||
|
if (knowledgeIds.size() > maxKnowledgeSources()) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"KNOWLEDGE_SOURCE_LIMIT_EXCEEDED",
|
||||||
|
"知识库节点选择数量超过平台上限",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
String legacyId = trimToNull(node.data.getString("knowledgeId"));
|
||||||
|
if (legacyId == null || !legacyId.matches("[0-9]+")) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"KNOWLEDGE_ID_INVALID",
|
||||||
|
"知识库节点需要选择知识库",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
knowledgeIds.add(legacyId);
|
||||||
|
}
|
||||||
|
|
||||||
|
String retrievalMode = trimToNull(
|
||||||
|
node.data.getString("retrievalMode"));
|
||||||
|
if (knowledgeIds.size() > 1
|
||||||
|
&& !"VECTOR".equalsIgnoreCase(retrievalMode)) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"MULTI_KNOWLEDGE_MODE_INVALID",
|
||||||
|
"多知识库检索仅支持向量检索",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
}
|
||||||
|
String limit = trimToNull(node.data.getString("limit"));
|
||||||
|
if (limit != null
|
||||||
|
&& !COMPLETE_VARIABLE_REFERENCE.matcher(limit).matches()) {
|
||||||
|
try {
|
||||||
|
int parsedLimit = Integer.parseInt(limit);
|
||||||
|
if (parsedLimit <= 0
|
||||||
|
|| (knowledgeIds.size() > 1
|
||||||
|
&& parsedLimit > maxMultiKnowledgeLimit())) {
|
||||||
|
throw new NumberFormatException("non-positive");
|
||||||
|
}
|
||||||
|
} catch (NumberFormatException exception) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"KNOWLEDGE_LIMIT_INVALID",
|
||||||
|
"知识库节点最终返回条数必须为正整数或有效变量引用",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识库节点只允许暴露稳定的 documents 输出及四个历史子字段。
|
||||||
|
*/
|
||||||
|
private void checkKnowledgeOutputContract(
|
||||||
|
NodeView node,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
if (!node.data.containsKey("outputDefs")) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Object rawOutputDefs = node.data.get("outputDefs");
|
||||||
|
if (rawOutputDefs instanceof JSONArray outputDefs
|
||||||
|
&& outputDefs.size() == 1
|
||||||
|
&& isCanonicalKnowledgeDocumentsOutput(
|
||||||
|
outputDefs.getJSONObject(0))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"KNOWLEDGE_OUTPUT_SCHEMA_INVALID",
|
||||||
|
"知识库节点输出参数必须为 documents,并仅包含 title、content、documentId、knowledgeId",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isCanonicalKnowledgeDocumentsOutput(JSONObject output) {
|
||||||
|
if (output == null
|
||||||
|
|| !"documents".equals(output.getString("name"))
|
||||||
|
|| !"Array".equalsIgnoreCase(output.getString("dataType"))) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
JSONArray children = output.getJSONArray("children");
|
||||||
|
if (children == null || children.size() != 4) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
Map<String, String> expectedTypes = Map.of(
|
||||||
|
"title", "String",
|
||||||
|
"content", "String",
|
||||||
|
"documentId", "Number",
|
||||||
|
"knowledgeId", "Number");
|
||||||
|
Set<String> names = new LinkedHashSet<>();
|
||||||
|
for (int index = 0; index < children.size(); index++) {
|
||||||
|
JSONObject child = children.getJSONObject(index);
|
||||||
|
if (child == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String name = trimToNull(child.getString("name"));
|
||||||
|
String expectedType = expectedTypes.get(name);
|
||||||
|
if (expectedType == null
|
||||||
|
|| !names.add(name)
|
||||||
|
|| !expectedType.equalsIgnoreCase(
|
||||||
|
child.getString("dataType"))) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names.equals(expectedTypes.keySet());
|
||||||
|
}
|
||||||
|
|
||||||
|
private int maxKnowledgeSources() {
|
||||||
|
return multiKnowledgeRetrievalProperties == null
|
||||||
|
? DEFAULT_MAX_KNOWLEDGE_SOURCES
|
||||||
|
: multiKnowledgeRetrievalProperties.getMaxSources();
|
||||||
|
}
|
||||||
|
|
||||||
|
private int maxMultiKnowledgeLimit() {
|
||||||
|
return multiKnowledgeRetrievalProperties == null
|
||||||
|
? DEFAULT_MAX_MULTI_KNOWLEDGE_LIMIT
|
||||||
|
: multiKnowledgeRetrievalProperties.getTotalCandidateLimit();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 校验节点汇聚模式及其静态可证明的到达安全性。
|
* 校验节点汇聚模式及其静态可证明的到达安全性。
|
||||||
*
|
*
|
||||||
@@ -514,6 +753,288 @@ public class WorkflowCheckService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验用户确认节点配置及其对外输出定义。
|
||||||
|
*/
|
||||||
|
private void checkConfirmConfigurations(
|
||||||
|
List<NodeView> nodes,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
for (NodeView node : nodes) {
|
||||||
|
if (!TYPE_CONFIRM.equals(node.type)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (node.data == null) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"CONFIRM_CONFIGURATION_INVALID",
|
||||||
|
"用户确认节点配置不能为空",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
ConfirmNode configuration;
|
||||||
|
try {
|
||||||
|
validateConfirmConfigurationTypes(node.data);
|
||||||
|
checkConfirmOutputDefinitions(
|
||||||
|
node,
|
||||||
|
Boolean.TRUE.equals(node.data.get("multiple")),
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
configuration = node.data.toJavaObject(ConfirmNode.class);
|
||||||
|
configuration.validateConfiguration();
|
||||||
|
} catch (Exception exception) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"CONFIRM_CONFIGURATION_INVALID",
|
||||||
|
"用户确认节点配置无效: " + shortError(exception),
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateConfirmConfigurationTypes(JSONObject data) {
|
||||||
|
for (String key : data.keySet()) {
|
||||||
|
if (!ConfirmNode.SUPPORTED_CONFIGURATION_KEYS.contains(key)) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"用户确认节点包含无效配置字段: " + key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!(data.get("message") instanceof String)) {
|
||||||
|
throw new IllegalArgumentException("用户确认节点提示内容必须为字符串");
|
||||||
|
}
|
||||||
|
if (!(data.get("multiple") instanceof Boolean)) {
|
||||||
|
throw new IllegalArgumentException("用户确认节点选择方式必须为布尔值");
|
||||||
|
}
|
||||||
|
|
||||||
|
Object optionsValue = data.get("options");
|
||||||
|
if (optionsValue == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!(optionsValue instanceof JSONArray options)) {
|
||||||
|
throw new IllegalArgumentException("用户确认节点选项必须为数组");
|
||||||
|
}
|
||||||
|
for (Object option : options) {
|
||||||
|
if (!(option instanceof String)) {
|
||||||
|
throw new IllegalArgumentException("用户确认节点选项内容必须为字符串");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkConfirmOutputDefinitions(
|
||||||
|
NodeView node,
|
||||||
|
boolean multiple,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
Object outputDefsValue = node.data.get("outputDefs");
|
||||||
|
if (!(outputDefsValue instanceof JSONArray outputDefs)
|
||||||
|
|| outputDefs.size() != 1) {
|
||||||
|
addConfirmOutputIssue(node, issues, issueKeys);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Object outputValue = outputDefs.get(0);
|
||||||
|
if (!(outputValue instanceof JSONObject output)) {
|
||||||
|
addConfirmOutputIssue(node, issues, issueKeys);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String expectedType = multiple
|
||||||
|
? DataType.Array_String.toString()
|
||||||
|
: DataType.String.toString();
|
||||||
|
if (trimToNull(output.getString("name")) == null
|
||||||
|
|| !expectedType.equals(
|
||||||
|
trimToNull(output.getString("dataType")))) {
|
||||||
|
addConfirmOutputIssue(node, issues, issueKeys);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addConfirmOutputIssue(
|
||||||
|
NodeView node,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"CONFIRM_OUTPUT_SCHEMA_INVALID",
|
||||||
|
"用户确认节点必须配置唯一非空输出参数,且类型与选择方式一致",
|
||||||
|
node.id,
|
||||||
|
null,
|
||||||
|
node.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 校验下游节点保存的确认输出引用仍然存在且类型一致。
|
||||||
|
*/
|
||||||
|
private void checkConfirmOutputReferences(
|
||||||
|
List<NodeView> nodes,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
ConfirmOutputIndex confirmOutputs = new ConfirmOutputIndex();
|
||||||
|
for (NodeView node : nodes) {
|
||||||
|
if (!TYPE_CONFIRM.equals(node.type) || node.data == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
ConfirmNode configuration = node.data.toJavaObject(ConfirmNode.class);
|
||||||
|
configuration.validateConfiguration();
|
||||||
|
confirmOutputs.put(
|
||||||
|
node.id,
|
||||||
|
configuration.resolveOutputName(),
|
||||||
|
configuration.isMultiple()
|
||||||
|
? DataType.Array_String.toString()
|
||||||
|
: DataType.String.toString());
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
// 配置错误已由 checkConfirmConfigurations 给出精确问题。
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (confirmOutputs.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (NodeView node : nodes) {
|
||||||
|
if (node.data != null) {
|
||||||
|
checkConfirmOutputReferences(
|
||||||
|
node.data,
|
||||||
|
node,
|
||||||
|
confirmOutputs,
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkConfirmOutputReferences(
|
||||||
|
Object value,
|
||||||
|
NodeView consumer,
|
||||||
|
ConfirmOutputIndex confirmOutputs,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
if (value instanceof JSONObject object) {
|
||||||
|
if ("ref".equals(trimToNull(object.getString("refType")))) {
|
||||||
|
checkConfirmOutputReference(
|
||||||
|
object.getString("ref"),
|
||||||
|
object.getString("dataType"),
|
||||||
|
true,
|
||||||
|
consumer,
|
||||||
|
confirmOutputs,
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
}
|
||||||
|
String leftType = checkConfirmOutputReference(
|
||||||
|
object.getString("leftRef"),
|
||||||
|
null,
|
||||||
|
false,
|
||||||
|
consumer,
|
||||||
|
confirmOutputs,
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
String operator = trimToNull(object.getString("operator"));
|
||||||
|
if (DataType.Array_String.toString().equals(leftType)
|
||||||
|
&& !CONFIRM_ARRAY_LEFT_OPERATORS.contains(operator)) {
|
||||||
|
addConfirmConditionTypeIssue(
|
||||||
|
object.getString("leftRef"),
|
||||||
|
operator,
|
||||||
|
consumer,
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
}
|
||||||
|
if ("ref".equals(trimToNull(object.getString("rightType")))) {
|
||||||
|
String rightType = checkConfirmOutputReference(
|
||||||
|
object.getString("rightRef"),
|
||||||
|
null,
|
||||||
|
false,
|
||||||
|
consumer,
|
||||||
|
confirmOutputs,
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
if (DataType.Array_String.toString().equals(rightType)) {
|
||||||
|
addConfirmConditionTypeIssue(
|
||||||
|
object.getString("rightRef"),
|
||||||
|
operator,
|
||||||
|
consumer,
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (Object child : object.values()) {
|
||||||
|
checkConfirmOutputReferences(
|
||||||
|
child,
|
||||||
|
consumer,
|
||||||
|
confirmOutputs,
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (value instanceof JSONArray array) {
|
||||||
|
for (Object child : array) {
|
||||||
|
checkConfirmOutputReferences(
|
||||||
|
child,
|
||||||
|
consumer,
|
||||||
|
confirmOutputs,
|
||||||
|
issues,
|
||||||
|
issueKeys);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String checkConfirmOutputReference(
|
||||||
|
String rawReference,
|
||||||
|
String rawActualType,
|
||||||
|
boolean requireActualType,
|
||||||
|
NodeView consumer,
|
||||||
|
ConfirmOutputIndex confirmOutputs,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
String reference = trimToNull(rawReference);
|
||||||
|
if (reference == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String expectedType = confirmOutputs.get(reference);
|
||||||
|
if (expectedType == null
|
||||||
|
&& !confirmOutputs.referencesConfirmNode(reference)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String actualType = trimToNull(rawActualType);
|
||||||
|
boolean invalidType = actualType == null
|
||||||
|
? requireActualType
|
||||||
|
: !Objects.equals(expectedType, actualType);
|
||||||
|
if (expectedType == null || invalidType) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"CONFIRM_OUTPUT_REFERENCE_INVALID",
|
||||||
|
"用户确认输出引用不存在或类型已变化: "
|
||||||
|
+ reference,
|
||||||
|
consumer.id,
|
||||||
|
null,
|
||||||
|
consumer.name);
|
||||||
|
}
|
||||||
|
return expectedType;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addConfirmConditionTypeIssue(
|
||||||
|
String reference,
|
||||||
|
String operator,
|
||||||
|
NodeView consumer,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
addIssue(
|
||||||
|
issues,
|
||||||
|
issueKeys,
|
||||||
|
"CONFIRM_OUTPUT_REFERENCE_INVALID",
|
||||||
|
"用户确认多选输出不支持当前条件操作符: "
|
||||||
|
+ trimToNull(reference) + " (" + operator + ")",
|
||||||
|
consumer.id,
|
||||||
|
null,
|
||||||
|
consumer.name);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 将 JSON 条件规则转换为运行时规则对象。
|
* 将 JSON 条件规则转换为运行时规则对象。
|
||||||
*
|
*
|
||||||
@@ -1674,6 +2195,65 @@ public class WorkflowCheckService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void logSlowCheck(
|
||||||
|
WorkflowCheckStage stage,
|
||||||
|
BigInteger workflowId,
|
||||||
|
ParsedWorkflow parsedWorkflow,
|
||||||
|
int issueCount,
|
||||||
|
long checkStartedAt,
|
||||||
|
long baseFinishedAt,
|
||||||
|
long schemaFinishedAt,
|
||||||
|
long checkFinishedAt) {
|
||||||
|
long totalMs = elapsedMillis(checkStartedAt, checkFinishedAt);
|
||||||
|
if (totalMs < SLOW_CHECK_THRESHOLD_MS) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LOGGER.warn(
|
||||||
|
"Workflow check is slow: stage={}, workflowId={}, nodes={}, "
|
||||||
|
+ "issues={}, baseMs={}, schemaMs={}, strictMs={}, totalMs={}",
|
||||||
|
stage,
|
||||||
|
workflowId,
|
||||||
|
parsedWorkflow == null ? 0 : parsedWorkflow.nodes.size(),
|
||||||
|
issueCount,
|
||||||
|
elapsedMillis(checkStartedAt, baseFinishedAt),
|
||||||
|
elapsedMillis(baseFinishedAt, schemaFinishedAt),
|
||||||
|
elapsedMillis(schemaFinishedAt, checkFinishedAt),
|
||||||
|
totalMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
private long elapsedMillis(long startedAt, long finishedAt) {
|
||||||
|
return Math.max(0L, (finishedAt - startedAt) / 1_000_000L);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class ConfirmOutputIndex {
|
||||||
|
private final Map<String, String> outputTypes = new HashMap<>();
|
||||||
|
private final Set<String> nodeIds = new HashSet<>();
|
||||||
|
|
||||||
|
private void put(String nodeId, String outputName, String dataType) {
|
||||||
|
nodeIds.add(nodeId);
|
||||||
|
outputTypes.put(nodeId + "." + outputName, dataType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String get(String reference) {
|
||||||
|
return outputTypes.get(reference);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isEmpty() {
|
||||||
|
return outputTypes.isEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean referencesConfirmNode(String reference) {
|
||||||
|
int separator = reference.indexOf('.');
|
||||||
|
while (separator > 0) {
|
||||||
|
if (nodeIds.contains(reference.substring(0, separator))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
separator = reference.indexOf('.', separator + 1);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void throwIfFailed(WorkflowCheckResult result) {
|
private void throwIfFailed(WorkflowCheckResult result) {
|
||||||
if (result == null) {
|
if (result == null) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package tech.easyflow.ai.easyagentsflow.service;
|
||||||
|
|
||||||
|
import com.easyagents.flow.core.chain.ChainResumeException;
|
||||||
|
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 统一校验并恢复暂停中的工作流。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
public class WorkflowResumeService {
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private ChainExecutor chainExecutor;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 恢复暂停实例,并将引擎校验失败转换为稳定的接口错误。
|
||||||
|
*
|
||||||
|
* @param executeId 工作流实例 ID
|
||||||
|
* @param variables 用户提交的确认参数
|
||||||
|
*/
|
||||||
|
public void resume(String executeId, Map<String, Object> variables) {
|
||||||
|
Map<String, Object> submitted = variables == null
|
||||||
|
? new LinkedHashMap<>()
|
||||||
|
: new LinkedHashMap<>(variables);
|
||||||
|
final boolean resumed;
|
||||||
|
try {
|
||||||
|
resumed = chainExecutor.resumeAsyncIfSuspended(
|
||||||
|
executeId, submitted);
|
||||||
|
} catch (ChainResumeException exception) {
|
||||||
|
throw new BusinessException(
|
||||||
|
400,
|
||||||
|
40031,
|
||||||
|
exception.getMessage(),
|
||||||
|
exception);
|
||||||
|
}
|
||||||
|
if (!resumed) {
|
||||||
|
throw new BusinessException(
|
||||||
|
409,
|
||||||
|
40901,
|
||||||
|
"当前执行状态不可恢复,仅暂停中的工作流允许恢复");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package tech.easyflow.ai.entity;
|
||||||
|
|
||||||
|
import com.mybatisflex.annotation.Column;
|
||||||
|
import com.mybatisflex.annotation.Id;
|
||||||
|
import com.mybatisflex.annotation.KeyType;
|
||||||
|
import com.mybatisflex.annotation.Table;
|
||||||
|
import tech.easyflow.common.entity.DateEntity;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文档分块检索索引同步任务。
|
||||||
|
*/
|
||||||
|
@Table("tb_document_chunk_sync_task")
|
||||||
|
public class DocumentChunkSyncTask extends DateEntity implements Serializable {
|
||||||
|
|
||||||
|
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||||
|
private BigInteger id;
|
||||||
|
private BigInteger chunkId;
|
||||||
|
private BigInteger documentId;
|
||||||
|
private BigInteger documentCollectionId;
|
||||||
|
private String vectorCollection;
|
||||||
|
private String operation;
|
||||||
|
private Long syncVersion;
|
||||||
|
private String status;
|
||||||
|
private Integer attemptCount;
|
||||||
|
private Date nextRetryAt;
|
||||||
|
private Date lastDispatchedAt;
|
||||||
|
private String executionToken;
|
||||||
|
private Date leaseUntil;
|
||||||
|
private String errorCode;
|
||||||
|
private String errorMessage;
|
||||||
|
private Date created;
|
||||||
|
private BigInteger createdBy;
|
||||||
|
private Date modified;
|
||||||
|
private BigInteger modifiedBy;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public BigInteger getChunkId() { return chunkId; }
|
||||||
|
public void setChunkId(BigInteger chunkId) { this.chunkId = chunkId; }
|
||||||
|
public BigInteger getDocumentId() { return documentId; }
|
||||||
|
public void setDocumentId(BigInteger documentId) { this.documentId = documentId; }
|
||||||
|
public BigInteger getDocumentCollectionId() { return documentCollectionId; }
|
||||||
|
public void setDocumentCollectionId(BigInteger documentCollectionId) { this.documentCollectionId = documentCollectionId; }
|
||||||
|
public String getVectorCollection() { return vectorCollection; }
|
||||||
|
public void setVectorCollection(String vectorCollection) { this.vectorCollection = vectorCollection; }
|
||||||
|
public String getOperation() { return operation; }
|
||||||
|
public void setOperation(String operation) { this.operation = operation; }
|
||||||
|
public Long getSyncVersion() { return syncVersion; }
|
||||||
|
public void setSyncVersion(Long syncVersion) { this.syncVersion = syncVersion; }
|
||||||
|
public String getStatus() { return status; }
|
||||||
|
public void setStatus(String status) { this.status = status; }
|
||||||
|
public Integer getAttemptCount() { return attemptCount; }
|
||||||
|
public void setAttemptCount(Integer attemptCount) { this.attemptCount = attemptCount; }
|
||||||
|
public Date getNextRetryAt() { return nextRetryAt; }
|
||||||
|
public void setNextRetryAt(Date nextRetryAt) { this.nextRetryAt = nextRetryAt; }
|
||||||
|
public Date getLastDispatchedAt() { return lastDispatchedAt; }
|
||||||
|
public void setLastDispatchedAt(Date lastDispatchedAt) { this.lastDispatchedAt = lastDispatchedAt; }
|
||||||
|
public String getExecutionToken() { return executionToken; }
|
||||||
|
public void setExecutionToken(String executionToken) { this.executionToken = executionToken; }
|
||||||
|
public Date getLeaseUntil() { return leaseUntil; }
|
||||||
|
public void setLeaseUntil(Date leaseUntil) { this.leaseUntil = leaseUntil; }
|
||||||
|
public String getErrorCode() { return errorCode; }
|
||||||
|
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
|
||||||
|
public String getErrorMessage() { return errorMessage; }
|
||||||
|
public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
|
||||||
|
@Override public Date getCreated() { return created; }
|
||||||
|
@Override public void setCreated(Date created) { this.created = created; }
|
||||||
|
public BigInteger getCreatedBy() { return createdBy; }
|
||||||
|
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||||
|
@Override public Date getModified() { return modified; }
|
||||||
|
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||||
|
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||||
|
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import com.easyagents.store.milvus.MilvusVectorStore;
|
|||||||
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
import com.easyagents.store.milvus.MilvusVectorStoreConfig;
|
||||||
import com.mybatisflex.annotation.Table;
|
import com.mybatisflex.annotation.Table;
|
||||||
import tech.easyflow.ai.config.AiMilvusConfig;
|
import tech.easyflow.ai.config.AiMilvusConfig;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusClientManager;
|
||||||
import tech.easyflow.ai.chattime.availability.ChatTimeToolAvailabilityContext;
|
import tech.easyflow.ai.chattime.availability.ChatTimeToolAvailabilityContext;
|
||||||
import tech.easyflow.ai.easyagents.tool.DocumentCollectionTool;
|
import tech.easyflow.ai.easyagents.tool.DocumentCollectionTool;
|
||||||
import tech.easyflow.ai.entity.base.DocumentCollectionBase;
|
import tech.easyflow.ai.entity.base.DocumentCollectionBase;
|
||||||
@@ -103,8 +104,10 @@ public class DocumentCollection extends DocumentCollectionBase implements Visibi
|
|||||||
|
|
||||||
private DocumentStore milvusStore() {
|
private DocumentStore milvusStore() {
|
||||||
AiMilvusConfig aiMilvusConfig = SpringContextUtil.getBean(AiMilvusConfig.class);
|
AiMilvusConfig aiMilvusConfig = SpringContextUtil.getBean(AiMilvusConfig.class);
|
||||||
|
AiMilvusClientManager clientManager = SpringContextUtil.getBean(AiMilvusClientManager.class);
|
||||||
|
clientManager.reconfigureIfNeeded(aiMilvusConfig);
|
||||||
MilvusVectorStoreConfig milvusVectorStoreConfig = aiMilvusConfig.copyForCollection(this.getVectorStoreCollection());
|
MilvusVectorStoreConfig milvusVectorStoreConfig = aiMilvusConfig.copyForCollection(this.getVectorStoreCollection());
|
||||||
return new MilvusVectorStore(milvusVectorStoreConfig);
|
return new MilvusVectorStore(milvusVectorStoreConfig, clientManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Tool toFunction(boolean needEnglishName) {
|
public Tool toFunction(boolean needEnglishName) {
|
||||||
|
|||||||
@@ -46,6 +46,18 @@ public class DocumentChunkBase implements Serializable {
|
|||||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展元信息")
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展元信息")
|
||||||
private Map<String, Object> options;
|
private Map<String, Object> options;
|
||||||
|
|
||||||
|
@Column(comment = "检索索引同步状态")
|
||||||
|
private String indexSyncStatus;
|
||||||
|
|
||||||
|
@Column(comment = "检索索引同步版本")
|
||||||
|
private Long indexSyncVersion;
|
||||||
|
|
||||||
|
@Column(comment = "脱敏同步错误码")
|
||||||
|
private String indexSyncErrorCode;
|
||||||
|
|
||||||
|
@Column(comment = "脱敏同步错误摘要")
|
||||||
|
private String indexSyncErrorMessage;
|
||||||
|
|
||||||
public BigInteger getId() {
|
public BigInteger getId() {
|
||||||
return id;
|
return id;
|
||||||
}
|
}
|
||||||
@@ -94,4 +106,36 @@ public class DocumentChunkBase implements Serializable {
|
|||||||
this.options = options;
|
this.options = options;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getIndexSyncStatus() {
|
||||||
|
return indexSyncStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexSyncStatus(String indexSyncStatus) {
|
||||||
|
this.indexSyncStatus = indexSyncStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getIndexSyncVersion() {
|
||||||
|
return indexSyncVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexSyncVersion(Long indexSyncVersion) {
|
||||||
|
this.indexSyncVersion = indexSyncVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getIndexSyncErrorCode() {
|
||||||
|
return indexSyncErrorCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexSyncErrorCode(String indexSyncErrorCode) {
|
||||||
|
this.indexSyncErrorCode = indexSyncErrorCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getIndexSyncErrorMessage() {
|
||||||
|
return indexSyncErrorMessage;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexSyncErrorMessage(String indexSyncErrorMessage) {
|
||||||
|
this.indexSyncErrorMessage = indexSyncErrorMessage;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,12 @@ package tech.easyflow.ai.mapper;
|
|||||||
|
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
import com.mybatisflex.core.BaseMapper;
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
import org.apache.ibatis.annotations.Update;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 映射层。
|
* 映射层。
|
||||||
@@ -11,4 +17,23 @@ import com.mybatisflex.core.BaseMapper;
|
|||||||
*/
|
*/
|
||||||
public interface DocumentChunkMapper extends BaseMapper<DocumentChunk> {
|
public interface DocumentChunkMapper extends BaseMapper<DocumentChunk> {
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk SET index_sync_status=#{status}, "
|
||||||
|
+ "index_sync_error_code=#{errorCode}, index_sync_error_message=#{errorMessage} "
|
||||||
|
+ "WHERE id=#{id} AND index_sync_version=#{version}")
|
||||||
|
int updateSyncState(@Param("id") BigInteger id,
|
||||||
|
@Param("version") long version,
|
||||||
|
@Param("status") String status,
|
||||||
|
@Param("errorCode") String errorCode,
|
||||||
|
@Param("errorMessage") String errorMessage);
|
||||||
|
|
||||||
|
@Select("<script>SELECT id, document_id AS documentId, "
|
||||||
|
+ "document_collection_id AS documentCollectionId, "
|
||||||
|
+ "index_sync_status AS indexSyncStatus, index_sync_version AS indexSyncVersion, "
|
||||||
|
+ "index_sync_error_code AS indexSyncErrorCode, "
|
||||||
|
+ "index_sync_error_message AS indexSyncErrorMessage "
|
||||||
|
+ "FROM tb_document_chunk WHERE document_id=#{documentId} AND id IN "
|
||||||
|
+ "<foreach collection='ids' item='id' open='(' separator=',' close=')'>#{id}</foreach>"
|
||||||
|
+ "</script>")
|
||||||
|
List<DocumentChunk> selectSyncStates(@Param("documentId") BigInteger documentId,
|
||||||
|
@Param("ids") List<BigInteger> ids);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package tech.easyflow.ai.mapper;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
import org.apache.ibatis.annotations.Update;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文档分块索引同步任务映射层。
|
||||||
|
*/
|
||||||
|
public interface DocumentChunkSyncTaskMapper extends BaseMapper<DocumentChunkSyncTask> {
|
||||||
|
|
||||||
|
String SELECT_COLUMNS = "id, chunk_id AS chunkId, document_id AS documentId, "
|
||||||
|
+ "document_collection_id AS documentCollectionId, "
|
||||||
|
+ "vector_collection AS vectorCollection, operation, sync_version AS syncVersion, "
|
||||||
|
+ "status, attempt_count AS attemptCount, next_retry_at AS nextRetryAt, "
|
||||||
|
+ "last_dispatched_at AS lastDispatchedAt, execution_token AS executionToken, "
|
||||||
|
+ "lease_until AS leaseUntil, error_code AS errorCode, error_message AS errorMessage, "
|
||||||
|
+ "created, created_by AS createdBy, modified, modified_by AS modifiedBy";
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='SUPERSEDED', "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, modified=#{now} "
|
||||||
|
+ "WHERE chunk_id=#{chunkId} AND sync_version < #{syncVersion} "
|
||||||
|
+ "AND status IN ('PENDING','RUNNING','FAILED')")
|
||||||
|
int supersedeOlder(@Param("chunkId") BigInteger chunkId,
|
||||||
|
@Param("syncVersion") long syncVersion,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='SUPERSEDED', "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, modified=#{now} "
|
||||||
|
+ "WHERE chunk_id=#{chunkId} AND status IN ('PENDING','RUNNING','FAILED')")
|
||||||
|
int supersedeChunk(@Param("chunkId") BigInteger chunkId,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 锁定同一分块的任务版本范围,保证外部索引写入与新版本建单按版本串行。
|
||||||
|
*/
|
||||||
|
@Select("SELECT id FROM tb_document_chunk_sync_task WHERE chunk_id=#{chunkId} "
|
||||||
|
+ "ORDER BY sync_version, id FOR UPDATE")
|
||||||
|
List<BigInteger> lockChunkTasks(@Param("chunkId") BigInteger chunkId);
|
||||||
|
|
||||||
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_chunk_sync_task "
|
||||||
|
+ "WHERE status='PENDING' AND next_retry_at <= #{now} "
|
||||||
|
+ "AND (last_dispatched_at IS NULL OR last_dispatched_at <= #{redispatchBefore}) "
|
||||||
|
+ "ORDER BY next_retry_at, id LIMIT #{limit}")
|
||||||
|
List<DocumentChunkSyncTask> selectPendingDue(@Param("now") Date now,
|
||||||
|
@Param("redispatchBefore") Date redispatchBefore,
|
||||||
|
@Param("limit") int limit);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET last_dispatched_at=#{now}, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND status='PENDING' AND next_retry_at <= #{now} "
|
||||||
|
+ "AND (last_dispatched_at IS NULL OR last_dispatched_at <= #{redispatchBefore})")
|
||||||
|
int markDispatched(@Param("id") BigInteger id,
|
||||||
|
@Param("now") Date now,
|
||||||
|
@Param("redispatchBefore") Date redispatchBefore);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='PENDING', "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, next_retry_at=#{now}, "
|
||||||
|
+ "last_dispatched_at=NULL, modified=#{now} "
|
||||||
|
+ "WHERE status='RUNNING' AND lease_until <= #{now}")
|
||||||
|
int recoverExpired(@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='RUNNING', "
|
||||||
|
+ "attempt_count=attempt_count + 1, execution_token=#{token}, "
|
||||||
|
+ "lease_until=#{leaseUntil}, error_code=NULL, error_message=NULL, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND status='PENDING' AND next_retry_at <= #{now}")
|
||||||
|
int claim(@Param("id") BigInteger id,
|
||||||
|
@Param("token") String token,
|
||||||
|
@Param("leaseUntil") Date leaseUntil,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status=#{status}, "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, error_code=#{errorCode}, "
|
||||||
|
+ "error_message=#{errorMessage}, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND status='RUNNING' AND execution_token=#{token}")
|
||||||
|
int finishOwned(@Param("id") BigInteger id,
|
||||||
|
@Param("token") String token,
|
||||||
|
@Param("status") String status,
|
||||||
|
@Param("errorCode") String errorCode,
|
||||||
|
@Param("errorMessage") String errorMessage,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status=#{status}, "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, next_retry_at=#{nextRetryAt}, "
|
||||||
|
+ "last_dispatched_at=NULL, error_code=#{errorCode}, "
|
||||||
|
+ "error_message=#{errorMessage}, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND status='RUNNING' AND execution_token=#{token}")
|
||||||
|
int failOrRetryOwned(@Param("id") BigInteger id,
|
||||||
|
@Param("token") String token,
|
||||||
|
@Param("status") String status,
|
||||||
|
@Param("nextRetryAt") Date nextRetryAt,
|
||||||
|
@Param("errorCode") String errorCode,
|
||||||
|
@Param("errorMessage") String errorMessage,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Update("UPDATE tb_document_chunk_sync_task SET status='PENDING', "
|
||||||
|
+ "attempt_count=0, next_retry_at=#{now}, last_dispatched_at=NULL, "
|
||||||
|
+ "execution_token=NULL, lease_until=NULL, error_code=NULL, error_message=NULL, modified=#{now} "
|
||||||
|
+ "WHERE id=#{id} AND chunk_id=#{chunkId} AND sync_version=#{syncVersion} "
|
||||||
|
+ "AND status='FAILED'")
|
||||||
|
int retryFailed(@Param("id") BigInteger id,
|
||||||
|
@Param("chunkId") BigInteger chunkId,
|
||||||
|
@Param("syncVersion") long syncVersion,
|
||||||
|
@Param("now") Date now);
|
||||||
|
|
||||||
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_chunk_sync_task WHERE chunk_id=#{chunkId} "
|
||||||
|
+ "AND sync_version=#{syncVersion} AND status='FAILED' ORDER BY id DESC LIMIT 1")
|
||||||
|
DocumentChunkSyncTask selectFailed(@Param("chunkId") BigInteger chunkId,
|
||||||
|
@Param("syncVersion") long syncVersion);
|
||||||
|
}
|
||||||
@@ -17,22 +17,49 @@ import java.util.Date;
|
|||||||
*/
|
*/
|
||||||
public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImportBatchItem> {
|
public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImportBatchItem> {
|
||||||
|
|
||||||
|
String SELECT_COLUMNS = "id, batch_id AS batchId, knowledge_id AS knowledgeId, "
|
||||||
|
+ "document_id AS documentId, replaced_document_id AS replacedDocumentId, "
|
||||||
|
+ "client_file_key AS clientFileKey, file_name AS fileName, "
|
||||||
|
+ "relative_path AS relativePath, file_size AS fileSize, "
|
||||||
|
+ "file_path AS filePath, storage_locator AS storageLocator, "
|
||||||
|
+ "cleanup_pending AS cleanupPending, content_sha256 AS contentSha256, "
|
||||||
|
+ "stage, status, error_summary AS errorSummary, "
|
||||||
|
+ "failure_code AS failureCode, applied_strategy_code AS appliedStrategyCode, "
|
||||||
|
+ "strategy_snapshot_json AS strategySnapshotJson, retryable, "
|
||||||
|
+ "attempt_count AS attemptCount, created, created_by AS createdBy, "
|
||||||
|
+ "modified, modified_by AS modifiedBy";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 锁定运行批次中的失败项并取得本轮恢复权。
|
* 锁定并读取批次文件项。
|
||||||
*
|
*
|
||||||
* <p>同时锁定批次和文件项,使恢复任务创建与批次熔断串行化。</p>
|
* <p>调用方必须先锁定所属批次,再调用本方法,避免批次计数更新时
|
||||||
|
* 出现共享锁升级死锁。</p>
|
||||||
*
|
*
|
||||||
* @param itemId 文件项 ID
|
* @param itemId 文件项 ID
|
||||||
* @return 可恢复文件项;状态已变化时返回 null
|
* @return 文件项;不存在时返回 null
|
||||||
*/
|
*/
|
||||||
@Select("SELECT item.* FROM tb_document_import_batch_item item "
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_batch_item "
|
||||||
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id "
|
+ "WHERE id=#{itemId} FOR UPDATE")
|
||||||
+ "WHERE item.id=#{itemId} AND item.status='FAILED' "
|
DocumentImportBatchItem selectForUpdate(
|
||||||
+ "AND batch.status='RUNNING' FOR UPDATE")
|
|
||||||
DocumentImportBatchItem selectFailedForRetry(
|
|
||||||
@Param("itemId") BigInteger itemId
|
@Param("itemId") BigInteger itemId
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 统计运行批次内已经由等价请求推进的活跃文件项。
|
||||||
|
*
|
||||||
|
* <p>调用方必须先锁定批次;所有文件项状态迁移遵循批次到文件项
|
||||||
|
* 的锁顺序,因此该当前状态检查到后续熔断之间不会新增活跃项。</p>
|
||||||
|
*
|
||||||
|
* @param batchId 批次 ID
|
||||||
|
* @return 活跃文件项数量
|
||||||
|
*/
|
||||||
|
@Select("SELECT COUNT(*) FROM tb_document_import_batch_item "
|
||||||
|
+ "WHERE batch_id=#{batchId} "
|
||||||
|
+ "AND status IN ('PENDING','UPLOADING','RUNNING','UPLOADED')")
|
||||||
|
int countActiveItems(
|
||||||
|
@Param("batchId") BigInteger batchId
|
||||||
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 将中断批次中尚未结束的文件项统一收口为可恢复失败。
|
* 将中断批次中尚未结束的文件项统一收口为可恢复失败。
|
||||||
*
|
*
|
||||||
@@ -100,14 +127,13 @@ public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImport
|
|||||||
* @param modified 修改时间
|
* @param modified 修改时间
|
||||||
* @return 更新行数
|
* @return 更新行数
|
||||||
*/
|
*/
|
||||||
@Update("UPDATE tb_document_import_batch_item item "
|
@Update("UPDATE tb_document_import_batch_item SET "
|
||||||
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id SET "
|
+ "stage=#{stage}, status=#{status}, "
|
||||||
+ "item.stage=#{stage}, item.status=#{status}, "
|
+ "error_summary=#{errorSummary}, failure_code=#{failureCode}, "
|
||||||
+ "item.error_summary=#{errorSummary}, item.failure_code=#{failureCode}, "
|
+ "retryable=#{retryable}, "
|
||||||
+ "item.retryable=#{retryable}, "
|
+ "attempt_count=attempt_count + #{attemptDelta}, "
|
||||||
+ "item.attempt_count=item.attempt_count + #{attemptDelta}, "
|
+ "modified=#{modified} WHERE id=#{id} "
|
||||||
+ "item.modified=#{modified} WHERE item.id=#{id} "
|
+ "AND status=#{expectedStatus}")
|
||||||
+ "AND item.status=#{expectedStatus} AND batch.status<>'INTERRUPTED'")
|
|
||||||
int transitionStatus(
|
int transitionStatus(
|
||||||
@Param("id") BigInteger id,
|
@Param("id") BigInteger id,
|
||||||
@Param("expectedStatus") String expectedStatus,
|
@Param("expectedStatus") String expectedStatus,
|
||||||
@@ -146,21 +172,46 @@ public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImport
|
|||||||
* @param modified 修改时间
|
* @param modified 修改时间
|
||||||
* @return 文件项仍属于运行批次且绑定成功时返回 1
|
* @return 文件项仍属于运行批次且绑定成功时返回 1
|
||||||
*/
|
*/
|
||||||
@Update("UPDATE tb_document_import_batch_item item "
|
@Update("UPDATE tb_document_import_batch_item SET "
|
||||||
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id SET "
|
+ "document_id=#{documentId}, stage='PARSE', "
|
||||||
+ "item.document_id=#{documentId}, item.stage='PARSE', "
|
+ "status='PENDING', error_summary=NULL, "
|
||||||
+ "item.status='PENDING', item.error_summary=NULL, "
|
+ "failure_code=NULL, retryable=0, "
|
||||||
+ "item.failure_code=NULL, item.retryable=0, "
|
+ "attempt_count=COALESCE(attempt_count, 0) + 1, "
|
||||||
+ "item.attempt_count=COALESCE(item.attempt_count, 0) + 1, "
|
+ "modified=#{modified} "
|
||||||
+ "item.modified=#{modified} "
|
+ "WHERE id=#{id} AND status='FAILED' "
|
||||||
+ "WHERE item.id=#{id} AND item.status='FAILED' "
|
+ "AND document_id IS NULL")
|
||||||
+ "AND item.document_id IS NULL AND batch.status='RUNNING'")
|
|
||||||
int bindFailedDocument(
|
int bindFailedDocument(
|
||||||
@Param("id") BigInteger id,
|
@Param("id") BigInteger id,
|
||||||
@Param("documentId") BigInteger documentId,
|
@Param("documentId") BigInteger documentId,
|
||||||
@Param("modified") Date modified
|
@Param("modified") Date modified
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将引用已丢失文档的失败项绑定到恢复创建的新文档。
|
||||||
|
*
|
||||||
|
* @param id 文件项 ID
|
||||||
|
* @param missingDocumentId 已不存在的旧文档 ID
|
||||||
|
* @param documentId 恢复创建的新文档 ID
|
||||||
|
* @param modified 修改时间
|
||||||
|
* @return 文件项仍处于预期失败状态且旧文档确实不存在时返回 1
|
||||||
|
*/
|
||||||
|
@Update("UPDATE tb_document_import_batch_item SET "
|
||||||
|
+ "document_id=#{documentId}, stage='PARSE', "
|
||||||
|
+ "status='PENDING', error_summary=NULL, "
|
||||||
|
+ "failure_code=NULL, retryable=0, "
|
||||||
|
+ "attempt_count=COALESCE(attempt_count, 0) + 1, "
|
||||||
|
+ "modified=#{modified} "
|
||||||
|
+ "WHERE id=#{id} AND status='FAILED' "
|
||||||
|
+ "AND document_id=#{missingDocumentId} "
|
||||||
|
+ "AND NOT EXISTS (SELECT 1 FROM tb_document "
|
||||||
|
+ "WHERE id=#{missingDocumentId})")
|
||||||
|
int replaceMissingDocument(
|
||||||
|
@Param("id") BigInteger id,
|
||||||
|
@Param("missingDocumentId") BigInteger missingDocumentId,
|
||||||
|
@Param("documentId") BigInteger documentId,
|
||||||
|
@Param("modified") Date modified
|
||||||
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 更新恢复失败项的业务错误,但不提前汇总批次终态。
|
* 更新恢复失败项的业务错误,但不提前汇总批次终态。
|
||||||
*
|
*
|
||||||
@@ -170,12 +221,11 @@ public interface DocumentImportBatchItemMapper extends BaseMapper<DocumentImport
|
|||||||
* @param modified 修改时间
|
* @param modified 修改时间
|
||||||
* @return 文件项仍属于运行批次且更新成功时返回 1
|
* @return 文件项仍属于运行批次且更新成功时返回 1
|
||||||
*/
|
*/
|
||||||
@Update("UPDATE tb_document_import_batch_item item "
|
@Update("UPDATE tb_document_import_batch_item SET "
|
||||||
+ "INNER JOIN tb_document_import_batch batch ON batch.id=item.batch_id SET "
|
+ "error_summary=#{errorSummary}, failure_code=NULL, "
|
||||||
+ "item.error_summary=#{errorSummary}, item.failure_code=NULL, "
|
+ "retryable=1, modified=#{modified} "
|
||||||
+ "item.retryable=1, item.modified=#{modified} "
|
+ "WHERE id=#{id} AND batch_id=#{batchId} "
|
||||||
+ "WHERE item.id=#{id} AND item.batch_id=#{batchId} "
|
+ "AND status='FAILED'")
|
||||||
+ "AND item.status='FAILED' AND batch.status='RUNNING'")
|
|
||||||
int updateFailedRetryError(
|
int updateFailedRetryError(
|
||||||
@Param("id") BigInteger id,
|
@Param("id") BigInteger id,
|
||||||
@Param("batchId") BigInteger batchId,
|
@Param("batchId") BigInteger batchId,
|
||||||
|
|||||||
@@ -18,6 +18,38 @@ import java.util.List;
|
|||||||
*/
|
*/
|
||||||
public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatch> {
|
public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatch> {
|
||||||
|
|
||||||
|
String SELECT_COLUMNS = "id, knowledge_id AS knowledgeId, "
|
||||||
|
+ "caller_type AS callerType, caller_id AS callerId, "
|
||||||
|
+ "idempotency_key_hash AS idempotencyKeyHash, "
|
||||||
|
+ "request_digest AS requestDigest, duplicate_policy AS duplicatePolicy, "
|
||||||
|
+ "requested_strategy_json AS requestedStrategyJson, "
|
||||||
|
+ "retry_generation AS retryGeneration, version, "
|
||||||
|
+ "import_mode AS importMode, status, total_count AS totalCount, "
|
||||||
|
+ "total_bytes AS totalBytes, completed_count AS completedCount, "
|
||||||
|
+ "processing_count AS processingCount, failed_count AS failedCount, "
|
||||||
|
+ "pending_count AS pendingCount, uploaded_count AS uploadedCount, "
|
||||||
|
+ "skipped_count AS skippedCount, cancelled_count AS cancelledCount, "
|
||||||
|
+ "retryable_failed_count AS retryableFailedCount, "
|
||||||
|
+ "interrupt_code AS interruptCode, interrupt_message AS interruptMessage, "
|
||||||
|
+ "interrupted_at AS interruptedAt, recovery_pending AS recoveryPending, "
|
||||||
|
+ "recovery_file_keys_json AS recoveryFileKeysJson, "
|
||||||
|
+ "recovery_token AS recoveryToken, "
|
||||||
|
+ "recovery_lease_until AS recoveryLeaseUntil, "
|
||||||
|
+ "started_at AS startedAt, finished_at AS finishedAt, created, "
|
||||||
|
+ "created_by AS createdBy, modified, modified_by AS modifiedBy";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 锁定并读取批次,统一批次计数与文件项状态更新的加锁顺序。
|
||||||
|
*
|
||||||
|
* @param batchId 批次 ID
|
||||||
|
* @return 批次;不存在时返回 null
|
||||||
|
*/
|
||||||
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_batch "
|
||||||
|
+ "WHERE id=#{batchId} FOR UPDATE")
|
||||||
|
DocumentImportBatch selectForUpdate(
|
||||||
|
@Param("batchId") BigInteger batchId
|
||||||
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 锁定并读取调用者拥有的批次,保证后续重试基于最新状态。
|
* 锁定并读取调用者拥有的批次,保证后续重试基于最新状态。
|
||||||
*
|
*
|
||||||
@@ -26,7 +58,7 @@ public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatc
|
|||||||
* @param callerId 调用者 ID
|
* @param callerId 调用者 ID
|
||||||
* @return 批次;不存在时返回 null
|
* @return 批次;不存在时返回 null
|
||||||
*/
|
*/
|
||||||
@Select("SELECT * FROM tb_document_import_batch "
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_batch "
|
||||||
+ "WHERE id=#{batchId} AND caller_type=#{callerType} "
|
+ "WHERE id=#{batchId} AND caller_type=#{callerType} "
|
||||||
+ "AND caller_id=#{callerId} FOR UPDATE")
|
+ "AND caller_id=#{callerId} FOR UPDATE")
|
||||||
DocumentImportBatch selectOwnedForUpdate(
|
DocumentImportBatch selectOwnedForUpdate(
|
||||||
@@ -195,7 +227,7 @@ public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatc
|
|||||||
* @param limit 最大批次数
|
* @param limit 最大批次数
|
||||||
* @return 待恢复批次
|
* @return 待恢复批次
|
||||||
*/
|
*/
|
||||||
@Select("SELECT * FROM tb_document_import_batch "
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_batch "
|
||||||
+ "WHERE status='RUNNING' AND recovery_pending=1 "
|
+ "WHERE status='RUNNING' AND recovery_pending=1 "
|
||||||
+ "AND (recovery_token IS NULL OR recovery_lease_until <= #{now}) "
|
+ "AND (recovery_token IS NULL OR recovery_lease_until <= #{now}) "
|
||||||
+ "ORDER BY modified, id LIMIT #{limit}")
|
+ "ORDER BY modified, id LIMIT #{limit}")
|
||||||
@@ -256,7 +288,7 @@ public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatc
|
|||||||
* @param recoveryToken 恢复调度令牌
|
* @param recoveryToken 恢复调度令牌
|
||||||
* @return 当前令牌持有的批次;令牌失效时返回空
|
* @return 当前令牌持有的批次;令牌失效时返回空
|
||||||
*/
|
*/
|
||||||
@Select("SELECT * FROM tb_document_import_batch "
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_batch "
|
||||||
+ "WHERE id=#{batchId} AND status='RUNNING' "
|
+ "WHERE id=#{batchId} AND status='RUNNING' "
|
||||||
+ "AND recovery_pending=1 AND recovery_token=#{recoveryToken}")
|
+ "AND recovery_pending=1 AND recovery_token=#{recoveryToken}")
|
||||||
DocumentImportBatch selectClaimedRecovery(
|
DocumentImportBatch selectClaimedRecovery(
|
||||||
@@ -383,7 +415,7 @@ public interface DocumentImportBatchMapper extends BaseMapper<DocumentImportBatc
|
|||||||
* @param limit 最大返回数量
|
* @param limit 最大返回数量
|
||||||
* @return 待回收批次
|
* @return 待回收批次
|
||||||
*/
|
*/
|
||||||
@Select("SELECT * FROM tb_document_import_batch "
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_batch "
|
||||||
+ "WHERE status IN ('UPLOADING','READY') "
|
+ "WHERE status IN ('UPLOADING','READY') "
|
||||||
+ "AND ((modified IS NOT NULL AND modified < #{incompleteCutoff}) "
|
+ "AND ((modified IS NOT NULL AND modified < #{incompleteCutoff}) "
|
||||||
+ "OR (modified IS NULL AND created < #{incompleteCutoff})) "
|
+ "OR (modified IS NULL AND created < #{incompleteCutoff})) "
|
||||||
|
|||||||
@@ -18,6 +18,26 @@ import java.util.List;
|
|||||||
*/
|
*/
|
||||||
public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask> {
|
public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask> {
|
||||||
|
|
||||||
|
String SELECT_COLUMNS = "id, document_id AS documentId, "
|
||||||
|
+ "knowledge_id AS knowledgeId, batch_id AS batchId, "
|
||||||
|
+ "batch_item_id AS batchItemId, phase, status, "
|
||||||
|
+ "provider_task_id AS providerTaskId, payload_json AS payloadJson, "
|
||||||
|
+ "error_summary AS errorSummary, failure_code AS failureCode, "
|
||||||
|
+ "attempt_no AS attemptNo, execution_token AS executionToken, "
|
||||||
|
+ "lease_until AS leaseUntil, version, started_at AS startedAt, "
|
||||||
|
+ "finished_at AS finishedAt, created, created_by AS createdBy, "
|
||||||
|
+ "modified, modified_by AS modifiedBy";
|
||||||
|
|
||||||
|
String QUALIFIED_SELECT_COLUMNS = "task.id, task.document_id AS documentId, "
|
||||||
|
+ "task.knowledge_id AS knowledgeId, task.batch_id AS batchId, "
|
||||||
|
+ "task.batch_item_id AS batchItemId, task.phase, task.status, "
|
||||||
|
+ "task.provider_task_id AS providerTaskId, task.payload_json AS payloadJson, "
|
||||||
|
+ "task.error_summary AS errorSummary, task.failure_code AS failureCode, "
|
||||||
|
+ "task.attempt_no AS attemptNo, task.execution_token AS executionToken, "
|
||||||
|
+ "task.lease_until AS leaseUntil, task.version, task.started_at AS startedAt, "
|
||||||
|
+ "task.finished_at AS finishedAt, task.created, task.created_by AS createdBy, "
|
||||||
|
+ "task.modified, task.modified_by AS modifiedBy";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 按任务阶段和批次轮转查询待投递任务,避免大批次独占投递窗口。
|
* 按任务阶段和批次轮转查询待投递任务,避免大批次独占投递窗口。
|
||||||
*
|
*
|
||||||
@@ -25,14 +45,16 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
|
|||||||
* @param limit 最大任务数
|
* @param limit 最大任务数
|
||||||
* @return 公平排序后的待投递任务
|
* @return 公平排序后的待投递任务
|
||||||
*/
|
*/
|
||||||
@Select("SELECT task.* FROM tb_document_import_task task JOIN ("
|
@Select("SELECT " + QUALIFIED_SELECT_COLUMNS
|
||||||
|
+ " FROM tb_document_import_task task JOIN ("
|
||||||
+ "SELECT task.id, ROW_NUMBER() OVER ("
|
+ "SELECT task.id, ROW_NUMBER() OVER ("
|
||||||
+ "PARTITION BY task.phase, COALESCE(task.batch_id, task.id) "
|
+ "PARTITION BY task.phase, COALESCE(task.batch_id, task.id) "
|
||||||
+ "ORDER BY task.created, task.id) AS lane_row "
|
+ "ORDER BY task.created, task.id) AS lane_row "
|
||||||
+ "FROM tb_document_import_task task "
|
+ "FROM tb_document_import_task task "
|
||||||
+ "LEFT JOIN tb_document_import_batch batch ON batch.id=task.batch_id "
|
+ "LEFT JOIN tb_document_import_batch batch ON batch.id=task.batch_id "
|
||||||
+ "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore} "
|
+ "WHERE task.status='PENDING' AND task.modified <= #{redispatchBefore} "
|
||||||
+ "AND (task.batch_id IS NULL OR batch.status='RUNNING')"
|
+ "AND (task.batch_id IS NULL OR batch.import_mode='MANUAL' "
|
||||||
|
+ "OR batch.status='RUNNING')"
|
||||||
+ ") ranked ON ranked.id=task.id "
|
+ ") ranked ON ranked.id=task.id "
|
||||||
+ "ORDER BY ranked.lane_row, task.created, task.id LIMIT #{limit}")
|
+ "ORDER BY ranked.lane_row, task.created, task.id LIMIT #{limit}")
|
||||||
List<DocumentImportTask> selectPendingFairly(
|
List<DocumentImportTask> selectPendingFairly(
|
||||||
@@ -52,11 +74,7 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
|
|||||||
@Update("UPDATE tb_document_import_task SET modified=#{now}, "
|
@Update("UPDATE tb_document_import_task SET modified=#{now}, "
|
||||||
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
||||||
+ "WHERE id=#{id} AND status='PENDING' "
|
+ "WHERE id=#{id} AND status='PENDING' "
|
||||||
+ "AND modified <= #{redispatchBefore} "
|
+ "AND modified <= #{redispatchBefore}")
|
||||||
+ "AND (batch_id IS NULL OR EXISTS ("
|
|
||||||
+ "SELECT 1 FROM tb_document_import_batch batch "
|
|
||||||
+ "WHERE batch.id=tb_document_import_task.batch_id "
|
|
||||||
+ "AND batch.status='RUNNING'))")
|
|
||||||
int touchPendingForDispatch(
|
int touchPendingForDispatch(
|
||||||
@Param("id") BigInteger id,
|
@Param("id") BigInteger id,
|
||||||
@Param("redispatchBefore") Date redispatchBefore,
|
@Param("redispatchBefore") Date redispatchBefore,
|
||||||
@@ -72,7 +90,7 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
|
|||||||
* @param limit 最大任务数
|
* @param limit 最大任务数
|
||||||
* @return 已失去执行租约的运行任务
|
* @return 已失去执行租约的运行任务
|
||||||
*/
|
*/
|
||||||
@Select("SELECT * FROM tb_document_import_task "
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_import_task "
|
||||||
+ "WHERE status='RUNNING' AND ("
|
+ "WHERE status='RUNNING' AND ("
|
||||||
+ "(lease_until IS NOT NULL AND lease_until <= #{now}) OR "
|
+ "(lease_until IS NOT NULL AND lease_until <= #{now}) OR "
|
||||||
+ "(lease_until IS NULL AND modified <= #{legacyCutoff})) "
|
+ "(lease_until IS NULL AND modified <= #{legacyCutoff})) "
|
||||||
@@ -99,11 +117,7 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
|
|||||||
+ "started_at=COALESCE(started_at, #{now}), finished_at=NULL, "
|
+ "started_at=COALESCE(started_at, #{now}), finished_at=NULL, "
|
||||||
+ "error_summary=NULL, failure_code=NULL, modified=#{now}, "
|
+ "error_summary=NULL, failure_code=NULL, modified=#{now}, "
|
||||||
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
||||||
+ "WHERE id=#{id} AND status='PENDING' "
|
+ "WHERE id=#{id} AND status='PENDING'")
|
||||||
+ "AND (batch_id IS NULL OR EXISTS ("
|
|
||||||
+ "SELECT 1 FROM tb_document_import_batch batch "
|
|
||||||
+ "WHERE batch.id=tb_document_import_task.batch_id "
|
|
||||||
+ "AND batch.status='RUNNING'))")
|
|
||||||
int claimPending(
|
int claimPending(
|
||||||
@Param("id") BigInteger id,
|
@Param("id") BigInteger id,
|
||||||
@Param("executionToken") String executionToken,
|
@Param("executionToken") String executionToken,
|
||||||
@@ -126,11 +140,7 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
|
|||||||
+ "modified=#{now}, modified_by=#{operatorId}, "
|
+ "modified=#{now}, modified_by=#{operatorId}, "
|
||||||
+ "version=COALESCE(version, 0) + 1 "
|
+ "version=COALESCE(version, 0) + 1 "
|
||||||
+ "WHERE id=#{id} AND status='RUNNING' "
|
+ "WHERE id=#{id} AND status='RUNNING' "
|
||||||
+ "AND execution_token=#{executionToken} "
|
+ "AND execution_token=#{executionToken}")
|
||||||
+ "AND (batch_id IS NULL OR EXISTS ("
|
|
||||||
+ "SELECT 1 FROM tb_document_import_batch batch "
|
|
||||||
+ "WHERE batch.id=tb_document_import_task.batch_id "
|
|
||||||
+ "AND batch.status='RUNNING'))")
|
|
||||||
int renewLease(
|
int renewLease(
|
||||||
@Param("id") BigInteger id,
|
@Param("id") BigInteger id,
|
||||||
@Param("executionToken") String executionToken,
|
@Param("executionToken") String executionToken,
|
||||||
@@ -156,11 +166,7 @@ public interface DocumentImportTaskMapper extends BaseMapper<DocumentImportTask>
|
|||||||
+ "lease_until=NULL, finished_at=#{now}, modified=#{now}, "
|
+ "lease_until=NULL, finished_at=#{now}, modified=#{now}, "
|
||||||
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
+ "modified_by=#{operatorId}, version=COALESCE(version, 0) + 1 "
|
||||||
+ "WHERE id=#{id} AND status='RUNNING' "
|
+ "WHERE id=#{id} AND status='RUNNING' "
|
||||||
+ "AND execution_token=#{executionToken} "
|
+ "AND execution_token=#{executionToken}")
|
||||||
+ "AND (batch_id IS NULL OR EXISTS ("
|
|
||||||
+ "SELECT 1 FROM tb_document_import_batch batch "
|
|
||||||
+ "WHERE batch.id=tb_document_import_task.batch_id "
|
|
||||||
+ "AND batch.status='RUNNING'))")
|
|
||||||
int finishOwned(
|
int finishOwned(
|
||||||
@Param("id") BigInteger id,
|
@Param("id") BigInteger id,
|
||||||
@Param("executionToken") String executionToken,
|
@Param("executionToken") String executionToken,
|
||||||
|
|||||||
@@ -26,10 +26,6 @@ public class DefaultReadService implements ReadDocService {
|
|||||||
@Override
|
@Override
|
||||||
public String read(String fileName, InputStream is) {
|
public String read(String fileName, InputStream is) {
|
||||||
String suffix = DocUtil.getSuffix(fileName);
|
String suffix = DocUtil.getSuffix(fileName);
|
||||||
if ("pdf".equals(suffix)) {
|
return DocUtil.readPreviewContent(suffix, is);
|
||||||
return DocUtil.readPdfFile(is);
|
|
||||||
} else {
|
|
||||||
return DocUtil.readWordFile(suffix, is);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import tech.easyflow.ai.document.model.DocumentSourceRef;
|
|||||||
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||||
import tech.easyflow.ai.document.service.DocumentParseBridgeService;
|
import tech.easyflow.ai.document.service.DocumentParseBridgeService;
|
||||||
import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
|
import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
|
||||||
|
import tech.easyflow.ai.document.support.DocumentParseFilePolicy;
|
||||||
import tech.easyflow.ai.document.support.DocumentParseSourceType;
|
import tech.easyflow.ai.document.support.DocumentParseSourceType;
|
||||||
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadedFileReader;
|
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadedFileReader;
|
||||||
import tech.easyflow.common.filestorage.FileStorageService;
|
import tech.easyflow.common.filestorage.FileStorageService;
|
||||||
@@ -206,6 +207,16 @@ public class DocNodeFileContentExtractor {
|
|||||||
if (!StringUtil.hasText(sourceRef.getFilePath())) {
|
if (!StringUtil.hasText(sourceRef.getFilePath())) {
|
||||||
throw new BusinessException("文件输入缺少 filePath");
|
throw new BusinessException("文件输入缺少 filePath");
|
||||||
}
|
}
|
||||||
|
if (!DocumentParseFilePolicy.isSupportedFileName(
|
||||||
|
sourceRef.getFileName())) {
|
||||||
|
throw new BusinessException(
|
||||||
|
"文件“"
|
||||||
|
+ sourceRef.getFileName()
|
||||||
|
+ "”格式不支持,文档解析仅支持 "
|
||||||
|
+ DocumentParseFilePolicy.supportedTypeLabel()
|
||||||
|
+ " 文件"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void collectFileValues(Object value, List<Object> result) {
|
private void collectFileValues(Object value, List<Object> result) {
|
||||||
@@ -285,8 +296,12 @@ public class DocNodeFileContentExtractor {
|
|||||||
"document:default-reader");
|
"document:default-reader");
|
||||||
InputStream inputStream =
|
InputStream inputStream =
|
||||||
Files.newInputStream(temporaryFile)) {
|
Files.newInputStream(temporaryFile)) {
|
||||||
return readerManager.getReader().read(
|
String content = readerManager.getReader().read(
|
||||||
sourceRef.getFileName(), inputStream);
|
sourceRef.getFileName(), inputStream);
|
||||||
|
if (!StringUtil.hasText(content)) {
|
||||||
|
throw new BusinessException("文档解析结果为空");
|
||||||
|
}
|
||||||
|
return content;
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
|
DocumentInputStreamSupport.SizeLimitExceededException sizeError =
|
||||||
|
|||||||
@@ -157,6 +157,15 @@ public abstract class AbstractAiResourceLifecycleHandler<T> implements ApprovalS
|
|||||||
protected void enrichOfflineSnapshot(T resource, Map<String, Object> snapshot) {
|
protected void enrichOfflineSnapshot(T resource, Map<String, Object> snapshot) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 在发布快照参与重复发布比较前补充资源专属契约。
|
||||||
|
*
|
||||||
|
* @param resource 资源
|
||||||
|
* @param snapshot 当前发布快照
|
||||||
|
*/
|
||||||
|
protected void enrichPublishSnapshot(T resource, Map<String, Object> snapshot) {
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除前额外校验。
|
* 删除前额外校验。
|
||||||
*
|
*
|
||||||
@@ -287,6 +296,7 @@ public abstract class AbstractAiResourceLifecycleHandler<T> implements ApprovalS
|
|||||||
throw new BusinessException("当前" + resourceLabel() + "状态不允许发布");
|
throw new BusinessException("当前" + resourceLabel() + "状态不允许发布");
|
||||||
}
|
}
|
||||||
Map<String, Object> snapshot = buildResourceSnapshot(resource);
|
Map<String, Object> snapshot = buildResourceSnapshot(resource);
|
||||||
|
enrichPublishSnapshot(resource, snapshot);
|
||||||
if (currentStatus == PublishStatus.PUBLISHED && isSameSnapshot(snapshot, getPublishedSnapshot(resource))) {
|
if (currentStatus == PublishStatus.PUBLISHED && isSameSnapshot(snapshot, getPublishedSnapshot(resource))) {
|
||||||
throw new BusinessException("当前内容与已发布版本一致,无需重新发布");
|
throw new BusinessException("当前内容与已发布版本一致,无需重新发布");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -64,6 +64,30 @@ public interface AiResourceLifecycleHandler {
|
|||||||
applyApprovedAction(actionType, resourceId, resourceSnapshot, operatorId);
|
applyApprovedAction(actionType, resourceId, resourceSnapshot, operatorId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 执行带审批申请人身份的通过回调。
|
||||||
|
*
|
||||||
|
* @param actionType 动作类型
|
||||||
|
* @param resourceId 资源 ID
|
||||||
|
* @param resourceSnapshot 审批冻结快照
|
||||||
|
* @param operatorId 审批操作人 ID
|
||||||
|
* @param approvalInstanceId 审批实例 ID
|
||||||
|
* @param applicantId 审批申请人 ID
|
||||||
|
*/
|
||||||
|
default void applyApprovedAction(String actionType,
|
||||||
|
BigInteger resourceId,
|
||||||
|
Map<String, Object> resourceSnapshot,
|
||||||
|
BigInteger operatorId,
|
||||||
|
BigInteger approvalInstanceId,
|
||||||
|
BigInteger applicantId) {
|
||||||
|
applyApprovedAction(
|
||||||
|
actionType,
|
||||||
|
resourceId,
|
||||||
|
resourceSnapshot,
|
||||||
|
operatorId,
|
||||||
|
approvalInstanceId);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 在实际提交或直接执行前持有冻结快照所需资源。
|
* 在实际提交或直接执行前持有冻结快照所需资源。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -110,7 +110,8 @@ public class AiResourceLifecycleServiceImpl implements AiResourceLifecycleServic
|
|||||||
instance.getResourceId(),
|
instance.getResourceId(),
|
||||||
readResourceSnapshot(instance.getSnapshotJson()),
|
readResourceSnapshot(instance.getSnapshotJson()),
|
||||||
operatorId,
|
operatorId,
|
||||||
instance.getId()
|
instance.getId(),
|
||||||
|
instance.getApplicantId()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ package tech.easyflow.ai.publish;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.knowledge.WorkflowKnowledgeContractService;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
import tech.easyflow.ai.enums.PublishStatus;
|
import tech.easyflow.ai.enums.PublishStatus;
|
||||||
import tech.easyflow.ai.plugin.workflow.binding.WorkflowPluginBindingService;
|
import tech.easyflow.ai.plugin.workflow.binding.WorkflowPluginBindingService;
|
||||||
import tech.easyflow.ai.plugin.workflow.snapshot.WorkflowPluginSnapshotResolver;
|
import tech.easyflow.ai.plugin.workflow.snapshot.WorkflowPluginSnapshotResolver;
|
||||||
@@ -13,16 +16,22 @@ import tech.easyflow.ai.service.WorkflowScheduleReferenceProvider;
|
|||||||
import tech.easyflow.ai.vo.OfflineImpactCheckVo;
|
import tech.easyflow.ai.vo.OfflineImpactCheckVo;
|
||||||
import tech.easyflow.ai.vo.OfflineImpactBindingVo;
|
import tech.easyflow.ai.vo.OfflineImpactBindingVo;
|
||||||
import tech.easyflow.approval.service.ApprovalInstanceService;
|
import tech.easyflow.approval.service.ApprovalInstanceService;
|
||||||
|
import tech.easyflow.approval.enums.ApprovalActionType;
|
||||||
import tech.easyflow.approval.enums.ApprovalResourceType;
|
import tech.easyflow.approval.enums.ApprovalResourceType;
|
||||||
|
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.system.entity.SysAccount;
|
||||||
import tech.easyflow.system.enums.CategoryResourceType;
|
import tech.easyflow.system.enums.CategoryResourceType;
|
||||||
import tech.easyflow.system.enums.ResourceAction;
|
import tech.easyflow.system.enums.ResourceAction;
|
||||||
import tech.easyflow.system.service.ResourceAccessService;
|
import tech.easyflow.system.service.ResourceAccessService;
|
||||||
|
import tech.easyflow.system.service.SysAccountService;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 工作流生命周期处理器。
|
* 工作流生命周期处理器。
|
||||||
@@ -37,6 +46,9 @@ public class WorkflowApprovalSubjectHandler extends AbstractAiResourceLifecycleH
|
|||||||
private final WorkflowPluginSnapshotResolver workflowPluginSnapshotResolver;
|
private final WorkflowPluginSnapshotResolver workflowPluginSnapshotResolver;
|
||||||
private final AgentResourceReferenceService agentResourceReferenceService;
|
private final AgentResourceReferenceService agentResourceReferenceService;
|
||||||
private final List<WorkflowScheduleReferenceProvider> workflowScheduleReferenceProviders;
|
private final List<WorkflowScheduleReferenceProvider> workflowScheduleReferenceProviders;
|
||||||
|
private final WorkflowCheckService workflowCheckService;
|
||||||
|
private final WorkflowKnowledgeContractService workflowKnowledgeContractService;
|
||||||
|
private final SysAccountService sysAccountService;
|
||||||
|
|
||||||
public WorkflowApprovalSubjectHandler(WorkflowService workflowService,
|
public WorkflowApprovalSubjectHandler(WorkflowService workflowService,
|
||||||
ResourceAccessService resourceAccessService,
|
ResourceAccessService resourceAccessService,
|
||||||
@@ -46,7 +58,10 @@ public class WorkflowApprovalSubjectHandler extends AbstractAiResourceLifecycleH
|
|||||||
WorkflowPluginSnapshotResolver workflowPluginSnapshotResolver,
|
WorkflowPluginSnapshotResolver workflowPluginSnapshotResolver,
|
||||||
AgentResourceReferenceService agentResourceReferenceService,
|
AgentResourceReferenceService agentResourceReferenceService,
|
||||||
ObjectMapper objectMapper,
|
ObjectMapper objectMapper,
|
||||||
List<WorkflowScheduleReferenceProvider> workflowScheduleReferenceProviders) {
|
List<WorkflowScheduleReferenceProvider> workflowScheduleReferenceProviders,
|
||||||
|
WorkflowCheckService workflowCheckService,
|
||||||
|
WorkflowKnowledgeContractService workflowKnowledgeContractService,
|
||||||
|
SysAccountService sysAccountService) {
|
||||||
super(approvalInstanceService, objectMapper);
|
super(approvalInstanceService, objectMapper);
|
||||||
this.workflowService = workflowService;
|
this.workflowService = workflowService;
|
||||||
this.resourceAccessService = resourceAccessService;
|
this.resourceAccessService = resourceAccessService;
|
||||||
@@ -57,6 +72,9 @@ public class WorkflowApprovalSubjectHandler extends AbstractAiResourceLifecycleH
|
|||||||
this.workflowScheduleReferenceProviders = workflowScheduleReferenceProviders == null
|
this.workflowScheduleReferenceProviders = workflowScheduleReferenceProviders == null
|
||||||
? List.of()
|
? List.of()
|
||||||
: List.copyOf(workflowScheduleReferenceProviders);
|
: List.copyOf(workflowScheduleReferenceProviders);
|
||||||
|
this.workflowCheckService = workflowCheckService;
|
||||||
|
this.workflowKnowledgeContractService = workflowKnowledgeContractService;
|
||||||
|
this.sysAccountService = sysAccountService;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -132,6 +150,12 @@ public class WorkflowApprovalSubjectHandler extends AbstractAiResourceLifecycleH
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Map<String, Object> buildPublishSnapshot(Workflow resource, PublishStatus currentStatus) {
|
protected Map<String, Object> buildPublishSnapshot(Workflow resource, PublishStatus currentStatus) {
|
||||||
|
workflowCheckService.checkOrThrow(
|
||||||
|
resource.getContent(),
|
||||||
|
WorkflowCheckStage.SAVE,
|
||||||
|
resource.getId());
|
||||||
|
assertKnowledgeUseAccess(
|
||||||
|
resource.getContent(), resource.getTenantId(), null);
|
||||||
Map<String, Object> snapshot = super.buildPublishSnapshot(resource, currentStatus);
|
Map<String, Object> snapshot = super.buildPublishSnapshot(resource, currentStatus);
|
||||||
OfflineImpactCheckVo impact = resourceOfflineImpactService.checkWorkflowImpact(resource.getId());
|
OfflineImpactCheckVo impact = resourceOfflineImpactService.checkWorkflowImpact(resource.getId());
|
||||||
if (impact.isHasPluginBindings()) {
|
if (impact.isHasPluginBindings()) {
|
||||||
@@ -140,6 +164,53 @@ public class WorkflowApprovalSubjectHandler extends AbstractAiResourceLifecycleH
|
|||||||
return snapshot;
|
return snapshot;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void enrichPublishSnapshot(
|
||||||
|
Workflow resource,
|
||||||
|
Map<String, Object> snapshot) {
|
||||||
|
snapshot.put(
|
||||||
|
WorkflowKnowledgeContractService.SNAPSHOT_KEY,
|
||||||
|
workflowKnowledgeContractService.buildSnapshotContracts(
|
||||||
|
resource.getContent(), resource.getTenantId()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void applyApprovedAction(
|
||||||
|
String actionType,
|
||||||
|
BigInteger resourceId,
|
||||||
|
Map<String, Object> resourceSnapshot,
|
||||||
|
BigInteger operatorId) {
|
||||||
|
if (ApprovalActionType.PUBLISH == ApprovalActionType.from(actionType)) {
|
||||||
|
assertKnowledgeUseAccess(
|
||||||
|
String.valueOf(resourceSnapshot.get("content")),
|
||||||
|
snapshotTenantId(resourceSnapshot),
|
||||||
|
null);
|
||||||
|
}
|
||||||
|
super.applyApprovedAction(
|
||||||
|
actionType, resourceId, resourceSnapshot, operatorId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void applyApprovedAction(
|
||||||
|
String actionType,
|
||||||
|
BigInteger resourceId,
|
||||||
|
Map<String, Object> resourceSnapshot,
|
||||||
|
BigInteger operatorId,
|
||||||
|
BigInteger approvalInstanceId,
|
||||||
|
BigInteger applicantId) {
|
||||||
|
if (ApprovalActionType.PUBLISH == ApprovalActionType.from(actionType)) {
|
||||||
|
BigInteger tenantId = snapshotTenantId(resourceSnapshot);
|
||||||
|
LoginAccount applicant = requireCurrentApplicant(
|
||||||
|
applicantId, tenantId);
|
||||||
|
assertKnowledgeUseAccess(
|
||||||
|
String.valueOf(resourceSnapshot.get("content")),
|
||||||
|
tenantId,
|
||||||
|
applicant);
|
||||||
|
}
|
||||||
|
super.applyApprovedAction(
|
||||||
|
actionType, resourceId, resourceSnapshot, operatorId);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void persistResourceState(BigInteger resourceId, PublishStatus publishStatus, BigInteger currentApprovalInstanceId) {
|
protected void persistResourceState(BigInteger resourceId, PublishStatus publishStatus, BigInteger currentApprovalInstanceId) {
|
||||||
Workflow update = new Workflow();
|
Workflow update = new Workflow();
|
||||||
@@ -151,6 +222,7 @@ public class WorkflowApprovalSubjectHandler extends AbstractAiResourceLifecycleH
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void publishResource(BigInteger resourceId, Map<String, Object> resourceSnapshot, BigInteger operatorId) {
|
protected void publishResource(BigInteger resourceId, Map<String, Object> resourceSnapshot, BigInteger operatorId) {
|
||||||
|
workflowKnowledgeContractService.assertSnapshotCurrent(resourceSnapshot);
|
||||||
Workflow update = new Workflow();
|
Workflow update = new Workflow();
|
||||||
update.setId(resourceId);
|
update.setId(resourceId);
|
||||||
update.setPublishStatus(PublishStatus.PUBLISHED.getCode());
|
update.setPublishStatus(PublishStatus.PUBLISHED.getCode());
|
||||||
@@ -162,6 +234,62 @@ public class WorkflowApprovalSubjectHandler extends AbstractAiResourceLifecycleH
|
|||||||
workflowPluginBindingService.syncByWorkflowId(resourceId);
|
workflowPluginBindingService.syncByWorkflowId(resourceId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void assertKnowledgeUseAccess(
|
||||||
|
String content,
|
||||||
|
BigInteger tenantId,
|
||||||
|
LoginAccount account) {
|
||||||
|
workflowKnowledgeContractService.resolveReferencedCollections(
|
||||||
|
content, tenantId)
|
||||||
|
.forEach(collection -> {
|
||||||
|
if (account == null) {
|
||||||
|
resourceAccessService.assertAccess(
|
||||||
|
CategoryResourceType.KNOWLEDGE,
|
||||||
|
collection,
|
||||||
|
ResourceAction.USE,
|
||||||
|
"无权限使用工作流知识库");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!resourceAccessService.canAccess(
|
||||||
|
account,
|
||||||
|
CategoryResourceType.KNOWLEDGE,
|
||||||
|
collection,
|
||||||
|
ResourceAction.USE)) {
|
||||||
|
throw new BusinessException(
|
||||||
|
403, 403, "无权限使用工作流知识库");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigInteger snapshotTenantId(Map<String, Object> resourceSnapshot) {
|
||||||
|
Object value = resourceSnapshot == null
|
||||||
|
? null
|
||||||
|
: resourceSnapshot.get("tenantId");
|
||||||
|
try {
|
||||||
|
BigInteger tenantId = new BigInteger(String.valueOf(value));
|
||||||
|
if (tenantId.signum() <= 0) {
|
||||||
|
throw new NumberFormatException("non-positive");
|
||||||
|
}
|
||||||
|
return tenantId;
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new BusinessException("工作流发布快照租户无效");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private LoginAccount requireCurrentApplicant(
|
||||||
|
BigInteger applicantId,
|
||||||
|
BigInteger tenantId) {
|
||||||
|
SysAccount account = applicantId == null
|
||||||
|
? null
|
||||||
|
: sysAccountService.getById(applicantId);
|
||||||
|
if (account == null
|
||||||
|
|| !EnumDataStatus.AVAILABLE.getCode().equals(account.getStatus())
|
||||||
|
|| !Objects.equals(tenantId, account.getTenantId())) {
|
||||||
|
throw new BusinessException(
|
||||||
|
403, 403, "审批申请人账号已失效或不属于当前租户");
|
||||||
|
}
|
||||||
|
return account.toLoginAccount();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void markResourceOffline(BigInteger resourceId) {
|
protected void markResourceOffline(BigInteger resourceId) {
|
||||||
Workflow update = new Workflow();
|
Workflow update = new Workflow();
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package tech.easyflow.ai.rag;
|
||||||
|
|
||||||
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 单知识库原始向量候选请求,仅供内部跨库编排使用。
|
||||||
|
*/
|
||||||
|
public class KnowledgeVectorCandidateRequest {
|
||||||
|
|
||||||
|
private BigInteger knowledgeId;
|
||||||
|
private DocumentCollection collection;
|
||||||
|
private String query;
|
||||||
|
private int limit;
|
||||||
|
private double minVectorScore;
|
||||||
|
private float[] queryVector;
|
||||||
|
private Long timeoutMillis;
|
||||||
|
private String callerType;
|
||||||
|
private String callerId;
|
||||||
|
|
||||||
|
public BigInteger getKnowledgeId() {
|
||||||
|
return knowledgeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setKnowledgeId(BigInteger knowledgeId) {
|
||||||
|
this.knowledgeId = knowledgeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DocumentCollection getCollection() {
|
||||||
|
return collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCollection(DocumentCollection collection) {
|
||||||
|
this.collection = collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getQuery() {
|
||||||
|
return query;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQuery(String query) {
|
||||||
|
this.query = query;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getLimit() {
|
||||||
|
return limit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setLimit(int limit) {
|
||||||
|
this.limit = limit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public double getMinVectorScore() {
|
||||||
|
return minVectorScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMinVectorScore(double minVectorScore) {
|
||||||
|
this.minVectorScore = minVectorScore;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float[] getQueryVector() {
|
||||||
|
return queryVector;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQueryVector(float[] queryVector) {
|
||||||
|
this.queryVector = queryVector;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getTimeoutMillis() {
|
||||||
|
return timeoutMillis;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTimeoutMillis(Long timeoutMillis) {
|
||||||
|
this.timeoutMillis = timeoutMillis;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCallerType() {
|
||||||
|
return callerType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCallerType(String callerType) {
|
||||||
|
this.callerType = callerType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCallerId() {
|
||||||
|
return callerId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCallerId(String callerId) {
|
||||||
|
this.callerId = callerId;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,10 +1,12 @@
|
|||||||
package tech.easyflow.ai.service;
|
package tech.easyflow.ai.service;
|
||||||
|
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
|
||||||
import com.mybatisflex.core.service.IService;
|
import com.mybatisflex.core.service.IService;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 服务层。
|
* 服务层。
|
||||||
@@ -14,5 +16,36 @@ import java.math.BigInteger;
|
|||||||
*/
|
*/
|
||||||
public interface DocumentChunkService extends IService<DocumentChunk> {
|
public interface DocumentChunkService extends IService<DocumentChunk> {
|
||||||
|
|
||||||
boolean removeChunk(DocumentCollection knowledge, BigInteger chunkId);
|
/**
|
||||||
|
* 更新分块的 Markdown 正文,并创建后台检索索引同步任务。
|
||||||
|
*
|
||||||
|
* @param knowledgeId 知识库 ID
|
||||||
|
* @param chunkId 分块 ID
|
||||||
|
* @param markdown Markdown 正文
|
||||||
|
* @return 更新后的分块
|
||||||
|
*/
|
||||||
|
DocumentChunk updateContent(BigInteger knowledgeId, BigInteger chunkId, String markdown);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除分块,并创建后台检索索引清理任务。
|
||||||
|
*
|
||||||
|
* @param knowledgeId 知识库 ID
|
||||||
|
* @param chunkId 分块 ID
|
||||||
|
* @return 删除结果
|
||||||
|
*/
|
||||||
|
DocumentChunkDeleteResult deleteChunk(BigInteger knowledgeId, BigInteger chunkId);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 手动重试当前版本的检索索引同步。
|
||||||
|
*/
|
||||||
|
DocumentChunk retryIndexSync(BigInteger knowledgeId, BigInteger chunkId, long syncVersion);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量读取文档内分块的索引同步状态。
|
||||||
|
*/
|
||||||
|
List<DocumentChunkSyncStatus> listIndexSyncStatus(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger documentId,
|
||||||
|
List<BigInteger> chunkIds
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package tech.easyflow.ai.service;
|
|||||||
import com.easyagents.core.document.Document;
|
import com.easyagents.core.document.Document;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeVectorCandidateRequest;
|
||||||
import com.mybatisflex.core.service.IService;
|
import com.mybatisflex.core.service.IService;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
@@ -20,6 +21,14 @@ public interface DocumentCollectionService extends IService<DocumentCollection>
|
|||||||
|
|
||||||
List<Document> search(KnowledgeRetrievalRequest request);
|
List<Document> search(KnowledgeRetrievalRequest request);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 查询未归一化、未取整、未重排的原始向量候选。
|
||||||
|
*
|
||||||
|
* @param request 原始向量候选请求
|
||||||
|
* @return 保留向量存储分数原始精度的候选
|
||||||
|
*/
|
||||||
|
List<Document> searchVectorCandidates(KnowledgeVectorCandidateRequest request);
|
||||||
|
|
||||||
DocumentCollection getDetail(String idOrAlias);
|
DocumentCollection getDetail(String idOrAlias);
|
||||||
|
|
||||||
DocumentCollection getByAlias(String idOrAlias);
|
DocumentCollection getByAlias(String idOrAlias);
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import tech.easyflow.ai.entity.Model;
|
|||||||
import tech.easyflow.ai.service.capability.ModelCapabilityResolution;
|
import tech.easyflow.ai.service.capability.ModelCapabilityResolution;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@@ -35,6 +36,14 @@ public interface ModelService extends IService<Model> {
|
|||||||
|
|
||||||
Model getModelInstance(BigInteger modelId);
|
Model getModelInstance(BigInteger modelId);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量读取已关联供应商并补齐供应商默认配置的模型。
|
||||||
|
*
|
||||||
|
* @param modelIds 模型 ID
|
||||||
|
* @return 实际运行配置模型
|
||||||
|
*/
|
||||||
|
List<Model> listModelInstances(Collection<BigInteger> modelIds);
|
||||||
|
|
||||||
Model getModelInstanceByInvokeCode(String invokeCode);
|
Model getModelInstanceByInvokeCode(String invokeCode);
|
||||||
|
|
||||||
void validateForSaveOrUpdate(Model entity, boolean isSave);
|
void validateForSaveOrUpdate(Model entity, boolean isSave);
|
||||||
|
|||||||
@@ -1,36 +1,365 @@
|
|||||||
package tech.easyflow.ai.service.impl;
|
package tech.easyflow.ai.service.impl;
|
||||||
|
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.rag.core.BgeM3ChunkSafety;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import com.easyagents.rag.core.RagDefaults;
|
||||||
import tech.easyflow.ai.config.SearcherFactory;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
|
||||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncState;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncTaskAppService;
|
||||||
|
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkDeleteResult;
|
||||||
|
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
||||||
|
import tech.easyflow.ai.entity.Document;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 服务层实现。
|
* 分块服务层实现。
|
||||||
*
|
|
||||||
* @author michael
|
|
||||||
* @since 2024-08-23
|
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
public class DocumentChunkServiceImpl extends ServiceImpl<DocumentChunkMapper, DocumentChunk> implements DocumentChunkService {
|
public class DocumentChunkServiceImpl
|
||||||
|
extends ServiceImpl<DocumentChunkMapper, DocumentChunk>
|
||||||
|
implements DocumentChunkService {
|
||||||
|
|
||||||
@Autowired
|
public static final int DOCUMENT_CHUNK_EMPTY_REQUIRES_DELETE = 42901;
|
||||||
private SearcherFactory searcherFactory;
|
public static final int DOCUMENT_CHUNK_LOCK_UNAVAILABLE = 42905;
|
||||||
|
|
||||||
|
private static final Duration LOCK_WAIT_TIMEOUT = Duration.ofSeconds(5);
|
||||||
|
private static final Duration LOCK_LEASE_TIMEOUT = Duration.ofSeconds(30);
|
||||||
|
private static final Pattern MARKDOWN_IMAGE_PATTERN = Pattern.compile(
|
||||||
|
"!\\[([^\\]]*)\\]\\((?:[^()\\r\\n]|\\([^()\\r\\n]*\\))*\\)"
|
||||||
|
);
|
||||||
|
private static final Pattern HTML_IMAGE_PATTERN = Pattern.compile(
|
||||||
|
"<img\\b[^>]*>",
|
||||||
|
Pattern.CASE_INSENSITIVE
|
||||||
|
);
|
||||||
|
private static final Pattern HTML_ALT_PATTERN = Pattern.compile(
|
||||||
|
"\\balt\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')",
|
||||||
|
Pattern.CASE_INSENSITIVE
|
||||||
|
);
|
||||||
|
|
||||||
|
private final DocumentChunkMapper documentChunkMapper;
|
||||||
|
private final DocumentMapper documentMapper;
|
||||||
|
private final DocumentCollectionService documentCollectionService;
|
||||||
|
private final DocumentChunkSyncTaskAppService syncTaskAppService;
|
||||||
|
private final TransactionTemplate transactionTemplate;
|
||||||
|
private final RedisLockExecutor redisLockExecutor;
|
||||||
|
|
||||||
|
public DocumentChunkServiceImpl(
|
||||||
|
DocumentChunkMapper documentChunkMapper,
|
||||||
|
DocumentMapper documentMapper,
|
||||||
|
DocumentCollectionService documentCollectionService,
|
||||||
|
DocumentChunkSyncTaskAppService syncTaskAppService,
|
||||||
|
PlatformTransactionManager transactionManager,
|
||||||
|
RedisLockExecutor redisLockExecutor
|
||||||
|
) {
|
||||||
|
this.documentChunkMapper = documentChunkMapper;
|
||||||
|
this.documentMapper = documentMapper;
|
||||||
|
this.documentCollectionService = documentCollectionService;
|
||||||
|
this.syncTaskAppService = syncTaskAppService;
|
||||||
|
this.transactionTemplate = new TransactionTemplate(transactionManager);
|
||||||
|
this.redisLockExecutor = redisLockExecutor;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean removeChunk(DocumentCollection knowledge, BigInteger chunkId) {
|
public DocumentChunk updateContent(
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher();
|
BigInteger knowledgeId,
|
||||||
// 删除搜索引擎中的数据
|
BigInteger chunkId,
|
||||||
if (searcher == null){
|
String markdown
|
||||||
return true;
|
) {
|
||||||
|
if (markdown == null || markdown.trim().isEmpty()) {
|
||||||
|
throw new BusinessException(
|
||||||
|
422,
|
||||||
|
DOCUMENT_CHUNK_EMPTY_REQUIRES_DELETE,
|
||||||
|
"分块内容为空,请删除该分块"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
return searcher.deleteDocument(chunkId);
|
DocumentChunk snapshot = requireChunk(knowledgeId, chunkId);
|
||||||
|
UpdateOutcome outcome = withDocumentLock(
|
||||||
|
snapshot.getDocumentId(),
|
||||||
|
() -> updateContentLocked(knowledgeId, chunkId, markdown)
|
||||||
|
);
|
||||||
|
syncTaskAppService.dispatchBestEffort(outcome.taskId());
|
||||||
|
return outcome.chunk();
|
||||||
|
}
|
||||||
|
|
||||||
|
private UpdateOutcome updateContentLocked(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger chunkId,
|
||||||
|
String markdown
|
||||||
|
) {
|
||||||
|
DocumentChunk current = requireChunk(knowledgeId, chunkId);
|
||||||
|
DocumentCollection collection = requireKnowledge(knowledgeId);
|
||||||
|
String searchableContent = toSearchableContent(markdown);
|
||||||
|
assertWithinEmbeddingLimit(chunkId, searchableContent);
|
||||||
|
Map<String, Object> options = current.getOptions() == null
|
||||||
|
? new HashMap<>()
|
||||||
|
: new HashMap<>(current.getOptions());
|
||||||
|
options.put(DocumentImportKeys.KEY_DOCUMENT_RENDER_MARKDOWN, markdown);
|
||||||
|
long nextVersion = current.getIndexSyncVersion() == null
|
||||||
|
? 1L
|
||||||
|
: current.getIndexSyncVersion() + 1L;
|
||||||
|
|
||||||
|
UpdateOutcome outcome = transactionTemplate.execute(status -> {
|
||||||
|
DocumentChunkSyncTask task = syncTaskAppService.createTask(
|
||||||
|
current,
|
||||||
|
collection,
|
||||||
|
DocumentChunkSyncState.OPERATION_UPSERT,
|
||||||
|
nextVersion,
|
||||||
|
new Date()
|
||||||
|
);
|
||||||
|
DocumentChunk update = new DocumentChunk();
|
||||||
|
update.setId(chunkId);
|
||||||
|
update.setContent(searchableContent);
|
||||||
|
update.setOptions(options);
|
||||||
|
update.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
update.setIndexSyncVersion(nextVersion);
|
||||||
|
update.setIndexSyncErrorCode(null);
|
||||||
|
update.setIndexSyncErrorMessage(null);
|
||||||
|
if (documentChunkMapper.update(update) <= 0) {
|
||||||
|
throw new BusinessException("分块更新失败");
|
||||||
|
}
|
||||||
|
touchDocument(current.getDocumentId(), null);
|
||||||
|
current.setContent(searchableContent);
|
||||||
|
current.setOptions(options);
|
||||||
|
current.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
current.setIndexSyncVersion(nextVersion);
|
||||||
|
current.setIndexSyncErrorCode(null);
|
||||||
|
current.setIndexSyncErrorMessage(null);
|
||||||
|
return new UpdateOutcome(current, task.getId());
|
||||||
|
});
|
||||||
|
if (outcome == null) {
|
||||||
|
throw new BusinessException("分块更新失败");
|
||||||
|
}
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DocumentChunkDeleteResult deleteChunk(BigInteger knowledgeId, BigInteger chunkId) {
|
||||||
|
DocumentChunk snapshot = requireChunk(knowledgeId, chunkId);
|
||||||
|
DeleteOutcome outcome = withDocumentLock(
|
||||||
|
snapshot.getDocumentId(),
|
||||||
|
() -> deleteChunkLocked(knowledgeId, chunkId)
|
||||||
|
);
|
||||||
|
syncTaskAppService.dispatchBestEffort(outcome.taskId());
|
||||||
|
return outcome.result();
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T> T withDocumentLock(BigInteger documentId, Supplier<T> action) {
|
||||||
|
RedisLockExecutor.LockHandle handle;
|
||||||
|
try {
|
||||||
|
handle = redisLockExecutor.tryAcquire(
|
||||||
|
DocumentChunkSyncState.parentLockKey(documentId),
|
||||||
|
LOCK_WAIT_TIMEOUT,
|
||||||
|
LOCK_LEASE_TIMEOUT
|
||||||
|
);
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new BusinessException(
|
||||||
|
503,
|
||||||
|
DOCUMENT_CHUNK_LOCK_UNAVAILABLE,
|
||||||
|
"分块正在处理中,请稍后重试",
|
||||||
|
exception
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (handle == null) {
|
||||||
|
throw new BusinessException(
|
||||||
|
503,
|
||||||
|
DOCUMENT_CHUNK_LOCK_UNAVAILABLE,
|
||||||
|
"分块正在处理中,请稍后重试"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
try (handle) {
|
||||||
|
return action.get();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private DeleteOutcome deleteChunkLocked(BigInteger knowledgeId, BigInteger chunkId) {
|
||||||
|
DocumentChunk current = requireChunk(knowledgeId, chunkId);
|
||||||
|
DocumentCollection collection = requireKnowledge(knowledgeId);
|
||||||
|
long nextVersion = current.getIndexSyncVersion() == null
|
||||||
|
? 1L
|
||||||
|
: current.getIndexSyncVersion() + 1L;
|
||||||
|
DeleteOutcome outcome = transactionTemplate.execute(status -> {
|
||||||
|
DocumentChunkSyncTask task = syncTaskAppService.createTask(
|
||||||
|
current,
|
||||||
|
collection,
|
||||||
|
DocumentChunkSyncState.OPERATION_DELETE,
|
||||||
|
nextVersion,
|
||||||
|
new Date()
|
||||||
|
);
|
||||||
|
if (documentChunkMapper.deleteById(chunkId) <= 0) {
|
||||||
|
throw new BusinessException("分块删除失败");
|
||||||
|
}
|
||||||
|
long remaining = documentChunkMapper.selectCountByQuery(
|
||||||
|
QueryWrapper.create().eq(DocumentChunk::getDocumentId, current.getDocumentId())
|
||||||
|
);
|
||||||
|
touchDocument(current.getDocumentId(), remaining);
|
||||||
|
return new DeleteOutcome(
|
||||||
|
new DocumentChunkDeleteResult(chunkId, current.getDocumentId(), remaining),
|
||||||
|
task.getId()
|
||||||
|
);
|
||||||
|
});
|
||||||
|
if (outcome == null) {
|
||||||
|
throw new BusinessException("分块删除失败");
|
||||||
|
}
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DocumentChunk retryIndexSync(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger chunkId,
|
||||||
|
long syncVersion
|
||||||
|
) {
|
||||||
|
requireChunk(knowledgeId, chunkId);
|
||||||
|
return syncTaskAppService.retry(chunkId, syncVersion);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DocumentChunkSyncStatus> listIndexSyncStatus(
|
||||||
|
BigInteger knowledgeId,
|
||||||
|
BigInteger documentId,
|
||||||
|
List<BigInteger> chunkIds
|
||||||
|
) {
|
||||||
|
if (documentId == null || chunkIds == null || chunkIds.isEmpty()) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
BigInteger effectiveKnowledgeId = knowledgeId;
|
||||||
|
if (effectiveKnowledgeId == null) {
|
||||||
|
Document document = documentMapper.selectOneById(documentId);
|
||||||
|
if (document == null || document.getCollectionId() == null) {
|
||||||
|
throw new BusinessException("文档不存在");
|
||||||
|
}
|
||||||
|
effectiveKnowledgeId = document.getCollectionId();
|
||||||
|
}
|
||||||
|
if (chunkIds.size() > 200) {
|
||||||
|
throw new BusinessException("单次最多查询 200 个分块状态");
|
||||||
|
}
|
||||||
|
List<DocumentChunk> chunks = documentChunkMapper.selectSyncStates(documentId, chunkIds);
|
||||||
|
for (DocumentChunk chunk : chunks) {
|
||||||
|
if (chunk.getDocumentCollectionId() == null
|
||||||
|
|| chunk.getDocumentCollectionId().compareTo(effectiveKnowledgeId) != 0) {
|
||||||
|
throw new BusinessException("分块不存在");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return chunks.stream().map(chunk -> new DocumentChunkSyncStatus(
|
||||||
|
chunk.getId(),
|
||||||
|
chunk.getIndexSyncStatus(),
|
||||||
|
chunk.getIndexSyncVersion(),
|
||||||
|
chunk.getIndexSyncErrorCode(),
|
||||||
|
chunk.getIndexSyncErrorMessage()
|
||||||
|
)).toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DocumentChunk requireChunk(BigInteger knowledgeId, BigInteger chunkId) {
|
||||||
|
if (knowledgeId == null || chunkId == null) {
|
||||||
|
throw new BusinessException("分块不存在");
|
||||||
|
}
|
||||||
|
DocumentChunk chunk = documentChunkMapper.selectOneById(chunkId);
|
||||||
|
if (chunk == null || chunk.getDocumentCollectionId() == null
|
||||||
|
|| chunk.getDocumentCollectionId().compareTo(knowledgeId) != 0) {
|
||||||
|
throw new BusinessException("分块不存在");
|
||||||
|
}
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DocumentCollection requireKnowledge(BigInteger knowledgeId) {
|
||||||
|
DocumentCollection knowledge = documentCollectionService.getById(knowledgeId);
|
||||||
|
if (knowledge == null) {
|
||||||
|
throw new BusinessException("知识库不存在");
|
||||||
|
}
|
||||||
|
return knowledge;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void touchDocument(BigInteger documentId, Long chunkCount) {
|
||||||
|
if (documentId == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Document update = new Document();
|
||||||
|
update.setId(documentId);
|
||||||
|
update.setModified(new Date());
|
||||||
|
if (chunkCount != null) {
|
||||||
|
int count = chunkCount > Integer.MAX_VALUE
|
||||||
|
? Integer.MAX_VALUE
|
||||||
|
: chunkCount.intValue();
|
||||||
|
update.setTotalChunks(count);
|
||||||
|
update.setCompletedChunks(count);
|
||||||
|
update.setFailedChunks(0);
|
||||||
|
update.setProgressPercent(100);
|
||||||
|
}
|
||||||
|
if (documentMapper.update(update) <= 0) {
|
||||||
|
throw new BusinessException("文档状态更新失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertWithinEmbeddingLimit(BigInteger chunkId, String content) {
|
||||||
|
int tokenEstimate = BgeM3ChunkSafety.estimateContentTokens(content);
|
||||||
|
if (tokenEstimate > RagDefaults.BGE_M3_HARD_CHUNK_TOKEN_LIMIT) {
|
||||||
|
throw new BusinessException(
|
||||||
|
"分块内容超过向量模型上下文上限,请缩短后保存:chunkId=" + chunkId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String toSearchableContent(String markdown) {
|
||||||
|
return replaceHtmlImages(replaceMarkdownImages(markdown)).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String replaceMarkdownImages(String markdown) {
|
||||||
|
Matcher matcher = MARKDOWN_IMAGE_PATTERN.matcher(markdown);
|
||||||
|
StringBuffer output = new StringBuffer();
|
||||||
|
while (matcher.find()) {
|
||||||
|
matcher.appendReplacement(
|
||||||
|
output,
|
||||||
|
Matcher.quoteReplacement(imageSearchText(matcher.group(1)))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
matcher.appendTail(output);
|
||||||
|
return output.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String replaceHtmlImages(String content) {
|
||||||
|
Matcher matcher = HTML_IMAGE_PATTERN.matcher(content);
|
||||||
|
StringBuffer output = new StringBuffer();
|
||||||
|
while (matcher.find()) {
|
||||||
|
Matcher altMatcher = HTML_ALT_PATTERN.matcher(matcher.group());
|
||||||
|
String alt = altMatcher.find()
|
||||||
|
? (altMatcher.group(1) == null ? altMatcher.group(2) : altMatcher.group(1))
|
||||||
|
: null;
|
||||||
|
matcher.appendReplacement(output, Matcher.quoteReplacement(imageSearchText(alt)));
|
||||||
|
}
|
||||||
|
matcher.appendTail(output);
|
||||||
|
return output.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String imageSearchText(String alt) {
|
||||||
|
return alt == null || alt.trim().isEmpty() ? "图片" : alt.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private record UpdateOutcome(DocumentChunk chunk, BigInteger taskId) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private record DeleteOutcome(DocumentChunkDeleteResult result, BigInteger taskId) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import tech.easyflow.ai.mapper.DocumentCollectionMapper;
|
|||||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
import tech.easyflow.ai.mapper.FaqItemMapper;
|
import tech.easyflow.ai.mapper.FaqItemMapper;
|
||||||
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||||
|
import tech.easyflow.ai.rag.KnowledgeVectorCandidateRequest;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
||||||
@@ -60,7 +61,6 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
private static final int MAX_FAQ_IMAGES_IN_PROMPT = 3;
|
private static final int MAX_FAQ_IMAGES_IN_PROMPT = 3;
|
||||||
private static final int INTERNAL_RECALL_MULTIPLIER = 5;
|
private static final int INTERNAL_RECALL_MULTIPLIER = 5;
|
||||||
private static final int MAX_INTERNAL_RECALL_LIMIT = 100;
|
private static final int MAX_INTERNAL_RECALL_LIMIT = 100;
|
||||||
private static final int LOG_TEXT_MAX_LENGTH = 300;
|
|
||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
private ModelService llmService;
|
private ModelService llmService;
|
||||||
@@ -204,6 +204,54 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
return formattedDocuments;
|
return formattedDocuments;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Document> searchVectorCandidates(
|
||||||
|
KnowledgeVectorCandidateRequest request) {
|
||||||
|
if (request == null || request.getKnowledgeId() == null) {
|
||||||
|
throw new BusinessException("知识库ID不能为空");
|
||||||
|
}
|
||||||
|
if (StringUtil.noText(request.getQuery())) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
if (request.getLimit() <= 0) {
|
||||||
|
throw new BusinessException("向量候选数量必须大于0");
|
||||||
|
}
|
||||||
|
if (!Double.isFinite(request.getMinVectorScore())
|
||||||
|
|| request.getMinVectorScore() < 0D
|
||||||
|
|| request.getMinVectorScore() > 1D) {
|
||||||
|
throw new BusinessException("向量相似度阈值无效");
|
||||||
|
}
|
||||||
|
DocumentCollection collection = request.getCollection();
|
||||||
|
if (collection == null
|
||||||
|
|| !Objects.equals(
|
||||||
|
request.getKnowledgeId(), collection.getId())) {
|
||||||
|
throw new BusinessException("知识库检索快照无效");
|
||||||
|
}
|
||||||
|
List<Document> documents = prepareSearchDocuments(
|
||||||
|
collection,
|
||||||
|
searchVectorDocuments(
|
||||||
|
collection,
|
||||||
|
request.getQuery(),
|
||||||
|
request.getLimit(),
|
||||||
|
request.getMinVectorScore(),
|
||||||
|
request.getQueryVector(),
|
||||||
|
request.getTimeoutMillis()));
|
||||||
|
for (Document document : documents) {
|
||||||
|
document.addMetadata("knowledgeId", collection.getId());
|
||||||
|
document.addMetadata("knowledgeName", collection.getTitle());
|
||||||
|
document.addMetadata("vectorScore", document.getScore());
|
||||||
|
}
|
||||||
|
LOG.info(
|
||||||
|
"Knowledge raw vector candidates completed, callerType={}, callerId={}, knowledgeId={}, limit={}, minVectorScore={}, hitCount={}",
|
||||||
|
request.getCallerType(),
|
||||||
|
request.getCallerId(),
|
||||||
|
request.getKnowledgeId(),
|
||||||
|
request.getLimit(),
|
||||||
|
request.getMinVectorScore(),
|
||||||
|
documents.size());
|
||||||
|
return documents;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*/
|
*/
|
||||||
@@ -279,38 +327,60 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
String keyword,
|
String keyword,
|
||||||
int docRecallMaxNum,
|
int docRecallMaxNum,
|
||||||
Float minSimilarity) {
|
Float minSimilarity) {
|
||||||
|
return searchVectorDocuments(
|
||||||
|
documentCollection,
|
||||||
|
keyword,
|
||||||
|
docRecallMaxNum,
|
||||||
|
minSimilarity == null
|
||||||
|
? null
|
||||||
|
: minSimilarity.doubleValue(),
|
||||||
|
null,
|
||||||
|
null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Document> searchVectorDocuments(DocumentCollection documentCollection,
|
||||||
|
String keyword,
|
||||||
|
int docRecallMaxNum,
|
||||||
|
Double minSimilarity,
|
||||||
|
float[] queryVector,
|
||||||
|
Long timeoutMillis) {
|
||||||
DocumentStore documentStore = documentCollection.toDocumentStore();
|
DocumentStore documentStore = documentCollection.toDocumentStore();
|
||||||
if (documentStore == null) {
|
if (documentStore == null) {
|
||||||
throw new BusinessException("知识库没有配置向量库");
|
throw new BusinessException("知识库没有配置向量库");
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (queryVector == null || queryVector.length == 0) {
|
||||||
Model model = llmService.getModelInstance(documentCollection.getVectorEmbedModelId());
|
Model model = llmService.getModelInstance(documentCollection.getVectorEmbedModelId());
|
||||||
if (model == null) {
|
if (model == null) {
|
||||||
throw new BusinessException("知识库没有配置向量模型");
|
throw new BusinessException("知识库没有配置向量模型");
|
||||||
}
|
}
|
||||||
|
|
||||||
documentStore.setEmbeddingModel(model.toEmbeddingModel());
|
documentStore.setEmbeddingModel(model.toEmbeddingModel());
|
||||||
|
}
|
||||||
SearchWrapper wrapper = new SearchWrapper();
|
SearchWrapper wrapper = new SearchWrapper();
|
||||||
wrapper.setMaxResults(docRecallMaxNum);
|
wrapper.setMaxResults(docRecallMaxNum);
|
||||||
|
if (queryVector != null && queryVector.length > 0) {
|
||||||
|
wrapper.setVector(queryVector);
|
||||||
|
}
|
||||||
if (minSimilarity != null) {
|
if (minSimilarity != null) {
|
||||||
wrapper.setMinScore((double) minSimilarity);
|
wrapper.setMinScore(minSimilarity);
|
||||||
}
|
}
|
||||||
wrapper.setText(keyword);
|
wrapper.setText(keyword);
|
||||||
|
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(documentCollection.getVectorStoreCollection());
|
StoreOptions options = StoreOptions.ofCollectionName(documentCollection.getVectorStoreCollection());
|
||||||
options.setIndexName(documentCollection.getVectorStoreCollection());
|
options.setIndexName(documentCollection.getVectorStoreCollection());
|
||||||
|
if (timeoutMillis != null) {
|
||||||
|
options.setTimeoutMillis(timeoutMillis);
|
||||||
|
}
|
||||||
List<Document> documents = documentStore.search(wrapper, options);
|
List<Document> documents = documentStore.search(wrapper, options);
|
||||||
List<Document> result = documents == null ? Collections.<Document>emptyList() : documents;
|
List<Document> result = documents == null ? Collections.<Document>emptyList() : documents;
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"Knowledge vector search completed, knowledgeId={}, collectionName={}, query={}, limit={}, minSimilarity={}, hitCount={}, hits={}",
|
"Knowledge vector search completed, knowledgeId={}, collectionName={}, limit={}, minSimilarity={}, hitCount={}",
|
||||||
documentCollection.getId(),
|
documentCollection.getId(),
|
||||||
documentCollection.getVectorStoreCollection(),
|
documentCollection.getVectorStoreCollection(),
|
||||||
keyword,
|
|
||||||
docRecallMaxNum,
|
docRecallMaxNum,
|
||||||
minSimilarity,
|
minSimilarity,
|
||||||
result.size(),
|
result.size()
|
||||||
summarizeDocuments(result)
|
|
||||||
);
|
);
|
||||||
return result;
|
return result;
|
||||||
} finally {
|
} finally {
|
||||||
@@ -496,6 +566,7 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
}
|
}
|
||||||
String sourceFileName = hitSnapshot.findSourceFileName(item.getId());
|
String sourceFileName = hitSnapshot.findSourceFileName(item.getId());
|
||||||
if (StringUtil.hasText(sourceFileName)) {
|
if (StringUtil.hasText(sourceFileName)) {
|
||||||
|
item.setTitle(sourceFileName);
|
||||||
item.addMetadata("sourceFileName", sourceFileName);
|
item.addMetadata("sourceFileName", sourceFileName);
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
@@ -594,6 +665,7 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
List<Map<String, String>> faqImages = readFaqImages(faqItem);
|
List<Map<String, String>> faqImages = readFaqImages(faqItem);
|
||||||
|
item.setTitle(faqItem.getQuestion());
|
||||||
item.setContent(buildFaqPromptContent(faqItem, faqImages));
|
item.setContent(buildFaqPromptContent(faqItem, faqImages));
|
||||||
|
|
||||||
Map<String, Object> metadataMap = item.getMetadataMap() == null
|
Map<String, Object> metadataMap = item.getMetadataMap() == null
|
||||||
@@ -646,6 +718,7 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
QueryWrapper chunkWrapper = QueryWrapper.create();
|
QueryWrapper chunkWrapper = QueryWrapper.create();
|
||||||
chunkWrapper.in(DocumentChunk::getId, chunkIds);
|
chunkWrapper.in(DocumentChunk::getId, chunkIds);
|
||||||
chunkWrapper.eq(DocumentChunk::getDocumentCollectionId, documentCollection.getId());
|
chunkWrapper.eq(DocumentChunk::getDocumentCollectionId, documentCollection.getId());
|
||||||
|
chunkWrapper.eq(DocumentChunk::getIndexSyncStatus, "SYNCED");
|
||||||
Map<String, DocumentChunk> chunkMap = documentChunkMapper.selectListByQuery(chunkWrapper).stream()
|
Map<String, DocumentChunk> chunkMap = documentChunkMapper.selectListByQuery(chunkWrapper).stream()
|
||||||
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
|
.collect(Collectors.toMap(item -> item.getId().toString(), item -> item, (a, b) -> a));
|
||||||
if (chunkMap.isEmpty()) {
|
if (chunkMap.isEmpty()) {
|
||||||
@@ -851,7 +924,7 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 构建 RAG 原始命中摘要,便于排查向量、关键词与融合阶段的召回情况。
|
* 构建不包含知识正文和元数据的 RAG 原始命中摘要。
|
||||||
*
|
*
|
||||||
* @param hits RAG 命中列表
|
* @param hits RAG 命中列表
|
||||||
* @return 命中摘要
|
* @return 命中摘要
|
||||||
@@ -865,21 +938,18 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
.map(hit -> {
|
.map(hit -> {
|
||||||
Map<String, Object> summary = new LinkedHashMap<>();
|
Map<String, Object> summary = new LinkedHashMap<>();
|
||||||
summary.put("id", hit.getDocumentId());
|
summary.put("id", hit.getDocumentId());
|
||||||
summary.put("title", hit.getTitle());
|
|
||||||
summary.put("source", hit.getHitSource());
|
summary.put("source", hit.getHitSource());
|
||||||
summary.put("score", hit.getScore());
|
summary.put("score", hit.getScore());
|
||||||
summary.put("vectorScore", hit.getVectorScore());
|
summary.put("vectorScore", hit.getVectorScore());
|
||||||
summary.put("keywordScore", hit.getKeywordScore());
|
summary.put("keywordScore", hit.getKeywordScore());
|
||||||
summary.put("rank", hit.getRank());
|
summary.put("rank", hit.getRank());
|
||||||
summary.put("content", truncate(hit.getContent()));
|
|
||||||
summary.put("metadata", hit.getMetadata());
|
|
||||||
return summary;
|
return summary;
|
||||||
})
|
})
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 构建文档命中摘要,避免完整知识库内容撑爆日志。
|
* 构建不包含知识正文、标题和元数据的文档命中摘要。
|
||||||
*
|
*
|
||||||
* @param documents 文档命中列表
|
* @param documents 文档命中列表
|
||||||
* @return 文档摘要
|
* @return 文档摘要
|
||||||
@@ -893,25 +963,10 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
|||||||
.map(document -> {
|
.map(document -> {
|
||||||
Map<String, Object> summary = new LinkedHashMap<>();
|
Map<String, Object> summary = new LinkedHashMap<>();
|
||||||
summary.put("id", document.getId());
|
summary.put("id", document.getId());
|
||||||
summary.put("title", document.getTitle());
|
|
||||||
summary.put("score", document.getScore());
|
summary.put("score", document.getScore());
|
||||||
summary.put("content", truncate(document.getContent()));
|
|
||||||
summary.put("metadata", document.getMetadataMap());
|
|
||||||
return summary;
|
return summary;
|
||||||
})
|
})
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 截断日志文本,保留足够排查上下文。
|
|
||||||
*
|
|
||||||
* @param text 原始文本
|
|
||||||
* @return 截断文本
|
|
||||||
*/
|
|
||||||
private String truncate(String text) {
|
|
||||||
if (text == null || text.length() <= LOG_TEXT_MAX_LENGTH) {
|
|
||||||
return text;
|
|
||||||
}
|
|
||||||
return text.substring(0, LOG_TEXT_MAX_LENGTH) + "...";
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import tech.easyflow.ai.config.SearcherFactory;
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
|
import tech.easyflow.ai.documentchunk.DocumentChunkSyncState;
|
||||||
import tech.easyflow.common.util.SearchKeywordUtil;
|
import tech.easyflow.common.util.SearchKeywordUtil;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportDtos;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
import tech.easyflow.ai.documentimport.DocumentImportKeys;
|
||||||
@@ -40,6 +41,7 @@ import tech.easyflow.ai.documentimport.task.KnowledgeDocumentImportTaskAppServic
|
|||||||
import tech.easyflow.ai.entity.*;
|
import tech.easyflow.ai.entity.*;
|
||||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||||
import tech.easyflow.ai.service.DocumentChunkService;
|
import tech.easyflow.ai.service.DocumentChunkService;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
@@ -47,6 +49,7 @@ import tech.easyflow.ai.service.DocumentService;
|
|||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
import tech.easyflow.ai.support.DocumentStoreLifecycleSupport;
|
||||||
import tech.easyflow.common.ai.rag.ExcelDocumentSplitter;
|
import tech.easyflow.common.ai.rag.ExcelDocumentSplitter;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.filestorage.FileStorageService;
|
import tech.easyflow.common.filestorage.FileStorageService;
|
||||||
import tech.easyflow.common.util.FileUtil;
|
import tech.easyflow.common.util.FileUtil;
|
||||||
@@ -58,8 +61,11 @@ import java.io.IOException;
|
|||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.math.BigDecimal;
|
import java.math.BigDecimal;
|
||||||
|
import java.time.Duration;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
import static tech.easyflow.ai.entity.DocumentCollection.KEY_CAN_UPDATE_EMBEDDING_MODEL;
|
||||||
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
|
import static tech.easyflow.ai.entity.table.DocumentChunkTableDef.DOCUMENT_CHUNK;
|
||||||
@@ -83,6 +89,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
@Resource
|
@Resource
|
||||||
private DocumentChunkMapper documentChunkMapper;
|
private DocumentChunkMapper documentChunkMapper;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DocumentChunkSyncTaskMapper documentChunkSyncTaskMapper;
|
||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
private DocumentCollectionService knowledgeService;
|
private DocumentCollectionService knowledgeService;
|
||||||
|
|
||||||
@@ -110,6 +119,9 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
@Autowired
|
@Autowired
|
||||||
private KnowledgeDocumentImportTaskAppService importTaskAppService;
|
private KnowledgeDocumentImportTaskAppService importTaskAppService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private RedisLockExecutor redisLockExecutor;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) {
|
public Page<Document> getDocumentList(String knowledgeId, int pageSize, int pageNum, String fileName) {
|
||||||
return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null, false);
|
return queryDocumentList(knowledgeId, pageSize, pageNum, fileName, null, false);
|
||||||
@@ -202,6 +214,49 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
@Override
|
@Override
|
||||||
@Transactional
|
@Transactional
|
||||||
public boolean removeDoc(String id) {
|
public boolean removeDoc(String id) {
|
||||||
|
AtomicBoolean removalCompleted = new AtomicBoolean();
|
||||||
|
AtomicReference<Boolean> removalResult = new AtomicReference<>();
|
||||||
|
try {
|
||||||
|
return redisLockExecutor.executeWithRenewingLock(
|
||||||
|
DocumentChunkSyncState.parentLockKey(id),
|
||||||
|
Duration.ofSeconds(5),
|
||||||
|
Duration.ofSeconds(30),
|
||||||
|
() -> {
|
||||||
|
try {
|
||||||
|
boolean result = removeDocLocked(id);
|
||||||
|
removalResult.set(result);
|
||||||
|
removalCompleted.set(true);
|
||||||
|
return result;
|
||||||
|
} catch (BusinessException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
throw new DocumentRemovalExecutionException(exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
} catch (BusinessException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (DocumentRemovalExecutionException exception) {
|
||||||
|
throw exception.getOriginalCause();
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
if (removalCompleted.get()) {
|
||||||
|
Log.warn(
|
||||||
|
"文档删除已完成但分布式锁在收尾阶段失效,继续提交数据库事务: documentId={}",
|
||||||
|
id,
|
||||||
|
exception
|
||||||
|
);
|
||||||
|
return Boolean.TRUE.equals(removalResult.get());
|
||||||
|
}
|
||||||
|
throw new BusinessException(
|
||||||
|
503,
|
||||||
|
DocumentChunkServiceImpl.DOCUMENT_CHUNK_LOCK_UNAVAILABLE,
|
||||||
|
"文档正在处理中,请稍后重试",
|
||||||
|
exception
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean removeDocLocked(String id) {
|
||||||
// 查询该文档对应哪些分割的字段,先删除
|
// 查询该文档对应哪些分割的字段,先删除
|
||||||
QueryWrapper queryWrapperDocument = QueryWrapper.create().eq(Document::getId, id);
|
QueryWrapper queryWrapperDocument = QueryWrapper.create().eq(Document::getId, id);
|
||||||
Document oneByQuery = documentMapper.selectOneByQuery(queryWrapperDocument);
|
Document oneByQuery = documentMapper.selectOneByQuery(queryWrapperDocument);
|
||||||
@@ -218,23 +273,38 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
|
|
||||||
QueryWrapper queryWrapper = QueryWrapper.create()
|
QueryWrapper queryWrapper = QueryWrapper.create()
|
||||||
.select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id);
|
.select(DOCUMENT_CHUNK.ID).eq(DocumentChunk::getDocumentId, id);
|
||||||
List<BigInteger> chunkIds = documentChunkMapper.selectListByQueryAs(
|
List<BigInteger> chunkIds = new ArrayList<>(
|
||||||
queryWrapper,
|
documentChunkMapper.selectListByQueryAs(queryWrapper, BigInteger.class)
|
||||||
BigInteger.class
|
|
||||||
);
|
);
|
||||||
|
chunkIds.sort(Comparator.naturalOrder());
|
||||||
DocumentStore documentStore = null;
|
DocumentStore documentStore = null;
|
||||||
try {
|
try {
|
||||||
|
Model model = null;
|
||||||
if (!chunkIds.isEmpty()) {
|
if (!chunkIds.isEmpty()) {
|
||||||
documentStore = knowledge.toDocumentStore();
|
documentStore = knowledge.toDocumentStore();
|
||||||
if (documentStore == null) {
|
if (documentStore == null) {
|
||||||
return false;
|
throw new BusinessException("文档向量存储不可用");
|
||||||
}
|
}
|
||||||
Model model = modelService.getById(
|
model = modelService.getById(
|
||||||
knowledge.getVectorEmbedModelId()
|
knowledge.getVectorEmbedModelId()
|
||||||
);
|
);
|
||||||
if (model == null) {
|
if (model == null) {
|
||||||
return false;
|
throw new BusinessException("文档向量模型不存在");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
Date supersededAt = new Date();
|
||||||
|
for (BigInteger chunkId : chunkIds) {
|
||||||
|
redisLockExecutor.executeWithRenewingLock(
|
||||||
|
DocumentChunkSyncState.syncLockKey(chunkId),
|
||||||
|
Duration.ofSeconds(5),
|
||||||
|
Duration.ofSeconds(30),
|
||||||
|
() -> {
|
||||||
|
documentChunkSyncTaskMapper.supersedeChunk(chunkId, supersededAt);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!chunkIds.isEmpty()) {
|
||||||
StoreOptions options = StoreOptions.ofCollectionName(
|
StoreOptions options = StoreOptions.ofCollectionName(
|
||||||
knowledge.getVectorStoreCollection()
|
knowledge.getVectorStoreCollection()
|
||||||
);
|
);
|
||||||
@@ -253,7 +323,16 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
// 删除搜索引擎中的数据
|
// 删除搜索引擎中的数据
|
||||||
DocumentSearcher searcher = searcherFactory.getSearcher();
|
DocumentSearcher searcher = searcherFactory.getSearcher();
|
||||||
if (searcher != null) {
|
if (searcher != null) {
|
||||||
chunkIds.forEach(searcher::deleteDocument);
|
for (BigInteger chunkId : chunkIds) {
|
||||||
|
if (!searcher.deleteDocument(chunkId)) {
|
||||||
|
Log.error(
|
||||||
|
"删除文档关键词索引失败: documentId={}, chunkId={}",
|
||||||
|
id,
|
||||||
|
chunkId
|
||||||
|
);
|
||||||
|
throw new BusinessException("文档关键词索引删除失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
int ck = documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, id));
|
int ck = documentChunkMapper.deleteByQuery(QueryWrapper.create().eq(DocumentChunk::getDocumentId, id));
|
||||||
if (ck < 0) {
|
if (ck < 0) {
|
||||||
@@ -1099,6 +1178,17 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static final class DocumentRemovalExecutionException extends RuntimeException {
|
||||||
|
|
||||||
|
private DocumentRemovalExecutionException(RuntimeException cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
private RuntimeException getOriginalCause() {
|
||||||
|
return (RuntimeException) getCause();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public DocumentSplitter getDocumentSplitter(DocumentCollectionSplitParams params) {
|
public DocumentSplitter getDocumentSplitter(DocumentCollectionSplitParams params) {
|
||||||
String splitterName = params.getSplitterName();
|
String splitterName = params.getSplitterName();
|
||||||
int chunkSize = params.getChunkSize();
|
int chunkSize = params.getChunkSize();
|
||||||
|
|||||||
@@ -216,6 +216,20 @@ public class ModelServiceImpl extends ServiceImpl<ModelMapper, Model> implements
|
|||||||
return fillProviderDefaults(model);
|
return fillProviderDefaults(model);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Model> listModelInstances(Collection<BigInteger> modelIds) {
|
||||||
|
if (modelIds == null || modelIds.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
return modelMapper.selectListWithRelationsByQuery(
|
||||||
|
QueryWrapper.create().in(Model::getId, modelIds))
|
||||||
|
.stream()
|
||||||
|
.map(model -> model.getModelProvider() == null
|
||||||
|
? model
|
||||||
|
: fillProviderDefaults(model))
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Model getModelInstanceByInvokeCode(String invokeCode) {
|
public Model getModelInstanceByInvokeCode(String invokeCode) {
|
||||||
if (StrUtil.isBlank(invokeCode)) {
|
if (StrUtil.isBlank(invokeCode)) {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.easyagents.document.core.entity.ParseResponse;
|
|||||||
import com.easyagents.document.core.entity.ParseResult;
|
import com.easyagents.document.core.entity.ParseResult;
|
||||||
import com.easyagents.document.core.entity.ParseTaskInfo;
|
import com.easyagents.document.core.entity.ParseTaskInfo;
|
||||||
import com.easyagents.document.core.entity.ParseTaskStatus;
|
import com.easyagents.document.core.entity.ParseTaskStatus;
|
||||||
|
import com.easyagents.document.core.exception.DocumentAsyncTaskNotFoundException;
|
||||||
import com.easyagents.document.pdf.PdfDocumentParseService;
|
import com.easyagents.document.pdf.PdfDocumentParseService;
|
||||||
import com.easyagents.document.pptx.PptxDocumentParseService;
|
import com.easyagents.document.pptx.PptxDocumentParseService;
|
||||||
import com.easyagents.document.xlsx.XlsxDocumentParseService;
|
import com.easyagents.document.xlsx.XlsxDocumentParseService;
|
||||||
@@ -128,6 +129,33 @@ public class DocumentParseBridgeServiceImplTest {
|
|||||||
Assert.assertEquals(0, pptxService.queryResultCallCount);
|
Assert.assertEquals(0, pptxService.queryResultCallCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证本地 Office 内存任务丢失会转换为稳定桥接错误码。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldExposeMissingOfficeTaskAsStableBridgeError() {
|
||||||
|
FakePptxDocumentParseService pptxService =
|
||||||
|
new FakePptxDocumentParseService();
|
||||||
|
pptxService.queryTaskInfoError =
|
||||||
|
new DocumentAsyncTaskNotFoundException("lost-task");
|
||||||
|
DocumentParseBridgeServiceImpl bridgeService =
|
||||||
|
buildBridgeService(null, pptxService, null, null);
|
||||||
|
|
||||||
|
try {
|
||||||
|
bridgeService.queryTaskInfo(
|
||||||
|
"lost-task",
|
||||||
|
buildSource(
|
||||||
|
"slides.pptx",
|
||||||
|
"application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
Assert.fail("expected DocumentParseBridgeException");
|
||||||
|
} catch (DocumentParseBridgeException error) {
|
||||||
|
Assert.assertEquals("task_not_found", error.getCode());
|
||||||
|
Assert.assertSame(pptxService.queryTaskInfoError, error.getCause());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证缺少底层服务时抛出稳定错误码。
|
* 验证缺少底层服务时抛出稳定错误码。
|
||||||
*/
|
*/
|
||||||
@@ -418,6 +446,7 @@ public class DocumentParseBridgeServiceImplTest {
|
|||||||
private int parseCallCount;
|
private int parseCallCount;
|
||||||
private int queryTaskInfoCallCount;
|
private int queryTaskInfoCallCount;
|
||||||
private int queryResultCallCount;
|
private int queryResultCallCount;
|
||||||
|
private RuntimeException queryTaskInfoError;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ParseResponse parse(ParseRequest request) {
|
public ParseResponse parse(ParseRequest request) {
|
||||||
@@ -450,6 +479,9 @@ public class DocumentParseBridgeServiceImplTest {
|
|||||||
@Override
|
@Override
|
||||||
public ParseTaskInfo queryTaskInfo(String taskId) {
|
public ParseTaskInfo queryTaskInfo(String taskId) {
|
||||||
queryTaskInfoCallCount++;
|
queryTaskInfoCallCount++;
|
||||||
|
if (queryTaskInfoError != null) {
|
||||||
|
throw queryTaskInfoError;
|
||||||
|
}
|
||||||
throw new UnsupportedOperationException();
|
throw new UnsupportedOperationException();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,233 @@
|
|||||||
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.mockito.InOrder;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.SimpleTransactionStatus;
|
||||||
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
import tech.easyflow.ai.config.AiMilvusConfig;
|
||||||
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
|
import tech.easyflow.ai.service.ModelService;
|
||||||
|
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分块索引同步任务恢复与版本语义测试。
|
||||||
|
*/
|
||||||
|
public class DocumentChunkSyncTaskAppServiceTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void dispatchPendingShouldRecoverExpiredTasksAndSendClaimedRows() {
|
||||||
|
Fixture fixture = fixture(1);
|
||||||
|
Mockito.when(fixture.taskMapper.selectPendingDue(
|
||||||
|
Mockito.any(), Mockito.any(), Mockito.anyInt()
|
||||||
|
)).thenReturn(List.of(fixture.task));
|
||||||
|
Mockito.when(fixture.taskMapper.markDispatched(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.any(), Mockito.any()
|
||||||
|
)).thenReturn(1);
|
||||||
|
|
||||||
|
fixture.service.dispatchPendingTasks();
|
||||||
|
|
||||||
|
Mockito.verify(fixture.taskMapper).recoverExpired(Mockito.any());
|
||||||
|
Mockito.verify(fixture.producer).send(fixture.taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void outdatedTaskShouldBeSupersededWithoutTouchingIndexes() {
|
||||||
|
Fixture fixture = fixture(1);
|
||||||
|
fixture.chunk.setIndexSyncVersion(2L);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
Mockito.verify(fixture.taskMapper).finishOwned(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.TASK_SUPERSEDED),
|
||||||
|
Mockito.isNull(), Mockito.isNull(), Mockito.any()
|
||||||
|
);
|
||||||
|
Mockito.verifyNoInteractions(fixture.collectionService);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void transientFailureShouldReturnTaskToPending() {
|
||||||
|
Fixture fixture = fixture(1);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
InOrder order = Mockito.inOrder(
|
||||||
|
fixture.taskMapper,
|
||||||
|
fixture.collectionService
|
||||||
|
);
|
||||||
|
order.verify(fixture.taskMapper).lockChunkTasks(fixture.chunkId);
|
||||||
|
order.verify(fixture.collectionService).getById(Mockito.any());
|
||||||
|
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.PENDING), Mockito.any(),
|
||||||
|
Mockito.eq("INDEX_SOURCE_MISSING"), Mockito.anyString(), Mockito.any()
|
||||||
|
);
|
||||||
|
Mockito.verify(fixture.chunkMapper, Mockito.never()).updateSyncState(
|
||||||
|
Mockito.any(), Mockito.anyLong(), Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void workerShouldLockChunkTaskRangeBeforeReadingOwnedTask() {
|
||||||
|
Fixture fixture = fixture(1);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
InOrder order = Mockito.inOrder(fixture.taskMapper);
|
||||||
|
order.verify(fixture.taskMapper).selectOneById(fixture.taskId);
|
||||||
|
order.verify(fixture.taskMapper).claim(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
);
|
||||||
|
order.verify(fixture.taskMapper).lockChunkTasks(fixture.chunkId);
|
||||||
|
order.verify(fixture.taskMapper).selectOneById(fixture.taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void exhaustedFailureShouldExposeFailedStateOnCurrentVersion() {
|
||||||
|
Fixture fixture = fixture(5);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.FAILED), Mockito.any(),
|
||||||
|
Mockito.eq("INDEX_SOURCE_MISSING"), Mockito.anyString(), Mockito.any()
|
||||||
|
);
|
||||||
|
Mockito.verify(fixture.chunkMapper).updateSyncState(
|
||||||
|
fixture.chunkId, 1L, DocumentChunkSyncState.FAILED,
|
||||||
|
"INDEX_SOURCE_MISSING", "分块或知识库不存在"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void exhaustedDeleteFailureShouldRemainPendingForDurableCleanup() {
|
||||||
|
Fixture fixture = fixture(5);
|
||||||
|
fixture.task.setOperation(DocumentChunkSyncState.OPERATION_DELETE);
|
||||||
|
Mockito.when(fixture.chunkMapper.selectOneById(fixture.chunkId)).thenReturn(null);
|
||||||
|
|
||||||
|
fixture.service.handleTask(fixture.taskId);
|
||||||
|
|
||||||
|
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.anyString(),
|
||||||
|
Mockito.eq(DocumentChunkSyncState.PENDING), Mockito.any(),
|
||||||
|
Mockito.eq("INDEX_SYNC_FAILED"), Mockito.anyString(), Mockito.any()
|
||||||
|
);
|
||||||
|
Mockito.verify(fixture.chunkMapper, Mockito.never()).updateSyncState(
|
||||||
|
Mockito.any(), Mockito.anyLong(), Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void manualRetryShouldResetTaskAndChunkBeforeDispatch() {
|
||||||
|
Fixture fixture = fixture(5);
|
||||||
|
fixture.chunk.setIndexSyncStatus(DocumentChunkSyncState.FAILED);
|
||||||
|
fixture.task.setStatus(DocumentChunkSyncState.FAILED);
|
||||||
|
Mockito.when(fixture.taskMapper.selectFailed(fixture.chunkId, 1L))
|
||||||
|
.thenReturn(fixture.task);
|
||||||
|
Mockito.when(fixture.taskMapper.retryFailed(
|
||||||
|
Mockito.eq(fixture.taskId), Mockito.eq(fixture.chunkId),
|
||||||
|
Mockito.eq(1L), Mockito.any()
|
||||||
|
)).thenReturn(1);
|
||||||
|
Mockito.when(fixture.chunkMapper.updateSyncState(
|
||||||
|
fixture.chunkId, 1L, DocumentChunkSyncState.PENDING, null, null
|
||||||
|
)).thenReturn(1);
|
||||||
|
|
||||||
|
fixture.service.retry(fixture.chunkId, 1L);
|
||||||
|
|
||||||
|
Mockito.verify(fixture.producer).send(fixture.taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Fixture fixture(int attemptCount) {
|
||||||
|
BigInteger taskId = BigInteger.valueOf(10);
|
||||||
|
BigInteger chunkId = BigInteger.valueOf(20);
|
||||||
|
DocumentChunkSyncTask task = new DocumentChunkSyncTask();
|
||||||
|
task.setId(taskId);
|
||||||
|
task.setChunkId(chunkId);
|
||||||
|
task.setDocumentId(BigInteger.valueOf(30));
|
||||||
|
task.setDocumentCollectionId(BigInteger.valueOf(40));
|
||||||
|
task.setVectorCollection("kb-test");
|
||||||
|
task.setOperation(DocumentChunkSyncState.OPERATION_UPSERT);
|
||||||
|
task.setSyncVersion(1L);
|
||||||
|
task.setStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
task.setAttemptCount(attemptCount);
|
||||||
|
|
||||||
|
DocumentChunk chunk = new DocumentChunk();
|
||||||
|
chunk.setId(chunkId);
|
||||||
|
chunk.setDocumentCollectionId(task.getDocumentCollectionId());
|
||||||
|
chunk.setIndexSyncVersion(1L);
|
||||||
|
chunk.setIndexSyncStatus(DocumentChunkSyncState.PENDING);
|
||||||
|
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper = Mockito.mock(DocumentChunkSyncTaskMapper.class);
|
||||||
|
Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
|
||||||
|
Mockito.when(taskMapper.claim(
|
||||||
|
Mockito.eq(taskId), Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
|
)).thenAnswer(invocation -> {
|
||||||
|
task.setStatus(DocumentChunkSyncState.TASK_RUNNING);
|
||||||
|
task.setExecutionToken(invocation.getArgument(1));
|
||||||
|
return 1;
|
||||||
|
});
|
||||||
|
Mockito.when(taskMapper.lockChunkTasks(chunkId)).thenReturn(List.of(taskId));
|
||||||
|
Mockito.when(taskMapper.failOrRetryOwned(
|
||||||
|
Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
|
||||||
|
Mockito.any(), Mockito.any(), Mockito.any()
|
||||||
|
)).thenReturn(1);
|
||||||
|
Mockito.when(taskMapper.finishOwned(
|
||||||
|
Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
|
||||||
|
Mockito.any(), Mockito.any()
|
||||||
|
)).thenReturn(1);
|
||||||
|
|
||||||
|
DocumentChunkMapper chunkMapper = Mockito.mock(DocumentChunkMapper.class);
|
||||||
|
Mockito.when(chunkMapper.selectOneById(chunkId)).thenReturn(chunk);
|
||||||
|
DocumentCollectionService collectionService = Mockito.mock(DocumentCollectionService.class);
|
||||||
|
ModelService modelService = Mockito.mock(ModelService.class);
|
||||||
|
SearcherFactory searcherFactory = Mockito.mock(SearcherFactory.class);
|
||||||
|
DocumentChunkSyncTaskProducer producer = Mockito.mock(DocumentChunkSyncTaskProducer.class);
|
||||||
|
PlatformTransactionManager transactionManager = Mockito.mock(PlatformTransactionManager.class);
|
||||||
|
Mockito.when(transactionManager.getTransaction(Mockito.any()))
|
||||||
|
.thenReturn(new SimpleTransactionStatus());
|
||||||
|
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
|
||||||
|
Mockito.doAnswer(invocation -> invocation.<Supplier<?>>getArgument(3).get())
|
||||||
|
.when(redisLockExecutor).executeWithRenewingLock(
|
||||||
|
Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.<Supplier<?>>any()
|
||||||
|
);
|
||||||
|
|
||||||
|
DocumentChunkSyncTaskAppService service = new DocumentChunkSyncTaskAppService(
|
||||||
|
taskMapper,
|
||||||
|
chunkMapper,
|
||||||
|
collectionService,
|
||||||
|
modelService,
|
||||||
|
searcherFactory,
|
||||||
|
producer,
|
||||||
|
transactionManager,
|
||||||
|
redisLockExecutor,
|
||||||
|
Mockito.mock(AiMilvusConfig.class),
|
||||||
|
Mockito.mock(ObjectProvider.class)
|
||||||
|
);
|
||||||
|
return new Fixture(service, taskMapper, chunkMapper, collectionService,
|
||||||
|
producer, task, chunk, taskId, chunkId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private record Fixture(
|
||||||
|
DocumentChunkSyncTaskAppService service,
|
||||||
|
DocumentChunkSyncTaskMapper taskMapper,
|
||||||
|
DocumentChunkMapper chunkMapper,
|
||||||
|
DocumentCollectionService collectionService,
|
||||||
|
DocumentChunkSyncTaskProducer producer,
|
||||||
|
DocumentChunkSyncTask task,
|
||||||
|
DocumentChunk chunk,
|
||||||
|
BigInteger taskId,
|
||||||
|
BigInteger chunkId
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package tech.easyflow.ai.documentimport.task;
|
|||||||
|
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import tech.easyflow.ai.document.support.DocumentParseFilePolicy;
|
||||||
|
|
||||||
import java.lang.reflect.Field;
|
import java.lang.reflect.Field;
|
||||||
import java.lang.reflect.Method;
|
import java.lang.reflect.Method;
|
||||||
@@ -20,6 +21,13 @@ public class CsvImportSupportPolicyTest {
|
|||||||
@Test
|
@Test
|
||||||
public void shouldAllowCsvAcrossAllKnowledgeImportEntrypoints()
|
public void shouldAllowCsvAcrossAllKnowledgeImportEntrypoints()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
Assert.assertFalse(
|
||||||
|
DocumentParseFilePolicy.isSupportedFileName("legacy.xls"));
|
||||||
|
Assert.assertTrue(
|
||||||
|
DocumentParseFilePolicy.isSupportedFileName("REPORT.XLSX"));
|
||||||
|
Assert.assertEquals(
|
||||||
|
"TXT、PDF、DOCX、MD、PPTX、XLSX、CSV",
|
||||||
|
DocumentParseFilePolicy.supportedTypeLabel());
|
||||||
KnowledgeDocumentImportTaskAppService taskService =
|
KnowledgeDocumentImportTaskAppService taskService =
|
||||||
new KnowledgeDocumentImportTaskAppService();
|
new KnowledgeDocumentImportTaskAppService();
|
||||||
Method assertSupported = KnowledgeDocumentImportTaskAppService.class
|
Method assertSupported = KnowledgeDocumentImportTaskAppService.class
|
||||||
@@ -43,10 +51,10 @@ public class CsvImportSupportPolicyTest {
|
|||||||
Assert.assertTrue(readSupportedExtensions(
|
Assert.assertTrue(readSupportedExtensions(
|
||||||
KnowledgeImportBatchFacade.class).contains("csv"));
|
KnowledgeImportBatchFacade.class).contains("csv"));
|
||||||
Assert.assertSame(
|
Assert.assertSame(
|
||||||
DocumentImportFormatPolicy.supportedExtensions(),
|
DocumentParseFilePolicy.supportedExtensions(),
|
||||||
readSupportedExtensions(DocumentImportBatchAppService.class));
|
readSupportedExtensions(DocumentImportBatchAppService.class));
|
||||||
Assert.assertSame(
|
Assert.assertSame(
|
||||||
DocumentImportFormatPolicy.supportedExtensions(),
|
DocumentParseFilePolicy.supportedExtensions(),
|
||||||
readSupportedExtensions(KnowledgeImportBatchFacade.class));
|
readSupportedExtensions(KnowledgeImportBatchFacade.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -873,8 +873,8 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
Mockito.anyString(),
|
Mockito.anyString(),
|
||||||
Mockito.same(recoveryError)
|
Mockito.same(recoveryError)
|
||||||
);
|
);
|
||||||
Mockito.verify(context.batchMapper, Mockito.never())
|
Mockito.verify(context.recoveryCoordinator, Mockito.never())
|
||||||
.finalizeRecoveryPending(
|
.finalizeRecovery(
|
||||||
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class));
|
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -892,8 +892,9 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
|
Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
|
||||||
Mockito.any(Date.class), Mockito.anyInt()
|
Mockito.any(Date.class), Mockito.anyInt()
|
||||||
)).thenReturn(List.of(batch));
|
)).thenReturn(List.of(batch));
|
||||||
Mockito.when(context.batchMapper.selectClaimedRecovery(
|
Mockito.when(context.recoveryCoordinator.claim(
|
||||||
Mockito.eq(batch.getId()), Mockito.anyString()
|
Mockito.eq(batch.getId()), Mockito.anyString(),
|
||||||
|
Mockito.any(Date.class), Mockito.any(Date.class)
|
||||||
)).thenReturn(null);
|
)).thenReturn(null);
|
||||||
|
|
||||||
int recovered = context.service.recoverPendingBatchRetries();
|
int recovered = context.service.recoverPendingBatchRetries();
|
||||||
@@ -924,8 +925,9 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
|
Mockito.when(context.batchMapper.selectRecoveryPendingBatches(
|
||||||
Mockito.any(Date.class), Mockito.anyInt()
|
Mockito.any(Date.class), Mockito.anyInt()
|
||||||
)).thenReturn(List.of(batch));
|
)).thenReturn(List.of(batch));
|
||||||
Mockito.when(context.batchMapper.selectClaimedRecovery(
|
Mockito.when(context.recoveryCoordinator.claim(
|
||||||
Mockito.eq(batch.getId()), Mockito.anyString()
|
Mockito.eq(batch.getId()), Mockito.anyString(),
|
||||||
|
Mockito.any(Date.class), Mockito.any(Date.class)
|
||||||
)).thenReturn(claimedBatch);
|
)).thenReturn(claimedBatch);
|
||||||
|
|
||||||
int recovered = context.service.recoverPendingBatchRetries();
|
int recovered = context.service.recoverPendingBatchRetries();
|
||||||
@@ -939,16 +941,14 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
);
|
);
|
||||||
ArgumentCaptor<String> recoveryToken =
|
ArgumentCaptor<String> recoveryToken =
|
||||||
ArgumentCaptor.forClass(String.class);
|
ArgumentCaptor.forClass(String.class);
|
||||||
Mockito.verify(context.batchMapper).claimRecoveryPending(
|
Mockito.verify(context.recoveryCoordinator).claim(
|
||||||
Mockito.eq(batch.getId()),
|
Mockito.eq(batch.getId()),
|
||||||
recoveryToken.capture(),
|
recoveryToken.capture(),
|
||||||
Mockito.any(Date.class),
|
Mockito.any(Date.class),
|
||||||
Mockito.any(Date.class)
|
Mockito.any(Date.class)
|
||||||
);
|
);
|
||||||
Mockito.verify(context.batchMapper).selectClaimedRecovery(
|
Mockito.verify(context.recoveryCoordinator)
|
||||||
batch.getId(), recoveryToken.getValue());
|
.finalizeRecovery(
|
||||||
Mockito.verify(context.batchMapper)
|
|
||||||
.finalizeRecoveryPending(
|
|
||||||
Mockito.eq(batch.getId()),
|
Mockito.eq(batch.getId()),
|
||||||
Mockito.eq(recoveryToken.getValue()),
|
Mockito.eq(recoveryToken.getValue()),
|
||||||
Mockito.any(Date.class));
|
Mockito.any(Date.class));
|
||||||
@@ -975,8 +975,8 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
|
|
||||||
Assert.assertEquals(1, context.service.recoverPendingBatchRetries());
|
Assert.assertEquals(1, context.service.recoverPendingBatchRetries());
|
||||||
|
|
||||||
Mockito.verify(context.batchMapper, Mockito.never())
|
Mockito.verify(context.recoveryCoordinator, Mockito.never())
|
||||||
.finalizeRecoveryPending(
|
.finalizeRecovery(
|
||||||
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class));
|
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class));
|
||||||
Mockito.verifyNoInteractions(context.circuitBreaker);
|
Mockito.verifyNoInteractions(context.circuitBreaker);
|
||||||
}
|
}
|
||||||
@@ -990,12 +990,12 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
BigInteger batchId = BigInteger.valueOf(84);
|
BigInteger batchId = BigInteger.valueOf(84);
|
||||||
String recoveryToken = "recovery-token";
|
String recoveryToken = "recovery-token";
|
||||||
AtomicLong now = new AtomicLong(1_000L);
|
AtomicLong now = new AtomicLong(1_000L);
|
||||||
Mockito.when(context.batchMapper.renewRecoveryPendingLease(
|
Mockito.when(context.recoveryCoordinator.renew(
|
||||||
Mockito.eq(batchId),
|
Mockito.eq(batchId),
|
||||||
Mockito.eq(recoveryToken),
|
Mockito.eq(recoveryToken),
|
||||||
Mockito.any(Date.class),
|
Mockito.any(Date.class),
|
||||||
Mockito.any(Date.class)
|
Mockito.any(Date.class)
|
||||||
)).thenReturn(1, 0);
|
)).thenReturn(true, false);
|
||||||
BooleanSupplier guard = context.service.createRecoveryLeaseGuard(
|
BooleanSupplier guard = context.service.createRecoveryLeaseGuard(
|
||||||
batchId,
|
batchId,
|
||||||
recoveryToken,
|
recoveryToken,
|
||||||
@@ -1004,8 +1004,8 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
);
|
);
|
||||||
|
|
||||||
Assert.assertTrue(guard.getAsBoolean());
|
Assert.assertTrue(guard.getAsBoolean());
|
||||||
Mockito.verify(context.batchMapper, Mockito.never())
|
Mockito.verify(context.recoveryCoordinator, Mockito.never())
|
||||||
.renewRecoveryPendingLease(
|
.renew(
|
||||||
Mockito.any(),
|
Mockito.any(),
|
||||||
Mockito.anyString(),
|
Mockito.anyString(),
|
||||||
Mockito.any(Date.class),
|
Mockito.any(Date.class),
|
||||||
@@ -1017,8 +1017,8 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
now.set(121_000L);
|
now.set(121_000L);
|
||||||
Assert.assertFalse(guard.getAsBoolean());
|
Assert.assertFalse(guard.getAsBoolean());
|
||||||
|
|
||||||
Mockito.verify(context.batchMapper, Mockito.times(2))
|
Mockito.verify(context.recoveryCoordinator, Mockito.times(2))
|
||||||
.renewRecoveryPendingLease(
|
.renew(
|
||||||
Mockito.eq(batchId),
|
Mockito.eq(batchId),
|
||||||
Mockito.eq(recoveryToken),
|
Mockito.eq(recoveryToken),
|
||||||
Mockito.any(Date.class),
|
Mockito.any(Date.class),
|
||||||
@@ -1198,6 +1198,8 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
|
RedisLockExecutor redisLockExecutor = Mockito.mock(RedisLockExecutor.class);
|
||||||
DocumentImportBatchCircuitBreaker circuitBreaker =
|
DocumentImportBatchCircuitBreaker circuitBreaker =
|
||||||
Mockito.mock(DocumentImportBatchCircuitBreaker.class);
|
Mockito.mock(DocumentImportBatchCircuitBreaker.class);
|
||||||
|
DocumentImportRecoveryCoordinator recoveryCoordinator =
|
||||||
|
Mockito.mock(DocumentImportRecoveryCoordinator.class);
|
||||||
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
|
RedisLockExecutor.LockHandle lockHandle = Mockito.mock(RedisLockExecutor.LockHandle.class);
|
||||||
Mockito.when(redisLockExecutor.tryAcquire(
|
Mockito.when(redisLockExecutor.tryAcquire(
|
||||||
Mockito.anyString(), Mockito.any(), Mockito.any()
|
Mockito.anyString(), Mockito.any(), Mockito.any()
|
||||||
@@ -1213,21 +1215,16 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
Mockito.when(batchMapper.selectOwnedForUpdate(
|
Mockito.when(batchMapper.selectOwnedForUpdate(
|
||||||
Mockito.any(), Mockito.anyString(), Mockito.any()
|
Mockito.any(), Mockito.anyString(), Mockito.any()
|
||||||
)).thenReturn(batch);
|
)).thenReturn(batch);
|
||||||
Mockito.when(batchMapper.claimRecoveryPending(
|
Mockito.when(recoveryCoordinator.claim(
|
||||||
Mockito.any(),
|
Mockito.any(), Mockito.anyString(),
|
||||||
Mockito.anyString(),
|
Mockito.any(Date.class), Mockito.any(Date.class)
|
||||||
Mockito.any(Date.class),
|
|
||||||
Mockito.any(Date.class)
|
|
||||||
)).thenReturn(1);
|
|
||||||
Mockito.when(batchMapper.selectClaimedRecovery(
|
|
||||||
Mockito.any(), Mockito.anyString()
|
|
||||||
)).thenReturn(batch);
|
)).thenReturn(batch);
|
||||||
Mockito.when(taskAppService.retryBatchFailures(
|
Mockito.when(taskAppService.retryBatchFailures(
|
||||||
Mockito.any(), Mockito.anySet(), Mockito.any(BooleanSupplier.class)
|
Mockito.any(), Mockito.anySet(), Mockito.any(BooleanSupplier.class)
|
||||||
)).thenReturn(true);
|
)).thenReturn(true);
|
||||||
Mockito.when(batchMapper.finalizeRecoveryPending(
|
Mockito.when(recoveryCoordinator.finalizeRecovery(
|
||||||
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class)
|
Mockito.any(), Mockito.anyString(), Mockito.any(Date.class)
|
||||||
)).thenReturn(1);
|
)).thenReturn(true);
|
||||||
|
|
||||||
DocumentImportBatchAppService service = new DocumentImportBatchAppService(
|
DocumentImportBatchAppService service = new DocumentImportBatchAppService(
|
||||||
batchService,
|
batchService,
|
||||||
@@ -1239,12 +1236,13 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
itemMapper,
|
itemMapper,
|
||||||
documentMapper,
|
documentMapper,
|
||||||
redisLockExecutor,
|
redisLockExecutor,
|
||||||
circuitBreaker
|
circuitBreaker,
|
||||||
|
recoveryCoordinator
|
||||||
);
|
);
|
||||||
return new TestContext(
|
return new TestContext(
|
||||||
service, batchService, itemService, batchTracker,
|
service, batchService, itemService, batchTracker,
|
||||||
taskAppService, batchMapper, itemMapper, documentMapper,
|
taskAppService, batchMapper, itemMapper, documentMapper,
|
||||||
circuitBreaker, lockHandle
|
circuitBreaker, recoveryCoordinator, lockHandle
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1356,6 +1354,7 @@ public class DocumentImportBatchAppServiceTest {
|
|||||||
DocumentImportBatchItemMapper itemMapper,
|
DocumentImportBatchItemMapper itemMapper,
|
||||||
DocumentMapper documentMapper,
|
DocumentMapper documentMapper,
|
||||||
DocumentImportBatchCircuitBreaker circuitBreaker,
|
DocumentImportBatchCircuitBreaker circuitBreaker,
|
||||||
|
DocumentImportRecoveryCoordinator recoveryCoordinator,
|
||||||
RedisLockExecutor.LockHandle lockHandle
|
RedisLockExecutor.LockHandle lockHandle
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ import org.springframework.dao.DataAccessResourceFailureException;
|
|||||||
import org.springframework.jdbc.BadSqlGrammarException;
|
import org.springframework.jdbc.BadSqlGrammarException;
|
||||||
import org.springframework.data.redis.RedisSystemException;
|
import org.springframework.data.redis.RedisSystemException;
|
||||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||||
|
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||||
import tech.easyflow.ai.entity.DocumentImportTask;
|
import tech.easyflow.ai.entity.DocumentImportTask;
|
||||||
|
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||||
import tech.easyflow.ai.enums.DocumentImportMode;
|
import tech.easyflow.ai.enums.DocumentImportMode;
|
||||||
import tech.easyflow.ai.enums.DocumentImportTaskPhase;
|
import tech.easyflow.ai.enums.DocumentImportTaskPhase;
|
||||||
@@ -50,7 +52,7 @@ public class DocumentImportBatchCircuitBreakerTest {
|
|||||||
DocumentImportTaskMapper taskMapper =
|
DocumentImportTaskMapper taskMapper =
|
||||||
Mockito.mock(DocumentImportTaskMapper.class);
|
Mockito.mock(DocumentImportTaskMapper.class);
|
||||||
Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
|
Mockito.when(taskMapper.selectOneById(taskId)).thenReturn(task);
|
||||||
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
|
Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch);
|
||||||
Mockito.when(batchMapper.interruptRunningBatchForActiveTask(
|
Mockito.when(batchMapper.interruptRunningBatchForActiveTask(
|
||||||
Mockito.eq(batchId),
|
Mockito.eq(batchId),
|
||||||
Mockito.eq(taskId),
|
Mockito.eq(taskId),
|
||||||
@@ -161,6 +163,7 @@ public class DocumentImportBatchCircuitBreakerTest {
|
|||||||
DocumentImportTaskMapper taskMapper =
|
DocumentImportTaskMapper taskMapper =
|
||||||
Mockito.mock(DocumentImportTaskMapper.class);
|
Mockito.mock(DocumentImportTaskMapper.class);
|
||||||
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
|
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
|
||||||
|
Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch);
|
||||||
Mockito.when(batchMapper.interruptOwnedRecoveryBatch(
|
Mockito.when(batchMapper.interruptOwnedRecoveryBatch(
|
||||||
Mockito.eq(batchId),
|
Mockito.eq(batchId),
|
||||||
Mockito.eq("stale-token"),
|
Mockito.eq("stale-token"),
|
||||||
@@ -183,6 +186,136 @@ public class DocumentImportBatchCircuitBreakerTest {
|
|||||||
Mockito.any(Date.class), Mockito.any());
|
Mockito.any(Date.class), Mockito.any());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证零重排熔断在批次与失败项锁内完成,且未发生并发推进时生效。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldInterruptRecoveryFailureWhenItemsRemainFailed() {
|
||||||
|
BigInteger batchId = BigInteger.valueOf(42);
|
||||||
|
BigInteger itemId = BigInteger.valueOf(43);
|
||||||
|
String recoveryToken = "current-token";
|
||||||
|
DocumentImportBatch batch = runningAutoBatch(batchId);
|
||||||
|
batch.setRecoveryPending(true);
|
||||||
|
batch.setRecoveryToken(recoveryToken);
|
||||||
|
batch.setRecoveryLeaseUntil(
|
||||||
|
new Date(System.currentTimeMillis() + 60_000));
|
||||||
|
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||||
|
item.setId(itemId);
|
||||||
|
item.setBatchId(batchId);
|
||||||
|
item.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||||
|
DocumentImportBatchMapper batchMapper =
|
||||||
|
Mockito.mock(DocumentImportBatchMapper.class);
|
||||||
|
DocumentImportBatchItemMapper itemMapper =
|
||||||
|
Mockito.mock(DocumentImportBatchItemMapper.class);
|
||||||
|
DocumentImportTaskMapper taskMapper =
|
||||||
|
Mockito.mock(DocumentImportTaskMapper.class);
|
||||||
|
Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch);
|
||||||
|
Mockito.when(itemMapper.selectForUpdate(itemId)).thenReturn(item);
|
||||||
|
Mockito.when(batchMapper.interruptOwnedRecoveryBatch(
|
||||||
|
Mockito.eq(batchId), Mockito.eq(recoveryToken),
|
||||||
|
Mockito.eq("document_import_recovery_failed"),
|
||||||
|
Mockito.anyString(), Mockito.any(Date.class)
|
||||||
|
)).thenReturn(1);
|
||||||
|
DocumentImportBatchCircuitBreaker circuitBreaker =
|
||||||
|
new DocumentImportBatchCircuitBreaker(
|
||||||
|
batchMapper, itemMapper, taskMapper);
|
||||||
|
|
||||||
|
Assert.assertTrue(circuitBreaker.interruptRecoveryBatch(
|
||||||
|
batchId,
|
||||||
|
recoveryToken,
|
||||||
|
new DocumentImportRecoveryException(java.util.List.of(itemId))
|
||||||
|
));
|
||||||
|
|
||||||
|
org.mockito.InOrder order = Mockito.inOrder(batchMapper, itemMapper);
|
||||||
|
order.verify(batchMapper).selectForUpdate(batchId);
|
||||||
|
order.verify(itemMapper).countActiveItems(batchId);
|
||||||
|
order.verify(itemMapper).selectForUpdate(itemId);
|
||||||
|
order.verify(batchMapper).interruptOwnedRecoveryBatch(
|
||||||
|
Mockito.eq(batchId), Mockito.eq(recoveryToken),
|
||||||
|
Mockito.eq("document_import_recovery_failed"),
|
||||||
|
Mockito.anyString(), Mockito.any(Date.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证错误写回提交后出现的等价并发推进会阻止旧请求熔断整批。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldKeepRunningBatchWhenRecoveryItemAdvanced() {
|
||||||
|
BigInteger batchId = BigInteger.valueOf(44);
|
||||||
|
BigInteger itemId = BigInteger.valueOf(45);
|
||||||
|
String recoveryToken = "current-token";
|
||||||
|
DocumentImportBatch batch = runningAutoBatch(batchId);
|
||||||
|
batch.setRecoveryPending(true);
|
||||||
|
batch.setRecoveryToken(recoveryToken);
|
||||||
|
batch.setRecoveryLeaseUntil(
|
||||||
|
new Date(System.currentTimeMillis() + 60_000));
|
||||||
|
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||||
|
item.setId(itemId);
|
||||||
|
item.setBatchId(batchId);
|
||||||
|
item.setStatus(DocumentImportBatchItemStatus.PENDING.name());
|
||||||
|
DocumentImportBatchMapper batchMapper =
|
||||||
|
Mockito.mock(DocumentImportBatchMapper.class);
|
||||||
|
DocumentImportBatchItemMapper itemMapper =
|
||||||
|
Mockito.mock(DocumentImportBatchItemMapper.class);
|
||||||
|
Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch);
|
||||||
|
Mockito.when(itemMapper.countActiveItems(batchId)).thenReturn(1);
|
||||||
|
DocumentImportBatchCircuitBreaker circuitBreaker =
|
||||||
|
new DocumentImportBatchCircuitBreaker(
|
||||||
|
batchMapper,
|
||||||
|
itemMapper,
|
||||||
|
Mockito.mock(DocumentImportTaskMapper.class));
|
||||||
|
|
||||||
|
Assert.assertFalse(circuitBreaker.interruptRecoveryBatch(
|
||||||
|
batchId,
|
||||||
|
recoveryToken,
|
||||||
|
new DocumentImportRecoveryException(java.util.List.of(itemId))
|
||||||
|
));
|
||||||
|
Mockito.verify(batchMapper, Mockito.never())
|
||||||
|
.interruptOwnedRecoveryBatch(
|
||||||
|
Mockito.any(), Mockito.anyString(), Mockito.anyString(),
|
||||||
|
Mockito.anyString(), Mockito.any(Date.class));
|
||||||
|
Mockito.verify(itemMapper, Mockito.never()).selectForUpdate(itemId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证失败快照外已有活跃项时,选择性恢复也不能熔断并发任务。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldKeepRunningBatchWhenAnotherItemAdvancedBeforeSnapshot() {
|
||||||
|
BigInteger batchId = BigInteger.valueOf(46);
|
||||||
|
BigInteger failedItemId = BigInteger.valueOf(47);
|
||||||
|
String recoveryToken = "current-token";
|
||||||
|
DocumentImportBatch batch = runningAutoBatch(batchId);
|
||||||
|
batch.setRecoveryPending(true);
|
||||||
|
batch.setRecoveryToken(recoveryToken);
|
||||||
|
batch.setRecoveryLeaseUntil(
|
||||||
|
new Date(System.currentTimeMillis() + 60_000));
|
||||||
|
DocumentImportBatchMapper batchMapper =
|
||||||
|
Mockito.mock(DocumentImportBatchMapper.class);
|
||||||
|
DocumentImportBatchItemMapper itemMapper =
|
||||||
|
Mockito.mock(DocumentImportBatchItemMapper.class);
|
||||||
|
Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch);
|
||||||
|
Mockito.when(itemMapper.countActiveItems(batchId)).thenReturn(1);
|
||||||
|
DocumentImportBatchCircuitBreaker circuitBreaker =
|
||||||
|
new DocumentImportBatchCircuitBreaker(
|
||||||
|
batchMapper,
|
||||||
|
itemMapper,
|
||||||
|
Mockito.mock(DocumentImportTaskMapper.class));
|
||||||
|
|
||||||
|
Assert.assertFalse(circuitBreaker.interruptRecoveryBatch(
|
||||||
|
batchId,
|
||||||
|
recoveryToken,
|
||||||
|
new DocumentImportRecoveryException(
|
||||||
|
java.util.List.of(failedItemId))
|
||||||
|
));
|
||||||
|
Mockito.verify(itemMapper, Mockito.never())
|
||||||
|
.selectForUpdate(failedItemId);
|
||||||
|
Mockito.verify(batchMapper, Mockito.never())
|
||||||
|
.interruptOwnedRecoveryBatch(
|
||||||
|
Mockito.any(), Mockito.anyString(), Mockito.anyString(),
|
||||||
|
Mockito.anyString(), Mockito.any(Date.class));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证 SQL 语法错误保留为通用系统异常,避免伪装成数据库不可用。
|
* 验证 SQL 语法错误保留为通用系统异常,避免伪装成数据库不可用。
|
||||||
*/
|
*/
|
||||||
@@ -192,7 +325,7 @@ public class DocumentImportBatchCircuitBreakerTest {
|
|||||||
DocumentImportBatch batch = runningAutoBatch(batchId);
|
DocumentImportBatch batch = runningAutoBatch(batchId);
|
||||||
DocumentImportBatchMapper batchMapper =
|
DocumentImportBatchMapper batchMapper =
|
||||||
Mockito.mock(DocumentImportBatchMapper.class);
|
Mockito.mock(DocumentImportBatchMapper.class);
|
||||||
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
|
Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch);
|
||||||
Mockito.when(batchMapper.interruptRunningBatch(
|
Mockito.when(batchMapper.interruptRunningBatch(
|
||||||
Mockito.eq(batchId),
|
Mockito.eq(batchId),
|
||||||
Mockito.eq("document_import_infrastructure_failure"),
|
Mockito.eq("document_import_infrastructure_failure"),
|
||||||
@@ -222,7 +355,7 @@ public class DocumentImportBatchCircuitBreakerTest {
|
|||||||
DocumentImportBatch batch = runningAutoBatch(batchId);
|
DocumentImportBatch batch = runningAutoBatch(batchId);
|
||||||
DocumentImportBatchMapper batchMapper =
|
DocumentImportBatchMapper batchMapper =
|
||||||
Mockito.mock(DocumentImportBatchMapper.class);
|
Mockito.mock(DocumentImportBatchMapper.class);
|
||||||
Mockito.when(batchMapper.selectOneById(batchId)).thenReturn(batch);
|
Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch);
|
||||||
Mockito.when(batchMapper.interruptRunningBatch(
|
Mockito.when(batchMapper.interruptRunningBatch(
|
||||||
Mockito.eq(batchId),
|
Mockito.eq(batchId),
|
||||||
Mockito.eq("database_unavailable"),
|
Mockito.eq("database_unavailable"),
|
||||||
@@ -242,6 +375,36 @@ public class DocumentImportBatchCircuitBreakerTest {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证没有文件重新排队时使用稳定恢复失败语义中断批次。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldExposeRecoveryFailureInsteadOfGenericInfrastructureError() {
|
||||||
|
BigInteger batchId = BigInteger.valueOf(71);
|
||||||
|
DocumentImportBatch batch = runningAutoBatch(batchId);
|
||||||
|
DocumentImportBatchMapper batchMapper =
|
||||||
|
Mockito.mock(DocumentImportBatchMapper.class);
|
||||||
|
Mockito.when(batchMapper.selectForUpdate(batchId)).thenReturn(batch);
|
||||||
|
Mockito.when(batchMapper.interruptRunningBatch(
|
||||||
|
Mockito.eq(batchId),
|
||||||
|
Mockito.eq("document_import_recovery_failed"),
|
||||||
|
Mockito.eq("未能重新排队任何失败文件,请查看文件错误后继续"),
|
||||||
|
Mockito.any(Date.class)
|
||||||
|
)).thenReturn(1);
|
||||||
|
DocumentImportBatchCircuitBreaker circuitBreaker =
|
||||||
|
new DocumentImportBatchCircuitBreaker(
|
||||||
|
batchMapper,
|
||||||
|
Mockito.mock(DocumentImportBatchItemMapper.class),
|
||||||
|
Mockito.mock(DocumentImportTaskMapper.class)
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertTrue(circuitBreaker.interruptBatch(
|
||||||
|
batchId,
|
||||||
|
new IllegalStateException(
|
||||||
|
"recovery failed", new DocumentImportRecoveryException())
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证恢复与任务熔断 SQL 都包含对应所有权围栏。
|
* 验证恢复与任务熔断 SQL 都包含对应所有权围栏。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package tech.easyflow.ai.documentimport.task;
|
package tech.easyflow.ai.documentimport.task;
|
||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.springframework.transaction.annotation.Propagation;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
import tech.easyflow.ai.documentimport.DocumentImportBatchDtos;
|
||||||
import tech.easyflow.ai.entity.DocumentImportBatch;
|
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||||
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
import tech.easyflow.ai.entity.DocumentImportBatchItem;
|
||||||
@@ -13,6 +15,7 @@ import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
|||||||
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
import tech.easyflow.ai.service.DocumentImportBatchItemService;
|
||||||
import tech.easyflow.ai.service.DocumentImportBatchService;
|
import tech.easyflow.ai.service.DocumentImportBatchService;
|
||||||
|
|
||||||
|
import java.lang.reflect.Method;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
|
|
||||||
@@ -25,6 +28,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
|
|||||||
import static org.mockito.ArgumentMatchers.eq;
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.never;
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.inOrder;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
@@ -62,6 +66,39 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
verify(batchService, never()).updateById(batch, false);
|
verify(batchService, never()).updateById(batch, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证任务变更始终先锁批次,但只有自动批次受运行态门禁约束。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldAllowManualTaskAfterUploadBatchCompleted() {
|
||||||
|
BigInteger automaticBatchId = BigInteger.valueOf(101);
|
||||||
|
BigInteger manualBatchId = BigInteger.valueOf(102);
|
||||||
|
DocumentImportBatch automatic = new DocumentImportBatch();
|
||||||
|
automatic.setId(automaticBatchId);
|
||||||
|
automatic.setImportMode(DocumentImportMode.AUTO.name());
|
||||||
|
automatic.setStatus(DocumentImportBatchStatus.COMPLETED.name());
|
||||||
|
DocumentImportBatch manual = new DocumentImportBatch();
|
||||||
|
manual.setId(manualBatchId);
|
||||||
|
manual.setImportMode(DocumentImportMode.MANUAL.name());
|
||||||
|
manual.setStatus(DocumentImportBatchStatus.COMPLETED.name());
|
||||||
|
DocumentImportBatchMapper batchMapper =
|
||||||
|
mock(DocumentImportBatchMapper.class);
|
||||||
|
when(batchMapper.selectForUpdate(automaticBatchId))
|
||||||
|
.thenReturn(automatic);
|
||||||
|
when(batchMapper.selectForUpdate(manualBatchId)).thenReturn(manual);
|
||||||
|
DocumentImportBatchTracker tracker = new DocumentImportBatchTracker(
|
||||||
|
mock(DocumentImportBatchService.class),
|
||||||
|
mock(DocumentImportBatchItemService.class),
|
||||||
|
batchMapper,
|
||||||
|
mock(DocumentImportBatchItemMapper.class)
|
||||||
|
);
|
||||||
|
|
||||||
|
assertFalse(tracker.lockBatchForTaskMutation(automaticBatchId));
|
||||||
|
assertTrue(tracker.lockBatchForTaskMutation(manualBatchId));
|
||||||
|
verify(batchMapper).selectForUpdate(automaticBatchId);
|
||||||
|
verify(batchMapper).selectForUpdate(manualBatchId);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证兼容可重试计数与人工可继续的失败总数保持一致。
|
* 验证兼容可重试计数与人工可继续的失败总数保持一致。
|
||||||
*/
|
*/
|
||||||
@@ -111,6 +148,8 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
item.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
item.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||||
item.setRetryable(true);
|
item.setRetryable(true);
|
||||||
when(itemService.getById(item.getId())).thenReturn(item);
|
when(itemService.getById(item.getId())).thenReturn(item);
|
||||||
|
when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch);
|
||||||
|
when(itemMapper.selectForUpdate(item.getId())).thenReturn(item);
|
||||||
when(itemMapper.transitionStatus(
|
when(itemMapper.transitionStatus(
|
||||||
eq(item.getId()),
|
eq(item.getId()),
|
||||||
eq(DocumentImportBatchItemStatus.FAILED.name()),
|
eq(DocumentImportBatchItemStatus.FAILED.name()),
|
||||||
@@ -143,6 +182,14 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
eq(-1),
|
eq(-1),
|
||||||
any(Date.class)
|
any(Date.class)
|
||||||
);
|
);
|
||||||
|
org.mockito.InOrder lockOrder = inOrder(batchMapper, itemMapper);
|
||||||
|
lockOrder.verify(batchMapper).selectForUpdate(batch.getId());
|
||||||
|
lockOrder.verify(itemMapper).selectForUpdate(item.getId());
|
||||||
|
lockOrder.verify(itemMapper).transitionStatus(
|
||||||
|
eq(item.getId()), eq(DocumentImportBatchItemStatus.FAILED.name()),
|
||||||
|
eq(DocumentImportBatchItemStage.INDEX.name()),
|
||||||
|
eq(DocumentImportBatchItemStatus.PENDING.name()), eq(null), eq(null),
|
||||||
|
eq(false), eq(1), any(Date.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -167,6 +214,8 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
item.setRetryable(true);
|
item.setRetryable(true);
|
||||||
BigInteger documentId = BigInteger.valueOf(99);
|
BigInteger documentId = BigInteger.valueOf(99);
|
||||||
when(itemService.getById(item.getId())).thenReturn(item);
|
when(itemService.getById(item.getId())).thenReturn(item);
|
||||||
|
when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch);
|
||||||
|
when(itemMapper.selectForUpdate(item.getId())).thenReturn(item);
|
||||||
when(itemMapper.bindFailedDocument(
|
when(itemMapper.bindFailedDocument(
|
||||||
eq(item.getId()), eq(documentId), any(Date.class)
|
eq(item.getId()), eq(documentId), any(Date.class)
|
||||||
)).thenReturn(1);
|
)).thenReturn(1);
|
||||||
@@ -184,6 +233,35 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
anyInt(), anyInt(), anyInt(), any(Date.class));
|
anyInt(), anyInt(), anyInt(), any(Date.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证失败项只能在运行批次内绑定恢复创建的文档。
|
||||||
|
*/
|
||||||
|
@Test(expected = tech.easyflow.common.web.exceptions.BusinessException.class)
|
||||||
|
public void shouldRejectRecoveredDocumentOutsideRunningBatch() {
|
||||||
|
DocumentImportBatchService batchService =
|
||||||
|
mock(DocumentImportBatchService.class);
|
||||||
|
DocumentImportBatchItemService itemService =
|
||||||
|
mock(DocumentImportBatchItemService.class);
|
||||||
|
DocumentImportBatchMapper batchMapper =
|
||||||
|
mock(DocumentImportBatchMapper.class);
|
||||||
|
DocumentImportBatchItemMapper itemMapper =
|
||||||
|
mock(DocumentImportBatchItemMapper.class);
|
||||||
|
DocumentImportBatch batch = batch(
|
||||||
|
DocumentImportBatchStatus.PARTIAL_SUCCEEDED, 1);
|
||||||
|
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||||
|
item.setId(BigInteger.TEN);
|
||||||
|
item.setBatchId(batch.getId());
|
||||||
|
item.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||||
|
when(itemService.getById(item.getId())).thenReturn(item);
|
||||||
|
when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch);
|
||||||
|
when(itemMapper.selectForUpdate(item.getId())).thenReturn(item);
|
||||||
|
DocumentImportBatchTracker tracker =
|
||||||
|
new DocumentImportBatchTracker(
|
||||||
|
batchService, itemService, batchMapper, itemMapper);
|
||||||
|
|
||||||
|
tracker.bindDocument(item.getId(), BigInteger.valueOf(99));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证迟到任务不能把已完成文件重新改为处理中。
|
* 验证迟到任务不能把已完成文件重新改为处理中。
|
||||||
*/
|
*/
|
||||||
@@ -198,6 +276,9 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
item.setBatchId(BigInteger.ONE);
|
item.setBatchId(BigInteger.ONE);
|
||||||
item.setStatus(DocumentImportBatchItemStatus.COMPLETED.name());
|
item.setStatus(DocumentImportBatchItemStatus.COMPLETED.name());
|
||||||
when(itemService.getById(item.getId())).thenReturn(item);
|
when(itemService.getById(item.getId())).thenReturn(item);
|
||||||
|
DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1);
|
||||||
|
when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch);
|
||||||
|
when(itemMapper.selectForUpdate(item.getId())).thenReturn(item);
|
||||||
|
|
||||||
DocumentImportBatchTracker tracker =
|
DocumentImportBatchTracker tracker =
|
||||||
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
|
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
|
||||||
@@ -210,6 +291,87 @@ public class DocumentImportBatchTrackerTest {
|
|||||||
any(), any(), any(), any(), any(), any(), anyBoolean(), anyInt(), any(Date.class));
|
any(), any(), any(), any(), any(), any(), anyBoolean(), anyInt(), any(Date.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证批次中断后,已到达的工作线程不能再迁移文件项。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void shouldFenceLateTransitionAfterBatchInterrupted() {
|
||||||
|
DocumentImportBatchService batchService = mock(DocumentImportBatchService.class);
|
||||||
|
DocumentImportBatchItemService itemService = mock(DocumentImportBatchItemService.class);
|
||||||
|
DocumentImportBatchMapper batchMapper = mock(DocumentImportBatchMapper.class);
|
||||||
|
DocumentImportBatchItemMapper itemMapper = mock(DocumentImportBatchItemMapper.class);
|
||||||
|
DocumentImportBatch batch = batch(DocumentImportBatchStatus.INTERRUPTED, 1);
|
||||||
|
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||||
|
item.setId(BigInteger.TEN);
|
||||||
|
item.setBatchId(batch.getId());
|
||||||
|
item.setStatus(DocumentImportBatchItemStatus.PENDING.name());
|
||||||
|
when(itemService.getById(item.getId())).thenReturn(item);
|
||||||
|
when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch);
|
||||||
|
when(itemMapper.selectForUpdate(item.getId())).thenReturn(item);
|
||||||
|
|
||||||
|
DocumentImportBatchTracker tracker =
|
||||||
|
new DocumentImportBatchTracker(batchService, itemService, batchMapper, itemMapper);
|
||||||
|
|
||||||
|
assertFalse(tracker.updateItem(item.getId(),
|
||||||
|
DocumentImportBatchItemStage.PARSE,
|
||||||
|
DocumentImportBatchItemStatus.RUNNING,
|
||||||
|
null));
|
||||||
|
verify(itemMapper, never()).transitionStatus(
|
||||||
|
any(), any(), any(), any(), any(), any(), anyBoolean(), anyInt(), any(Date.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证恢复失败原因在提交后回调内仍使用独立事务持久化。
|
||||||
|
*
|
||||||
|
* @throws Exception 方法不存在时抛出
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void failedRetryErrorShouldUseIndependentTransaction()
|
||||||
|
throws Exception {
|
||||||
|
Method method = DocumentImportBatchTracker.class.getMethod(
|
||||||
|
"updateFailedRetryError",
|
||||||
|
BigInteger.class,
|
||||||
|
BigInteger.class,
|
||||||
|
String.class
|
||||||
|
);
|
||||||
|
Transactional transactional = method.getAnnotation(Transactional.class);
|
||||||
|
|
||||||
|
assertEquals(Propagation.REQUIRES_NEW, transactional.propagation());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 验证失败原因写回与等价并发推进在同一锁内完成判定。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void failedRetryErrorShouldRecognizeConcurrentAdvance() {
|
||||||
|
DocumentImportBatchService batchService =
|
||||||
|
mock(DocumentImportBatchService.class);
|
||||||
|
DocumentImportBatchItemService itemService =
|
||||||
|
mock(DocumentImportBatchItemService.class);
|
||||||
|
DocumentImportBatchMapper batchMapper =
|
||||||
|
mock(DocumentImportBatchMapper.class);
|
||||||
|
DocumentImportBatchItemMapper itemMapper =
|
||||||
|
mock(DocumentImportBatchItemMapper.class);
|
||||||
|
DocumentImportBatch batch = batch(DocumentImportBatchStatus.RUNNING, 1);
|
||||||
|
DocumentImportBatchItem item = new DocumentImportBatchItem();
|
||||||
|
item.setId(BigInteger.TEN);
|
||||||
|
item.setBatchId(batch.getId());
|
||||||
|
item.setStatus(DocumentImportBatchItemStatus.PENDING.name());
|
||||||
|
when(itemService.getById(item.getId())).thenReturn(item);
|
||||||
|
when(batchMapper.selectForUpdate(batch.getId())).thenReturn(batch);
|
||||||
|
when(itemMapper.selectForUpdate(item.getId())).thenReturn(item);
|
||||||
|
DocumentImportBatchTracker tracker =
|
||||||
|
new DocumentImportBatchTracker(
|
||||||
|
batchService, itemService, batchMapper, itemMapper);
|
||||||
|
|
||||||
|
assertEquals(
|
||||||
|
DocumentImportBatchTracker.FailedRetryErrorOutcome.ALREADY_ADVANCED,
|
||||||
|
tracker.updateFailedRetryError(
|
||||||
|
item.getId(), batch.getId(), "状态已变化"));
|
||||||
|
verify(itemMapper, never()).updateFailedRetryError(
|
||||||
|
any(), any(), any(), any(Date.class));
|
||||||
|
}
|
||||||
|
|
||||||
private DocumentImportBatch batch(DocumentImportBatchStatus status, int totalCount) {
|
private DocumentImportBatch batch(DocumentImportBatchStatus status, int totalCount) {
|
||||||
DocumentImportBatch batch = new DocumentImportBatch();
|
DocumentImportBatch batch = new DocumentImportBatch();
|
||||||
batch.setId(BigInteger.ONE);
|
batch.setId(BigInteger.ONE);
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
package tech.easyflow.ai.documentimport.task;
|
||||||
|
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.springframework.transaction.annotation.Propagation;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import tech.easyflow.ai.entity.DocumentImportBatch;
|
||||||
|
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||||
|
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link DocumentImportRecoveryCoordinator} 恢复令牌事务测试。
|
||||||
|
*/
|
||||||
|
public class DocumentImportRecoveryCoordinatorTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void claimShouldReturnOnlyCurrentTokenBatch() {
|
||||||
|
DocumentImportBatchMapper mapper =
|
||||||
|
Mockito.mock(DocumentImportBatchMapper.class);
|
||||||
|
DocumentImportRecoveryCoordinator coordinator =
|
||||||
|
new DocumentImportRecoveryCoordinator(mapper);
|
||||||
|
BigInteger batchId = BigInteger.valueOf(81);
|
||||||
|
String token = "token";
|
||||||
|
Date claimedAt = new Date(1_000L);
|
||||||
|
Date leaseUntil = new Date(121_000L);
|
||||||
|
DocumentImportBatch batch = new DocumentImportBatch();
|
||||||
|
batch.setId(batchId);
|
||||||
|
Mockito.when(mapper.claimRecoveryPending(
|
||||||
|
batchId, token, leaseUntil, claimedAt)).thenReturn(1);
|
||||||
|
Mockito.when(mapper.selectClaimedRecovery(batchId, token))
|
||||||
|
.thenReturn(batch);
|
||||||
|
|
||||||
|
Assert.assertSame(batch, coordinator.claim(
|
||||||
|
batchId, token, leaseUntil, claimedAt));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void claimShouldSkipReadWhenTokenWasNotAcquired() {
|
||||||
|
DocumentImportBatchMapper mapper =
|
||||||
|
Mockito.mock(DocumentImportBatchMapper.class);
|
||||||
|
DocumentImportRecoveryCoordinator coordinator =
|
||||||
|
new DocumentImportRecoveryCoordinator(mapper);
|
||||||
|
|
||||||
|
Assert.assertNull(coordinator.claim(
|
||||||
|
BigInteger.ONE, "token", new Date(2_000L), new Date(1_000L)));
|
||||||
|
Mockito.verify(mapper, Mockito.never())
|
||||||
|
.selectClaimedRecovery(Mockito.any(), Mockito.anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void tokenMutationsShouldAlwaysUseIndependentTransactions()
|
||||||
|
throws Exception {
|
||||||
|
assertRequiresNew("claim", BigInteger.class, String.class,
|
||||||
|
Date.class, Date.class);
|
||||||
|
assertRequiresNew("renew", BigInteger.class, String.class,
|
||||||
|
Date.class, Date.class);
|
||||||
|
assertRequiresNew("finalizeRecovery", BigInteger.class,
|
||||||
|
String.class, Date.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertRequiresNew(String methodName, Class<?>... parameterTypes)
|
||||||
|
throws Exception {
|
||||||
|
Method method = DocumentImportRecoveryCoordinator.class.getMethod(
|
||||||
|
methodName, parameterTypes);
|
||||||
|
Transactional transactional = method.getAnnotation(Transactional.class);
|
||||||
|
Assert.assertNotNull(transactional);
|
||||||
|
Assert.assertEquals(Propagation.REQUIRES_NEW,
|
||||||
|
transactional.propagation());
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user