Compare commits
20 Commits
main
...
263f5f4b8b
| Author | SHA1 | Date | |
|---|---|---|---|
| 263f5f4b8b | |||
| 6daf805cd0 | |||
| 4386b2a1a6 | |||
| f28e3919ac | |||
| 0cedf85729 | |||
| 155af9989c | |||
| 8c174e5c02 | |||
| 17ef189862 | |||
| e6b2e2798f | |||
| 4823b0741f | |||
| 1c68e3582c | |||
| 619b60600d | |||
| 240b84063a | |||
| ec6e03587a | |||
| 1ccdafdb47 | |||
| e38821e48a | |||
| 3e79e99925 | |||
| 98b34bd4bb | |||
| c27e97bcc2 | |||
| 9068d42f4d |
@@ -40,6 +40,10 @@
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-module-job</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-module-dataspace</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-captcha</artifactId>
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package tech.easyflow.admin.controller.ai;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaIgnore;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
import tech.easyflow.admin.service.ai.WorkflowPublicChatService;
|
||||
import tech.easyflow.ai.share.WorkflowSharePolicy;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.vo.UploadResVo;
|
||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 工作流对话匿名分享接口。
|
||||
*/
|
||||
@SaIgnore
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/workflowChat/public")
|
||||
public class WorkflowPublicChatController {
|
||||
|
||||
private final WorkflowPublicChatService publicChatService;
|
||||
|
||||
public WorkflowPublicChatController(
|
||||
WorkflowPublicChatService publicChatService
|
||||
) {
|
||||
this.publicChatService = publicChatService;
|
||||
}
|
||||
|
||||
@GetMapping("/descriptor")
|
||||
public Result<Map<String, Object>> descriptor(HttpServletRequest request) {
|
||||
return Result.ok(publicChatService.descriptor(
|
||||
shareKey(request),
|
||||
visitorId(request)
|
||||
));
|
||||
}
|
||||
|
||||
@PostMapping(value = "/run", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
|
||||
public SseEmitter run(
|
||||
@JsonBody("variables") Map<String, Object> variables,
|
||||
HttpServletRequest request
|
||||
) {
|
||||
return publicChatService.run(
|
||||
shareKey(request),
|
||||
visitorId(request),
|
||||
variables
|
||||
);
|
||||
}
|
||||
|
||||
@GetMapping("/execution")
|
||||
public Result<Map<String, Object>> execution(
|
||||
@RequestParam String executeId,
|
||||
HttpServletRequest request
|
||||
) {
|
||||
return Result.ok(publicChatService.detail(
|
||||
shareKey(request),
|
||||
visitorId(request),
|
||||
executeId
|
||||
));
|
||||
}
|
||||
|
||||
@PostMapping("/cancel")
|
||||
public Result<Boolean> cancel(
|
||||
@JsonBody(value = "executeId", required = true) String executeId,
|
||||
HttpServletRequest request
|
||||
) {
|
||||
return Result.ok(publicChatService.cancel(
|
||||
shareKey(request),
|
||||
visitorId(request),
|
||||
executeId
|
||||
));
|
||||
}
|
||||
|
||||
@PostMapping("/resume")
|
||||
public Result<Void> resume(
|
||||
@JsonBody(value = "executeId", required = true) String executeId,
|
||||
@JsonBody("confirmParams") Map<String, Object> confirmParams,
|
||||
HttpServletRequest request
|
||||
) {
|
||||
publicChatService.resume(
|
||||
shareKey(request),
|
||||
visitorId(request),
|
||||
executeId,
|
||||
confirmParams
|
||||
);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
@PostMapping(value = "/upload", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public Result<UploadResVo> upload(
|
||||
@RequestParam("file") MultipartFile file,
|
||||
@RequestParam("parameterName") String parameterName,
|
||||
HttpServletRequest request
|
||||
) {
|
||||
return Result.ok(publicChatService.upload(
|
||||
shareKey(request),
|
||||
visitorId(request),
|
||||
parameterName,
|
||||
file
|
||||
));
|
||||
}
|
||||
|
||||
private String shareKey(HttpServletRequest request) {
|
||||
return request.getHeader(WorkflowSharePolicy.CHAT_SHARE_KEY_HEADER);
|
||||
}
|
||||
|
||||
private String visitorId(HttpServletRequest request) {
|
||||
return request.getHeader(WorkflowSharePolicy.CHAT_VISITOR_HEADER);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package tech.easyflow.admin.controller.ai;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import cn.dev33.satoken.annotation.SaIgnore;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
@@ -96,11 +97,10 @@ public class WorkflowShareController {
|
||||
* @return 工作流标识
|
||||
*/
|
||||
@GetMapping("/resolve")
|
||||
@SaIgnore
|
||||
public Result<Map<String, BigInteger>> resolveUrlShare(HttpServletRequest request) {
|
||||
LoginAccount loginAccount = SaTokenUtil.getLoginAccount();
|
||||
WorkflowShare share = workflowShareService.resolveChatShare(
|
||||
request.getHeader(WorkflowSharePolicy.CHAT_SHARE_KEY_HEADER),
|
||||
loginAccount.getTenantId()
|
||||
WorkflowShare share = workflowShareService.resolvePublicChatShare(
|
||||
request.getHeader(WorkflowSharePolicy.CHAT_SHARE_KEY_HEADER)
|
||||
);
|
||||
return Result.ok(Map.of("workflowId", share.getWorkflowId()));
|
||||
}
|
||||
|
||||
@@ -24,6 +24,11 @@ import java.util.List;
|
||||
@RequestMapping("/api/v1/datacenterDataset")
|
||||
public class DatacenterDatasetController {
|
||||
|
||||
/** 对外 Schema 接口的默认字段页码。 */
|
||||
private static final long DEFAULT_FIELD_PAGE_NUMBER = 1L;
|
||||
/** 对外 Schema 接口的默认字段页大小。 */
|
||||
private static final long DEFAULT_FIELD_PAGE_SIZE = 200L;
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
@Resource
|
||||
@@ -32,13 +37,18 @@ public class DatacenterDatasetController {
|
||||
@PostMapping("/queryPage")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<Page<Row>> queryPage(@RequestBody DatacenterQueryRequest request) {
|
||||
return Result.ok(queryService.queryPage(request));
|
||||
return Result.ok(queryService.queryPage(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
@GetMapping("/schema")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterSchemaResponse> schema(DatasetRef datasetRef) {
|
||||
return Result.ok(queryService.getSchema(datasetRef));
|
||||
public Result<DatacenterSchemaResponse> schema(
|
||||
DatasetRef datasetRef,
|
||||
@RequestParam(defaultValue = "1") Long fieldPageNumber,
|
||||
@RequestParam(defaultValue = "200") Long fieldPageSize) {
|
||||
return Result.ok(queryService.getSchema(
|
||||
datasetRef, fieldPageNumber, fieldPageSize));
|
||||
}
|
||||
|
||||
@GetMapping("/managedTables")
|
||||
@@ -63,6 +73,13 @@ public class DatacenterDatasetController {
|
||||
request == null ? List.of() : request.getFields(),
|
||||
account
|
||||
);
|
||||
return Result.ok(queryService.getSchema(registryService.resolveDatasetRef(table.getId())));
|
||||
return Result.ok(queryService.getSchema(
|
||||
registryService.resolveDatasetRef(table.getId()),
|
||||
request == null || request.getFieldPageNumber() == null
|
||||
? DEFAULT_FIELD_PAGE_NUMBER
|
||||
: request.getFieldPageNumber(),
|
||||
request == null || request.getFieldPageSize() == null
|
||||
? DEFAULT_FIELD_PAGE_SIZE
|
||||
: request.getFieldPageSize()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package tech.easyflow.admin.controller.datacenter;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlCancelRequest;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
/**
|
||||
* 数据中枢管理端只读 SQL 控制台。
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterQuery")
|
||||
public class DatacenterQueryController {
|
||||
|
||||
private final DatacenterDatasetRegistryService registryService;
|
||||
private final DatacenterFederationQueryService queryService;
|
||||
private final DatacenterFederationQueryCancellationService cancellationService;
|
||||
|
||||
/**
|
||||
* 创建查询 Controller。
|
||||
*
|
||||
* @param registryService 数据集注册服务
|
||||
* @param queryService Federation 查询服务
|
||||
* @param cancellationService 跨节点查询取消服务
|
||||
*/
|
||||
public DatacenterQueryController(
|
||||
DatacenterDatasetRegistryService registryService,
|
||||
DatacenterFederationQueryService queryService,
|
||||
DatacenterFederationQueryCancellationService cancellationService) {
|
||||
this.registryService = registryService;
|
||||
this.queryService = queryService;
|
||||
this.cancellationService = cancellationService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行一条受 Calcite 与业务 Policy 校验的只读 SQL。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @return 有界查询结果
|
||||
*/
|
||||
@PostMapping("/execute")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterSqlConsoleResult> execute(
|
||||
@RequestBody DatacenterSqlConsoleRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
DatacenterSource source = registryService.getSourceRequired(
|
||||
request == null ? null : request.sourceId());
|
||||
return Result.ok(queryService.execute(
|
||||
source,
|
||||
request == null ? null : request.sql(),
|
||||
java.util.List.of(),
|
||||
request == null ? null : request.maxRows(),
|
||||
account,
|
||||
"MANUAL",
|
||||
account == null || account.getId() == null
|
||||
? null : account.getId().toString(),
|
||||
request == null ? null : request.queryId()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消当前租户在任一节点执行的 SQL 查询。
|
||||
*
|
||||
* @param request 取消请求
|
||||
* @return 是否在本地或集群中接受取消提示
|
||||
*/
|
||||
@PostMapping("/cancel")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<Boolean> cancel(@RequestBody DatacenterSqlCancelRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(cancellationService.cancel(
|
||||
request == null ? null : request.queryId(), account));
|
||||
}
|
||||
}
|
||||
@@ -8,10 +8,16 @@ import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateMetadataRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateCatalogRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||
|
||||
@@ -19,6 +25,9 @@ import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据源绑定、生命周期与元数据浏览接口。
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterSource")
|
||||
public class DatacenterSourceController {
|
||||
@@ -26,23 +35,100 @@ public class DatacenterSourceController {
|
||||
@Resource
|
||||
private DatacenterSourceService sourceService;
|
||||
|
||||
@PostMapping("/testConnection")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterConnectionTestResult> testConnection(@RequestBody DatacenterSource source) {
|
||||
@PostMapping("/draft")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> saveDraft(@RequestBody DatacenterSourceDraftRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.testConnection(source, account));
|
||||
return Result.ok(sourceService.saveDraft(request, account));
|
||||
}
|
||||
|
||||
@PostMapping("/save")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSource> save(@RequestBody DatacenterSource source) {
|
||||
@PostMapping("/{sourceId}/probe")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterConnectionTestResult> probe(@PathVariable BigInteger sourceId) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.saveSource(source, account));
|
||||
return Result.ok(sourceService.probe(sourceId, account));
|
||||
}
|
||||
|
||||
@PostMapping("/activate")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> activate(@RequestBody DatacenterSourceActivateRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.activate(request, account));
|
||||
}
|
||||
|
||||
/**
|
||||
* 探测活动数据源的未发布候选配置。
|
||||
*
|
||||
* @param request 候选连接配置
|
||||
* @return 连接探测结果
|
||||
*/
|
||||
@PostMapping("/candidate/probe")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterConnectionTestResult> probeCandidate(
|
||||
@RequestBody DatacenterSourceDraftRequest request) {
|
||||
return Result.ok(sourceService.probeCandidate(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览活动数据源候选配置可访问的命名空间。
|
||||
*
|
||||
* @param request 候选连接配置
|
||||
* @return 命名空间列表
|
||||
*/
|
||||
@PostMapping("/candidate/catalogs")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<List<DatacenterCatalogMeta>> candidateCatalogs(
|
||||
@RequestBody DatacenterSourceDraftRequest request) {
|
||||
return Result.ok(sourceService.listCandidateCatalogs(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页浏览候选配置可访问的命名空间。
|
||||
*
|
||||
* @param request 候选配置和分页条件
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
@PostMapping("/candidate/catalogs/page")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterMetadataPage<DatacenterCatalogMeta>> candidateCatalogsPage(
|
||||
@RequestBody DatacenterSourceCandidateCatalogRequest request) {
|
||||
return Result.ok(sourceService.listCandidateCatalogsPage(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页浏览活动数据源候选配置可访问的表。
|
||||
*
|
||||
* @param request 候选配置与分页条件
|
||||
* @return 有界表列表
|
||||
*/
|
||||
@PostMapping("/candidate/tables")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterMetadataPage<DatacenterTable>> candidateTables(
|
||||
@RequestBody DatacenterSourceCandidateMetadataRequest request) {
|
||||
return Result.ok(sourceService.listCandidateTables(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 原地发布活动数据源的新连接配置和纳管范围。
|
||||
*
|
||||
* @param request 重配置请求
|
||||
* @return 发布后的数据源视图
|
||||
*/
|
||||
@PostMapping("/reconfigure")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> reconfigure(
|
||||
@RequestBody DatacenterSourceReconfigureRequest request) {
|
||||
return Result.ok(sourceService.reconfigure(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
@GetMapping("/page")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<Page<DatacenterSource>> page(Long pageNumber, Long pageSize) {
|
||||
public Result<Page<DatacenterSourceView>> page(Long pageNumber, Long pageSize) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.pageSources(pageNumber, pageSize, account));
|
||||
}
|
||||
@@ -54,19 +140,53 @@ public class DatacenterSourceController {
|
||||
return Result.ok(sourceService.listCatalogs(sourceId, account));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页浏览当前数据源的命名空间。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param keyword 名称搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
@GetMapping("/catalogs/page")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterMetadataPage<DatacenterCatalogMeta>> catalogsPage(
|
||||
BigInteger sourceId,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
return Result.ok(sourceService.listCatalogsPage(
|
||||
sourceId, keyword, pageNumber, pageSize,
|
||||
SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
@GetMapping("/tables")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<List<DatacenterTable>> tables(BigInteger sourceId, String catalogName) {
|
||||
public Result<DatacenterMetadataPage<DatacenterTable>> tables(
|
||||
BigInteger sourceId,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.listTables(sourceId, catalogName, account));
|
||||
return Result.ok(sourceService.listTables(
|
||||
sourceId, catalogName, keyword, pageNumber, pageSize, account));
|
||||
}
|
||||
|
||||
@GetMapping("/tableDetail")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterTableDetailMeta> tableDetail(BigInteger sourceId, String catalogName, String tableName,
|
||||
@RequestParam(defaultValue = "false") boolean register) {
|
||||
public Result<DatacenterTableDetailMeta> tableDetail(
|
||||
BigInteger sourceId,
|
||||
String catalogName,
|
||||
String tableName,
|
||||
@RequestParam(defaultValue = "false") boolean register,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.getTableDetail(sourceId, catalogName, tableName, register, account));
|
||||
return Result.ok(sourceService.getTableDetail(
|
||||
sourceId, catalogName, tableName, register,
|
||||
fieldPageNumber, fieldPageSize, account));
|
||||
}
|
||||
|
||||
@PostMapping("/registerBatch")
|
||||
@@ -83,4 +203,44 @@ public class DatacenterSourceController {
|
||||
sourceService.removeSource(request == null ? null : request.getSourceId(), account);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
/**
|
||||
* 停用活动数据源。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @return 停用后的数据源视图
|
||||
*/
|
||||
@PostMapping("/{sourceId}/disable")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> disable(@PathVariable BigInteger sourceId) {
|
||||
return Result.ok(sourceService.disable(
|
||||
sourceId, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 重新启用已停用数据源。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @return 启用后的数据源视图
|
||||
*/
|
||||
@PostMapping("/{sourceId}/enable")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> enable(@PathVariable BigInteger sourceId) {
|
||||
return Result.ok(sourceService.enable(
|
||||
sourceId, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新已纳管对象的 JDBC 元数据观测状态。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @return 刷新后的数据源视图
|
||||
*/
|
||||
@PostMapping("/{sourceId}/metadata/refresh")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> refreshMetadata(
|
||||
@PathVariable BigInteger sourceId) {
|
||||
return Result.ok(sourceService.refreshMetadata(
|
||||
sourceId, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package tech.easyflow.admin.controller.dataspace;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.dataspace.model.ConnectionDefinition;
|
||||
import tech.easyflow.dataspace.model.ConnectionView;
|
||||
import tech.easyflow.dataspace.model.ObjectView;
|
||||
import tech.easyflow.dataspace.provider.DataspaceProbe;
|
||||
import tech.easyflow.dataspace.service.DataspaceConnectionService;
|
||||
|
||||
/**
|
||||
* 数据空间物理连接管理端 API。
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/dataspaceConnection")
|
||||
public class DataspaceConnectionController {
|
||||
|
||||
private final DataspaceConnectionService connectionService;
|
||||
|
||||
/**
|
||||
* 创建连接控制器。
|
||||
*
|
||||
* @param connectionService 连接服务
|
||||
*/
|
||||
public DataspaceConnectionController(DataspaceConnectionService connectionService) {
|
||||
this.connectionService = connectionService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询当前租户连接列表。
|
||||
*
|
||||
* @param keyword 搜索关键词
|
||||
* @return 连接列表
|
||||
*/
|
||||
@GetMapping("/list")
|
||||
@SaCheckPermission("/api/v1/dataspaceConnection/query")
|
||||
public Result<List<ConnectionView>> list(String keyword) {
|
||||
return Result.ok(connectionService.list(keyword));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取连接详情。
|
||||
*
|
||||
* @param id 连接 ID
|
||||
* @return 连接详情
|
||||
*/
|
||||
@GetMapping("/detail")
|
||||
@SaCheckPermission("/api/v1/dataspaceConnection/query")
|
||||
public Result<ConnectionView> detail(BigInteger id) {
|
||||
return Result.ok(connectionService.detail(id));
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试候选或已保存连接。
|
||||
*
|
||||
* @param definition 连接定义
|
||||
* @return 测试结果
|
||||
*/
|
||||
@PostMapping("/test")
|
||||
@SaCheckPermission("/api/v1/dataspaceConnection/test")
|
||||
public Result<DataspaceProbe> test(
|
||||
@JsonBody(required = true, skipConvertError = false) ConnectionDefinition definition) {
|
||||
return Result.ok(connectionService.test(definition));
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建或更新连接。
|
||||
*
|
||||
* @param definition 连接定义
|
||||
* @return 保存后的连接
|
||||
*/
|
||||
@PostMapping("/save")
|
||||
@SaCheckPermission("/api/v1/dataspaceConnection/save")
|
||||
public Result<ConnectionView> save(
|
||||
@JsonBody(required = true, skipConvertError = false) ConnectionDefinition definition) {
|
||||
return Result.ok(connectionService.save(definition));
|
||||
}
|
||||
|
||||
/**
|
||||
* 启用或禁用连接。
|
||||
*
|
||||
* @param request 状态变更请求
|
||||
* @return 变更后的连接
|
||||
*/
|
||||
@PostMapping("/status")
|
||||
@SaCheckPermission("/api/v1/dataspaceConnection/save")
|
||||
public Result<ConnectionView> status(
|
||||
@JsonBody(required = true, skipConvertError = false) StatusRequest request) {
|
||||
if (request == null || request.enabled() == null) {
|
||||
throw new BusinessException("连接状态不能为空");
|
||||
}
|
||||
return Result.ok(connectionService.setEnabled(request.id(), request.enabled()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询当前连接的对象树数据。
|
||||
*
|
||||
* @param connectionId 连接 ID
|
||||
* @param keyword Schema 或表名关键词
|
||||
* @return 对象列表
|
||||
*/
|
||||
@GetMapping("/objects")
|
||||
@SaCheckPermission("/api/v1/dataspaceConnection/query")
|
||||
public Result<List<ObjectView>> objects(BigInteger connectionId, String keyword) {
|
||||
return Result.ok(connectionService.objects(connectionId, keyword));
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新连接元数据。
|
||||
*
|
||||
* @param request 刷新请求
|
||||
* @return 新 revision 对象列表
|
||||
*/
|
||||
@PostMapping("/refreshMetadata")
|
||||
@SaCheckPermission("/api/v1/dataspaceConnection/metadata")
|
||||
public Result<List<ObjectView>> refreshMetadata(
|
||||
@JsonBody(required = true, skipConvertError = false) RefreshRequest request) {
|
||||
return Result.ok(connectionService.refreshMetadata(
|
||||
request.connectionId(), request.expectedRevision()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除未被引用的连接。
|
||||
*
|
||||
* @param id 连接 ID
|
||||
* @return 成功结果
|
||||
*/
|
||||
@PostMapping("/remove")
|
||||
@SaCheckPermission("/api/v1/dataspaceConnection/remove")
|
||||
public Result<Void> remove(
|
||||
@JsonBody(value = "id", required = true, skipConvertError = false) BigInteger id) {
|
||||
connectionService.remove(id);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
/**
|
||||
* 元数据刷新请求。
|
||||
*
|
||||
* @param connectionId 连接 ID
|
||||
* @param expectedRevision 期望 revision
|
||||
*/
|
||||
public record RefreshRequest(BigInteger connectionId, long expectedRevision) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接状态变更请求。
|
||||
*
|
||||
* @param id 连接 ID
|
||||
* @param enabled 是否启用
|
||||
*/
|
||||
public record StatusRequest(BigInteger id, Boolean enabled) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package tech.easyflow.admin.controller.dataspace;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.dataspace.model.DataspaceDefinition;
|
||||
import tech.easyflow.dataspace.model.DataspaceSummary;
|
||||
import tech.easyflow.dataspace.model.DataspaceView;
|
||||
import tech.easyflow.dataspace.service.DataspaceService;
|
||||
|
||||
/**
|
||||
* 虚拟数据空间管理端 API。
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/dataspace")
|
||||
public class DataspaceController {
|
||||
|
||||
private final DataspaceService dataspaceService;
|
||||
|
||||
/**
|
||||
* 创建数据空间控制器。
|
||||
*
|
||||
* @param dataspaceService 数据空间服务
|
||||
*/
|
||||
public DataspaceController(DataspaceService dataspaceService) {
|
||||
this.dataspaceService = dataspaceService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询数据空间列表。
|
||||
*
|
||||
* @param keyword 搜索关键词
|
||||
* @return 数据空间摘要
|
||||
*/
|
||||
@GetMapping("/list")
|
||||
@SaCheckPermission("/api/v1/dataspace/query")
|
||||
public Result<List<DataspaceSummary>> list(String keyword) {
|
||||
return Result.ok(dataspaceService.list(keyword));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取数据空间当前 revision 详情。
|
||||
*
|
||||
* @param id 数据空间 ID
|
||||
* @return 数据空间详情
|
||||
*/
|
||||
@GetMapping("/detail")
|
||||
@SaCheckPermission("/api/v1/dataspace/detail")
|
||||
public Result<DataspaceView> detail(BigInteger id) {
|
||||
return Result.ok(dataspaceService.detail(id));
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存数据空间并生成新 revision。
|
||||
*
|
||||
* @param definition 数据空间定义
|
||||
* @return 保存后的详情
|
||||
*/
|
||||
@PostMapping("/save")
|
||||
@SaCheckPermission("/api/v1/dataspace/save")
|
||||
public Result<DataspaceView> save(
|
||||
@RequestBody DataspaceDefinition definition) {
|
||||
return Result.ok(dataspaceService.save(definition));
|
||||
}
|
||||
|
||||
/**
|
||||
* 启用或禁用数据空间。
|
||||
*
|
||||
* @param request 状态变更请求
|
||||
* @return 成功结果
|
||||
*/
|
||||
@PostMapping("/status")
|
||||
@SaCheckPermission("/api/v1/dataspace/save")
|
||||
public Result<Void> status(
|
||||
@JsonBody(required = true, skipConvertError = false) StatusRequest request) {
|
||||
if (request == null || request.enabled() == null) {
|
||||
throw new BusinessException("数据空间状态不能为空");
|
||||
}
|
||||
dataspaceService.setEnabled(request.id(), request.enabled());
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
/**
|
||||
* 逻辑删除数据空间。
|
||||
*
|
||||
* @param id 数据空间 ID
|
||||
* @return 成功结果
|
||||
*/
|
||||
@PostMapping("/remove")
|
||||
@SaCheckPermission("/api/v1/dataspace/remove")
|
||||
public Result<Void> remove(
|
||||
@JsonBody(value = "id", required = true, skipConvertError = false) BigInteger id) {
|
||||
dataspaceService.remove(id);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
/**
|
||||
* 数据空间状态变更请求。
|
||||
*
|
||||
* @param id 数据空间 ID
|
||||
* @param enabled 是否启用
|
||||
*/
|
||||
public record StatusRequest(BigInteger id, Boolean enabled) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package tech.easyflow.admin.controller.dataspace;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||
import tech.easyflow.dataspace.model.DataspaceExplainResult;
|
||||
import tech.easyflow.dataspace.model.DataspaceQueryRequest;
|
||||
import tech.easyflow.dataspace.model.DataspaceQueryResult;
|
||||
import tech.easyflow.dataspace.model.DataspaceSqlCompletionRequest;
|
||||
import tech.easyflow.dataspace.model.DataspaceSqlCompletionResult;
|
||||
import tech.easyflow.dataspace.service.DataspaceQueryService;
|
||||
|
||||
/**
|
||||
* 数据空间 SQL 工作台 Query、Explain、Complete 与 Cancel API。
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/dataspaceSql")
|
||||
public class DataspaceSqlController {
|
||||
|
||||
private final DataspaceQueryService queryService;
|
||||
|
||||
/**
|
||||
* 创建 SQL 控制器。
|
||||
*
|
||||
* @param queryService 查询服务
|
||||
*/
|
||||
public DataspaceSqlController(DataspaceQueryService queryService) {
|
||||
this.queryService = queryService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行只读 SQL。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @return 查询结果与指标
|
||||
*/
|
||||
@PostMapping("/query")
|
||||
@SaCheckPermission("/api/v1/dataspaceSql/query")
|
||||
public Result<DataspaceQueryResult> query(
|
||||
@JsonBody(required = true, skipConvertError = false) DataspaceQueryRequest request) {
|
||||
return Result.ok(queryService.query(request));
|
||||
}
|
||||
|
||||
/**
|
||||
* 显式执行非 ANALYZE Explain。
|
||||
*
|
||||
* @param request Explain 请求
|
||||
* @return Explain 与索引信息
|
||||
*/
|
||||
@PostMapping("/explain")
|
||||
@SaCheckPermission("/api/v1/dataspaceSql/explain")
|
||||
public Result<DataspaceExplainResult> explain(
|
||||
@JsonBody(required = true, skipConvertError = false) DataspaceQueryRequest request) {
|
||||
return Result.ok(queryService.explain(request));
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回当前数据空间内的 Calcite SQL 补全候选。
|
||||
*
|
||||
* @param request 补全请求
|
||||
* @return 补全替换区间与候选
|
||||
*/
|
||||
@PostMapping("/complete")
|
||||
@SaCheckPermission("/api/v1/dataspaceSql/query")
|
||||
public Result<DataspaceSqlCompletionResult> complete(
|
||||
@JsonBody(required = true, skipConvertError = false)
|
||||
DataspaceSqlCompletionRequest request) {
|
||||
return Result.ok(queryService.complete(request));
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试取消当前节点查询。
|
||||
*
|
||||
* @param queryId 查询 ID
|
||||
* @return 是否找到并发起取消
|
||||
*/
|
||||
@PostMapping("/cancel")
|
||||
@SaCheckPermission("/api/v1/dataspaceSql/query")
|
||||
public Result<Boolean> cancel(
|
||||
@JsonBody(value = "queryId", required = true, skipConvertError = false) String queryId) {
|
||||
return Result.ok(queryService.cancel(queryId));
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,31 @@
|
||||
package tech.easyflow.admin.controller.job;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import cn.hutool.core.date.DateUtil;
|
||||
import com.easyagents.flow.core.chain.Parameter;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.quartz.CronExpression;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.enums.PublishStatus;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
|
||||
import tech.easyflow.admin.model.SysJobWorkflowOptionView;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
import tech.easyflow.job.service.SysJobService;
|
||||
@@ -32,7 +37,8 @@ import tech.easyflow.system.service.ResourceAccessService;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
@@ -61,6 +67,9 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
/** 工作流运行参数解析器。 */
|
||||
private final WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
||||
|
||||
/** 与调度计算一致的 Cron 预览格式化器。 */
|
||||
private final DateTimeFormatter jobTimeFormatter;
|
||||
|
||||
/**
|
||||
* 创建定时任务控制器。
|
||||
*
|
||||
@@ -69,17 +78,21 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
* @param workflowUsageAuthorizationService 工作流使用权限校验服务
|
||||
* @param resourceAccessService 资源访问控制服务
|
||||
* @param workflowRunningParameterResolver 工作流运行参数解析器
|
||||
* @param jobTimezone 定时任务业务时区
|
||||
*/
|
||||
public SysJobController(SysJobService service,
|
||||
WorkflowService workflowService,
|
||||
WorkflowUsageAuthorizationService workflowUsageAuthorizationService,
|
||||
ResourceAccessService resourceAccessService,
|
||||
WorkflowRunningParameterResolver workflowRunningParameterResolver) {
|
||||
WorkflowRunningParameterResolver workflowRunningParameterResolver,
|
||||
@Value("${easyflow.job.timezone:Asia/Shanghai}") String jobTimezone) {
|
||||
super(service);
|
||||
this.workflowService = workflowService;
|
||||
this.workflowUsageAuthorizationService = workflowUsageAuthorizationService;
|
||||
this.resourceAccessService = resourceAccessService;
|
||||
this.workflowRunningParameterResolver = workflowRunningParameterResolver;
|
||||
this.jobTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
|
||||
.withZone(ZoneId.of(jobTimezone));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -111,18 +124,43 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
@GetMapping("/trigger")
|
||||
@SaCheckPermission("/api/v1/sysJob/save")
|
||||
@LogRecord("立即执行定时任务")
|
||||
public Result<String> trigger(BigInteger id) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
SysJob job = requireExistingJob(id);
|
||||
validateWorkflowReference(job, account);
|
||||
return Result.ok(service.triggerNow(id));
|
||||
}
|
||||
|
||||
@GetMapping("/getNextTimes")
|
||||
@SaCheckPermission("/api/v1/sysJob/save")
|
||||
public Result<List<String>> getNextTimes(String cronExpression) throws Exception{
|
||||
CronExpression ex = new CronExpression(cronExpression);
|
||||
List<String> times = new ArrayList<>();
|
||||
Date date = new Date();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
Date next = ex.getNextValidTimeAfter(date);
|
||||
times.add(DateUtil.formatDateTime(next));
|
||||
date = next;
|
||||
public Result<List<String>> getNextTimes(String cronExpression) {
|
||||
return Result.ok(service.nextFireTimes(cronExpression, 5).stream()
|
||||
.map(Date::toInstant)
|
||||
.map(jobTimeFormatter::format)
|
||||
.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
@PostMapping("remove")
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
public Result<?> remove(@JsonBody(value = "id", required = true) Serializable id) {
|
||||
service.deleteJob(List.of(id));
|
||||
return Result.ok(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
@PostMapping("removeBatch")
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
public Result<?> removeBatch(
|
||||
@JsonBody(value = "ids", required = true) Collection<Serializable> ids) {
|
||||
if (ids == null || ids.isEmpty()) {
|
||||
return Result.fail("id不能为空");
|
||||
}
|
||||
return Result.ok(times);
|
||||
service.deleteJob(ids);
|
||||
return Result.ok(true);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -136,15 +174,19 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
List<SysJobWorkflowOptionView> options = workflowService.list(QueryWrapper.create()
|
||||
.eq(Workflow::getTenantId, account.getTenantId())
|
||||
.eq(Workflow::getStatus, EnumDataStatus.AVAILABLE.getCode())
|
||||
.eq(Workflow::getPublishStatus, PublishStatus.PUBLISHED.getCode())
|
||||
.orderBy(Workflow::getModified, false))
|
||||
.stream()
|
||||
.filter(workflow -> Objects.equals(workflow.getTenantId(), account.getTenantId()))
|
||||
.filter(workflow -> workflow.getPublishedSnapshotJson() != null
|
||||
&& !workflow.getPublishedSnapshotJson().isEmpty())
|
||||
.filter(workflow -> resourceAccessService.canAccess(
|
||||
account,
|
||||
CategoryResourceType.WORKFLOW,
|
||||
workflow,
|
||||
ResourceAction.USE))
|
||||
.map(workflowService::toPublishedView)
|
||||
.filter(Objects::nonNull)
|
||||
.map(workflow -> new SysJobWorkflowOptionView(
|
||||
workflow.getId(),
|
||||
workflow.getTitle(),
|
||||
@@ -166,7 +208,7 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||
id,
|
||||
SaTokenUtil.getLoginAccount(),
|
||||
"工作流不存在、已禁用或无权运行");
|
||||
"工作流不存在、未发布或无权运行");
|
||||
Map<String, Object> result = workflowRunningParameterResolver.buildRunningParametersView(workflow);
|
||||
if (result == null) {
|
||||
throw new BusinessException("工作流参数配置无效,请检查工作流后重试");
|
||||
@@ -182,6 +224,9 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
||||
SysJob effectiveEntity = entity;
|
||||
if (isSave) {
|
||||
// 新任务固定从 STOP 和第 0 代开始,禁止请求绕过启动协议。
|
||||
entity.setStatus(EnumJobStatus.STOP.getCode());
|
||||
entity.setScheduleGeneration(0L);
|
||||
commonFiled(entity,loginUser.getId(),loginUser.getTenantId(), loginUser.getDeptId());
|
||||
} else {
|
||||
SysJob existing = requireExistingJob(entity.getId());
|
||||
@@ -191,9 +236,25 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
entity.setModifiedBy(loginUser.getId());
|
||||
}
|
||||
validateWorkflowReference(effectiveEntity, loginUser);
|
||||
validateCronExpression(effectiveEntity.getCronExpression());
|
||||
validateMisfirePolicy(effectiveEntity.getMisfirePolicy());
|
||||
return super.onSaveOrUpdateBefore(entity, isSave);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onSaveOrUpdateAfter(SysJob entity, boolean isSave) {
|
||||
service.syncJob(entity.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
@PostMapping("update")
|
||||
public Result<?> update(@JsonBody SysJob entity) {
|
||||
Result<?> result = onSaveOrUpdateBefore(entity, false);
|
||||
if (result != null) return result;
|
||||
service.updateJobDefinition(entity);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验工作流类型任务引用的工作流可被当前用户运行。
|
||||
*
|
||||
@@ -210,7 +271,7 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||
workflowId,
|
||||
account,
|
||||
"工作流不存在、已禁用或无权运行");
|
||||
"工作流不存在、未发布或无权运行");
|
||||
validateRequiredWorkflowParams(entity, workflow);
|
||||
}
|
||||
|
||||
@@ -243,6 +304,8 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
entity.setDeptId(existing.getDeptId());
|
||||
entity.setCreated(existing.getCreated());
|
||||
entity.setCreatedBy(existing.getCreatedBy());
|
||||
entity.setStatus(existing.getStatus());
|
||||
entity.setScheduleGeneration(existing.getScheduleGeneration());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -260,9 +323,30 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
effective.setJobParams(entity.getJobParams() == null
|
||||
? existing.getJobParams()
|
||||
: entity.getJobParams());
|
||||
effective.setCronExpression(entity.getCronExpression() == null
|
||||
? existing.getCronExpression()
|
||||
: entity.getCronExpression());
|
||||
effective.setMisfirePolicy(entity.getMisfirePolicy() == null
|
||||
? existing.getMisfirePolicy()
|
||||
: entity.getMisfirePolicy());
|
||||
return effective;
|
||||
}
|
||||
|
||||
private void validateMisfirePolicy(Integer misfirePolicy) {
|
||||
if (!Integer.valueOf(EnumMisfirePolicy.FIRE_ONCE_NOW.getCode()).equals(misfirePolicy)
|
||||
&& !Integer.valueOf(EnumMisfirePolicy.SKIP.getCode()).equals(misfirePolicy)) {
|
||||
throw new BusinessException("错过策略只支持恢复后补执行一次或跳过本次");
|
||||
}
|
||||
}
|
||||
|
||||
private void validateCronExpression(String cronExpression) {
|
||||
try {
|
||||
service.nextFireTimes(cronExpression, 1);
|
||||
} catch (RuntimeException exception) {
|
||||
throw new BusinessException(400, 1, "Cron 表达式无效", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验定时任务已填写工作流的全部必填运行参数。
|
||||
*
|
||||
@@ -322,9 +406,4 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Result onRemoveBefore(Collection<Serializable> ids) {
|
||||
service.deleteJob(ids);
|
||||
return super.onRemoveBefore(ids);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,29 @@
|
||||
package tech.easyflow.admin.controller.job;
|
||||
|
||||
import tech.easyflow.common.annotation.UsePermission;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.annotation.UsePermission;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.util.StringUtil;
|
||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
import tech.easyflow.job.service.SysJobLogService;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 系统任务日志 控制层。
|
||||
*
|
||||
@@ -20,16 +34,112 @@ import tech.easyflow.job.service.SysJobLogService;
|
||||
@RequestMapping("/api/v1/sysJobLog")
|
||||
@UsePermission(moduleName = "/api/v1/sysJob")
|
||||
public class SysJobLogController extends BaseCurdController<SysJobLogService, SysJobLog> {
|
||||
public SysJobLogController(SysJobLogService service) {
|
||||
private static final long DEFAULT_PAGE_SIZE = 10L;
|
||||
private static final long MAX_PAGE_SIZE = 100L;
|
||||
private static final DateTimeFormatter QUERY_TIME_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
private final ZoneId jobZoneId;
|
||||
|
||||
public SysJobLogController(
|
||||
SysJobLogService service,
|
||||
@Value("${easyflow.job.timezone:Asia/Shanghai}") String jobTimezone) {
|
||||
super(service);
|
||||
this.jobZoneId = ZoneId.of(jobTimezone);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造日志筛选条件,并追加计划触发时间和实际触发时间范围。
|
||||
*/
|
||||
@Override
|
||||
protected QueryWrapper buildQueryWrapper(HttpServletRequest request) {
|
||||
QueryWrapper queryWrapper = super.buildQueryWrapper(request);
|
||||
Date scheduledStart = parseQueryTime(
|
||||
request.getParameter("scheduledStart"), "计划触发开始时间");
|
||||
Date scheduledEnd = parseQueryTime(
|
||||
request.getParameter("scheduledEnd"), "计划触发结束时间");
|
||||
Date actualStart = parseQueryTime(
|
||||
request.getParameter("actualStart"), "实际触发开始时间");
|
||||
Date actualEnd = parseQueryTime(
|
||||
request.getParameter("actualEnd"), "实际触发结束时间");
|
||||
|
||||
validateTimeRange(scheduledStart, scheduledEnd, "计划触发时间");
|
||||
validateTimeRange(actualStart, actualEnd, "实际触发时间");
|
||||
if (scheduledStart != null) {
|
||||
queryWrapper.ge(SysJobLog::getScheduledFireTime, scheduledStart);
|
||||
}
|
||||
if (scheduledEnd != null) {
|
||||
queryWrapper.le(SysJobLog::getScheduledFireTime, scheduledEnd);
|
||||
}
|
||||
if (actualStart != null) {
|
||||
queryWrapper.ge(SysJobLog::getActualFireTime, actualStart);
|
||||
}
|
||||
if (actualEnd != null) {
|
||||
queryWrapper.le(SysJobLog::getActualFireTime, actualEnd);
|
||||
}
|
||||
return queryWrapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 自动刷新只读取当前第一页,不执行分页总数统计。
|
||||
*/
|
||||
@GetMapping("refresh")
|
||||
public Result<List<SysJobLog>> refresh(HttpServletRequest request, Long pageSize) {
|
||||
QueryWrapper queryWrapper = buildQueryWrapper(request);
|
||||
queryWrapper.orderBy(buildOrderBy(null, null, getDefaultOrderBy()));
|
||||
queryWrapper.limit(resolvePageSize(pageSize));
|
||||
return Result.ok(service.list(queryWrapper));
|
||||
}
|
||||
|
||||
/**
|
||||
* 最新计划触发记录优先,并用主键保证毫秒时间相同时顺序稳定。
|
||||
*/
|
||||
@Override
|
||||
protected String getDefaultOrderBy() {
|
||||
return "scheduled_fire_time desc, id desc";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Page<SysJobLog> queryPage(
|
||||
Page<SysJobLog> page, QueryWrapper queryWrapper) {
|
||||
page.setPageSize(resolvePageSize(page.getPageSize()));
|
||||
return super.queryPage(page, queryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Result onSaveOrUpdateBefore(SysJobLog entity, boolean isSave) {
|
||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
||||
if (isSave) {
|
||||
commonFiled(entity,loginUser.getId(),loginUser.getTenantId(), loginUser.getDeptId());
|
||||
}
|
||||
return super.onSaveOrUpdateBefore(entity, isSave);
|
||||
throw new IllegalStateException("定时任务执行记录由系统维护,禁止外部写入");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Result onRemoveBefore(Collection<Serializable> ids) {
|
||||
service.requireTerminal(ids);
|
||||
return super.onRemoveBefore(ids);
|
||||
}
|
||||
|
||||
private long resolvePageSize(Long pageSize) {
|
||||
if (pageSize == null || pageSize < 1) {
|
||||
return DEFAULT_PAGE_SIZE;
|
||||
}
|
||||
return Math.min(pageSize, MAX_PAGE_SIZE);
|
||||
}
|
||||
|
||||
private Date parseQueryTime(String value, String fieldName) {
|
||||
if (!StringUtil.hasText(value)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
LocalDateTime dateTime = LocalDateTime.parse(value, QUERY_TIME_FORMATTER);
|
||||
return Date.from(dateTime.atZone(jobZoneId).toInstant());
|
||||
} catch (DateTimeParseException exception) {
|
||||
throw new BusinessException(
|
||||
400, 400, fieldName + "格式不正确", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private void validateTimeRange(Date start, Date end, String fieldName) {
|
||||
if (start != null && end != null && start.after(end)) {
|
||||
throw new BusinessException(400, 400, fieldName + "范围不正确");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,6 +126,7 @@ public record WorkflowDesignerOptionsView(
|
||||
* 已接入数据集安全选项。
|
||||
*
|
||||
* @param id 数据集 ID
|
||||
* @param tenantId 租户 ID
|
||||
* @param sourceId 数据源 ID
|
||||
* @param catalogId 目录 ID
|
||||
* @param tableName 数据表名称
|
||||
@@ -133,6 +134,7 @@ public record WorkflowDesignerOptionsView(
|
||||
*/
|
||||
public record DatasetOption(
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger id,
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger tenantId,
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger sourceId,
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger catalogId,
|
||||
String tableName,
|
||||
|
||||
@@ -19,11 +19,17 @@ import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@@ -40,6 +46,15 @@ public class WorkflowChatEventStream {
|
||||
private final ChainExecutor chainExecutor;
|
||||
private final Map<String, StreamSession> sessions =
|
||||
new ConcurrentHashMap<>();
|
||||
private final ScheduledExecutorService detachedSessionCleaner =
|
||||
Executors.newSingleThreadScheduledExecutor(task -> {
|
||||
Thread thread = new Thread(
|
||||
task,
|
||||
"workflow-chat-detached-session-cleaner"
|
||||
);
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
/**
|
||||
* 创建工作流对话事件流服务。
|
||||
@@ -59,6 +74,15 @@ public class WorkflowChatEventStream {
|
||||
chainExecutor.addErrorListener(this::onChainError);
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭断开会话清理线程并释放残留外部资源。
|
||||
*/
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
sessions.values().forEach(this::removeSession);
|
||||
detachedSessionCleaner.shutdownNow();
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动工作流并返回其 SSE 连接。
|
||||
*
|
||||
@@ -67,11 +91,53 @@ public class WorkflowChatEventStream {
|
||||
* @return SSE 连接
|
||||
*/
|
||||
public SseEmitter start(String definitionId, Map<String, Object> variables) {
|
||||
SseEmitter emitter = new SseEmitter(SSE_TIMEOUT_MILLIS);
|
||||
StreamSession session = new StreamSession(emitter);
|
||||
emitter.onTimeout(() -> disconnect(session, "运行连接超时"));
|
||||
emitter.onError(error -> disconnect(session, "运行连接已断开"));
|
||||
emitter.onCompletion(() -> removeSession(session));
|
||||
return start(definitionId, variables, () -> {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动工作流并在流会话结束时执行清理回调。
|
||||
*
|
||||
* @param definitionId 工作流定义 ID
|
||||
* @param variables 运行变量
|
||||
* @param cleanup 终态、启动失败或连接断开后的幂等清理任务
|
||||
* @return SSE 连接
|
||||
*/
|
||||
public SseEmitter start(
|
||||
String definitionId,
|
||||
Map<String, Object> variables,
|
||||
Runnable cleanup
|
||||
) {
|
||||
return start(definitionId, variables, cleanup, Duration.ZERO);
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动工作流并将浏览器连接与 Runtime 生命周期分离。
|
||||
*
|
||||
* <p>浏览器断开后不取消工作流;在保留期内继续监听真实终态并执行清理,
|
||||
* 超过保留期时由租约兜底释放资源。</p>
|
||||
*
|
||||
* @param definitionId 工作流定义 ID
|
||||
* @param variables 运行变量
|
||||
* @param cleanup 终态、启动失败或保留期结束后的幂等清理任务
|
||||
* @param detachedRetention 浏览器断开后的监听保留时长
|
||||
* @return SSE 连接
|
||||
*/
|
||||
public SseEmitter start(
|
||||
String definitionId,
|
||||
Map<String, Object> variables,
|
||||
Runnable cleanup,
|
||||
Duration detachedRetention
|
||||
) {
|
||||
SseEmitter emitter = createEmitter();
|
||||
StreamSession session = new StreamSession(
|
||||
emitter,
|
||||
cleanup,
|
||||
detachedRetention
|
||||
);
|
||||
emitter.onTimeout(() -> detach(session));
|
||||
emitter.onError(error -> detach(session));
|
||||
emitter.onCompletion(() -> detach(session));
|
||||
|
||||
try {
|
||||
chainExecutor.executeAsync(
|
||||
@@ -79,6 +145,9 @@ public class WorkflowChatEventStream {
|
||||
variables,
|
||||
executeId -> {
|
||||
session.attach(executeId);
|
||||
if (session.cleaned.get()) {
|
||||
return;
|
||||
}
|
||||
sessions.put(executeId, session);
|
||||
session.send("execution_started", Map.of(
|
||||
"executeId", executeId
|
||||
@@ -92,6 +161,13 @@ public class WorkflowChatEventStream {
|
||||
return emitter;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 SSE 发送器,便于验证连接生命周期。
|
||||
*/
|
||||
SseEmitter createEmitter() {
|
||||
return new SseEmitter(SSE_TIMEOUT_MILLIS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将工作流事件转发到对应执行流。
|
||||
*
|
||||
@@ -162,20 +238,21 @@ public class WorkflowChatEventStream {
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理 SSE 连接异常,并取消尚未结束的工作流。
|
||||
* 分离已经断开的浏览器传输,不影响工作流 Runtime。
|
||||
*
|
||||
* @param session 流会话
|
||||
* @param message 取消原因
|
||||
*/
|
||||
private void disconnect(StreamSession session, String message) {
|
||||
private void detach(StreamSession session) {
|
||||
if (session == null || session.terminal.get()) {
|
||||
return;
|
||||
}
|
||||
String executeId = session.executeId;
|
||||
removeSession(session);
|
||||
if (executeId != null) {
|
||||
chainExecutor.cancel(executeId, message);
|
||||
session.detachTransport();
|
||||
if (session.detachedRetention.isZero()
|
||||
|| session.detachedRetention.isNegative()) {
|
||||
removeSession(session);
|
||||
return;
|
||||
}
|
||||
session.scheduleDetachedCleanup();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -187,6 +264,9 @@ public class WorkflowChatEventStream {
|
||||
if (session != null && session.executeId != null) {
|
||||
sessions.remove(session.executeId, session);
|
||||
}
|
||||
if (session != null) {
|
||||
session.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -231,6 +311,11 @@ public class WorkflowChatEventStream {
|
||||
private final SseEmitter emitter;
|
||||
private final AtomicLong sequence = new AtomicLong();
|
||||
private final AtomicBoolean terminal = new AtomicBoolean(false);
|
||||
private final AtomicBoolean cleaned = new AtomicBoolean(false);
|
||||
private final AtomicBoolean connected = new AtomicBoolean(true);
|
||||
private final Runnable cleanup;
|
||||
private final Duration detachedRetention;
|
||||
private volatile ScheduledFuture<?> detachedCleanup;
|
||||
private volatile String executeId;
|
||||
|
||||
/**
|
||||
@@ -238,8 +323,36 @@ public class WorkflowChatEventStream {
|
||||
*
|
||||
* @param emitter SSE 发送器
|
||||
*/
|
||||
private StreamSession(SseEmitter emitter) {
|
||||
private StreamSession(
|
||||
SseEmitter emitter,
|
||||
Runnable cleanup,
|
||||
Duration detachedRetention
|
||||
) {
|
||||
this.emitter = emitter;
|
||||
this.cleanup = cleanup == null ? () -> {
|
||||
} : cleanup;
|
||||
this.detachedRetention = detachedRetention == null
|
||||
? Duration.ZERO
|
||||
: detachedRetention;
|
||||
}
|
||||
|
||||
/**
|
||||
* 幂等释放当前流持有的外部资源。
|
||||
*/
|
||||
private void cleanup() {
|
||||
if (!cleaned.compareAndSet(false, true)) {
|
||||
return;
|
||||
}
|
||||
cancelDetachedCleanup();
|
||||
try {
|
||||
cleanup.run();
|
||||
} catch (RuntimeException error) {
|
||||
log.warn(
|
||||
"workflow chat stream cleanup failed, executeId={}",
|
||||
executeId,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -251,6 +364,38 @@ public class WorkflowChatEventStream {
|
||||
this.executeId = executeId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记浏览器传输已经断开,后续事件只推进 Runtime 清理。
|
||||
*/
|
||||
private void detachTransport() {
|
||||
connected.set(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览器断开后按活动租约安排会话兜底清理。
|
||||
*/
|
||||
private synchronized void scheduleDetachedCleanup() {
|
||||
if (detachedCleanup != null || cleaned.get()) {
|
||||
return;
|
||||
}
|
||||
detachedCleanup = detachedSessionCleaner.schedule(
|
||||
() -> removeSession(this),
|
||||
Math.max(1L, detachedRetention.toMillis()),
|
||||
TimeUnit.MILLISECONDS
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消尚未触发的断开会话兜底任务。
|
||||
*/
|
||||
private synchronized void cancelDetachedCleanup() {
|
||||
if (detachedCleanup == null) {
|
||||
return;
|
||||
}
|
||||
detachedCleanup.cancel(false);
|
||||
detachedCleanup = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理节点开始事件。
|
||||
*
|
||||
@@ -424,7 +569,9 @@ public class WorkflowChatEventStream {
|
||||
}
|
||||
send(eventType, data);
|
||||
removeSession(this);
|
||||
emitter.complete();
|
||||
if (connected.compareAndSet(true, false)) {
|
||||
emitter.complete();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -434,6 +581,9 @@ public class WorkflowChatEventStream {
|
||||
* @param data 事件数据
|
||||
*/
|
||||
private void send(String type, Map<String, ?> data) {
|
||||
if (!connected.get()) {
|
||||
return;
|
||||
}
|
||||
long nextSequence = sequence.incrementAndGet();
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("eventId", executeId + ":" + nextSequence);
|
||||
@@ -453,7 +603,7 @@ public class WorkflowChatEventStream {
|
||||
executeId,
|
||||
error
|
||||
);
|
||||
disconnect(this, "运行连接已断开");
|
||||
detach(this);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -468,7 +618,9 @@ public class WorkflowChatEventStream {
|
||||
"message", safeErrorMessage(error)
|
||||
));
|
||||
removeSession(this);
|
||||
emitter.completeWithError(error);
|
||||
if (connected.compareAndSet(true, false)) {
|
||||
emitter.completeWithError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -318,7 +318,7 @@ public class WorkflowDesignerOptionService {
|
||||
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||
childWorkflowId,
|
||||
account,
|
||||
"子流程不存在、已禁用或无权使用");
|
||||
"子流程不存在、未发布或无权使用");
|
||||
assertContentReferences(workflow.getContent());
|
||||
|
||||
ChainDefinition definition = chainParser.parse(
|
||||
@@ -518,7 +518,7 @@ public class WorkflowDesignerOptionService {
|
||||
workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||
workflowId,
|
||||
account,
|
||||
"子流程不存在、已禁用或无权使用");
|
||||
"子流程不存在、未发布或无权使用");
|
||||
}
|
||||
|
||||
private void assertDatasetReference(
|
||||
@@ -653,6 +653,7 @@ public class WorkflowDesignerOptionService {
|
||||
private WorkflowDesignerOptionsView.DatasetOption toDatasetOption(DatacenterTable table) {
|
||||
return new WorkflowDesignerOptionsView.DatasetOption(
|
||||
table.getId(),
|
||||
table.getTenantId(),
|
||||
table.getSourceId(),
|
||||
table.getCatalogId(),
|
||||
table.getTableName(),
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.script.DefaultRedisScript;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 工作流匿名分享的限流与活动执行互斥保护。
|
||||
*/
|
||||
@Component
|
||||
public class WorkflowPublicChatAccessGuard {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(
|
||||
WorkflowPublicChatAccessGuard.class);
|
||||
private static final String KEY_PREFIX = "easyflow:workflow-public-share:";
|
||||
private static final DefaultRedisScript<Long> RATE_LIMIT_SCRIPT;
|
||||
|
||||
static {
|
||||
RATE_LIMIT_SCRIPT = new DefaultRedisScript<>();
|
||||
RATE_LIMIT_SCRIPT.setScriptText(
|
||||
"local visitor = redis.call('incr', KEYS[1]); "
|
||||
+ "if visitor == 1 then redis.call('pexpire', KEYS[1], ARGV[3]); end; "
|
||||
+ "local share = redis.call('incr', KEYS[2]); "
|
||||
+ "if share == 1 then redis.call('pexpire', KEYS[2], ARGV[3]); end; "
|
||||
+ "if visitor > tonumber(ARGV[1]) or share > tonumber(ARGV[2]) "
|
||||
+ "then return 0 else return 1 end"
|
||||
);
|
||||
RATE_LIMIT_SCRIPT.setResultType(Long.class);
|
||||
}
|
||||
|
||||
private final StringRedisTemplate redisTemplate;
|
||||
private final RedisLockExecutor redisLockExecutor;
|
||||
private final WorkflowPublicShareProperties properties;
|
||||
|
||||
public WorkflowPublicChatAccessGuard(
|
||||
StringRedisTemplate redisTemplate,
|
||||
RedisLockExecutor redisLockExecutor,
|
||||
WorkflowPublicShareProperties properties
|
||||
) {
|
||||
this.redisTemplate = redisTemplate;
|
||||
this.redisLockExecutor = redisLockExecutor;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查匿名运行固定窗口限流。
|
||||
*/
|
||||
public void checkRun(BigInteger shareId, String visitorDigest) {
|
||||
checkRate(
|
||||
shareId,
|
||||
visitorDigest,
|
||||
"run",
|
||||
properties.getRunVisitorLimit(),
|
||||
properties.getRunShareLimit()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查匿名上传固定窗口限流。
|
||||
*/
|
||||
public void checkUpload(BigInteger shareId, String visitorDigest) {
|
||||
checkRate(
|
||||
shareId,
|
||||
visitorDigest,
|
||||
"upload",
|
||||
properties.getUploadVisitorLimit(),
|
||||
properties.getUploadShareLimit()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取同一分享访客的活动执行锁。
|
||||
*
|
||||
* @return 由 SSE 生命周期显式释放的锁句柄
|
||||
*/
|
||||
public RedisLockExecutor.LockHandle acquireActivity(
|
||||
BigInteger shareId,
|
||||
String visitorDigest
|
||||
) {
|
||||
try {
|
||||
RedisLockExecutor.LockHandle handle = redisLockExecutor.tryAcquire(
|
||||
KEY_PREFIX + "{" + shareId + "}:active:" + visitorDigest,
|
||||
Duration.ZERO,
|
||||
properties.getActiveLease()
|
||||
);
|
||||
if (handle == null) {
|
||||
throw new BusinessException(
|
||||
409,
|
||||
40931,
|
||||
"当前分享访客已有工作流正在运行"
|
||||
);
|
||||
}
|
||||
return handle;
|
||||
} catch (BusinessException exception) {
|
||||
throw exception;
|
||||
} catch (RuntimeException exception) {
|
||||
log.error("匿名工作流活动锁暂不可用,shareId={}", shareId, exception);
|
||||
throw unavailable(exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取匿名活动执行锁的租约,用作浏览器断开后的监听保留上限。
|
||||
*/
|
||||
public Duration activityLease() {
|
||||
return properties.getActiveLease();
|
||||
}
|
||||
|
||||
private void checkRate(
|
||||
BigInteger shareId,
|
||||
String visitorDigest,
|
||||
String action,
|
||||
int visitorLimit,
|
||||
int shareLimit
|
||||
) {
|
||||
String slot = "{" + shareId + "}";
|
||||
List<String> keys = List.of(
|
||||
KEY_PREFIX + slot + ":rate:" + action + ":visitor:" + visitorDigest,
|
||||
KEY_PREFIX + slot + ":rate:" + action + ":share"
|
||||
);
|
||||
try {
|
||||
Long allowed = redisTemplate.execute(
|
||||
RATE_LIMIT_SCRIPT,
|
||||
keys,
|
||||
String.valueOf(visitorLimit),
|
||||
String.valueOf(shareLimit),
|
||||
String.valueOf(properties.getRateWindow().toMillis())
|
||||
);
|
||||
if (allowed == null) {
|
||||
throw unavailable(new IllegalStateException(
|
||||
"Redis 未返回匿名工作流限流结果"));
|
||||
}
|
||||
if (!Long.valueOf(1L).equals(allowed)) {
|
||||
throw new BusinessException(
|
||||
429,
|
||||
42931,
|
||||
"匿名工作流请求过于频繁,请稍后重试"
|
||||
);
|
||||
}
|
||||
} catch (BusinessException exception) {
|
||||
throw exception;
|
||||
} catch (RuntimeException exception) {
|
||||
log.error("匿名工作流限流暂不可用,shareId={}, action={}",
|
||||
shareId, action, exception);
|
||||
throw unavailable(exception);
|
||||
}
|
||||
}
|
||||
|
||||
private BusinessException unavailable(RuntimeException cause) {
|
||||
return new BusinessException(
|
||||
503,
|
||||
50331,
|
||||
"匿名工作流保护服务暂不可用,请稍后重试",
|
||||
cause
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.entity.WorkflowShare;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
|
||||
/**
|
||||
* 完成匿名分享边界校验后的运行上下文。
|
||||
*/
|
||||
public record WorkflowPublicChatContext(
|
||||
WorkflowShare share,
|
||||
Workflow workflow,
|
||||
LoginAccount creator,
|
||||
String shareKey,
|
||||
String visitorDigest
|
||||
) {
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.entity.WorkflowShare;
|
||||
import tech.easyflow.ai.enums.PublishStatus;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
import tech.easyflow.ai.service.WorkflowShareService;
|
||||
import tech.easyflow.ai.share.WorkflowSharePolicy;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.system.entity.SysAccount;
|
||||
import tech.easyflow.system.service.SysAccountService;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 解析并校验工作流匿名分享上下文。
|
||||
*/
|
||||
@Service
|
||||
public class WorkflowPublicChatContextResolver {
|
||||
|
||||
private static final Pattern VISITOR_PATTERN = Pattern.compile("[a-f0-9]{32}");
|
||||
|
||||
private final WorkflowShareService shareService;
|
||||
private final WorkflowService workflowService;
|
||||
private final SysAccountService accountService;
|
||||
|
||||
public WorkflowPublicChatContextResolver(
|
||||
WorkflowShareService shareService,
|
||||
WorkflowService workflowService,
|
||||
SysAccountService accountService
|
||||
) {
|
||||
this.shareService = shareService;
|
||||
this.workflowService = workflowService;
|
||||
this.accountService = accountService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析新运行、恢复与上传所需的当前有效上下文。
|
||||
*/
|
||||
public WorkflowPublicChatContext resolveActive(
|
||||
String shareKey,
|
||||
String visitorId
|
||||
) {
|
||||
String normalizedVisitor = requireVisitor(visitorId);
|
||||
WorkflowShare share = shareService.resolvePublicChatShare(shareKey);
|
||||
Workflow workflow = TenantManager.withoutTenantCondition(
|
||||
() -> workflowService.getPublishedById(share.getWorkflowId()));
|
||||
if (!isStrictlyPublished(workflow)
|
||||
|| !Objects.equals(share.getTenantId(), workflow.getTenantId())) {
|
||||
throw new BusinessException(409, 409, "工作流尚未发布或已下线");
|
||||
}
|
||||
SysAccount account = TenantManager.withoutTenantCondition(
|
||||
() -> accountService.getById(share.getCreatedBy()));
|
||||
if (account == null
|
||||
|| !EnumDataStatus.AVAILABLE.getCode().equals(account.getStatus())
|
||||
|| !Objects.equals(share.getTenantId(), account.getTenantId())) {
|
||||
throw new BusinessException(
|
||||
403,
|
||||
40331,
|
||||
"工作流分享创建者账号当前不可用"
|
||||
);
|
||||
}
|
||||
LoginAccount creator = account.toLoginAccount();
|
||||
return new WorkflowPublicChatContext(
|
||||
share,
|
||||
workflow,
|
||||
creator,
|
||||
shareKey,
|
||||
WorkflowSharePolicy.hashChatVisitor(
|
||||
shareKey,
|
||||
normalizedVisitor
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析已发起执行的详情与取消所需历史上下文。
|
||||
*/
|
||||
public WorkflowPublicChatContext resolveHistorical(
|
||||
String shareKey,
|
||||
String visitorId
|
||||
) {
|
||||
String normalizedVisitor = requireVisitor(visitorId);
|
||||
WorkflowShare share = shareService.resolveHistoricalChatShare(shareKey);
|
||||
return new WorkflowPublicChatContext(
|
||||
share,
|
||||
null,
|
||||
null,
|
||||
shareKey,
|
||||
WorkflowSharePolicy.hashChatVisitor(
|
||||
shareKey,
|
||||
normalizedVisitor
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private String requireVisitor(String visitorId) {
|
||||
String normalized = visitorId == null ? "" : visitorId.trim();
|
||||
if (!VISITOR_PATTERN.matcher(normalized).matches()) {
|
||||
throw new BusinessException(
|
||||
400,
|
||||
40031,
|
||||
"工作流分享访客标识无效"
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private boolean isStrictlyPublished(Workflow workflow) {
|
||||
return workflow != null
|
||||
&& PublishStatus.PUBLISHED.getCode().equals(
|
||||
workflow.getPublishStatus())
|
||||
&& workflow.getPublishedSnapshotJson() != null
|
||||
&& !workflow.getPublishedSnapshotJson().isEmpty();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import com.easyagents.flow.core.chain.ChainStatus;
|
||||
import com.easyagents.flow.core.chain.ChainState;
|
||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||
import tech.easyflow.ai.entity.WorkflowExecResult;
|
||||
import tech.easyflow.ai.entity.WorkflowExecStep;
|
||||
import tech.easyflow.ai.service.WorkflowExecResultService;
|
||||
import tech.easyflow.ai.service.WorkflowExecStepService;
|
||||
import tech.easyflow.ai.utils.WorkFlowUtil;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.constant.Constants;
|
||||
import tech.easyflow.common.vo.UploadResVo;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* 工作流匿名分享对话应用服务。
|
||||
*/
|
||||
@Service
|
||||
public class WorkflowPublicChatService {
|
||||
|
||||
private static final Logger log =
|
||||
LoggerFactory.getLogger(WorkflowPublicChatService.class);
|
||||
|
||||
private final WorkflowPublicChatContextResolver contextResolver;
|
||||
private final WorkflowCheckService workflowCheckService;
|
||||
private final WorkflowRunningParameterResolver parameterResolver;
|
||||
private final WorkflowPublicChatUploadService uploadService;
|
||||
private final WorkflowPublicChatAccessGuard accessGuard;
|
||||
private final WorkflowChatEventStream eventStream;
|
||||
private final ChainExecutor chainExecutor;
|
||||
private final WorkflowExecResultService execResultService;
|
||||
private final WorkflowExecStepService execStepService;
|
||||
|
||||
public WorkflowPublicChatService(
|
||||
WorkflowPublicChatContextResolver contextResolver,
|
||||
WorkflowCheckService workflowCheckService,
|
||||
WorkflowRunningParameterResolver parameterResolver,
|
||||
WorkflowPublicChatUploadService uploadService,
|
||||
WorkflowPublicChatAccessGuard accessGuard,
|
||||
WorkflowChatEventStream eventStream,
|
||||
ChainExecutor chainExecutor,
|
||||
WorkflowExecResultService execResultService,
|
||||
WorkflowExecStepService execStepService
|
||||
) {
|
||||
this.contextResolver = contextResolver;
|
||||
this.workflowCheckService = workflowCheckService;
|
||||
this.parameterResolver = parameterResolver;
|
||||
this.uploadService = uploadService;
|
||||
this.accessGuard = accessGuard;
|
||||
this.eventStream = eventStream;
|
||||
this.chainExecutor = chainExecutor;
|
||||
this.execResultService = execResultService;
|
||||
this.execStepService = execStepService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取匿名分享的发布工作流描述。
|
||||
*/
|
||||
public Map<String, Object> descriptor(String shareKey, String visitorId) {
|
||||
WorkflowPublicChatContext context = contextResolver.resolveActive(
|
||||
shareKey, visitorId);
|
||||
checkWorkflow(context);
|
||||
Map<String, Object> descriptor = parameterResolver
|
||||
.buildRunningParametersView(context.workflow());
|
||||
if (descriptor == null) {
|
||||
throw new BusinessException("工作流输入配置无法解析");
|
||||
}
|
||||
descriptor.put("workflowId", context.workflow().getId());
|
||||
descriptor.put("publishStatus", context.workflow().getPublishStatus());
|
||||
descriptor.put("shareable", false);
|
||||
return descriptor;
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动匿名分享工作流并返回 SSE。
|
||||
*/
|
||||
public SseEmitter run(
|
||||
String shareKey,
|
||||
String visitorId,
|
||||
Map<String, Object> variables
|
||||
) {
|
||||
WorkflowPublicChatContext context = contextResolver.resolveActive(
|
||||
shareKey, visitorId);
|
||||
accessGuard.checkRun(
|
||||
context.share().getId(),
|
||||
context.visitorDigest()
|
||||
);
|
||||
checkWorkflow(context);
|
||||
Map<String, Object> normalized = parameterResolver
|
||||
.normalizeRuntimeVariables(
|
||||
context.workflow().getContent(),
|
||||
variables
|
||||
);
|
||||
uploadService.assertOwnedUploads(context, normalized);
|
||||
normalized.put(Constants.LOGIN_USER_KEY, context.creator());
|
||||
normalized.put(
|
||||
WorkFlowUtil.CREATED_KEY_MEMORY_KEY,
|
||||
WorkFlowUtil.publicChatShareCreatedKey(
|
||||
context.share().getId())
|
||||
);
|
||||
normalized.put(
|
||||
WorkFlowUtil.CREATED_BY_MEMORY_KEY,
|
||||
context.visitorDigest()
|
||||
);
|
||||
|
||||
RedisLockExecutor.LockHandle activity = accessGuard.acquireActivity(
|
||||
context.share().getId(),
|
||||
context.visitorDigest()
|
||||
);
|
||||
try {
|
||||
return eventStream.start(
|
||||
PublishedWorkflowDefinitionIds.published(
|
||||
context.workflow().getId().toString()),
|
||||
normalized,
|
||||
activity::release,
|
||||
accessGuard.activityLease()
|
||||
);
|
||||
} catch (RuntimeException | Error error) {
|
||||
activity.release();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前匿名访客发起的执行详情。
|
||||
*/
|
||||
public Map<String, Object> detail(
|
||||
String shareKey,
|
||||
String visitorId,
|
||||
String executeId
|
||||
) {
|
||||
WorkflowPublicChatContext context = contextResolver.resolveHistorical(
|
||||
shareKey, visitorId);
|
||||
WorkflowExecResult record = assertExecutionOwnership(
|
||||
context, executeId);
|
||||
List<WorkflowExecStep> steps = TenantManager.withoutTenantCondition(
|
||||
() -> execStepService.list(
|
||||
QueryWrapper.create()
|
||||
.eq(WorkflowExecStep::getRecordId, record.getId())
|
||||
.orderBy(WorkflowExecStep::getStartTime, true)
|
||||
));
|
||||
return buildExecutionDetail(record, steps, runtimeView(executeId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消当前匿名访客发起的执行。
|
||||
*/
|
||||
public boolean cancel(
|
||||
String shareKey,
|
||||
String visitorId,
|
||||
String executeId
|
||||
) {
|
||||
WorkflowPublicChatContext context = contextResolver.resolveHistorical(
|
||||
shareKey, visitorId);
|
||||
assertExecutionOwnership(context, executeId);
|
||||
return chainExecutor.cancel(executeId, "匿名访客已中止运行");
|
||||
}
|
||||
|
||||
/**
|
||||
* 恢复当前有效分享访客等待确认的执行。
|
||||
*/
|
||||
public void resume(
|
||||
String shareKey,
|
||||
String visitorId,
|
||||
String executeId,
|
||||
Map<String, Object> confirmParams
|
||||
) {
|
||||
WorkflowPublicChatContext context = contextResolver.resolveActive(
|
||||
shareKey, visitorId);
|
||||
WorkflowExecResult record = assertExecutionOwnership(
|
||||
context, executeId);
|
||||
if (isTerminal(record.getStatus())) {
|
||||
throw new BusinessException("当前工作流执行已结束");
|
||||
}
|
||||
chainExecutor.resumeAsync(
|
||||
executeId,
|
||||
confirmParams == null
|
||||
? new LinkedHashMap<>()
|
||||
: new LinkedHashMap<>(confirmParams)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 上传当前发布快照声明的匿名输入文件。
|
||||
*/
|
||||
public UploadResVo upload(
|
||||
String shareKey,
|
||||
String visitorId,
|
||||
String parameterName,
|
||||
MultipartFile file
|
||||
) {
|
||||
WorkflowPublicChatContext context = contextResolver.resolveActive(
|
||||
shareKey, visitorId);
|
||||
return uploadService.upload(context, parameterName, file);
|
||||
}
|
||||
|
||||
private void checkWorkflow(WorkflowPublicChatContext context) {
|
||||
TenantManager.withoutTenantCondition(() -> {
|
||||
workflowCheckService.checkOrThrow(
|
||||
context.workflow().getContent(),
|
||||
WorkflowCheckStage.PRE_EXECUTE,
|
||||
context.workflow().getId()
|
||||
);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private WorkflowExecResult assertExecutionOwnership(
|
||||
WorkflowPublicChatContext context,
|
||||
String executeId
|
||||
) {
|
||||
if (executeId == null || executeId.isBlank()) {
|
||||
throw new BusinessException("执行ID不能为空");
|
||||
}
|
||||
WorkflowExecResult record = TenantManager.withoutTenantCondition(
|
||||
() -> execResultService.getByExecKey(executeId));
|
||||
if (record == null) {
|
||||
throw new BusinessException("工作流执行记录不存在,请稍后重试");
|
||||
}
|
||||
String expectedSource = WorkFlowUtil.publicChatShareCreatedKey(
|
||||
context.share().getId());
|
||||
if (!Objects.equals(expectedSource, record.getCreatedKey())
|
||||
|| !Objects.equals(
|
||||
context.visitorDigest(),
|
||||
record.getCreatedBy())
|
||||
|| !Objects.equals(
|
||||
context.share().getWorkflowId(),
|
||||
record.getWorkflowId())) {
|
||||
throw new BusinessException(
|
||||
403,
|
||||
40333,
|
||||
"无权限访问当前工作流执行记录"
|
||||
);
|
||||
}
|
||||
return record;
|
||||
}
|
||||
|
||||
private boolean isTerminal(Integer status) {
|
||||
return status != null
|
||||
&& (status == ChainStatus.SUCCEEDED.getValue()
|
||||
|| status == ChainStatus.FAILED.getValue()
|
||||
|| status == ChainStatus.CANCELLED.getValue());
|
||||
}
|
||||
|
||||
private Map<String, Object> buildExecutionDetail(
|
||||
WorkflowExecResult record,
|
||||
List<WorkflowExecStep> steps,
|
||||
Map<String, Object> runtime
|
||||
) {
|
||||
List<Map<String, Object>> stepViews = new ArrayList<>(steps.size());
|
||||
for (WorkflowExecStep step : steps) {
|
||||
Map<String, Object> view = new LinkedHashMap<>();
|
||||
view.put("id", step.getId());
|
||||
view.put("attemptKey", step.getExecKey());
|
||||
view.put("nodeId", step.getNodeId());
|
||||
view.put("nodeName", step.getNodeName());
|
||||
view.put("input", step.getInput());
|
||||
view.put("output", step.getOutput());
|
||||
view.put("status", step.getStatus());
|
||||
view.put("errorInfo", step.getErrorInfo());
|
||||
view.put("startTime", step.getStartTime());
|
||||
view.put("endTime", step.getEndTime());
|
||||
view.put("execTime", step.getExecTime());
|
||||
stepViews.add(view);
|
||||
}
|
||||
|
||||
Map<String, Object> recordView = new LinkedHashMap<>();
|
||||
recordView.put("executeId", record.getExecKey());
|
||||
recordView.put("workflowId", record.getWorkflowId());
|
||||
recordView.put("title", record.getTitle());
|
||||
recordView.put("status", record.getStatus());
|
||||
recordView.put("input", record.getInput());
|
||||
recordView.put("output", record.getOutput());
|
||||
recordView.put("errorInfo", record.getErrorInfo());
|
||||
recordView.put("startTime", record.getStartTime());
|
||||
recordView.put("endTime", record.getEndTime());
|
||||
recordView.put("execTime", record.getExecTime());
|
||||
|
||||
Map<String, Object> detail = new LinkedHashMap<>();
|
||||
detail.put("record", recordView);
|
||||
detail.put("steps", stepViews);
|
||||
detail.put("runtime", runtime);
|
||||
return detail;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建刷新恢复所需的最小 Runtime 视图。
|
||||
*/
|
||||
private Map<String, Object> runtimeView(String executeId) {
|
||||
try {
|
||||
ChainState state = chainExecutor.getChainStateRepository()
|
||||
.load(executeId);
|
||||
if (state == null || state.getStatus() == null) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, Object> view = new LinkedHashMap<>();
|
||||
view.put("status", state.getStatus().name());
|
||||
view.put("statusValue", state.getStatus().getValue());
|
||||
view.put("message", state.getMessage());
|
||||
if (state.getStatus() == ChainStatus.SUSPEND) {
|
||||
view.put("parameters", state.getSuspendForParameters());
|
||||
}
|
||||
if (state.getStatus() == ChainStatus.SUCCEEDED) {
|
||||
view.put(
|
||||
"output",
|
||||
WorkflowChatEventStream.visibleFinalOutput(
|
||||
state.getExecuteResult())
|
||||
);
|
||||
}
|
||||
return view;
|
||||
} catch (RuntimeException error) {
|
||||
log.warn(
|
||||
"failed to load public workflow runtime state, executeId={}",
|
||||
executeId,
|
||||
error
|
||||
);
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||
import tech.easyflow.ai.share.WorkflowSharePolicy;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.vo.UploadResVo;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collection;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 工作流匿名分享的隔离上传与运行引用校验。
|
||||
*/
|
||||
@Service
|
||||
public class WorkflowPublicChatUploadService {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(
|
||||
WorkflowPublicChatUploadService.class);
|
||||
private static final long FILE_MAX_SIZE = 100L * 1024L * 1024L;
|
||||
private static final long IMAGE_MAX_SIZE = 10L * 1024L * 1024L;
|
||||
private static final Set<String> IMAGE_MIME_TYPES = Set.of(
|
||||
"image/bmp", "image/gif", "image/jpeg", "image/png", "image/webp");
|
||||
private static final Set<String> IMAGE_EXTENSIONS = Set.of(
|
||||
"bmp", "gif", "jpeg", "jpg", "png", "webp");
|
||||
private static final String GRANT_PREFIX = "easyflow:workflow-public-share:upload:";
|
||||
|
||||
private final WorkflowRunningParameterResolver parameterResolver;
|
||||
private final WorkflowPublicChatAccessGuard accessGuard;
|
||||
private final WorkflowPublicShareProperties properties;
|
||||
private final StringRedisTemplate redisTemplate;
|
||||
private final FileStorageService storageService;
|
||||
|
||||
public WorkflowPublicChatUploadService(
|
||||
WorkflowRunningParameterResolver parameterResolver,
|
||||
WorkflowPublicChatAccessGuard accessGuard,
|
||||
WorkflowPublicShareProperties properties,
|
||||
StringRedisTemplate redisTemplate,
|
||||
@Qualifier("default") FileStorageService storageService
|
||||
) {
|
||||
this.parameterResolver = parameterResolver;
|
||||
this.accessGuard = accessGuard;
|
||||
this.properties = properties;
|
||||
this.redisTemplate = redisTemplate;
|
||||
this.storageService = storageService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 上传发布快照声明的文件或图片参数。
|
||||
*/
|
||||
public UploadResVo upload(
|
||||
WorkflowPublicChatContext context,
|
||||
String parameterName,
|
||||
MultipartFile file
|
||||
) {
|
||||
String normalizedName = requireParameterName(parameterName);
|
||||
String contentType = resolveUploadContentType(context, normalizedName);
|
||||
validateFile(file, contentType);
|
||||
accessGuard.checkUpload(
|
||||
context.share().getId(),
|
||||
context.visitorDigest()
|
||||
);
|
||||
|
||||
String path = storageService.save(
|
||||
file,
|
||||
"workflow-chat-share/" + context.share().getId()
|
||||
+ "/" + context.visitorDigest()
|
||||
);
|
||||
if (!StringUtils.hasText(path)) {
|
||||
throw new BusinessException(503, 50332, "匿名文件上传失败,请稍后重试");
|
||||
}
|
||||
try {
|
||||
redisTemplate.opsForValue().set(
|
||||
grantKey(context, normalizedName, path),
|
||||
contentType,
|
||||
grantTtl(context).toMillis(),
|
||||
TimeUnit.MILLISECONDS
|
||||
);
|
||||
} catch (RuntimeException exception) {
|
||||
try {
|
||||
storageService.delete(path);
|
||||
} catch (RuntimeException cleanupError) {
|
||||
log.warn("匿名上传授权写入失败后清理文件失败,path={}",
|
||||
path, cleanupError);
|
||||
}
|
||||
throw new BusinessException(
|
||||
503,
|
||||
50332,
|
||||
"匿名上传保护服务暂不可用,请稍后重试",
|
||||
exception
|
||||
);
|
||||
}
|
||||
|
||||
UploadResVo response = new UploadResVo();
|
||||
response.setPath(path);
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验公开运行引用的上传文件均属于当前分享访客和参数。
|
||||
*/
|
||||
public void assertOwnedUploads(
|
||||
WorkflowPublicChatContext context,
|
||||
Map<String, Object> variables
|
||||
) {
|
||||
Map<String, String> uploadFields = resolveUploadFields(context);
|
||||
for (Map.Entry<String, String> entry : uploadFields.entrySet()) {
|
||||
Object value = variables.get(entry.getKey());
|
||||
if (value == null) {
|
||||
continue;
|
||||
}
|
||||
if ("image".equals(entry.getValue())) {
|
||||
assertOwnedImage(context, entry.getKey(), value);
|
||||
} else {
|
||||
assertOwnedFiles(context, entry.getKey(), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void assertOwnedImage(
|
||||
WorkflowPublicChatContext context,
|
||||
String parameterName,
|
||||
Object value
|
||||
) {
|
||||
if (!(value instanceof Map<?, ?> image)) {
|
||||
throw invalidUploadReference(parameterName);
|
||||
}
|
||||
String sourceType = trim(image.get("sourceType"));
|
||||
if ("url".equals(sourceType)) {
|
||||
String url = trim(image.get("url"));
|
||||
if (isHttpUrl(url)) {
|
||||
return;
|
||||
}
|
||||
throw invalidUploadReference(parameterName);
|
||||
}
|
||||
if (!"upload".equals(sourceType)) {
|
||||
throw invalidUploadReference(parameterName);
|
||||
}
|
||||
assertGrant(
|
||||
context,
|
||||
parameterName,
|
||||
trim(image.get("filePath")),
|
||||
"image"
|
||||
);
|
||||
}
|
||||
|
||||
private void assertOwnedFiles(
|
||||
WorkflowPublicChatContext context,
|
||||
String parameterName,
|
||||
Object value
|
||||
) {
|
||||
if (!(value instanceof Collection<?> files)) {
|
||||
throw invalidUploadReference(parameterName);
|
||||
}
|
||||
for (Object item : files) {
|
||||
if (!(item instanceof Map<?, ?> file)) {
|
||||
throw invalidUploadReference(parameterName);
|
||||
}
|
||||
assertGrant(
|
||||
context,
|
||||
parameterName,
|
||||
trim(file.get("filePath")),
|
||||
"file"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private void assertGrant(
|
||||
WorkflowPublicChatContext context,
|
||||
String parameterName,
|
||||
String path,
|
||||
String expectedContentType
|
||||
) {
|
||||
if (!StringUtils.hasText(path)) {
|
||||
throw invalidUploadReference(parameterName);
|
||||
}
|
||||
try {
|
||||
String grantedContentType = redisTemplate.opsForValue().get(
|
||||
grantKey(context, parameterName, path));
|
||||
if (!expectedContentType.equals(grantedContentType)) {
|
||||
throw invalidUploadReference(parameterName);
|
||||
}
|
||||
} catch (BusinessException exception) {
|
||||
throw exception;
|
||||
} catch (RuntimeException exception) {
|
||||
throw new BusinessException(
|
||||
503,
|
||||
50332,
|
||||
"匿名上传保护服务暂不可用,请稍后重试",
|
||||
exception
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveUploadContentType(
|
||||
WorkflowPublicChatContext context,
|
||||
String parameterName
|
||||
) {
|
||||
String contentType = resolveUploadFields(context).get(parameterName);
|
||||
if (contentType == null) {
|
||||
throw new BusinessException(
|
||||
400,
|
||||
40032,
|
||||
"当前发布工作流未声明该上传参数"
|
||||
);
|
||||
}
|
||||
return contentType;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private Map<String, String> resolveUploadFields(
|
||||
WorkflowPublicChatContext context
|
||||
) {
|
||||
Map<String, Object> descriptor = parameterResolver
|
||||
.buildRunningParametersView(context.workflow());
|
||||
if (descriptor == null) {
|
||||
throw new BusinessException("工作流输入配置无法解析");
|
||||
}
|
||||
Map<String, String> fields = new java.util.LinkedHashMap<>();
|
||||
Object rawSchema = descriptor.get("startFormSchema");
|
||||
if (!(rawSchema instanceof Collection<?> schema)) {
|
||||
return fields;
|
||||
}
|
||||
for (Object item : schema) {
|
||||
if (!(item instanceof Map<?, ?> field)) {
|
||||
continue;
|
||||
}
|
||||
String name = trim(field.get("key"));
|
||||
String contentType = trim(field.get("contentType"));
|
||||
if (StringUtils.hasText(name)
|
||||
&& ("file".equals(contentType)
|
||||
|| "image".equals(contentType))) {
|
||||
fields.put(name, contentType);
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
private void validateFile(MultipartFile file, String contentType) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("上传文件不能为空");
|
||||
}
|
||||
long maxSize = "image".equals(contentType)
|
||||
? IMAGE_MAX_SIZE
|
||||
: FILE_MAX_SIZE;
|
||||
if (file.getSize() > maxSize) {
|
||||
throw new BusinessException(
|
||||
"image".equals(contentType)
|
||||
? "单张图片不能超过 10 MiB"
|
||||
: "单个文件不能超过 100 MiB"
|
||||
);
|
||||
}
|
||||
if (!"image".equals(contentType)) {
|
||||
return;
|
||||
}
|
||||
String mimeType = trim(file.getContentType()).toLowerCase(Locale.ROOT);
|
||||
String filename = trim(file.getOriginalFilename());
|
||||
int dot = filename.lastIndexOf('.');
|
||||
String extension = dot < 0
|
||||
? ""
|
||||
: filename.substring(dot + 1).toLowerCase(Locale.ROOT);
|
||||
if (!IMAGE_MIME_TYPES.contains(mimeType)
|
||||
&& !IMAGE_EXTENSIONS.contains(extension)) {
|
||||
throw new BusinessException("仅支持 PNG、JPEG、WebP、GIF、BMP 图片");
|
||||
}
|
||||
}
|
||||
|
||||
private Duration grantTtl(WorkflowPublicChatContext context) {
|
||||
long expiresIn = context.share().getExpiresAt().getTime()
|
||||
- System.currentTimeMillis();
|
||||
long ttl = Math.min(
|
||||
properties.getUploadGrantTtl().toMillis(),
|
||||
expiresIn
|
||||
);
|
||||
return Duration.ofMillis(Math.max(1L, ttl));
|
||||
}
|
||||
|
||||
private String grantKey(
|
||||
WorkflowPublicChatContext context,
|
||||
String parameterName,
|
||||
String path
|
||||
) {
|
||||
return GRANT_PREFIX + "{" + context.share().getId() + "}:"
|
||||
+ context.visitorDigest() + ":"
|
||||
+ WorkflowSharePolicy.hashShareKey(parameterName) + ":"
|
||||
+ WorkflowSharePolicy.hashShareKey(path);
|
||||
}
|
||||
|
||||
private String requireParameterName(String value) {
|
||||
String normalized = value == null ? "" : value.trim();
|
||||
if (!StringUtils.hasText(normalized)) {
|
||||
throw new BusinessException("上传参数名不能为空");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String trim(Object value) {
|
||||
return value == null ? "" : String.valueOf(value).trim();
|
||||
}
|
||||
|
||||
private boolean isHttpUrl(String value) {
|
||||
String normalized = value == null ? "" : value.toLowerCase(Locale.ROOT);
|
||||
return normalized.startsWith("http://")
|
||||
|| normalized.startsWith("https://");
|
||||
}
|
||||
|
||||
private BusinessException invalidUploadReference(String parameterName) {
|
||||
return new BusinessException(
|
||||
403,
|
||||
40332,
|
||||
"上传参数 " + parameterName + " 不属于当前分享访客"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* 工作流匿名分享运行保护参数。
|
||||
*/
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "easyflow.workflow.public-share")
|
||||
public class WorkflowPublicShareProperties {
|
||||
|
||||
private Duration rateWindow = Duration.ofMinutes(1);
|
||||
private int runVisitorLimit = 5;
|
||||
private int runShareLimit = 60;
|
||||
private int uploadVisitorLimit = 10;
|
||||
private int uploadShareLimit = 60;
|
||||
private Duration activeLease = Duration.ofMinutes(35);
|
||||
private Duration uploadGrantTtl = Duration.ofDays(7);
|
||||
|
||||
public Duration getRateWindow() {
|
||||
return rateWindow;
|
||||
}
|
||||
|
||||
public void setRateWindow(Duration rateWindow) {
|
||||
this.rateWindow = requirePositive(rateWindow, "rateWindow");
|
||||
}
|
||||
|
||||
public int getRunVisitorLimit() {
|
||||
return runVisitorLimit;
|
||||
}
|
||||
|
||||
public void setRunVisitorLimit(int runVisitorLimit) {
|
||||
this.runVisitorLimit = requirePositive(runVisitorLimit, "runVisitorLimit");
|
||||
}
|
||||
|
||||
public int getRunShareLimit() {
|
||||
return runShareLimit;
|
||||
}
|
||||
|
||||
public void setRunShareLimit(int runShareLimit) {
|
||||
this.runShareLimit = requirePositive(runShareLimit, "runShareLimit");
|
||||
}
|
||||
|
||||
public int getUploadVisitorLimit() {
|
||||
return uploadVisitorLimit;
|
||||
}
|
||||
|
||||
public void setUploadVisitorLimit(int uploadVisitorLimit) {
|
||||
this.uploadVisitorLimit = requirePositive(uploadVisitorLimit, "uploadVisitorLimit");
|
||||
}
|
||||
|
||||
public int getUploadShareLimit() {
|
||||
return uploadShareLimit;
|
||||
}
|
||||
|
||||
public void setUploadShareLimit(int uploadShareLimit) {
|
||||
this.uploadShareLimit = requirePositive(uploadShareLimit, "uploadShareLimit");
|
||||
}
|
||||
|
||||
public Duration getActiveLease() {
|
||||
return activeLease;
|
||||
}
|
||||
|
||||
public void setActiveLease(Duration activeLease) {
|
||||
this.activeLease = requirePositive(activeLease, "activeLease");
|
||||
}
|
||||
|
||||
public Duration getUploadGrantTtl() {
|
||||
return uploadGrantTtl;
|
||||
}
|
||||
|
||||
public void setUploadGrantTtl(Duration uploadGrantTtl) {
|
||||
this.uploadGrantTtl = requirePositive(uploadGrantTtl, "uploadGrantTtl");
|
||||
}
|
||||
|
||||
private static int requirePositive(int value, String name) {
|
||||
if (value <= 0) {
|
||||
throw new IllegalArgumentException(name + " 必须大于 0");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static Duration requirePositive(Duration value, String name) {
|
||||
if (value == null || value.isZero() || value.isNegative()) {
|
||||
throw new IllegalArgumentException(name + " 必须大于 0");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -3,17 +3,50 @@ package tech.easyflow.admin.controller.ai;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import tech.easyflow.ai.entity.WorkflowShare;
|
||||
import tech.easyflow.ai.service.WorkflowShareService;
|
||||
import tech.easyflow.ai.share.WorkflowSharePolicy;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.math.BigInteger;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Proxy;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link WorkflowShareController} 分享地址构建测试。
|
||||
*/
|
||||
public class WorkflowShareControllerTest {
|
||||
|
||||
/**
|
||||
* 验证分享解析仅依赖分享密钥,不读取当前浏览器登录租户。
|
||||
*/
|
||||
@Test
|
||||
public void shouldResolvePublicChatShareWithoutLoginContext()
|
||||
throws Exception {
|
||||
WorkflowShareService shareService = mock(WorkflowShareService.class);
|
||||
WorkflowShare share = new WorkflowShare();
|
||||
share.setWorkflowId(BigInteger.valueOf(11));
|
||||
when(shareService.resolvePublicChatShare("share-key"))
|
||||
.thenReturn(share);
|
||||
WorkflowShareController controller = new WorkflowShareController();
|
||||
setField(controller, "workflowShareService", shareService);
|
||||
|
||||
BigInteger workflowId = controller.resolveUrlShare(request(Map.of(
|
||||
WorkflowSharePolicy.CHAT_SHARE_KEY_HEADER
|
||||
.toLowerCase(Locale.ROOT),
|
||||
"share-key"
|
||||
))).getData().get("workflowId");
|
||||
|
||||
Assert.assertEquals(workflowId, BigInteger.valueOf(11));
|
||||
verify(shareService).resolvePublicChatShare("share-key");
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证分享地址保留前端部署基路径。
|
||||
*
|
||||
@@ -120,4 +153,11 @@ public class WorkflowShareControllerTest {
|
||||
}
|
||||
return 0D;
|
||||
}
|
||||
|
||||
private void setField(Object target, String name, Object value)
|
||||
throws Exception {
|
||||
Field field = target.getClass().getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package tech.easyflow.admin.controller.dataspace;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertNotNull;
|
||||
import static org.testng.Assert.assertNull;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Parameter;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.testng.annotations.Test;
|
||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
||||
import tech.easyflow.dataspace.model.DataspaceDefinition;
|
||||
|
||||
/**
|
||||
* 数据空间管理接口请求绑定契约测试。
|
||||
*/
|
||||
public class DataspaceControllerContractTest {
|
||||
|
||||
/**
|
||||
* 验证保存接口使用 Jackson 请求体绑定,避免嵌套定义残留为 JSONObject。
|
||||
*
|
||||
* @throws Exception 反射或 JSON 转换失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void shouldBindNestedDataspaceDefinitionWithJackson() throws Exception {
|
||||
Method method = DataspaceController.class.getMethod(
|
||||
"save", DataspaceDefinition.class);
|
||||
Parameter parameter = method.getParameters()[0];
|
||||
assertNotNull(parameter.getAnnotation(RequestBody.class));
|
||||
assertNull(parameter.getAnnotation(JsonBody.class));
|
||||
|
||||
String request = """
|
||||
{
|
||||
"name": "网点经营分析",
|
||||
"tables": [
|
||||
{
|
||||
"clientKey": "table:outlet",
|
||||
"objectId": "1001",
|
||||
"sourceAlias": "MYSQL_1",
|
||||
"schemaAlias": "MAIN",
|
||||
"tableAlias": "outlet",
|
||||
"positionX": 80,
|
||||
"positionY": 120
|
||||
},
|
||||
{
|
||||
"clientKey": "table:region",
|
||||
"objectId": "1002",
|
||||
"sourceAlias": "PG_1",
|
||||
"schemaAlias": "PUBLIC",
|
||||
"tableAlias": "outlet_region",
|
||||
"positionX": 420,
|
||||
"positionY": 120
|
||||
}
|
||||
],
|
||||
"relations": [
|
||||
{
|
||||
"leftClientKey": "table:outlet",
|
||||
"rightClientKey": "table:region",
|
||||
"joinType": "INNER",
|
||||
"leftColumn": "institution_id",
|
||||
"rightColumn": "institution_id"
|
||||
}
|
||||
]
|
||||
}
|
||||
""";
|
||||
|
||||
DataspaceDefinition definition = new ObjectMapper().readValue(
|
||||
request, DataspaceDefinition.class);
|
||||
|
||||
assertEquals(2, definition.tables().size());
|
||||
assertEquals("outlet", definition.tables().get(0).tableAlias());
|
||||
assertEquals("outlet_region", definition.tables().get(1).tableAlias());
|
||||
assertEquals(1, definition.relations().size());
|
||||
assertEquals("institution_id", definition.relations().get(0).leftColumn());
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,39 @@
|
||||
package tech.easyflow.admin.controller.job;
|
||||
|
||||
import com.easyagents.flow.core.chain.Parameter;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.enums.PublishStatus;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
|
||||
import tech.easyflow.admin.model.SysJobWorkflowOptionView;
|
||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
import tech.easyflow.job.service.SysJobService;
|
||||
import tech.easyflow.system.enums.CategoryResourceType;
|
||||
import tech.easyflow.system.enums.ResourceAction;
|
||||
import tech.easyflow.system.service.ResourceAccessService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Instant;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockStatic;
|
||||
import static org.mockito.Mockito.verify;
|
||||
@@ -31,6 +44,114 @@ import static org.mockito.Mockito.when;
|
||||
*/
|
||||
public class SysJobControllerTest {
|
||||
|
||||
@Test
|
||||
public void shouldQueryPublishedWorkflowOptionsAndReturnPublishedMetadata() {
|
||||
BigInteger workflowId = BigInteger.valueOf(501);
|
||||
LoginAccount account = account();
|
||||
Workflow raw = new Workflow();
|
||||
raw.setId(workflowId);
|
||||
raw.setTenantId(account.getTenantId());
|
||||
raw.setPublishStatus(PublishStatus.PUBLISHED.getCode());
|
||||
raw.setPublishedSnapshotJson(Map.of("title", "发布标题"));
|
||||
raw.setTitle("草稿标题");
|
||||
Workflow withoutSnapshot = new Workflow();
|
||||
withoutSnapshot.setId(BigInteger.valueOf(502));
|
||||
withoutSnapshot.setTenantId(account.getTenantId());
|
||||
withoutSnapshot.setPublishStatus(PublishStatus.PUBLISHED.getCode());
|
||||
Workflow published = new Workflow();
|
||||
published.setId(workflowId);
|
||||
published.setTitle("发布标题");
|
||||
published.setDescription("发布描述");
|
||||
WorkflowService workflowService = mock(WorkflowService.class);
|
||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||
when(workflowService.list(any(QueryWrapper.class)))
|
||||
.thenReturn(List.of(raw, withoutSnapshot));
|
||||
when(resourceAccessService.canAccess(
|
||||
account,
|
||||
CategoryResourceType.WORKFLOW,
|
||||
raw,
|
||||
ResourceAction.USE)).thenReturn(true);
|
||||
when(workflowService.toPublishedView(raw)).thenReturn(published);
|
||||
SysJobController controller = new SysJobController(
|
||||
mock(SysJobService.class),
|
||||
workflowService,
|
||||
mock(WorkflowUsageAuthorizationService.class),
|
||||
resourceAccessService,
|
||||
mock(WorkflowRunningParameterResolver.class),
|
||||
"Asia/Shanghai");
|
||||
|
||||
List<SysJobWorkflowOptionView> options;
|
||||
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
||||
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account);
|
||||
options = controller.workflowOptions().getData();
|
||||
}
|
||||
|
||||
Assert.assertEquals(options.size(), 1);
|
||||
Assert.assertEquals(options.get(0).title(), "发布标题");
|
||||
ArgumentCaptor<QueryWrapper> queryCaptor = ArgumentCaptor.forClass(QueryWrapper.class);
|
||||
verify(workflowService).list(queryCaptor.capture());
|
||||
String sql = queryCaptor.getValue().toSQL().toLowerCase(Locale.ROOT);
|
||||
Assert.assertTrue(sql.contains("publish_status"));
|
||||
Assert.assertFalse(sql.replace("publish_status", "").matches("(?s).*\\bstatus\\b.*"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldFormatCronPreviewWithConfiguredTimezone() {
|
||||
SysJobService service = mock(SysJobService.class);
|
||||
when(service.nextFireTimes("0 0 9 * * ?", 5))
|
||||
.thenReturn(List.of(Date.from(Instant.parse("2026-01-01T01:00:00Z"))));
|
||||
TimeZone previous = TimeZone.getDefault();
|
||||
try {
|
||||
TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
|
||||
Assert.assertEquals(
|
||||
controller(service).getNextTimes("0 0 9 * * ?").getData().get(0),
|
||||
"2026-01-01 09:00:00");
|
||||
} finally {
|
||||
TimeZone.setDefault(previous);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldForceNewJobToStoppedGenerationZero() {
|
||||
SysJobController controller = controller(mock(SysJobService.class));
|
||||
SysJob job = validJavaJob();
|
||||
job.setStatus(EnumJobStatus.RUNNING.getCode());
|
||||
job.setScheduleGeneration(99L);
|
||||
LoginAccount account = account();
|
||||
|
||||
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
||||
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account);
|
||||
controller.onSaveOrUpdateBefore(job, true);
|
||||
}
|
||||
|
||||
Assert.assertEquals(job.getStatus(), Integer.valueOf(EnumJobStatus.STOP.getCode()));
|
||||
Assert.assertEquals(job.getScheduleGeneration(), Long.valueOf(0L));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRejectStatusAndGenerationMutationThroughOrdinaryUpdate() {
|
||||
BigInteger id = BigInteger.valueOf(401);
|
||||
SysJobService service = mock(SysJobService.class);
|
||||
SysJob existing = validJavaJob();
|
||||
existing.setId(id);
|
||||
existing.setStatus(EnumJobStatus.STOP.getCode());
|
||||
existing.setScheduleGeneration(8L);
|
||||
when(service.getById(id)).thenReturn(existing);
|
||||
|
||||
SysJob update = validJavaJob();
|
||||
update.setId(id);
|
||||
update.setStatus(EnumJobStatus.RUNNING.getCode());
|
||||
update.setScheduleGeneration(100L);
|
||||
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
||||
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account());
|
||||
controller(service).update(update);
|
||||
}
|
||||
|
||||
Assert.assertEquals(update.getStatus(), Integer.valueOf(EnumJobStatus.STOP.getCode()));
|
||||
Assert.assertEquals(update.getScheduleGeneration(), Long.valueOf(8L));
|
||||
verify(service).updateJobDefinition(update);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证缺少工作流必填参数时拒绝保存定时任务。
|
||||
*/
|
||||
@@ -63,7 +184,8 @@ public class SysJobControllerTest {
|
||||
workflowService,
|
||||
workflowAuthorizationService,
|
||||
resourceAccessService,
|
||||
parameterResolver
|
||||
parameterResolver,
|
||||
"Asia/Shanghai"
|
||||
);
|
||||
SysJob job = new SysJob();
|
||||
job.setJobType(EnumJobType.TINY_FLOW.getCode());
|
||||
@@ -130,7 +252,8 @@ public class SysJobControllerTest {
|
||||
workflowService,
|
||||
workflowAuthorizationService,
|
||||
resourceAccessService,
|
||||
parameterResolver
|
||||
parameterResolver,
|
||||
"Asia/Shanghai"
|
||||
);
|
||||
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
||||
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account);
|
||||
@@ -147,4 +270,34 @@ public class SysJobControllerTest {
|
||||
org.mockito.ArgumentMatchers.anyString());
|
||||
}
|
||||
}
|
||||
|
||||
private static SysJobController controller(SysJobService service) {
|
||||
return new SysJobController(
|
||||
service,
|
||||
mock(WorkflowService.class),
|
||||
mock(WorkflowUsageAuthorizationService.class),
|
||||
mock(ResourceAccessService.class),
|
||||
mock(WorkflowRunningParameterResolver.class),
|
||||
"Asia/Shanghai");
|
||||
}
|
||||
|
||||
private static SysJob validJavaJob() {
|
||||
SysJob job = new SysJob();
|
||||
job.setJobName("generation-test");
|
||||
job.setJobType(EnumJobType.JAVA_CLASS.getCode());
|
||||
job.setCronExpression("0 0 0 1 1 ? 2099");
|
||||
job.setMisfirePolicy(EnumMisfirePolicy.SKIP.getCode());
|
||||
job.setAllowConcurrent(0);
|
||||
job.setJobParams(Map.of(JobConstant.JAVA_METHOD_KEY,
|
||||
"tech.easyflow.job.util.JobUtil.test()"));
|
||||
return job;
|
||||
}
|
||||
|
||||
private static LoginAccount account() {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.ONE);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
account.setDeptId(BigInteger.ONE);
|
||||
return account;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package tech.easyflow.admin.controller.job;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
import tech.easyflow.job.service.SysJobLogService;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link SysJobLogController} 查询与轻量刷新边界测试。
|
||||
*/
|
||||
public class SysJobLogControllerTest {
|
||||
|
||||
@Test
|
||||
public void shouldBuildBothFireTimeRanges() {
|
||||
SysJobLogController controller = controller(mock(SysJobLogService.class));
|
||||
HttpServletRequest request = emptyRequest();
|
||||
when(request.getParameter("scheduledStart")).thenReturn("2026-08-31 10:00:00");
|
||||
when(request.getParameter("scheduledEnd")).thenReturn("2026-08-31 11:00:00");
|
||||
when(request.getParameter("actualStart")).thenReturn("2026-08-31 10:00:01");
|
||||
when(request.getParameter("actualEnd")).thenReturn("2026-08-31 11:00:01");
|
||||
|
||||
String sql = controller.buildQueryWrapper(request).toSQL().toLowerCase(Locale.ROOT);
|
||||
|
||||
Assert.assertEquals(countOccurrences(sql, "scheduled_fire_time"), 2);
|
||||
Assert.assertEquals(countOccurrences(sql, "actual_fire_time"), 2);
|
||||
}
|
||||
|
||||
@Test(expectedExceptions = BusinessException.class)
|
||||
public void shouldRejectInvalidFireTime() {
|
||||
SysJobLogController controller = controller(mock(SysJobLogService.class));
|
||||
HttpServletRequest request = emptyRequest();
|
||||
when(request.getParameter("scheduledStart")).thenReturn("2026/08/31 10:00:00");
|
||||
|
||||
controller.buildQueryWrapper(request);
|
||||
}
|
||||
|
||||
@Test(expectedExceptions = BusinessException.class)
|
||||
public void shouldRejectReversedActualFireTimeRange() {
|
||||
SysJobLogController controller = controller(mock(SysJobLogService.class));
|
||||
HttpServletRequest request = emptyRequest();
|
||||
when(request.getParameter("actualStart")).thenReturn("2026-08-31 11:00:00");
|
||||
when(request.getParameter("actualEnd")).thenReturn("2026-08-31 10:00:00");
|
||||
|
||||
controller.buildQueryWrapper(request);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldClampRefreshAndPageSize() {
|
||||
SysJobLogService service = mock(SysJobLogService.class);
|
||||
when(service.list(any(QueryWrapper.class))).thenReturn(List.of());
|
||||
when(service.page(any(Page.class), any(QueryWrapper.class)))
|
||||
.thenAnswer(invocation -> invocation.getArgument(0));
|
||||
SysJobLogController controller = controller(service);
|
||||
HttpServletRequest request = emptyRequest();
|
||||
|
||||
controller.refresh(request, 500L);
|
||||
Page<SysJobLog> page = controller.queryPage(
|
||||
new Page<>(1, 500), QueryWrapper.create());
|
||||
|
||||
ArgumentCaptor<QueryWrapper> queryCaptor =
|
||||
ArgumentCaptor.forClass(QueryWrapper.class);
|
||||
verify(service).list(queryCaptor.capture());
|
||||
String refreshSql = queryCaptor.getValue().toSQL().toLowerCase(Locale.ROOT);
|
||||
Assert.assertTrue(refreshSql.contains("limit 100"));
|
||||
Assert.assertEquals(page.getPageSize(), 100L);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldUseStableScheduledFireTimeOrder() {
|
||||
Assert.assertEquals(
|
||||
controller(mock(SysJobLogService.class)).getDefaultOrderBy(),
|
||||
"scheduled_fire_time desc, id desc");
|
||||
}
|
||||
|
||||
private static SysJobLogController controller(SysJobLogService service) {
|
||||
return new SysJobLogController(service, "Asia/Shanghai");
|
||||
}
|
||||
|
||||
private static HttpServletRequest emptyRequest() {
|
||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
||||
when(request.getParameterMap()).thenReturn(Collections.emptyMap());
|
||||
return request;
|
||||
}
|
||||
|
||||
private static int countOccurrences(String source, String expected) {
|
||||
return (source.length() - source.replace(expected, "").length())
|
||||
/ expected.length();
|
||||
}
|
||||
}
|
||||
@@ -4,13 +4,19 @@ import com.easyagents.flow.core.chain.ChainConsts;
|
||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
@@ -69,4 +75,81 @@ public class WorkflowChatEventStreamTest {
|
||||
WorkflowChatEventStream.visibleFinalOutput(null).isEmpty()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证工作流启动异常也会释放匿名活动执行租约。
|
||||
*/
|
||||
@Test
|
||||
public void shouldCleanupExternalResourceWhenStartFails() {
|
||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||
doThrow(new IllegalStateException("start failed"))
|
||||
.when(chainExecutor)
|
||||
.executeAsync(any(), any(), any());
|
||||
WorkflowChatEventStream eventStream =
|
||||
new WorkflowChatEventStream(chainExecutor);
|
||||
AtomicInteger cleanupCount = new AtomicInteger();
|
||||
|
||||
Assert.expectThrows(
|
||||
IllegalStateException.class,
|
||||
() -> eventStream.start(
|
||||
"definition",
|
||||
Map.of(),
|
||||
cleanupCount::incrementAndGet
|
||||
)
|
||||
);
|
||||
|
||||
Assert.assertEquals(cleanupCount.get(), 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证浏览器断开只分离 SSE,不取消仍在运行的工作流。
|
||||
*/
|
||||
@Test
|
||||
public void shouldKeepRuntimeRunningWhenBrowserDisconnects() {
|
||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||
doAnswer(invocation -> {
|
||||
@SuppressWarnings("unchecked")
|
||||
Consumer<String> beforeStart = invocation.getArgument(2);
|
||||
beforeStart.accept("execution-1");
|
||||
return "execution-1";
|
||||
}).when(chainExecutor).executeAsync(any(), any(), any());
|
||||
CapturingSseEmitter emitter = new CapturingSseEmitter();
|
||||
WorkflowChatEventStream eventStream =
|
||||
new WorkflowChatEventStream(chainExecutor) {
|
||||
@Override
|
||||
SseEmitter createEmitter() {
|
||||
return emitter;
|
||||
}
|
||||
};
|
||||
AtomicInteger cleanupCount = new AtomicInteger();
|
||||
|
||||
eventStream.start(
|
||||
"definition",
|
||||
Map.of(),
|
||||
cleanupCount::incrementAndGet,
|
||||
Duration.ofMinutes(35)
|
||||
);
|
||||
emitter.disconnect();
|
||||
|
||||
verify(chainExecutor, never()).cancel(any(), any());
|
||||
Assert.assertEquals(cleanupCount.get(), 0);
|
||||
|
||||
eventStream.shutdown();
|
||||
Assert.assertEquals(cleanupCount.get(), 1);
|
||||
}
|
||||
|
||||
private static final class CapturingSseEmitter extends SseEmitter {
|
||||
|
||||
private Runnable completion;
|
||||
|
||||
@Override
|
||||
public synchronized void onCompletion(Runnable callback) {
|
||||
this.completion = callback;
|
||||
}
|
||||
|
||||
private void disconnect() {
|
||||
Assert.assertNotNull(completion);
|
||||
completion.run();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.script.DefaultRedisScript;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link WorkflowPublicChatAccessGuard} Redis 失败关闭测试。
|
||||
*/
|
||||
public class WorkflowPublicChatAccessGuardTest {
|
||||
|
||||
@Test
|
||||
public void shouldExposeDocumentedProtectionDefaults() {
|
||||
WorkflowPublicShareProperties properties =
|
||||
new WorkflowPublicShareProperties();
|
||||
|
||||
Assert.assertEquals(properties.getRunVisitorLimit(), 5);
|
||||
Assert.assertEquals(properties.getRunShareLimit(), 60);
|
||||
Assert.assertEquals(properties.getUploadVisitorLimit(), 10);
|
||||
Assert.assertEquals(properties.getUploadShareLimit(), 60);
|
||||
Assert.assertEquals(properties.getRateWindow(), Duration.ofMinutes(1));
|
||||
Assert.assertEquals(properties.getActiveLease(), Duration.ofMinutes(35));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReturn429WhenFixedWindowIsExceeded() {
|
||||
StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
|
||||
RedisLockExecutor lockExecutor = mock(RedisLockExecutor.class);
|
||||
when(redisTemplate.execute(
|
||||
any(DefaultRedisScript.class),
|
||||
anyList(),
|
||||
anyString(),
|
||||
anyString(),
|
||||
anyString()
|
||||
)).thenReturn(0L);
|
||||
WorkflowPublicChatAccessGuard guard = new WorkflowPublicChatAccessGuard(
|
||||
redisTemplate,
|
||||
lockExecutor,
|
||||
new WorkflowPublicShareProperties()
|
||||
);
|
||||
|
||||
BusinessException error = Assert.expectThrows(
|
||||
BusinessException.class,
|
||||
() -> guard.checkRun(BigInteger.ONE, "visitor")
|
||||
);
|
||||
|
||||
Assert.assertEquals(error.getHttpStatus(), 429);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReturn503WhenRedisRateLimitIsUnavailable() {
|
||||
StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
|
||||
RedisLockExecutor lockExecutor = mock(RedisLockExecutor.class);
|
||||
when(redisTemplate.execute(
|
||||
any(DefaultRedisScript.class),
|
||||
anyList(),
|
||||
anyString(),
|
||||
anyString(),
|
||||
anyString()
|
||||
)).thenThrow(new IllegalStateException("redis unavailable"));
|
||||
WorkflowPublicChatAccessGuard guard = new WorkflowPublicChatAccessGuard(
|
||||
redisTemplate,
|
||||
lockExecutor,
|
||||
new WorkflowPublicShareProperties()
|
||||
);
|
||||
|
||||
BusinessException error = Assert.expectThrows(
|
||||
BusinessException.class,
|
||||
() -> guard.checkUpload(BigInteger.ONE, "visitor")
|
||||
);
|
||||
|
||||
Assert.assertEquals(error.getHttpStatus(), 503);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.entity.WorkflowShare;
|
||||
import tech.easyflow.ai.enums.PublishStatus;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
import tech.easyflow.ai.service.WorkflowShareService;
|
||||
import tech.easyflow.ai.share.WorkflowSharePolicy;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.system.entity.SysAccount;
|
||||
import tech.easyflow.system.service.SysAccountService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link WorkflowPublicChatContextResolver} 匿名主体边界测试。
|
||||
*/
|
||||
public class WorkflowPublicChatContextResolverTest {
|
||||
|
||||
@Test
|
||||
public void shouldUseCurrentShareCreatorAsPermissionSubject() {
|
||||
Fixture fixture = fixture(EnumDataStatus.AVAILABLE.getCode());
|
||||
|
||||
WorkflowPublicChatContext context = fixture.resolver.resolveActive(
|
||||
"share-key",
|
||||
"00112233445566778899aabbccddeeff"
|
||||
);
|
||||
|
||||
Assert.assertEquals(context.creator().getId(), BigInteger.TEN);
|
||||
Assert.assertEquals(context.creator().getTenantId(), BigInteger.ONE);
|
||||
Assert.assertEquals(
|
||||
context.visitorDigest(),
|
||||
WorkflowSharePolicy.hashChatVisitor(
|
||||
"share-key",
|
||||
"00112233445566778899aabbccddeeff"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRejectDisabledShareCreator() {
|
||||
Fixture fixture = fixture(EnumDataStatus.UNAVAILABLE.getCode());
|
||||
|
||||
BusinessException error = Assert.expectThrows(
|
||||
BusinessException.class,
|
||||
() -> fixture.resolver.resolveActive(
|
||||
"share-key",
|
||||
"00112233445566778899aabbccddeeff"
|
||||
)
|
||||
);
|
||||
|
||||
Assert.assertEquals(error.getHttpStatus(), 403);
|
||||
Assert.assertTrue(error.getMessage().contains("创建者账号"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldResolveHistoricalShareWithoutCurrentCreatorCheck() {
|
||||
Fixture fixture = fixture(EnumDataStatus.UNAVAILABLE.getCode());
|
||||
|
||||
WorkflowPublicChatContext context = fixture.resolver.resolveHistorical(
|
||||
"share-key",
|
||||
"00112233445566778899aabbccddeeff"
|
||||
);
|
||||
|
||||
Assert.assertNull(context.creator());
|
||||
Assert.assertNull(context.workflow());
|
||||
verify(fixture.accountService, never()).getById(BigInteger.TEN);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRejectMalformedVisitorIdentity() {
|
||||
Fixture fixture = fixture(EnumDataStatus.AVAILABLE.getCode());
|
||||
|
||||
BusinessException error = Assert.expectThrows(
|
||||
BusinessException.class,
|
||||
() -> fixture.resolver.resolveActive("share-key", "short")
|
||||
);
|
||||
|
||||
Assert.assertEquals(error.getErrorCode(), 40031);
|
||||
}
|
||||
|
||||
private Fixture fixture(Integer accountStatus) {
|
||||
WorkflowShareService shareService = mock(WorkflowShareService.class);
|
||||
WorkflowService workflowService = mock(WorkflowService.class);
|
||||
SysAccountService accountService = mock(SysAccountService.class);
|
||||
|
||||
WorkflowShare share = new WorkflowShare();
|
||||
share.setId(BigInteger.valueOf(7));
|
||||
share.setWorkflowId(BigInteger.valueOf(11));
|
||||
share.setTenantId(BigInteger.ONE);
|
||||
share.setCreatedBy(BigInteger.TEN);
|
||||
|
||||
Workflow workflow = new Workflow();
|
||||
workflow.setId(BigInteger.valueOf(11));
|
||||
workflow.setTenantId(BigInteger.ONE);
|
||||
workflow.setPublishStatus(PublishStatus.PUBLISHED.getCode());
|
||||
workflow.setPublishedSnapshotJson(Map.of("content", "{}"));
|
||||
|
||||
SysAccount account = new SysAccount();
|
||||
account.setId(BigInteger.TEN);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
account.setStatus(accountStatus);
|
||||
|
||||
when(shareService.resolvePublicChatShare("share-key"))
|
||||
.thenReturn(share);
|
||||
when(shareService.resolveHistoricalChatShare("share-key"))
|
||||
.thenReturn(share);
|
||||
when(workflowService.getPublishedById(BigInteger.valueOf(11)))
|
||||
.thenReturn(workflow);
|
||||
when(accountService.getById(BigInteger.TEN)).thenReturn(account);
|
||||
|
||||
return new Fixture(
|
||||
new WorkflowPublicChatContextResolver(
|
||||
shareService,
|
||||
workflowService,
|
||||
accountService
|
||||
),
|
||||
accountService
|
||||
);
|
||||
}
|
||||
|
||||
private record Fixture(
|
||||
WorkflowPublicChatContextResolver resolver,
|
||||
SysAccountService accountService
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import com.easyagents.flow.core.chain.ChainState;
|
||||
import com.easyagents.flow.core.chain.ChainStatus;
|
||||
import com.easyagents.flow.core.chain.Parameter;
|
||||
import com.easyagents.flow.core.chain.repository.ChainStateRepository;
|
||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.entity.WorkflowExecResult;
|
||||
import tech.easyflow.ai.entity.WorkflowShare;
|
||||
import tech.easyflow.ai.service.WorkflowExecResultService;
|
||||
import tech.easyflow.ai.service.WorkflowExecStepService;
|
||||
import tech.easyflow.ai.utils.WorkFlowUtil;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.constant.Constants;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyMap;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link WorkflowPublicChatService} 匿名执行归属测试。
|
||||
*/
|
||||
public class WorkflowPublicChatServiceTest {
|
||||
|
||||
@Test
|
||||
public void shouldSeparatePermissionSubjectFromExecutionOwner() {
|
||||
Fixture fixture = fixture();
|
||||
RedisLockExecutor.LockHandle activity = mock(
|
||||
RedisLockExecutor.LockHandle.class);
|
||||
when(fixture.parameterResolver.normalizeRuntimeVariables(
|
||||
eq("{}"), anyMap())).thenReturn(new LinkedHashMap<>());
|
||||
when(fixture.accessGuard.acquireActivity(
|
||||
BigInteger.valueOf(7), "visitor-digest"))
|
||||
.thenReturn(activity);
|
||||
when(fixture.accessGuard.activityLease())
|
||||
.thenReturn(Duration.ofMinutes(35));
|
||||
when(fixture.eventStream.start(
|
||||
eq(PublishedWorkflowDefinitionIds.published("11")),
|
||||
anyMap(),
|
||||
any(Runnable.class),
|
||||
eq(Duration.ofMinutes(35))
|
||||
)).thenReturn(new SseEmitter());
|
||||
|
||||
fixture.service.run("share-key", visitorId(), Map.of());
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<Map<String, Object>> variables = ArgumentCaptor
|
||||
.forClass((Class) Map.class);
|
||||
verify(fixture.eventStream).start(
|
||||
eq(PublishedWorkflowDefinitionIds.published("11")),
|
||||
variables.capture(),
|
||||
any(Runnable.class),
|
||||
eq(Duration.ofMinutes(35))
|
||||
);
|
||||
Assert.assertSame(
|
||||
variables.getValue().get(Constants.LOGIN_USER_KEY),
|
||||
fixture.context.creator()
|
||||
);
|
||||
Assert.assertEquals(
|
||||
variables.getValue().get(WorkFlowUtil.CREATED_KEY_MEMORY_KEY),
|
||||
"WORKFLOW_CHAT_SHARE:7"
|
||||
);
|
||||
Assert.assertEquals(
|
||||
variables.getValue().get(WorkFlowUtil.CREATED_BY_MEMORY_KEY),
|
||||
"visitor-digest"
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRejectExecutionOwnedByAnotherVisitor() {
|
||||
Fixture fixture = fixture();
|
||||
WorkflowExecResult record = new WorkflowExecResult();
|
||||
record.setWorkflowId(BigInteger.valueOf(11));
|
||||
record.setCreatedKey("WORKFLOW_CHAT_SHARE:7");
|
||||
record.setCreatedBy("another-visitor");
|
||||
when(fixture.execResultService.getByExecKey("execution-1"))
|
||||
.thenReturn(record);
|
||||
|
||||
BusinessException error = Assert.expectThrows(
|
||||
BusinessException.class,
|
||||
() -> fixture.service.detail(
|
||||
"share-key", visitorId(), "execution-1")
|
||||
);
|
||||
|
||||
Assert.assertEquals(error.getHttpStatus(), 403);
|
||||
Assert.assertEquals(error.getErrorCode(), 40333);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldExposeMinimalRuntimeStateForRefreshRecovery() {
|
||||
Fixture fixture = fixture();
|
||||
WorkflowExecResult record = ownedRecord();
|
||||
when(fixture.execResultService.getByExecKey("execution-1"))
|
||||
.thenReturn(record);
|
||||
when(fixture.execStepService.list(any(QueryWrapper.class)))
|
||||
.thenReturn(List.of());
|
||||
ChainStateRepository repository = mock(ChainStateRepository.class);
|
||||
ChainState state = new ChainState();
|
||||
state.setStatus(ChainStatus.SUSPEND);
|
||||
state.setMessage("请确认是否继续");
|
||||
state.setSuspendForParameters(List.of(new Parameter("approved")));
|
||||
when(fixture.chainExecutor.getChainStateRepository())
|
||||
.thenReturn(repository);
|
||||
when(repository.load("execution-1")).thenReturn(state);
|
||||
|
||||
Map<String, Object> detail = fixture.service.detail(
|
||||
"share-key", visitorId(), "execution-1");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> runtime =
|
||||
(Map<String, Object>) detail.get("runtime");
|
||||
Assert.assertEquals(runtime.get("status"), "SUSPEND");
|
||||
Assert.assertEquals(runtime.get("statusValue"), 5);
|
||||
Assert.assertEquals(runtime.get("message"), "请确认是否继续");
|
||||
Assert.assertEquals(
|
||||
((List<?>) runtime.get("parameters")).size(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
private WorkflowExecResult ownedRecord() {
|
||||
WorkflowExecResult record = new WorkflowExecResult();
|
||||
record.setId(BigInteger.valueOf(31));
|
||||
record.setWorkflowId(BigInteger.valueOf(11));
|
||||
record.setExecKey("execution-1");
|
||||
record.setCreatedKey("WORKFLOW_CHAT_SHARE:7");
|
||||
record.setCreatedBy("visitor-digest");
|
||||
return record;
|
||||
}
|
||||
|
||||
private Fixture fixture() {
|
||||
WorkflowPublicChatContextResolver contextResolver = mock(
|
||||
WorkflowPublicChatContextResolver.class);
|
||||
WorkflowCheckService workflowCheckService = mock(
|
||||
WorkflowCheckService.class);
|
||||
WorkflowRunningParameterResolver parameterResolver = mock(
|
||||
WorkflowRunningParameterResolver.class);
|
||||
WorkflowPublicChatUploadService uploadService = mock(
|
||||
WorkflowPublicChatUploadService.class);
|
||||
WorkflowPublicChatAccessGuard accessGuard = mock(
|
||||
WorkflowPublicChatAccessGuard.class);
|
||||
WorkflowChatEventStream eventStream = mock(
|
||||
WorkflowChatEventStream.class);
|
||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||
WorkflowExecResultService execResultService = mock(
|
||||
WorkflowExecResultService.class);
|
||||
WorkflowExecStepService execStepService = mock(
|
||||
WorkflowExecStepService.class);
|
||||
|
||||
WorkflowShare share = new WorkflowShare();
|
||||
share.setId(BigInteger.valueOf(7));
|
||||
share.setWorkflowId(BigInteger.valueOf(11));
|
||||
Workflow workflow = new Workflow();
|
||||
workflow.setId(BigInteger.valueOf(11));
|
||||
workflow.setContent("{}");
|
||||
LoginAccount creator = new LoginAccount();
|
||||
creator.setId(BigInteger.TEN);
|
||||
creator.setTenantId(BigInteger.ONE);
|
||||
WorkflowPublicChatContext context = new WorkflowPublicChatContext(
|
||||
share,
|
||||
workflow,
|
||||
creator,
|
||||
"share-key",
|
||||
"visitor-digest"
|
||||
);
|
||||
when(contextResolver.resolveActive("share-key", visitorId()))
|
||||
.thenReturn(context);
|
||||
when(contextResolver.resolveHistorical("share-key", visitorId()))
|
||||
.thenReturn(context);
|
||||
|
||||
WorkflowPublicChatService service = new WorkflowPublicChatService(
|
||||
contextResolver,
|
||||
workflowCheckService,
|
||||
parameterResolver,
|
||||
uploadService,
|
||||
accessGuard,
|
||||
eventStream,
|
||||
chainExecutor,
|
||||
execResultService,
|
||||
execStepService
|
||||
);
|
||||
return new Fixture(
|
||||
service,
|
||||
context,
|
||||
parameterResolver,
|
||||
accessGuard,
|
||||
eventStream,
|
||||
chainExecutor,
|
||||
execResultService,
|
||||
execStepService
|
||||
);
|
||||
}
|
||||
|
||||
private String visitorId() {
|
||||
return "00112233445566778899aabbccddeeff";
|
||||
}
|
||||
|
||||
private record Fixture(
|
||||
WorkflowPublicChatService service,
|
||||
WorkflowPublicChatContext context,
|
||||
WorkflowRunningParameterResolver parameterResolver,
|
||||
WorkflowPublicChatAccessGuard accessGuard,
|
||||
WorkflowChatEventStream eventStream,
|
||||
ChainExecutor chainExecutor,
|
||||
WorkflowExecResultService execResultService,
|
||||
WorkflowExecStepService execStepService
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package tech.easyflow.admin.service.ai;
|
||||
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.entity.WorkflowShare;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link WorkflowPublicChatUploadService} 上传边界测试。
|
||||
*/
|
||||
public class WorkflowPublicChatUploadServiceTest {
|
||||
|
||||
@Test
|
||||
public void shouldStoreDeclaredFileUnderVisitorScope() {
|
||||
Fixture fixture = fixture("file");
|
||||
MultipartFile file = mock(MultipartFile.class);
|
||||
when(file.isEmpty()).thenReturn(false);
|
||||
when(file.getSize()).thenReturn(1024L);
|
||||
when(file.getOriginalFilename()).thenReturn("input.pdf");
|
||||
when(fixture.storageService.save(
|
||||
eq(file), anyString())).thenReturn("/files/input.pdf");
|
||||
|
||||
fixture.service.upload(fixture.context, "attachment", file);
|
||||
|
||||
verify(fixture.accessGuard).checkUpload(
|
||||
BigInteger.valueOf(7), "visitor-digest");
|
||||
verify(fixture.storageService).save(
|
||||
file,
|
||||
"workflow-chat-share/7/visitor-digest"
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRejectReferenceWithoutCurrentVisitorGrant() {
|
||||
Fixture fixture = fixture("file");
|
||||
|
||||
BusinessException error = Assert.expectThrows(
|
||||
BusinessException.class,
|
||||
() -> fixture.service.assertOwnedUploads(
|
||||
fixture.context,
|
||||
Map.of("attachment", List.of(Map.of(
|
||||
"fileName", "input.pdf",
|
||||
"filePath", "/files/other.pdf"
|
||||
)))
|
||||
)
|
||||
);
|
||||
|
||||
Assert.assertEquals(error.getHttpStatus(), 403);
|
||||
Assert.assertEquals(error.getErrorCode(), 40332);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRejectGrantCreatedForDifferentParameterType() {
|
||||
Fixture fixture = fixture("image");
|
||||
when(fixture.valueOperations.get(anyString())).thenReturn("file");
|
||||
|
||||
BusinessException error = Assert.expectThrows(
|
||||
BusinessException.class,
|
||||
() -> fixture.service.assertOwnedUploads(
|
||||
fixture.context,
|
||||
Map.of("attachment", Map.of(
|
||||
"sourceType", "upload",
|
||||
"filePath", "/files/input.png"
|
||||
))
|
||||
)
|
||||
);
|
||||
|
||||
Assert.assertEquals(error.getHttpStatus(), 403);
|
||||
Assert.assertEquals(error.getErrorCode(), 40332);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRejectUnsupportedImageType() {
|
||||
Fixture fixture = fixture("image");
|
||||
MultipartFile file = mock(MultipartFile.class);
|
||||
when(file.isEmpty()).thenReturn(false);
|
||||
when(file.getSize()).thenReturn(1024L);
|
||||
when(file.getContentType()).thenReturn("image/svg+xml");
|
||||
when(file.getOriginalFilename()).thenReturn("input.svg");
|
||||
|
||||
BusinessException error = Assert.expectThrows(
|
||||
BusinessException.class,
|
||||
() -> fixture.service.upload(
|
||||
fixture.context, "attachment", file)
|
||||
);
|
||||
|
||||
Assert.assertTrue(error.getMessage().contains("PNG"));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private Fixture fixture(String contentType) {
|
||||
WorkflowRunningParameterResolver parameterResolver = mock(
|
||||
WorkflowRunningParameterResolver.class);
|
||||
WorkflowPublicChatAccessGuard accessGuard = mock(
|
||||
WorkflowPublicChatAccessGuard.class);
|
||||
StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
|
||||
ValueOperations<String, String> valueOperations = mock(
|
||||
ValueOperations.class);
|
||||
FileStorageService storageService = mock(FileStorageService.class);
|
||||
when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
|
||||
Workflow workflow = new Workflow();
|
||||
workflow.setId(BigInteger.valueOf(11));
|
||||
when(parameterResolver.buildRunningParametersView(workflow))
|
||||
.thenReturn(Map.of(
|
||||
"startFormSchema",
|
||||
List.of(Map.of(
|
||||
"key", "attachment",
|
||||
"contentType", contentType
|
||||
))
|
||||
));
|
||||
|
||||
WorkflowShare share = new WorkflowShare();
|
||||
share.setId(BigInteger.valueOf(7));
|
||||
share.setExpiresAt(new Date(
|
||||
System.currentTimeMillis() + 60_000L));
|
||||
WorkflowPublicChatContext context = new WorkflowPublicChatContext(
|
||||
share,
|
||||
workflow,
|
||||
null,
|
||||
"share-key",
|
||||
"visitor-digest"
|
||||
);
|
||||
WorkflowPublicChatUploadService service =
|
||||
new WorkflowPublicChatUploadService(
|
||||
parameterResolver,
|
||||
accessGuard,
|
||||
new WorkflowPublicShareProperties(),
|
||||
redisTemplate,
|
||||
storageService
|
||||
);
|
||||
return new Fixture(
|
||||
service,
|
||||
context,
|
||||
accessGuard,
|
||||
redisTemplate,
|
||||
valueOperations,
|
||||
storageService
|
||||
);
|
||||
}
|
||||
|
||||
private record Fixture(
|
||||
WorkflowPublicChatUploadService service,
|
||||
WorkflowPublicChatContext context,
|
||||
WorkflowPublicChatAccessGuard accessGuard,
|
||||
StringRedisTemplate redisTemplate,
|
||||
ValueOperations<String, String> valueOperations,
|
||||
FileStorageService storageService
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,12 @@ import tech.easyflow.common.annotation.DictDef;
|
||||
@DictDef(name = "任务执行结果", code = "jobResult", keyField = "code", labelField = "text")
|
||||
public enum EnumJobResult {
|
||||
|
||||
|
||||
SUCCESS(1,"成功"),
|
||||
FAIL(0,"失败"),
|
||||
PENDING(2,"等待执行"),
|
||||
RUNNING(3,"执行中"),
|
||||
DEAD(4,"需人工处理"),
|
||||
CANCELLED(5,"已取消"),
|
||||
;
|
||||
|
||||
private final int code;
|
||||
|
||||
@@ -5,10 +5,8 @@ import tech.easyflow.common.annotation.DictDef;
|
||||
@DictDef(name = "错过策略", code = "misfirePolicy", keyField = "code", labelField = "text")
|
||||
public enum EnumMisfirePolicy {
|
||||
|
||||
DEFAULT(0,"默认"),
|
||||
MISFIRE_IGNORE_MISFIRES(1,"立即触发"),
|
||||
MISFIRE_FIRE_AND_PROCEED(2,"立即触发一次"),
|
||||
MISFIRE_DO_NOTHING(3,"忽略");
|
||||
FIRE_ONCE_NOW(2,"恢复后补执行一次"),
|
||||
SKIP(3,"跳过本次");
|
||||
;
|
||||
|
||||
private final int code;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package tech.easyflow.common.cache;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -10,6 +11,11 @@ import org.springframework.stereotype.Component;
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
@@ -27,6 +33,13 @@ public class RedisLockExecutor {
|
||||
private static final DefaultRedisScript<Long> NEXT_FENCING_TOKEN_SCRIPT;
|
||||
private static final DefaultRedisScript<Long> ACQUIRE_FENCED_LOCK_SCRIPT;
|
||||
|
||||
private final ScheduledExecutorService lockRenewalExecutor =
|
||||
Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "easyflow-redis-lock-renewal");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
static {
|
||||
RELEASE_LOCK_SCRIPT = new DefaultRedisScript<>();
|
||||
RELEASE_LOCK_SCRIPT.setScriptText(
|
||||
@@ -94,6 +107,66 @@ public class RedisLockExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 在自动续租的分布式锁保护下执行任务。
|
||||
*
|
||||
* <p>适用于包含数据库锁等待或外部持久化操作、无法由固定租约严格覆盖的管理命令。
|
||||
* 若执行期间确认锁已丢失,则不向调用方返回成功。</p>
|
||||
*/
|
||||
public void executeWithRenewingLock(
|
||||
String lockKey,
|
||||
Duration waitTimeout,
|
||||
Duration leaseTimeout,
|
||||
Runnable task) {
|
||||
executeWithRenewingLock(lockKey, waitTimeout, leaseTimeout, () -> {
|
||||
task.run();
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 在自动续租的分布式锁保护下执行有返回值任务。
|
||||
*/
|
||||
public <T> T executeWithRenewingLock(
|
||||
String lockKey,
|
||||
Duration waitTimeout,
|
||||
Duration leaseTimeout,
|
||||
Supplier<T> task) {
|
||||
LockHandle handle = acquire(lockKey, waitTimeout, leaseTimeout);
|
||||
AtomicBoolean lost = new AtomicBoolean();
|
||||
long renewalIntervalMillis = Math.max(1L, leaseTimeout.toMillis() / 3L);
|
||||
ScheduledFuture<?> renewal = lockRenewalExecutor.scheduleWithFixedDelay(
|
||||
() -> {
|
||||
try {
|
||||
if (!handle.renew()) {
|
||||
lost.set(true);
|
||||
}
|
||||
} catch (RuntimeException exception) {
|
||||
lost.set(true);
|
||||
log.warn("分布式锁续租失败,当前命令不得返回成功: lockKey={}",
|
||||
lockKey, exception);
|
||||
}
|
||||
},
|
||||
renewalIntervalMillis,
|
||||
renewalIntervalMillis,
|
||||
TimeUnit.MILLISECONDS);
|
||||
try {
|
||||
T result = task.get();
|
||||
if (lost.get()) {
|
||||
throw new IllegalStateException("执行期间分布式锁已丢失,lockKey=" + lockKey);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
renewal.cancel(false);
|
||||
handle.release();
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdownLockRenewalExecutor() {
|
||||
lockRenewalExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取显式释放的分布式锁句柄。
|
||||
*
|
||||
|
||||
@@ -11,6 +11,9 @@ import org.springframework.data.redis.core.script.RedisScript;
|
||||
import java.lang.reflect.Field;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
* {@link RedisLockExecutor} 回归测试。
|
||||
@@ -147,6 +150,96 @@ public class RedisLockExecutorTest {
|
||||
String.valueOf(Duration.ofDays(4).toMillis())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void renewingLockShouldRenewBeforeLongRunningCommandCompletes() throws Exception {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
ValueOperations<String, String> valueOperations = mockValueOperations(true);
|
||||
CountDownLatch renewed = new CountDownLatch(1);
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
Mockito.when(redisTemplate.execute(
|
||||
ArgumentMatchers.<RedisScript<Long>>any(),
|
||||
ArgumentMatchers.<List<String>>any(),
|
||||
ArgumentMatchers.anyString(),
|
||||
ArgumentMatchers.anyString()
|
||||
)).thenAnswer(invocation -> {
|
||||
renewed.countDown();
|
||||
return 1L;
|
||||
});
|
||||
|
||||
RedisLockExecutor executor = new RedisLockExecutor();
|
||||
setRedisTemplate(executor, redisTemplate);
|
||||
try {
|
||||
executor.executeWithRenewingLock(
|
||||
"easyflow:test:renewing-lock",
|
||||
Duration.ZERO,
|
||||
Duration.ofMillis(60),
|
||||
() -> {
|
||||
try {
|
||||
Assert.assertTrue(renewed.await(1, TimeUnit.SECONDS));
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new AssertionError("等待锁续租时被中断", exception);
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
executor.shutdownLockRenewalExecutor();
|
||||
}
|
||||
|
||||
Mockito.verify(redisTemplate, Mockito.atLeastOnce()).execute(
|
||||
ArgumentMatchers.<RedisScript<Long>>any(),
|
||||
ArgumentMatchers.eq(List.of("easyflow:test:renewing-lock")),
|
||||
ArgumentMatchers.anyString(),
|
||||
ArgumentMatchers.eq("60"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void renewingLockMustNotReturnSuccessAfterRenewalThrows() throws Exception {
|
||||
StringRedisTemplate redisTemplate = Mockito.mock(StringRedisTemplate.class);
|
||||
ValueOperations<String, String> valueOperations = mockValueOperations(true);
|
||||
CountDownLatch renewalAttempted = new CountDownLatch(1);
|
||||
AtomicInteger scriptCalls = new AtomicInteger();
|
||||
Mockito.when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
Mockito.when(redisTemplate.execute(
|
||||
ArgumentMatchers.<RedisScript<Long>>any(),
|
||||
ArgumentMatchers.<List<String>>any(),
|
||||
ArgumentMatchers.anyString(),
|
||||
ArgumentMatchers.anyString()
|
||||
)).thenAnswer(invocation -> {
|
||||
if (scriptCalls.incrementAndGet() == 1) {
|
||||
renewalAttempted.countDown();
|
||||
throw new IllegalStateException("redis unavailable");
|
||||
}
|
||||
return 1L;
|
||||
});
|
||||
|
||||
RedisLockExecutor executor = new RedisLockExecutor();
|
||||
setRedisTemplate(executor, redisTemplate);
|
||||
try {
|
||||
try {
|
||||
executor.executeWithRenewingLock(
|
||||
"easyflow:test:renewal-failure",
|
||||
Duration.ZERO,
|
||||
Duration.ofMillis(60),
|
||||
() -> {
|
||||
try {
|
||||
Assert.assertTrue(renewalAttempted.await(1, TimeUnit.SECONDS));
|
||||
// 等待续租线程把失败结果发布到调用线程;业务任务与续租
|
||||
// 同时完成时,锁仍处于原租约内且 callback 已结束。
|
||||
Thread.sleep(50L);
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new AssertionError(exception);
|
||||
}
|
||||
});
|
||||
Assert.fail("续租异常后不应返回成功");
|
||||
} catch (IllegalStateException exception) {
|
||||
Assert.assertTrue(exception.getMessage().contains("分布式锁已丢失"));
|
||||
}
|
||||
} finally {
|
||||
executor.shutdownLockRenewalExecutor();
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private ValueOperations<String, String> mockValueOperations(boolean acquired) {
|
||||
ValueOperations<String, String> valueOperations = Mockito.mock(ValueOperations.class);
|
||||
|
||||
@@ -11,6 +11,7 @@ import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
@@ -151,6 +152,19 @@ public class FileStorageManager implements FileStorageService {
|
||||
return serviceForHandle(handle).readRecoverable(handle);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用当前后端解析服务端可信文件引用。
|
||||
*
|
||||
* @param reference 文件 URL 或其他后端可识别引用
|
||||
* @return 可信文件的物理读取句柄;引用无法确认时为空
|
||||
* @throws IOException 文件记录无法安全解析时抛出
|
||||
*/
|
||||
@Override
|
||||
public Optional<FileStorageWriteHandle> resolveTrustedFile(String reference)
|
||||
throws IOException {
|
||||
return currentService().resolveTrustedFile(reference);
|
||||
}
|
||||
|
||||
/**
|
||||
* 严格按句柄中的后端精确删除物理对象。
|
||||
*
|
||||
|
||||
@@ -5,6 +5,7 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* EasyFlow 文件存储统一接口。
|
||||
@@ -105,6 +106,21 @@ public interface FileStorageService {
|
||||
throw unsupportedRecoverableOperation("readRecoverable");
|
||||
}
|
||||
|
||||
/**
|
||||
* 将服务端可信文件引用解析为物理读取句柄。
|
||||
*
|
||||
* <p>实现必须以服务端持久化记录或存储平台配置为信任来源,并要求外部引用与可信来源
|
||||
* 精确匹配;不得仅根据客户端传入的 URL、路径或 locator 构造句柄。</p>
|
||||
*
|
||||
* @param reference 文件 URL 或其他后端可识别引用
|
||||
* @return 可信文件的物理读取句柄;引用无法确认时为空
|
||||
* @throws IOException 文件记录损坏或存储配置不兼容时抛出
|
||||
*/
|
||||
default Optional<FileStorageWriteHandle> resolveTrustedFile(String reference)
|
||||
throws IOException {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 精确且幂等地删除句柄对应的物理对象。
|
||||
*
|
||||
|
||||
@@ -21,6 +21,7 @@ import java.io.*;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* 基于 x-file-storage 的 EasyFlow 文件存储实现。
|
||||
@@ -268,6 +269,47 @@ public class XFIleStorageServiceImpl implements FileStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 x-file-storage 文件记录或服务端平台配置恢复可信物理读取句柄。
|
||||
*
|
||||
* <p>优先使用 recorder 的精确记录。记录不存在时,仅允许与已配置平台 domain、basePath
|
||||
* 及重建后的完整 URL 完全一致的引用。其他 URL 返回空,由上层继续执行公网地址安全校验。</p>
|
||||
*
|
||||
* @param reference 文件 URL
|
||||
* @return 可信文件的物理读取句柄;引用无法确认时为空
|
||||
* @throws IOException 文件记录损坏或平台配置不兼容时抛出
|
||||
*/
|
||||
@Override
|
||||
public Optional<FileStorageWriteHandle> resolveTrustedFile(String reference)
|
||||
throws IOException {
|
||||
if (!StringUtils.hasText(reference)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
FileInfo fileInfo = null;
|
||||
try {
|
||||
fileInfo = fileStorageService.getFileInfoByUrl(reference);
|
||||
} catch (RuntimeException exception) {
|
||||
// 存储平台配置本身仍可提供精确可信边界,记录器异常不应阻断内部对象读取。
|
||||
LOG.warn("查询 x-file-storage 文件记录失败,继续按服务端存储配置识别,reference={}",
|
||||
reference, exception);
|
||||
}
|
||||
if (fileInfo != null) {
|
||||
if (!reference.equals(fileInfo.getUrl())) {
|
||||
throw new IOException("x-file-storage 文件记录 URL 与请求引用不一致");
|
||||
}
|
||||
try {
|
||||
FileStorageWriteHandle handle = handleFromFileInfo(fileInfo);
|
||||
FileStorage storage = requireStorage(handle);
|
||||
requirePersistedBasePathSupport(storage, handle);
|
||||
verifyRecordedLocation(fileInfo, handle);
|
||||
return Optional.of(handle);
|
||||
} catch (RuntimeException exception) {
|
||||
throw new IOException("x-file-storage 文件记录无法恢复为安全读取位置", exception);
|
||||
}
|
||||
}
|
||||
return resolveConfiguredStorageReference(reference);
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接调用句柄指定平台的物理删除与存在检查,绕过依赖 URL 记录的聚合删除路径。
|
||||
*
|
||||
@@ -391,6 +433,108 @@ public class XFIleStorageServiceImpl implements FileStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验 recorder 中的物理定位字段可由恢复句柄无损重建。
|
||||
*
|
||||
* @param fileInfo 服务端文件记录
|
||||
* @param handle 恢复出的物理读取句柄
|
||||
*/
|
||||
private void verifyRecordedLocation(FileInfo fileInfo, FileStorageWriteHandle handle) {
|
||||
String actualBasePath = fileInfo.getBasePath() == null ? "" : fileInfo.getBasePath();
|
||||
String actualPath = fileInfo.getPath() == null ? "" : fileInfo.getPath();
|
||||
if (!handle.getPlatform().equals(fileInfo.getPlatform())
|
||||
|| !handle.getBasePath().equals(actualBasePath)
|
||||
|| !physicalPath(handle).equals(actualPath)
|
||||
|| !handle.getFilename().equals(fileInfo.getFilename())) {
|
||||
throw new IllegalStateException("x-file-storage 文件记录包含非规范物理位置");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 FileInfo 的物理定位字段构造严格校验的读取句柄。
|
||||
*
|
||||
* @param fileInfo 服务端文件信息
|
||||
* @return 可信物理读取句柄
|
||||
*/
|
||||
private FileStorageWriteHandle handleFromFileInfo(FileInfo fileInfo) {
|
||||
String basePath = fileInfo.getBasePath() == null ? "" : fileInfo.getBasePath();
|
||||
return new FileStorageWriteHandle(
|
||||
RECOVERABLE_BACKEND,
|
||||
fileInfo.getPlatform(),
|
||||
basePath,
|
||||
recordedRelativePath(basePath, fileInfo.getPath()),
|
||||
fileInfo.getFilename());
|
||||
}
|
||||
|
||||
/**
|
||||
* 按服务端配置的平台 domain 与 basePath 识别内部存储 URL。
|
||||
*
|
||||
* <p>解析后会再次通过平台自身的 getFileKey 重建完整 URL 并进行精确比较,避免仅凭
|
||||
* host 或字符串前缀放行其他私网目标。</p>
|
||||
*
|
||||
* @param reference 待识别 URL
|
||||
* @return 精确匹配配置的读取句柄;不匹配任何平台时为空
|
||||
* @throws IOException 匹配平台前缀但路径无法安全恢复时抛出
|
||||
*/
|
||||
private Optional<FileStorageWriteHandle> resolveConfiguredStorageReference(
|
||||
String reference) throws IOException {
|
||||
if (fileStorageService.getFileStorageList() == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
for (FileStorage storage : fileStorageService.getFileStorageList()) {
|
||||
String domain = readDomainBestEffort(storage);
|
||||
if (!StringUtils.hasText(domain)) {
|
||||
continue;
|
||||
}
|
||||
String basePath = readRequiredBasePath(storage);
|
||||
String prefix = domain + basePath;
|
||||
if (!reference.startsWith(prefix)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String remainder = reference.substring(prefix.length());
|
||||
int filenameIndex = remainder.lastIndexOf('/') + 1;
|
||||
FileInfo fileInfo = new FileInfo()
|
||||
.setUrl(reference)
|
||||
.setPlatform(storage.getPlatform())
|
||||
.setBasePath(basePath)
|
||||
.setPath(remainder.substring(0, filenameIndex))
|
||||
.setFilename(remainder.substring(filenameIndex));
|
||||
FileStorageWriteHandle handle = handleFromFileInfo(fileInfo);
|
||||
requirePersistedBasePathSupport(storage, handle);
|
||||
verifyRecordedLocation(fileInfo, handle);
|
||||
if (!reference.equals(deriveUrlBestEffort(storage, toFileInfo(handle)))) {
|
||||
throw new IllegalArgumentException("重建 URL 与请求引用不一致");
|
||||
}
|
||||
return Optional.of(handle);
|
||||
} catch (RuntimeException exception) {
|
||||
throw new IOException("服务端存储 URL 无法恢复为安全读取位置", exception);
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 recorder 保存的 x-file-storage 物理路径还原为句柄相对路径。
|
||||
*
|
||||
* @param basePath 平台基础路径
|
||||
* @param recordedPath recorder 中保存的物理目录
|
||||
* @return 不带前导斜杠的相对目录
|
||||
*/
|
||||
private String recordedRelativePath(String basePath, String recordedPath) {
|
||||
String path = recordedPath == null ? "" : recordedPath;
|
||||
if (basePath.isEmpty() || basePath.endsWith("/")) {
|
||||
if (path.startsWith("/")) {
|
||||
throw new IllegalArgumentException("文件记录路径与平台基础路径格式不一致");
|
||||
}
|
||||
return path;
|
||||
}
|
||||
if (!path.startsWith("/")) {
|
||||
throw new IllegalArgumentException("文件记录路径缺少必要的前导斜杠");
|
||||
}
|
||||
return path.substring(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造仅包含精确物理定位字段的 FileInfo。
|
||||
*
|
||||
@@ -487,16 +631,26 @@ public class XFIleStorageServiceImpl implements FileStorageService {
|
||||
* @return 可推导 URL;平台不支持时返回 null
|
||||
*/
|
||||
private String deriveUrlBestEffort(FileStorage storage, FileInfo fileInfo) {
|
||||
String domain = readDomainBestEffort(storage);
|
||||
if (domain == null) {
|
||||
return null;
|
||||
}
|
||||
return domain + storage.getFileKey(fileInfo);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用平台公开的 getDomain 方法读取文件访问域名。
|
||||
*
|
||||
* @param storage 具体平台存储
|
||||
* @return 平台访问域名;平台不支持时返回 null
|
||||
*/
|
||||
private String readDomainBestEffort(FileStorage storage) {
|
||||
try {
|
||||
Method method = storage.getClass().getMethod("getDomain");
|
||||
if (!String.class.equals(method.getReturnType())) {
|
||||
return null;
|
||||
}
|
||||
String domain = (String) method.invoke(storage);
|
||||
if (domain == null) {
|
||||
return null;
|
||||
}
|
||||
return domain + storage.getFileKey(fileInfo);
|
||||
return (String) method.invoke(storage);
|
||||
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | RuntimeException exception) {
|
||||
LOG.debug("当前 x-file-storage 平台无法推导 recorder URL: {}", storage.getClass().getName());
|
||||
return null;
|
||||
|
||||
@@ -7,6 +7,7 @@ import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
@@ -49,6 +50,27 @@ public class FileStorageManagerTest {
|
||||
assertFalse(exists);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证服务端文件记录解析使用当前配置的具体存储后端。
|
||||
*
|
||||
* @throws IOException 文件记录解析失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void recordedFileResolutionUsesCurrentBackend() throws IOException {
|
||||
RecordingStorage local = new RecordingStorage("local");
|
||||
RecordingStorage xFile = new RecordingStorage("xFileStorage");
|
||||
FileStorageManager manager = new FileStorageManager(
|
||||
() -> "xFileStorage",
|
||||
backend -> Map.of("local", local, "xFileStorage", xFile).get(backend));
|
||||
|
||||
Optional<FileStorageWriteHandle> resolved = manager.resolveTrustedFile(
|
||||
"http://127.0.0.1:39000/easyflow/attachment/demo.pdf");
|
||||
|
||||
assertSame(xFile.recordedHandle, resolved.orElseThrow());
|
||||
assertEquals(1, xFile.resolveCalls);
|
||||
assertEquals(0, local.resolveCalls);
|
||||
}
|
||||
|
||||
/**
|
||||
* 可记录可恢复调用的存储测试替身。
|
||||
*/
|
||||
@@ -57,6 +79,8 @@ public class FileStorageManagerTest {
|
||||
private final String backend;
|
||||
/** 固定结果。 */
|
||||
private final FileStorageWriteResult result;
|
||||
/** 固定服务端文件记录句柄。 */
|
||||
private final FileStorageWriteHandle recordedHandle;
|
||||
/** 固定可恢复读取流。 */
|
||||
private final InputStream recoverableInput = InputStream.nullInputStream();
|
||||
/** prepare 调用次数。 */
|
||||
@@ -69,6 +93,8 @@ public class FileStorageManagerTest {
|
||||
private int deleteCalls;
|
||||
/** exists 调用次数。 */
|
||||
private int existsCalls;
|
||||
/** 服务端文件记录解析调用次数。 */
|
||||
private int resolveCalls;
|
||||
|
||||
/**
|
||||
* 创建指定名称的存储替身。
|
||||
@@ -80,6 +106,8 @@ public class FileStorageManagerTest {
|
||||
FileStorageWriteHandle handle = new FileStorageWriteHandle(
|
||||
backend, "", "/tmp/easyflow", "skill-content", "content.bin");
|
||||
this.result = new FileStorageWriteResult("/files/content.bin", handle.encodeLocator());
|
||||
this.recordedHandle = new FileStorageWriteHandle(
|
||||
backend, "", "/tmp/easyflow", "attachment", "demo.pdf");
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@@ -114,6 +142,13 @@ public class FileStorageManagerTest {
|
||||
return recoverableInput;
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public Optional<FileStorageWriteHandle> resolveTrustedFile(String reference) {
|
||||
resolveCalls++;
|
||||
return Optional.of(recordedHandle);
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public void deleteRecoverable(FileStorageWriteHandle handle) {
|
||||
|
||||
@@ -212,6 +212,123 @@ public class XFIleStorageServiceImplTest {
|
||||
client.lastArgs.object());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 recorder 登记的回环地址附件可恢复为可信句柄并通过 MinIO 客户端直读。
|
||||
*
|
||||
* @throws Exception 测试替身配置或流读取失败
|
||||
*/
|
||||
@Test
|
||||
public void recordedLoopbackUrlUsesExactMinioObject() throws Exception {
|
||||
byte[] content = "workflow-content".getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||
RecordingMinioClient client = new RecordingMinioClient(content);
|
||||
MinioFileStorage platform = new MinioFileStorage();
|
||||
platform.setPlatform("minio-main");
|
||||
platform.setBucketName("easyflow");
|
||||
platform.setBasePath("attachment");
|
||||
platform.setDomain("http://127.0.0.1:39000/easyflow/");
|
||||
platform.setClientFactory(new FixedMinioClientFactory(client));
|
||||
RecoverableStorageService delegate = new RecoverableStorageService(platform);
|
||||
String fileUrl = "http://127.0.0.1:39000/easyflow/attachment/1/2026/8/26/"
|
||||
+ "d6186b17-4ab7-4f99-9299-b19df7ff0a3b/投标文件否决(废标)违规事项汇总.pdf";
|
||||
delegate.recordedFileInfo = new FileInfo()
|
||||
.setUrl(fileUrl)
|
||||
.setPlatform("minio-main")
|
||||
.setBasePath("attachment")
|
||||
.setPath("/1/2026/8/26/d6186b17-4ab7-4f99-9299-b19df7ff0a3b/")
|
||||
.setFilename("投标文件否决(废标)违规事项汇总.pdf");
|
||||
XFIleStorageServiceImpl service = createService(delegate);
|
||||
|
||||
FileStorageWriteHandle handle = service.resolveTrustedFile(fileUrl).orElseThrow();
|
||||
byte[] actual;
|
||||
try (InputStream inputStream = service.readRecoverable(handle)) {
|
||||
actual = inputStream.readAllBytes();
|
||||
}
|
||||
|
||||
assertEquals("attachment", handle.getBasePath());
|
||||
assertEquals(
|
||||
"1/2026/8/26/d6186b17-4ab7-4f99-9299-b19df7ff0a3b/",
|
||||
handle.getPath());
|
||||
assertArrayEquals(content, actual);
|
||||
assertEquals("easyflow", client.lastArgs.bucket());
|
||||
assertEquals(
|
||||
"attachment/1/2026/8/26/d6186b17-4ab7-4f99-9299-b19df7ff0a3b/"
|
||||
+ "投标文件否决(废标)违规事项汇总.pdf",
|
||||
client.lastArgs.object());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 recorder 没有记录时,服务端配置的存储 URL 仍可通过 MinIO 客户端直读。
|
||||
*
|
||||
* @throws Exception 测试替身配置或流读取失败
|
||||
*/
|
||||
@Test
|
||||
public void configuredStorageUrlWithoutRecorderUsesExactMinioObject() throws Exception {
|
||||
byte[] content = "configured-content".getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||
RecordingMinioClient client = new RecordingMinioClient(content);
|
||||
MinioFileStorage platform = new MinioFileStorage();
|
||||
platform.setPlatform("minio-main");
|
||||
platform.setBucketName("easyflow");
|
||||
platform.setBasePath("attachment");
|
||||
platform.setDomain("http://127.0.0.1:39000/easyflow/");
|
||||
platform.setClientFactory(new FixedMinioClientFactory(client));
|
||||
XFIleStorageServiceImpl service = createService(new RecoverableStorageService(platform));
|
||||
String fileUrl = "http://127.0.0.1:39000/easyflow/attachment/1/2026/8/26/"
|
||||
+ "0f0db465-7fa0-46b1-9fae-4f5a8c85f881/投标文件否决(废标)违规事项汇总.pdf";
|
||||
|
||||
FileStorageWriteHandle handle = service.resolveTrustedFile(fileUrl).orElseThrow();
|
||||
byte[] actual;
|
||||
try (InputStream inputStream = service.readRecoverable(handle)) {
|
||||
actual = inputStream.readAllBytes();
|
||||
}
|
||||
|
||||
assertArrayEquals(content, actual);
|
||||
assertEquals(
|
||||
"attachment/1/2026/8/26/0f0db465-7fa0-46b1-9fae-4f5a8c85f881/"
|
||||
+ "投标文件否决(废标)违规事项汇总.pdf",
|
||||
client.lastArgs.object());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证未配置为存储地址的回环 URL 不会被识别为可信附件。
|
||||
*
|
||||
* @throws Exception 测试替身注入失败
|
||||
*/
|
||||
@Test
|
||||
public void unconfiguredLoopbackUrlIsNotResolved() throws Exception {
|
||||
RecoverablePlatform platform = new RecoverablePlatform(
|
||||
"minio-main", "attachment", "http://127.0.0.1:39000/easyflow/");
|
||||
XFIleStorageServiceImpl service = createService(new RecoverableStorageService(platform));
|
||||
|
||||
assertTrue(service.resolveTrustedFile(
|
||||
"http://127.0.0.1:39000/other/unconfigured.pdf").isEmpty());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 recorder 中非规范物理路径会失败关闭。
|
||||
*
|
||||
* @throws Exception 测试替身注入失败
|
||||
*/
|
||||
@Test
|
||||
public void corruptedRecordedLocationIsRejected() throws Exception {
|
||||
RecoverablePlatform platform = new RecoverablePlatform(
|
||||
"minio-main", "attachment", "http://127.0.0.1:39000/easyflow/");
|
||||
RecoverableStorageService delegate = new RecoverableStorageService(platform);
|
||||
String fileUrl = "http://127.0.0.1:39000/easyflow/attachment/demo.pdf";
|
||||
delegate.recordedFileInfo = new FileInfo()
|
||||
.setUrl(fileUrl)
|
||||
.setPlatform("minio-main")
|
||||
.setBasePath("attachment")
|
||||
.setPath("missing-leading-slash/")
|
||||
.setFilename("demo.pdf");
|
||||
XFIleStorageServiceImpl service = createService(delegate);
|
||||
|
||||
IOException exception = assertThrows(
|
||||
IOException.class,
|
||||
() -> service.resolveTrustedFile(fileUrl));
|
||||
|
||||
assertTrue(exception.getMessage().contains("无法恢复"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 recorder 完全缺失目标记录时,精确删除仍直接作用于物理平台并成功。
|
||||
*
|
||||
@@ -471,6 +588,8 @@ public class XFIleStorageServiceImplTest {
|
||||
private int recorderDeleteCalls;
|
||||
/** recorder 删除是否抛出异常。 */
|
||||
private boolean recorderDeleteThrows;
|
||||
/** recorder 返回的服务端文件记录。 */
|
||||
private FileInfo recordedFileInfo;
|
||||
|
||||
/**
|
||||
* 创建聚合服务替身。
|
||||
@@ -479,10 +598,17 @@ public class XFIleStorageServiceImplTest {
|
||||
*/
|
||||
private RecoverableStorageService(FileStorage platform) {
|
||||
this.platform = platform;
|
||||
setFileStorageList(new java.util.concurrent.CopyOnWriteArrayList<>(
|
||||
java.util.List.of(platform)));
|
||||
setFileRecorder(new FileRecorder() {
|
||||
@Override public boolean save(FileInfo fileInfo) { return true; }
|
||||
@Override public void update(FileInfo fileInfo) { }
|
||||
@Override public FileInfo getByUrl(String url) { return null; }
|
||||
@Override public FileInfo getByUrl(String url) {
|
||||
return recordedFileInfo != null
|
||||
&& url.equals(recordedFileInfo.getUrl())
|
||||
? recordedFileInfo
|
||||
: null;
|
||||
}
|
||||
@Override public boolean delete(String url) {
|
||||
recorderDeleteCalls++;
|
||||
if (recorderDeleteThrows) {
|
||||
@@ -506,6 +632,15 @@ public class XFIleStorageServiceImplTest {
|
||||
return platform.getPlatform().equals(name) ? (T) platform : null;
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public FileInfo getFileInfoByUrl(String url) {
|
||||
return recordedFileInfo != null
|
||||
&& url.equals(recordedFileInfo.getUrl())
|
||||
? recordedFileInfo
|
||||
: null;
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public org.dromara.x.file.storage.core.upload.UploadPretreatment of(Object file) {
|
||||
|
||||
@@ -423,7 +423,7 @@ public class AgentRunService {
|
||||
chatContext.getExt().put(DOCUMENT_CONTEXT_TOKEN_ESTIMATE_EXT_KEY,
|
||||
documentContext.tokenEstimate());
|
||||
String runtimePrompt = effectivePrompt(prompt, !boundDocuments.isEmpty(), !boundMedia.isEmpty());
|
||||
AgentMessage userMessage = buildAgentMessage(runtimePrompt, boundMedia);
|
||||
AgentMessage userMessage = buildAgentMessage(runtimePrompt, boundMedia, documentContext);
|
||||
threadPoolTaskExecutor.execute(() -> startRuntime(
|
||||
agent, userMessage, documentContext, account, requestId, traceId, runtimeSessionId,
|
||||
assistantCode, chatContext, runOutput, persistChatlog, runtimeSessionStore, lockHandle));
|
||||
@@ -646,24 +646,6 @@ public class AgentRunService {
|
||||
return agentDocumentService.bindDraft(documentUploads);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将本轮文档正文追加到临时运行定义的系统提示词中。
|
||||
*
|
||||
* <p>正文只存在于本轮模型调用定义,不写入 chatlog 或 AgentScope 消息记忆。</p>
|
||||
*
|
||||
* @param bundle 临时运行时编译结果
|
||||
* @param documentContext 本轮文档上下文
|
||||
*/
|
||||
private void appendDocumentContext(AgentRuntimeBundle bundle, AgentDocumentContext documentContext) {
|
||||
if (bundle == null || bundle.getDefinition() == null
|
||||
|| documentContext == null || documentContext.text().isBlank()) {
|
||||
return;
|
||||
}
|
||||
String current = bundle.getDefinition().getSystemPrompt();
|
||||
bundle.getDefinition().setSystemPrompt(
|
||||
(current == null ? "" : current) + documentContext.text());
|
||||
}
|
||||
|
||||
/**
|
||||
* 为仅附件输入生成可持久化的最小用户意图。
|
||||
*
|
||||
@@ -1187,6 +1169,8 @@ public class AgentRunService {
|
||||
StringBuilder answer = new StringBuilder();
|
||||
ChatAssistantAccumulator assistantAccumulator = new ChatAssistantAccumulator();
|
||||
LegacyThinkingTagParser legacyThinkingTagParser = new LegacyThinkingTagParser();
|
||||
KnowledgeRetrievalStatusTracker knowledgeRetrievalStatusTracker =
|
||||
new KnowledgeRetrievalStatusTracker();
|
||||
// 注册 emit 服务
|
||||
registerEmitterCancellation(requestId, runOutput, chatContext, answer,
|
||||
assistantAccumulator, legacyThinkingTagParser, finished, persistChatlog);
|
||||
@@ -1195,7 +1179,7 @@ public class AgentRunService {
|
||||
if (isAguiCancellationRequested(runOutput)) {
|
||||
handleRuntimeEvent(cancellationEvent("用户已停止生成"), requestId, runOutput,
|
||||
answer, assistantAccumulator, legacyThinkingTagParser,
|
||||
chatContext, finished, persistChatlog);
|
||||
knowledgeRetrievalStatusTracker, chatContext, finished, persistChatlog);
|
||||
if (lockHandle != null) {
|
||||
releaseRunLockQuietly(lockHandle, requestId);
|
||||
}
|
||||
@@ -1206,7 +1190,6 @@ public class AgentRunService {
|
||||
}
|
||||
AgentRuntimeContext runtimeContext = buildAgentRuntimeContext(chatContext, traceId, runtimeSessionId);
|
||||
AgentRuntimeBundle bundle = agentRuntimeCompiler.compile(agent, runtimeContext, !persistChatlog);
|
||||
appendDocumentContext(bundle, documentContext);
|
||||
AgentRuntime runtime = agentRuntimeFactory.create();
|
||||
// 会话初始化请求
|
||||
AgentInitRequest request = new AgentInitRequest();
|
||||
@@ -1214,7 +1197,7 @@ public class AgentRunService {
|
||||
request.setAgentDefinition(bundle.getDefinition());
|
||||
request.setRuntimeContext(runtimeContext);
|
||||
request.setToolInvokers(bundle.getToolInvokers());
|
||||
request.setKnowledgeRetrievers(bundle.getKnowledgeRetrievers());
|
||||
request.setKnowledgeRegistrations(bundle.getKnowledgeRegistrations());
|
||||
request.setSessionStore(runtimeSessionStore);
|
||||
request.setMediaResolver(agentMediaService.runtimeResolver(account));
|
||||
request.getMetadata().put("assistantCode", assistantCode);
|
||||
@@ -1243,6 +1226,7 @@ public class AgentRunService {
|
||||
runRuntimeCallbackSafely(
|
||||
() -> handleRuntimeEvent(event, requestId, runOutput, answer,
|
||||
assistantAccumulator, legacyThinkingTagParser,
|
||||
knowledgeRetrievalStatusTracker,
|
||||
chatContext, finished, persistChatlog),
|
||||
requestId, runOutput, chatContext, finished, persistChatlog);
|
||||
}
|
||||
@@ -1532,7 +1516,8 @@ public class AgentRunService {
|
||||
AtomicBoolean finished,
|
||||
boolean persistChatlog) {
|
||||
handleRuntimeEvent(event, requestId, runOutput, answer, assistantAccumulator,
|
||||
new LegacyThinkingTagParser(), chatContext, finished, persistChatlog);
|
||||
new LegacyThinkingTagParser(), new KnowledgeRetrievalStatusTracker(),
|
||||
chatContext, finished, persistChatlog);
|
||||
}
|
||||
|
||||
private void handleRuntimeEvent(AgentRuntimeEvent event,
|
||||
@@ -1544,6 +1529,35 @@ public class AgentRunService {
|
||||
ChatRuntimeContext chatContext,
|
||||
AtomicBoolean finished,
|
||||
boolean persistChatlog) {
|
||||
handleRuntimeEvent(event, requestId, runOutput, answer, assistantAccumulator,
|
||||
legacyThinkingTagParser, new KnowledgeRetrievalStatusTracker(),
|
||||
chatContext, finished, persistChatlog);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将单个 Runtime 事件投影到聊天协议,并复用本轮知识库工具状态追踪器。
|
||||
*
|
||||
* @param event Runtime 事件
|
||||
* @param requestId 请求 ID
|
||||
* @param runOutput 运行输出
|
||||
* @param answer 回答累积器
|
||||
* @param assistantAccumulator Assistant 结构化累积器
|
||||
* @param legacyThinkingTagParser 旧思考标签解析器
|
||||
* @param knowledgeRetrievalStatusTracker 知识库工具状态追踪器
|
||||
* @param chatContext 聊天上下文
|
||||
* @param finished 终态仲裁标记
|
||||
* @param persistChatlog 是否持久化聊天日志
|
||||
*/
|
||||
private void handleRuntimeEvent(AgentRuntimeEvent event,
|
||||
String requestId,
|
||||
AgentRunOutput runOutput,
|
||||
StringBuilder answer,
|
||||
ChatAssistantAccumulator assistantAccumulator,
|
||||
LegacyThinkingTagParser legacyThinkingTagParser,
|
||||
KnowledgeRetrievalStatusTracker knowledgeRetrievalStatusTracker,
|
||||
ChatRuntimeContext chatContext,
|
||||
AtomicBoolean finished,
|
||||
boolean persistChatlog) {
|
||||
if (event == null || event.getEventType() == null) {
|
||||
return;
|
||||
}
|
||||
@@ -1661,6 +1675,17 @@ public class AgentRunService {
|
||||
return;
|
||||
}
|
||||
Map<String, Object> toolPayload = toolStatus;
|
||||
if (isKnowledgeToolEvent(event)) {
|
||||
Map<String, Object> statusPayload = buildKnowledgeRetrievalStatusPayload(
|
||||
knowledgeRetrievalStatusTracker.update(event));
|
||||
LOG.info("Agent runtime knowledge tool call, requestId={}, toolCallId={}, toolName={}",
|
||||
requestId, event.getToolCallId(), stringValue(toolPayload, "toolName"));
|
||||
if (!sendEnvelope(runOutput, ChatDomain.BUSINESS, ChatType.STATUS, statusPayload)) {
|
||||
cancelDisconnectedRun(requestId, chatContext, answer, assistantAccumulator,
|
||||
legacyThinkingTagParser, finished, persistChatlog);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!runOutput.emitRuntimeEvent(publicRuntimeEvent(event, toolPayload))) {
|
||||
cancelDisconnectedRun(requestId, chatContext, answer, assistantAccumulator,
|
||||
legacyThinkingTagParser, finished, persistChatlog);
|
||||
@@ -1683,6 +1708,20 @@ public class AgentRunService {
|
||||
}
|
||||
if (event.getEventType() == AgentRuntimeEventType.TOOL_RESULT) {
|
||||
Map<String, Object> toolPayload = toolStatus;
|
||||
if (isKnowledgeToolEvent(event)) {
|
||||
Map<String, Object> statusPayload = buildKnowledgeRetrievalStatusPayload(
|
||||
knowledgeRetrievalStatusTracker.update(event));
|
||||
LOG.info("Agent runtime knowledge tool result, requestId={}, toolCallId={}, toolName={}, status={}",
|
||||
requestId, event.getToolCallId(), stringValue(toolPayload, "toolName"),
|
||||
stringValue(statusPayload, "status"));
|
||||
if (!sendEnvelope(runOutput, ChatDomain.BUSINESS, ChatType.STATUS, statusPayload)) {
|
||||
cancelDisconnectedRun(requestId, chatContext, answer, assistantAccumulator,
|
||||
legacyThinkingTagParser, finished, persistChatlog);
|
||||
return;
|
||||
}
|
||||
legacyThinkingTagParser.reset();
|
||||
return;
|
||||
}
|
||||
LOG.info("Agent runtime tool result, requestId={}, toolCallId={}, toolName={}, status={}",
|
||||
requestId, event.getToolCallId(), stringValue(toolPayload, "toolName"),
|
||||
stringValue(toolPayload, "status"));
|
||||
@@ -1708,10 +1747,7 @@ public class AgentRunService {
|
||||
if (event.getEventType() == AgentRuntimeEventType.KNOWLEDGE_RETRIEVAL) {
|
||||
LOG.info("Agent runtime knowledge retrieval, requestId={}, payload={}, metadata={}",
|
||||
requestId, event.getPayload(), event.getMetadata());
|
||||
if (!sendEnvelope(runOutput, ChatDomain.BUSINESS, ChatType.STATUS, buildKnowledgeRetrievalStatusPayload(event))) {
|
||||
cancelDisconnectedRun(requestId, chatContext, answer, assistantAccumulator,
|
||||
legacyThinkingTagParser, finished, persistChatlog);
|
||||
}
|
||||
// 文档摘要事件用于引用与监察;UI 完成态统一以 TOOL_RESULT 为准。
|
||||
return;
|
||||
}
|
||||
if (event.getEventType() == AgentRuntimeEventType.MEMORY_COMPRESSION_STARTED
|
||||
@@ -1769,6 +1805,10 @@ public class AgentRunService {
|
||||
if (event.getEventType() == AgentRuntimeEventType.FAILED) {
|
||||
emitAssistantSegments(legacyThinkingTagParser.finish(), requestId, runOutput, chatContext,
|
||||
answer, assistantAccumulator, legacyThinkingTagParser, finished, persistChatlog);
|
||||
if (knowledgeRetrievalStatusTracker.failActiveCalls()) {
|
||||
sendEnvelope(runOutput, ChatDomain.BUSINESS, ChatType.STATUS,
|
||||
buildKnowledgeRetrievalStatusPayload("error"));
|
||||
}
|
||||
runOutput.emitRuntimeEvent(event);
|
||||
assistantAccumulator.finalizePendingSkillInvocations("FAILED", "技能调用失败");
|
||||
if (persistChatlog) {
|
||||
@@ -2340,13 +2380,30 @@ public class AgentRunService {
|
||||
return message;
|
||||
}
|
||||
|
||||
private AgentMessage buildAgentMessage(String prompt, List<AgentBoundMedia> media) {
|
||||
/**
|
||||
* 构建发送给 AgentScope 的用户消息。
|
||||
*
|
||||
* <p>文档正文属于用户提供的不可信材料,作为用户内容块进入本轮模型调用和 AgentScope
|
||||
* memory。聊天记录仍单独保存原始输入与附件引用,页面不会展示正文内容块。</p>
|
||||
*
|
||||
* @param prompt 用户输入
|
||||
* @param media 图片附件
|
||||
* @param documentContext 本轮选中的文档上下文
|
||||
* @return 可持久化的运行时用户消息
|
||||
*/
|
||||
private AgentMessage buildAgentMessage(String prompt,
|
||||
List<AgentBoundMedia> media,
|
||||
AgentDocumentContext documentContext) {
|
||||
AgentMessage message = new AgentMessage();
|
||||
message.setRole(AgentMessageRole.USER);
|
||||
List<com.easyagents.agent.runtime.message.AgentContentBlock> blocks = new ArrayList<>();
|
||||
if (prompt != null && !prompt.isBlank()) {
|
||||
blocks.add(new AgentTextBlock(prompt));
|
||||
}
|
||||
if (documentContext != null && documentContext.text() != null
|
||||
&& !documentContext.text().isBlank()) {
|
||||
blocks.add(new AgentTextBlock(documentContext.text()));
|
||||
}
|
||||
if (media != null) {
|
||||
for (AgentBoundMedia item : media) {
|
||||
AgentMediaBlock image = new AgentMediaBlock("image");
|
||||
@@ -2815,7 +2872,8 @@ public class AgentRunService {
|
||||
Map<String, Object> rawPayload = event.getPayload() == null ? Map.of() : event.getPayload();
|
||||
Map<String, Object> payload = selectPayload(rawPayload,
|
||||
"name", "status", "success", "toolDisplayName", "toolName",
|
||||
"skillDisplayName", "skillId");
|
||||
"skillDisplayName", "skillId", "toolCategory",
|
||||
"knowledgeId", "knowledgeName", "knowledgeRuntimeName");
|
||||
String toolCallId = firstText(event.getToolCallId(), stringValue(rawPayload, "toolCallId"));
|
||||
if (toolCallId != null && !toolCallId.isBlank()) {
|
||||
payload.put("toolCallId", toolCallId);
|
||||
@@ -2951,17 +3009,110 @@ public class AgentRunService {
|
||||
/**
|
||||
* 构建知识库检索状态载荷,确保前端可按稳定 key 合并同一轮状态行。
|
||||
*
|
||||
* @param event 知识库检索运行时事件
|
||||
* @param status running、done 或 error
|
||||
* @return 知识库检索状态载荷
|
||||
*/
|
||||
private Map<String, Object> buildKnowledgeRetrievalStatusPayload(AgentRuntimeEvent event) {
|
||||
private Map<String, Object> buildKnowledgeRetrievalStatusPayload(String status) {
|
||||
String normalizedStatus = "running".equals(status) || "error".equals(status)
|
||||
? status : "done";
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("statusKey", "knowledge-retrieval");
|
||||
payload.put("status", "done");
|
||||
payload.put("label", "已检索知识库");
|
||||
payload.put("status", normalizedStatus);
|
||||
payload.put("label", switch (normalizedStatus) {
|
||||
case "running" -> "正在检索知识库";
|
||||
case "error" -> "知识库检索失败";
|
||||
default -> "已检索知识库";
|
||||
});
|
||||
return payload;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断标准工具生命周期事件是否属于知识库工具。
|
||||
*
|
||||
* @param event 运行时工具事件
|
||||
* @return 知识库工具事件时为 true
|
||||
*/
|
||||
private boolean isKnowledgeToolEvent(AgentRuntimeEvent event) {
|
||||
String category = stringPayload(event, "toolCategory");
|
||||
if ("KNOWLEDGE".equalsIgnoreCase(category)) {
|
||||
return true;
|
||||
}
|
||||
String toolName = firstText(stringPayload(event, "toolName"), stringPayload(event, "name"));
|
||||
if (toolName == null) {
|
||||
return false;
|
||||
}
|
||||
String normalizedName = toolName.trim().toLowerCase(Locale.ROOT);
|
||||
return "retrieve_knowledge".equals(normalizedName)
|
||||
|| normalizedName.startsWith("retrieve_knowledge_");
|
||||
}
|
||||
|
||||
/**
|
||||
* 聚合同一批知识库工具调用,避免并行检索中首个结果提前结束 UI 状态。
|
||||
*/
|
||||
static final class KnowledgeRetrievalStatusTracker {
|
||||
|
||||
private final Set<String> activeToolCallIds = new LinkedHashSet<>();
|
||||
private boolean failed;
|
||||
|
||||
/**
|
||||
* 应用一次知识库工具生命周期事件。
|
||||
*
|
||||
* @param event TOOL_CALL 或 TOOL_RESULT 事件
|
||||
* @return 聚合后的 running、done 或 error 状态
|
||||
*/
|
||||
String update(AgentRuntimeEvent event) {
|
||||
String toolCallId = toolCallIdentity(event);
|
||||
if (event.getEventType() == AgentRuntimeEventType.TOOL_CALL) {
|
||||
if (activeToolCallIds.isEmpty()) {
|
||||
failed = false;
|
||||
}
|
||||
activeToolCallIds.add(toolCallId);
|
||||
return "running";
|
||||
}
|
||||
if (event.getEventType() == AgentRuntimeEventType.TOOL_RESULT) {
|
||||
activeToolCallIds.remove(toolCallId);
|
||||
failed = failed || !toolSucceeded(event);
|
||||
if (!activeToolCallIds.isEmpty()) {
|
||||
return "running";
|
||||
}
|
||||
return failed ? "error" : "done";
|
||||
}
|
||||
throw new IllegalArgumentException("Knowledge status only accepts TOOL_CALL or TOOL_RESULT events.");
|
||||
}
|
||||
|
||||
/**
|
||||
* 将运行失败时仍未结束的知识库调用收口为失败。
|
||||
*
|
||||
* @return 存在未结束调用时为 true
|
||||
*/
|
||||
boolean failActiveCalls() {
|
||||
if (activeToolCallIds.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
activeToolCallIds.clear();
|
||||
failed = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
private String toolCallIdentity(AgentRuntimeEvent event) {
|
||||
String toolCallId = event.getToolCallId();
|
||||
if (toolCallId == null || toolCallId.isBlank()) {
|
||||
Object payloadId = event.getPayload() == null ? null : event.getPayload().get("toolCallId");
|
||||
toolCallId = payloadId == null ? event.getEventId() : String.valueOf(payloadId);
|
||||
}
|
||||
return toolCallId;
|
||||
}
|
||||
|
||||
private boolean toolSucceeded(AgentRuntimeEvent event) {
|
||||
Map<String, Object> payload = event.getPayload() == null ? Map.of() : event.getPayload();
|
||||
if (Boolean.FALSE.equals(payload.get("success"))) {
|
||||
return false;
|
||||
}
|
||||
Object status = payload.get("status");
|
||||
return status == null || !"FAILED".equalsIgnoreCase(String.valueOf(status));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建不含 Runtime 原始上下文的内存压缩公开状态载荷。
|
||||
*
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package tech.easyflow.agent.runtime;
|
||||
|
||||
import com.easyagents.agent.runtime.AgentDefinition;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRetriever;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRegistration;
|
||||
import com.easyagents.agent.runtime.tool.AgentToolInvoker;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
@@ -14,7 +16,7 @@ public class AgentRuntimeBundle {
|
||||
|
||||
private AgentDefinition definition;
|
||||
private Map<String, AgentToolInvoker> toolInvokers = new LinkedHashMap<>();
|
||||
private Map<String, AgentKnowledgeRetriever> knowledgeRetrievers = new LinkedHashMap<>();
|
||||
private List<AgentKnowledgeRegistration> knowledgeRegistrations = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 获取 Agent 定义。
|
||||
@@ -57,16 +59,18 @@ public class AgentRuntimeBundle {
|
||||
*
|
||||
* @return 知识库检索器
|
||||
*/
|
||||
public Map<String, AgentKnowledgeRetriever> getKnowledgeRetrievers() {
|
||||
return knowledgeRetrievers;
|
||||
public List<AgentKnowledgeRegistration> getKnowledgeRegistrations() {
|
||||
return knowledgeRegistrations;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置知识库检索器。
|
||||
*
|
||||
* @param knowledgeRetrievers 知识库检索器
|
||||
* @param knowledgeRegistrations 知识库运行时绑定
|
||||
*/
|
||||
public void setKnowledgeRetrievers(Map<String, AgentKnowledgeRetriever> knowledgeRetrievers) {
|
||||
this.knowledgeRetrievers = knowledgeRetrievers == null ? new LinkedHashMap<>() : knowledgeRetrievers;
|
||||
public void setKnowledgeRegistrations(List<AgentKnowledgeRegistration> knowledgeRegistrations) {
|
||||
this.knowledgeRegistrations = knowledgeRegistrations == null
|
||||
? new ArrayList<>()
|
||||
: new ArrayList<>(knowledgeRegistrations);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,9 +6,10 @@ import com.easyagents.agent.runtime.AgentRuntimeContext;
|
||||
import com.easyagents.agent.runtime.event.AgentRuntimeEvent;
|
||||
import com.easyagents.agent.runtime.event.AgentRuntimeEventType;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeDocument;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgePolicy;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRegistration;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRetrievalResult;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeSpec;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeToolNames;
|
||||
import com.easyagents.agent.runtime.memory.AgentMemoryCompressionParameter;
|
||||
import com.easyagents.agent.runtime.memory.AgentMemoryPolicy;
|
||||
import com.easyagents.agent.runtime.memory.AgentMemoryType;
|
||||
@@ -118,11 +119,11 @@ public class AgentRuntimeCompiler {
|
||||
bundle.setDefinition(definition);
|
||||
|
||||
compileTools(agent, definition, bundle);
|
||||
compileKnowledge(agent, definition, bundle);
|
||||
if (agentBuiltinToolsConfigResolver != null) {
|
||||
validateBuiltinTools(definition,
|
||||
agentBuiltinToolsConfigResolver.resolvePublishedRuntime(agent.getExecutionConfigJson()));
|
||||
}
|
||||
compileKnowledge(agent, definition, bundle);
|
||||
return bundle;
|
||||
}
|
||||
|
||||
@@ -294,7 +295,7 @@ public class AgentRuntimeCompiler {
|
||||
if (config.artifactPublish().enabled()) {
|
||||
specs.add(buildArtifactPublishSpec(config.artifactPublish()));
|
||||
}
|
||||
assertToolBudget(specs, definition.getMcpSpecs());
|
||||
assertToolBudget(specs, definition.getMcpSpecs(), definition.getKnowledgeSpecs().size());
|
||||
}
|
||||
|
||||
private void attachBuiltinTools(Agent agent,
|
||||
@@ -510,11 +511,21 @@ public class AgentRuntimeCompiler {
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验内置工具与普通工具、知识库工具及 MCP 工具不存在运行名冲突。
|
||||
*
|
||||
* @param definition 已编译 Agent 定义
|
||||
* @param builtinNames 待启用内置工具名称
|
||||
* @throws BusinessException 工具名称冲突时抛出
|
||||
*/
|
||||
private void assertNoBuiltinNameConflict(AgentDefinition definition, Set<String> builtinNames) {
|
||||
Set<String> existing = new LinkedHashSet<>();
|
||||
for (AgentToolSpec spec : definition.getToolSpecs()) {
|
||||
existing.add(spec.getName());
|
||||
}
|
||||
for (AgentKnowledgeSpec spec : definition.getKnowledgeSpecs()) {
|
||||
existing.add(AgentKnowledgeToolNames.build(spec.getRuntimeName()));
|
||||
}
|
||||
for (McpSpec mcp : definition.getMcpSpecs()) {
|
||||
if (mcp.getFrozenToolManifest() != null) {
|
||||
mcp.getFrozenToolManifest().forEach(entry -> existing.add(entry.getName()));
|
||||
@@ -540,6 +551,14 @@ public class AgentRuntimeCompiler {
|
||||
assertToolBudget(toolSpecs, mcpSpecs, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验最终工具数量和 Schema 大小预算。
|
||||
*
|
||||
* @param toolSpecs 静态 Tool 声明
|
||||
* @param mcpSpecs MCP 声明
|
||||
* @param additionalToolCount 知识库等额外工具数量
|
||||
* @throws BusinessException 超出预算时抛出
|
||||
*/
|
||||
private void assertToolBudget(List<AgentToolSpec> toolSpecs,
|
||||
List<McpSpec> mcpSpecs,
|
||||
int additionalToolCount) {
|
||||
@@ -570,7 +589,7 @@ public class AgentRuntimeCompiler {
|
||||
}
|
||||
}
|
||||
if (toolCount > MAX_RUNTIME_TOOL_COUNT) {
|
||||
throw new BusinessException("Agent Runtime Tool 数量超过 128 个,请减少直接工具或 Skill 绑定");
|
||||
throw new BusinessException("Agent Runtime Tool 数量超过 128 个,请减少工具、知识库或 Skill 绑定");
|
||||
}
|
||||
if (schemaBytes > MAX_RUNTIME_SCHEMA_BYTES) {
|
||||
throw new BusinessException("Agent Runtime Tool Schema 超过 2 MiB,请减少工具或精简 Schema");
|
||||
@@ -591,12 +610,27 @@ public class AgentRuntimeCompiler {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 EasyFlow 知识库绑定编译为一库一工具所需的声明和 Retriever 绑定。
|
||||
*
|
||||
* @param agent Agent 发布视图
|
||||
* @param definition 中立 Agent 定义
|
||||
* @param bundle 运行时编译结果
|
||||
* @throws BusinessException 知识库不存在、英文运行名非法或工具名冲突时抛出
|
||||
*/
|
||||
private void compileKnowledge(Agent agent, AgentDefinition definition, AgentRuntimeBundle bundle) {
|
||||
if (agent.getKnowledgeBindings() == null) {
|
||||
return;
|
||||
}
|
||||
List<AgentKnowledgeSpec> specs = new ArrayList<>();
|
||||
Map<String, com.easyagents.agent.runtime.knowledge.AgentKnowledgeRetriever> retrievers = new LinkedHashMap<>();
|
||||
List<AgentKnowledgeRegistration> registrations = new ArrayList<>();
|
||||
Set<String> knowledgeToolNames = new LinkedHashSet<>();
|
||||
Set<String> existingToolNames = new LinkedHashSet<>();
|
||||
definition.getToolSpecs().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(AgentToolSpec::getName)
|
||||
.filter(Objects::nonNull)
|
||||
.forEach(existingToolNames::add);
|
||||
for (AgentKnowledgeBinding binding : agent.getKnowledgeBindings()) {
|
||||
if (!Boolean.TRUE.equals(binding.getEnabled())) {
|
||||
continue;
|
||||
@@ -607,9 +641,9 @@ public class AgentRuntimeCompiler {
|
||||
}
|
||||
AgentKnowledgeSpec spec = new AgentKnowledgeSpec();
|
||||
spec.setKnowledgeId(binding.getKnowledgeId().toString());
|
||||
spec.setRuntimeName(requireKnowledgeRuntimeName(knowledge));
|
||||
spec.setName(knowledge.getTitle());
|
||||
spec.setDescription(knowledge.getDescription());
|
||||
spec.setRetrievalMode(AgentKnowledgePolicy.AGENTIC);
|
||||
spec.getMetadata().put("knowledgeType", knowledge.getCollectionType());
|
||||
spec.getMetadata().put("faqCollection", knowledge.isFaqCollection());
|
||||
Integer limit = intValue(binding.getOptionsJson(), "limit");
|
||||
@@ -618,11 +652,37 @@ public class AgentRuntimeCompiler {
|
||||
if (threshold != null) {
|
||||
spec.setScoreThreshold(threshold);
|
||||
}
|
||||
String toolName = AgentKnowledgeToolNames.build(spec.getRuntimeName());
|
||||
if (!knowledgeToolNames.add(toolName) || existingToolNames.contains(toolName)) {
|
||||
throw new BusinessException("Agent 知识库工具运行名冲突:" + toolName);
|
||||
}
|
||||
specs.add(spec);
|
||||
retrievers.put(spec.getKnowledgeId(), request -> retrieveKnowledge(binding, request.getQuery(), request.getLimit(), request.getScoreThreshold()));
|
||||
registrations.add(new AgentKnowledgeRegistration(spec,
|
||||
request -> retrieveKnowledge(binding, request.getQuery(), request.getLimit(), request.getScoreThreshold())));
|
||||
}
|
||||
definition.setKnowledgeSpecs(specs);
|
||||
bundle.setKnowledgeRetrievers(retrievers);
|
||||
bundle.setKnowledgeRegistrations(registrations);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取并校验知识库英文运行名。
|
||||
*
|
||||
* @param knowledge 知识库发布视图
|
||||
* @return 合法英文运行名
|
||||
* @throws BusinessException 英文运行名缺失或非法时抛出
|
||||
*/
|
||||
private String requireKnowledgeRuntimeName(DocumentCollection knowledge) {
|
||||
String runtimeName = knowledge == null ? null : knowledge.getEnglishName();
|
||||
try {
|
||||
AgentKnowledgeToolNames.build(runtimeName);
|
||||
return runtimeName.trim();
|
||||
} catch (RuntimeException exception) {
|
||||
String knowledgeName = knowledge == null || knowledge.getTitle() == null
|
||||
? "未知知识库"
|
||||
: knowledge.getTitle();
|
||||
throw new BusinessException(400, 400, "知识库“" + knowledgeName
|
||||
+ "”的英文名称不能为空,且只能包含字母、数字、下划线和连字符", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private AgentKnowledgeRetrievalResult retrieveKnowledge(AgentKnowledgeBinding binding, String query, int limit, double scoreThreshold) {
|
||||
|
||||
@@ -18,6 +18,7 @@ import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
@@ -563,7 +564,9 @@ public final class AguiAgentRunOutput implements AgentRunOutput {
|
||||
}
|
||||
|
||||
private static boolean isHiddenToolName(String toolName) {
|
||||
return "retrieve_knowledge".equalsIgnoreCase(toolName)
|
||||
String normalizedName = toolName == null ? "" : toolName.trim().toLowerCase(Locale.ROOT);
|
||||
return "retrieve_knowledge".equals(normalizedName)
|
||||
|| normalizedName.startsWith("retrieve_knowledge_")
|
||||
|| "context_reload".equalsIgnoreCase(toolName)
|
||||
|| "__fragment__".equalsIgnoreCase(toolName);
|
||||
}
|
||||
|
||||
@@ -51,6 +51,9 @@ public class AgentSkillReferenceProvider implements SkillReferenceProvider {
|
||||
ids.add(agent.getId());
|
||||
}
|
||||
}
|
||||
if (ids.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> result = new ArrayList<>();
|
||||
for (Agent agent : agentService.listByIds(ids)) {
|
||||
result.add("智能体“" + (agent.getName() == null ? "未命名智能体" : agent.getName()) + "”");
|
||||
|
||||
@@ -8,6 +8,7 @@ import org.junit.Test;
|
||||
import tech.easyflow.agent.entity.Agent;
|
||||
import tech.easyflow.agent.entity.AgentToolBinding;
|
||||
import tech.easyflow.agent.enums.AgentToolType;
|
||||
import tech.easyflow.agent.runtime.skill.AgentSkillRuntimeCompiler;
|
||||
import tech.easyflow.agent.runtime.tool.AgentToolRuntimeCompiler;
|
||||
import tech.easyflow.ai.entity.Mcp;
|
||||
import tech.easyflow.ai.entity.Model;
|
||||
@@ -44,6 +45,8 @@ public class AgentDefinitionCompilerMcpTest {
|
||||
setField(toolCompiler, "objectMapper", new com.fasterxml.jackson.databind.ObjectMapper());
|
||||
setField(toolCompiler, "mcpService", mcpService(mcp));
|
||||
setField(compiler, "agentToolRuntimeCompiler", toolCompiler);
|
||||
setField(compiler, "agentSkillRuntimeCompiler", new AgentSkillRuntimeCompiler(
|
||||
null, toolCompiler, new com.fasterxml.jackson.databind.ObjectMapper()));
|
||||
|
||||
Agent agent = agent(modelId, mcpId);
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import com.easyagents.agent.runtime.event.AgentRuntimeEventType;
|
||||
import com.easyagents.agent.runtime.message.AgentKnowledgeReference;
|
||||
import com.easyagents.agent.runtime.message.AgentMessage;
|
||||
import com.easyagents.agent.runtime.message.AgentMessageRole;
|
||||
import com.easyagents.agent.runtime.message.AgentTextBlock;
|
||||
import com.easyagents.agent.runtime.persistence.session.AgentSessionStore;
|
||||
import com.easyagents.agent.runtime.persistence.session.memory.InMemoryAgentSessionStore;
|
||||
import org.junit.Assert;
|
||||
@@ -69,6 +70,28 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||
*/
|
||||
public class AgentRunServiceDraftAndHitlTest {
|
||||
|
||||
/**
|
||||
* 验证文档上下文随用户消息进入可持久化 memory,同时保持独立内容块边界。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void buildAgentMessageShouldIncludeDocumentContext() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
AgentDocumentContext documentContext = new AgentDocumentContext(
|
||||
"\n<<<DOCUMENT name=\"demo.docx\">>>\n正文\n<<<END_DOCUMENT>>>", 8, List.of());
|
||||
|
||||
AgentMessage message = invoke(service, "buildAgentMessage",
|
||||
new Class<?>[]{String.class, List.class, AgentDocumentContext.class},
|
||||
"请介绍文档", List.of(), documentContext);
|
||||
|
||||
Assert.assertEquals(2, message.getContentBlocks().size());
|
||||
Assert.assertEquals("请介绍文档",
|
||||
((AgentTextBlock) message.getContentBlocks().get(0)).getText());
|
||||
Assert.assertEquals(documentContext.text(),
|
||||
((AgentTextBlock) message.getContentBlocks().get(1)).getText());
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建用于 owner 恢复测试的运行描述。
|
||||
*
|
||||
@@ -453,18 +476,48 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证知识检索状态不会携带命中文档和内部 metadata。
|
||||
* 验证知识库工具开始事件会投影为脱敏的检索中状态。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void handleRuntimeEventShouldWhitelistKnowledgeStatusPayload() throws Exception {
|
||||
public void handleRuntimeEventShouldProjectKnowledgeToolCallAsRunningStatus() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingChatSseEmitter emitter = new RecordingChatSseEmitter();
|
||||
AgentRuntimeEvent event = AgentRuntimeEvent.of(AgentRuntimeEventType.KNOWLEDGE_RETRIEVAL);
|
||||
AgentRuntimeEvent event = AgentRuntimeEvent.of(AgentRuntimeEventType.TOOL_CALL);
|
||||
event.setToolCallId("knowledge-call-1");
|
||||
event.getPayload().put("toolCallId", "knowledge-call-1");
|
||||
event.getPayload().put("toolName", "retrieve_knowledge_homeinn_faq");
|
||||
event.getPayload().put("toolCategory", "KNOWLEDGE");
|
||||
event.getPayload().put("documents", List.of(Map.of("chunkContent", "private chunk")));
|
||||
event.getPayload().put("metadata", Map.of("sourceUri", "private://document"));
|
||||
|
||||
invoke(service, "handleRuntimeEvent",
|
||||
runtimeEventParameterTypes(),
|
||||
event, "request-knowledge", legacyOutput(emitter), new StringBuilder(),
|
||||
new ChatAssistantAccumulator(), chatContext(), new AtomicBoolean(false), false);
|
||||
|
||||
Assert.assertEquals(1, emitter.envelopes.size());
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> payload = (Map<String, Object>) emitter.envelopes.get(0).getPayload();
|
||||
Assert.assertEquals(Map.of(
|
||||
"label", "正在检索知识库",
|
||||
"status", "running",
|
||||
"statusKey", "knowledge-retrieval"), payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证知识库工具结果事件会投影为完成状态。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void handleRuntimeEventShouldProjectKnowledgeToolResultAsDoneStatus() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingChatSseEmitter emitter = new RecordingChatSseEmitter();
|
||||
AgentRuntimeEvent event = knowledgeToolEvent(
|
||||
AgentRuntimeEventType.TOOL_RESULT, "knowledge-call-1", true);
|
||||
|
||||
invoke(service, "handleRuntimeEvent",
|
||||
runtimeEventParameterTypes(),
|
||||
event, "request-knowledge", legacyOutput(emitter), new StringBuilder(),
|
||||
@@ -479,6 +532,48 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
"statusKey", "knowledge-retrieval"), payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文档摘要事件不会抢先把知识库工具状态标记为完成。
|
||||
*
|
||||
* @throws Exception 反射调用失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void handleRuntimeEventShouldNotCompleteKnowledgeStatusFromDocumentEvent() throws Exception {
|
||||
AgentRunService service = new AgentRunService();
|
||||
RecordingChatSseEmitter emitter = new RecordingChatSseEmitter();
|
||||
AgentRuntimeEvent event = AgentRuntimeEvent.of(AgentRuntimeEventType.KNOWLEDGE_RETRIEVAL);
|
||||
event.getPayload().put("documents", List.of(Map.of("chunkContent", "private chunk")));
|
||||
|
||||
invoke(service, "handleRuntimeEvent",
|
||||
runtimeEventParameterTypes(),
|
||||
event, "request-knowledge", legacyOutput(emitter), new StringBuilder(),
|
||||
new ChatAssistantAccumulator(), chatContext(), new AtomicBoolean(false), false);
|
||||
|
||||
Assert.assertTrue(emitter.envelopes.isEmpty());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证并行知识库调用全部结束后才进入终态,并保留任一调用失败结果。
|
||||
*/
|
||||
@Test
|
||||
public void knowledgeStatusTrackerShouldAggregateParallelToolCalls() {
|
||||
AgentRunService.KnowledgeRetrievalStatusTracker tracker =
|
||||
new AgentRunService.KnowledgeRetrievalStatusTracker();
|
||||
AgentRuntimeEvent firstCall = knowledgeToolEvent(
|
||||
AgentRuntimeEventType.TOOL_CALL, "knowledge-call-1", true);
|
||||
AgentRuntimeEvent secondCall = knowledgeToolEvent(
|
||||
AgentRuntimeEventType.TOOL_CALL, "knowledge-call-2", true);
|
||||
AgentRuntimeEvent firstResult = knowledgeToolEvent(
|
||||
AgentRuntimeEventType.TOOL_RESULT, "knowledge-call-1", false);
|
||||
AgentRuntimeEvent secondResult = knowledgeToolEvent(
|
||||
AgentRuntimeEventType.TOOL_RESULT, "knowledge-call-2", true);
|
||||
|
||||
Assert.assertEquals("running", tracker.update(firstCall));
|
||||
Assert.assertEquals("running", tracker.update(secondCall));
|
||||
Assert.assertEquals("running", tracker.update(firstResult));
|
||||
Assert.assertEquals("error", tracker.update(secondResult));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证完成事件不会再次发送正文消息,只用于最终收口。
|
||||
*
|
||||
@@ -1556,6 +1651,29 @@ public class AgentRunServiceDraftAndHitlTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建知识库工具生命周期测试事件。
|
||||
*
|
||||
* @param eventType 工具开始或结果事件类型
|
||||
* @param toolCallId 工具调用 ID
|
||||
* @param success 工具结果是否成功
|
||||
* @return 知识库工具事件
|
||||
*/
|
||||
private AgentRuntimeEvent knowledgeToolEvent(AgentRuntimeEventType eventType,
|
||||
String toolCallId,
|
||||
boolean success) {
|
||||
AgentRuntimeEvent event = AgentRuntimeEvent.of(eventType);
|
||||
event.setToolCallId(toolCallId);
|
||||
event.getPayload().put("toolCallId", toolCallId);
|
||||
event.getPayload().put("toolName", "retrieve_knowledge_homeinn_faq");
|
||||
event.getPayload().put("toolCategory", "KNOWLEDGE");
|
||||
if (eventType == AgentRuntimeEventType.TOOL_RESULT) {
|
||||
event.getPayload().put("success", success);
|
||||
event.getPayload().put("status", success ? "SUCCESS" : "FAILED");
|
||||
}
|
||||
return event;
|
||||
}
|
||||
|
||||
private Class<?>[] runtimeEventParameterTypes() {
|
||||
return new Class<?>[]{AgentRuntimeEvent.class, String.class, AgentRunOutput.class, StringBuilder.class,
|
||||
ChatAssistantAccumulator.class,
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
package tech.easyflow.agent.runtime;
|
||||
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeDocument;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRegistration;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRetrievalRequest;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeRetrievalResult;
|
||||
import com.easyagents.agent.runtime.knowledge.AgentKnowledgeSpec;
|
||||
import com.easyagents.core.document.Document;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.agent.entity.Agent;
|
||||
import tech.easyflow.agent.entity.AgentKnowledgeBinding;
|
||||
import tech.easyflow.agent.runtime.skill.AgentSkillRuntimeCompiler;
|
||||
import tech.easyflow.agent.runtime.tool.AgentToolRuntimeCompiler;
|
||||
import tech.easyflow.ai.entity.Model;
|
||||
import tech.easyflow.ai.entity.ModelProvider;
|
||||
import tech.easyflow.ai.rag.KnowledgeRetrievalRequest;
|
||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||
import tech.easyflow.ai.service.ModelService;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Proxy;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
* Agent 知识库一库一工具运行时编译测试。
|
||||
*/
|
||||
public class AgentRuntimeCompilerKnowledgeTest {
|
||||
|
||||
/**
|
||||
* 验证知识库英文名称、描述和检索配置会编译到中立声明及独立 Retriever。
|
||||
*
|
||||
* @throws Exception 反射注入依赖失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void compileShouldBuildOneKnowledgeRegistrationWithEnglishRuntimeName() throws Exception {
|
||||
AtomicReference<KnowledgeRetrievalRequest> capturedRequest = new AtomicReference<>();
|
||||
Document document = new Document("如家酒店通常在入住日 14:00 后办理入住。");
|
||||
document.setId("chunk-1");
|
||||
document.setTitle("如家 FAQ");
|
||||
document.setScore(0.92D);
|
||||
document.addMetadata("documentId", "faq-document-1");
|
||||
document.addMetadata("chunkId", "faq-chunk-1");
|
||||
AgentRuntimeCompiler compiler = compiler(capturedRequest, List.of(document));
|
||||
Agent agent = agent(knowledgeBinding("homeinn_faq", BigInteger.valueOf(20L)));
|
||||
|
||||
AgentRuntimeBundle bundle = compiler.compile(agent);
|
||||
|
||||
Assert.assertEquals(1, bundle.getDefinition().getKnowledgeSpecs().size());
|
||||
AgentKnowledgeSpec spec = bundle.getDefinition().getKnowledgeSpecs().get(0);
|
||||
Assert.assertEquals("homeinn_faq", spec.getRuntimeName());
|
||||
Assert.assertEquals("如家 FAQ", spec.getName());
|
||||
Assert.assertTrue(spec.getDescription().contains("入住"));
|
||||
Assert.assertEquals(7, spec.getLimit());
|
||||
Assert.assertEquals(0.55D, spec.getScoreThreshold(), 0.0001D);
|
||||
Assert.assertEquals(1, bundle.getKnowledgeRegistrations().size());
|
||||
|
||||
AgentKnowledgeRegistration registration = bundle.getKnowledgeRegistrations().get(0);
|
||||
AgentKnowledgeRetrievalRequest retrievalRequest = new AgentKnowledgeRetrievalRequest();
|
||||
retrievalRequest.setQuery("如家几点入住");
|
||||
retrievalRequest.setLimit(spec.getLimit());
|
||||
retrievalRequest.setScoreThreshold(spec.getScoreThreshold());
|
||||
AgentKnowledgeRetrievalResult result = registration.getRetriever().retrieve(retrievalRequest);
|
||||
|
||||
Assert.assertEquals("如家几点入住", capturedRequest.get().getQuery());
|
||||
Assert.assertEquals(Integer.valueOf(7), capturedRequest.get().getLimit());
|
||||
Assert.assertEquals(Double.valueOf(0.55D), capturedRequest.get().getMinSimilarity());
|
||||
Assert.assertEquals("AGENT_KNOWLEDGE", capturedRequest.get().getCallerType());
|
||||
Assert.assertEquals(1, result.getDocuments().size());
|
||||
AgentKnowledgeDocument mapped = result.getDocuments().get(0);
|
||||
Assert.assertEquals("faq-document-1", mapped.getDocumentId());
|
||||
Assert.assertEquals("faq-chunk-1", mapped.getChunkId());
|
||||
Assert.assertEquals(0.92D, mapped.getScore(), 0.0001D);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证缺失知识库英文名称时在发布编译阶段明确失败。
|
||||
*
|
||||
* @throws Exception 反射注入依赖失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void compileShouldRejectMissingKnowledgeEnglishName() throws Exception {
|
||||
AgentRuntimeCompiler compiler = compiler(new AtomicReference<>(), List.of());
|
||||
Agent agent = agent(knowledgeBinding(null, BigInteger.valueOf(20L)));
|
||||
|
||||
try {
|
||||
compiler.compile(agent);
|
||||
Assert.fail("缺失英文名称时应拒绝编译");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("英文名称不能为空"));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证多个知识库生成相同工具名时在编译阶段拒绝发布。
|
||||
*
|
||||
* @throws Exception 反射注入依赖失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void compileShouldRejectDuplicateKnowledgeToolNames() throws Exception {
|
||||
AgentRuntimeCompiler compiler = compiler(new AtomicReference<>(), List.of());
|
||||
AgentKnowledgeBinding first = knowledgeBinding("homeinn_faq", BigInteger.valueOf(20L));
|
||||
AgentKnowledgeBinding second = knowledgeBinding("homeinn_faq", BigInteger.valueOf(21L));
|
||||
Agent agent = agent(first, second);
|
||||
|
||||
try {
|
||||
compiler.compile(agent);
|
||||
Assert.fail("重复知识库工具名时应拒绝编译");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("retrieve_knowledge_homeinn_faq"));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建仅含测试模型与知识库服务的运行时编译器。
|
||||
*
|
||||
* @param capturedRequest 检索请求捕获器
|
||||
* @param documents 检索服务返回文档
|
||||
* @return 已注入依赖的编译器
|
||||
* @throws Exception 反射注入失败时抛出
|
||||
*/
|
||||
private AgentRuntimeCompiler compiler(AtomicReference<KnowledgeRetrievalRequest> capturedRequest,
|
||||
List<Document> documents) throws Exception {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
AgentToolRuntimeCompiler toolCompiler = new AgentToolRuntimeCompiler();
|
||||
AgentRuntimeCompiler compiler = new AgentRuntimeCompiler();
|
||||
setField(compiler, "objectMapper", objectMapper);
|
||||
setField(compiler, "modelService", modelService(model()));
|
||||
setField(compiler, "documentCollectionService", documentCollectionService(capturedRequest, documents));
|
||||
setField(compiler, "agentToolRuntimeCompiler", toolCompiler);
|
||||
setField(compiler, "agentSkillRuntimeCompiler",
|
||||
new AgentSkillRuntimeCompiler(null, toolCompiler, objectMapper));
|
||||
return compiler;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建带知识库绑定的 Agent。
|
||||
*
|
||||
* @param bindings 知识库绑定
|
||||
* @return Agent 测试对象
|
||||
*/
|
||||
private Agent agent(AgentKnowledgeBinding... bindings) {
|
||||
Agent agent = new Agent();
|
||||
agent.setId(BigInteger.ONE);
|
||||
agent.setName("如家助手");
|
||||
agent.setModelId(BigInteger.TEN);
|
||||
agent.setKnowledgeBindings(List.of(bindings));
|
||||
return agent;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建冻结知识库绑定。
|
||||
*
|
||||
* @param englishName 知识库英文名称
|
||||
* @param knowledgeId 知识库 ID
|
||||
* @return 知识库绑定
|
||||
*/
|
||||
private AgentKnowledgeBinding knowledgeBinding(String englishName, BigInteger knowledgeId) {
|
||||
AgentKnowledgeBinding binding = new AgentKnowledgeBinding();
|
||||
binding.setAgentId(BigInteger.ONE);
|
||||
binding.setKnowledgeId(knowledgeId);
|
||||
binding.setRetrievalMode("HYBRID");
|
||||
binding.setEnabled(true);
|
||||
binding.setOptionsJson(Map.of("limit", 7, "scoreThreshold", 0.55D));
|
||||
binding.setResourceSnapshot(Map.of(
|
||||
"id", knowledgeId,
|
||||
"title", "如家 FAQ",
|
||||
"description", "如家酒店入住、退房和会员服务常见问题",
|
||||
"collectionType", "FAQ",
|
||||
"englishName", englishName == null ? "" : englishName));
|
||||
return binding;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建模型服务代理。
|
||||
*
|
||||
* @param model 测试模型
|
||||
* @return 模型服务代理
|
||||
*/
|
||||
private ModelService modelService(Model model) {
|
||||
return (ModelService) Proxy.newProxyInstance(
|
||||
ModelService.class.getClassLoader(),
|
||||
new Class<?>[]{ModelService.class},
|
||||
(proxy, method, args) -> "getModelInstance".equals(method.getName())
|
||||
? model
|
||||
: defaultValue(method.getReturnType()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建知识库服务代理。
|
||||
*
|
||||
* @param capturedRequest 检索请求捕获器
|
||||
* @param documents 返回文档
|
||||
* @return 知识库服务代理
|
||||
*/
|
||||
private DocumentCollectionService documentCollectionService(
|
||||
AtomicReference<KnowledgeRetrievalRequest> capturedRequest,
|
||||
List<Document> documents) {
|
||||
return (DocumentCollectionService) Proxy.newProxyInstance(
|
||||
DocumentCollectionService.class.getClassLoader(),
|
||||
new Class<?>[]{DocumentCollectionService.class},
|
||||
(proxy, method, args) -> {
|
||||
if ("search".equals(method.getName()) && args != null && args.length == 1
|
||||
&& args[0] instanceof KnowledgeRetrievalRequest request) {
|
||||
capturedRequest.set(request);
|
||||
return documents;
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建可映射为 AgentScope 模型配置的测试模型。
|
||||
*
|
||||
* @return 测试模型
|
||||
*/
|
||||
private Model model() {
|
||||
ModelProvider provider = new ModelProvider();
|
||||
provider.setProviderType("openai");
|
||||
provider.setProviderName("OpenAI");
|
||||
Model model = new Model();
|
||||
model.setId(BigInteger.TEN);
|
||||
model.setModelProvider(provider);
|
||||
model.setModelName("gpt-test");
|
||||
model.setEndpoint("https://example.com");
|
||||
model.setRequestPath("/v1/chat/completions");
|
||||
model.setApiKey("test-key");
|
||||
return model;
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回代理方法所需的默认值。
|
||||
*
|
||||
* @param type 返回类型
|
||||
* @return 对应默认值
|
||||
*/
|
||||
private Object defaultValue(Class<?> type) {
|
||||
if (type == boolean.class) {
|
||||
return false;
|
||||
}
|
||||
if (type == int.class || type == long.class || type == short.class || type == byte.class) {
|
||||
return 0;
|
||||
}
|
||||
if (type == double.class || type == float.class) {
|
||||
return 0D;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 反射注入测试依赖。
|
||||
*
|
||||
* @param target 目标对象
|
||||
* @param fieldName 字段名称
|
||||
* @param value 字段值
|
||||
* @throws Exception 字段不存在或不可写时抛出
|
||||
*/
|
||||
private void setField(Object target, String fieldName, Object value) throws Exception {
|
||||
Field field = target.getClass().getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,22 @@ public class AgentSkillReferenceProviderTest {
|
||||
"智能体“线上引用智能体”"), references);
|
||||
}
|
||||
|
||||
/**
|
||||
* 没有 Agent 引用 Skill 时不应执行空主键集合查询。
|
||||
*/
|
||||
@Test
|
||||
public void shouldSkipEntityQueryWhenSkillHasNoReferences() {
|
||||
AgentService agentService = Mockito.mock(AgentService.class);
|
||||
AgentSkillBindingService bindingService = Mockito.mock(AgentSkillBindingService.class);
|
||||
Mockito.when(bindingService.list(Mockito.any(QueryWrapper.class))).thenReturn(List.of());
|
||||
Mockito.when(agentService.list(Mockito.any(QueryWrapper.class))).thenReturn(List.of());
|
||||
AgentSkillReferenceProvider provider = new AgentSkillReferenceProvider(
|
||||
agentService, bindingService);
|
||||
|
||||
Assert.assertTrue(provider.listReferences(BigInteger.TEN).isEmpty());
|
||||
Mockito.verify(agentService, Mockito.never()).listByIds(Mockito.anyCollection());
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 Agent 摘要。
|
||||
*
|
||||
|
||||
@@ -10,6 +10,7 @@ import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||
import tech.easyflow.ai.document.model.DocumentSourceRef;
|
||||
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadedFileReader;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.filestorage.FileStorageWriteHandle;
|
||||
import tech.easyflow.common.filestorage.utils.PathGeneratorUtil;
|
||||
|
||||
import java.io.IOException;
|
||||
@@ -164,7 +165,7 @@ public class DocumentSourceLoader {
|
||||
}
|
||||
|
||||
/**
|
||||
* 优先打开经过上传记录验证的受管 URL,再执行普通公网 URL 校验与下载。
|
||||
* 优先打开经过上传记录验证的受管 URL 和服务端已登记附件,再执行普通公网 URL 校验与下载。
|
||||
*
|
||||
* @param remoteUrl 远端 URL
|
||||
* @param maxBytes 最大允许读取字节数
|
||||
@@ -176,6 +177,13 @@ public class DocumentSourceLoader {
|
||||
if (managed.isPresent()) {
|
||||
return DocumentInputStreamSupport.limit(managed.get(), maxBytes);
|
||||
}
|
||||
Optional<FileStorageWriteHandle> trusted =
|
||||
fileStorageService.resolveTrustedFile(remoteUrl);
|
||||
if (trusted.isPresent()) {
|
||||
return DocumentInputStreamSupport.limit(
|
||||
fileStorageService.readRecoverable(trusted.get()),
|
||||
maxBytes);
|
||||
}
|
||||
return DocumentInputStreamSupport.openRemote(remoteUrl, maxBytes);
|
||||
}
|
||||
|
||||
|
||||
@@ -252,6 +252,7 @@ public final class DocumentImportBatchDtos {
|
||||
private BigInteger batchId;
|
||||
private String importMode;
|
||||
private String status;
|
||||
private Boolean actualCompleted;
|
||||
private Integer totalCount;
|
||||
private Long totalBytes;
|
||||
private Integer completedCount;
|
||||
@@ -292,6 +293,24 @@ public final class DocumentImportBatchDtos {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取批次关联任务是否已经按真实文档状态全部完成。
|
||||
*
|
||||
* @return 全部完成时返回 {@code true}
|
||||
*/
|
||||
public Boolean getActualCompleted() {
|
||||
return actualCompleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置批次关联任务是否已经按真实文档状态全部完成。
|
||||
*
|
||||
* @param actualCompleted 是否全部完成
|
||||
*/
|
||||
public void setActualCompleted(Boolean actualCompleted) {
|
||||
this.actualCompleted = actualCompleted;
|
||||
}
|
||||
|
||||
public Integer getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportMode;
|
||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||
@@ -435,7 +436,79 @@ public class DocumentImportBatchAppService {
|
||||
.orderBy(DocumentImportBatch::getCreated, false)
|
||||
.limit(1)
|
||||
);
|
||||
return batch == null ? null : batchTracker.toStatusResponse(batch);
|
||||
if (batch == null) {
|
||||
return null;
|
||||
}
|
||||
DocumentImportBatchDtos.StatusResponse response =
|
||||
batchTracker.toStatusResponse(batch);
|
||||
response.setActualCompleted(isAutoBatchActuallyCompleted(batch));
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据批次项及其关联文档的真实状态判断失败批次是否已经完成。
|
||||
*
|
||||
* <p>正常完成批次直接返回成功;仅对部分失败或中断批次执行补充查询,
|
||||
* 避免运行中轮询产生额外数据库压力。批次项数量不完整、文档缺失、
|
||||
* 跨知识库或文档仍未完成时均保持失败提示。</p>
|
||||
*
|
||||
* @param batch 自动导入批次
|
||||
* @return 批次关联任务是否已经全部完成
|
||||
*/
|
||||
private boolean isAutoBatchActuallyCompleted(DocumentImportBatch batch) {
|
||||
if (DocumentImportBatchStatus.COMPLETED.name().equals(batch.getStatus())) {
|
||||
return true;
|
||||
}
|
||||
if (!DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name().equals(batch.getStatus())
|
||||
&& !DocumentImportBatchStatus.INTERRUPTED.name().equals(batch.getStatus())) {
|
||||
return false;
|
||||
}
|
||||
List<DocumentImportBatchItem> items = itemService.list(
|
||||
QueryWrapper.create()
|
||||
.eq(DocumentImportBatchItem::getBatchId, batch.getId())
|
||||
.orderBy(DocumentImportBatchItem::getId, true)
|
||||
);
|
||||
int totalCount = valueOrZero(batch.getTotalCount());
|
||||
if (items == null || totalCount <= 0 || items.size() != totalCount) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Set<BigInteger> unresolvedDocumentIds = new LinkedHashSet<BigInteger>();
|
||||
for (DocumentImportBatchItem item : items) {
|
||||
String itemStatus = item.getStatus();
|
||||
if (DocumentImportBatchItemStatus.COMPLETED.name().equals(itemStatus)
|
||||
|| DocumentImportBatchItemStatus.SKIPPED.name().equals(itemStatus)
|
||||
|| DocumentImportBatchItemStatus.CANCELLED.name().equals(itemStatus)) {
|
||||
continue;
|
||||
}
|
||||
if (item.getDocumentId() == null
|
||||
|| !batch.getKnowledgeId().equals(item.getKnowledgeId())) {
|
||||
return false;
|
||||
}
|
||||
unresolvedDocumentIds.add(item.getDocumentId());
|
||||
}
|
||||
if (unresolvedDocumentIds.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
List<tech.easyflow.ai.entity.Document> completedDocuments =
|
||||
documentMapper.selectListByQuery(
|
||||
QueryWrapper.create()
|
||||
.select(tech.easyflow.ai.entity.Document::getId)
|
||||
.eq(tech.easyflow.ai.entity.Document::getCollectionId,
|
||||
batch.getKnowledgeId())
|
||||
.eq(tech.easyflow.ai.entity.Document::getProcessStatus,
|
||||
DocumentProcessStatus.COMPLETED.name())
|
||||
.in(tech.easyflow.ai.entity.Document::getId,
|
||||
unresolvedDocumentIds)
|
||||
);
|
||||
if (completedDocuments == null) {
|
||||
return false;
|
||||
}
|
||||
Set<BigInteger> completedDocumentIds = completedDocuments.stream()
|
||||
.map(tech.easyflow.ai.entity.Document::getId)
|
||||
.collect(java.util.stream.Collectors.toSet());
|
||||
return completedDocumentIds.containsAll(unresolvedDocumentIds);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
||||
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.chain.ChainStatus;
|
||||
import com.easyagents.flow.core.chain.Event;
|
||||
import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent;
|
||||
import com.easyagents.flow.core.chain.listener.ChainEventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 将工作流取消终态桥接到活动数据集查询。
|
||||
*/
|
||||
@Component
|
||||
public class WorkflowDatasetQueryCancellationListener
|
||||
implements ChainEventListener {
|
||||
|
||||
private final WorkflowDatasetQueryCancellationRegistry registry;
|
||||
|
||||
/**
|
||||
* 创建工作流查询取消监听器。
|
||||
*
|
||||
* @param registry 工作流查询取消登记表
|
||||
*/
|
||||
public WorkflowDatasetQueryCancellationListener(
|
||||
WorkflowDatasetQueryCancellationRegistry registry) {
|
||||
this.registry = registry;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在工作流进入取消终态后取消该实例的活动查询。
|
||||
*
|
||||
* @param event 工作流事件
|
||||
* @param chain 工作流实例
|
||||
*/
|
||||
@Override
|
||||
public void onEvent(Event event, Chain chain) {
|
||||
if (event instanceof ChainStatusChangeEvent statusChangeEvent
|
||||
&& statusChangeEvent.getStatus() == ChainStatus.CANCELLED) {
|
||||
registry.cancelExecution(chain.getStateInstanceId());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
||||
|
||||
/**
|
||||
* 维护工作流实例到活动数据集查询的跨节点取消映射。
|
||||
*/
|
||||
@Component
|
||||
public class WorkflowDatasetQueryCancellationRegistry {
|
||||
|
||||
private static final String ACTIVE_KEY_PREFIX =
|
||||
"easyflow:workflow:dataset-query:active:";
|
||||
private static final String CANCELLED_KEY_PREFIX =
|
||||
"easyflow:workflow:dataset-query:cancelled:";
|
||||
private static final Duration STATE_TTL = Duration.ofHours(24);
|
||||
private static final Logger log = LoggerFactory.getLogger(
|
||||
WorkflowDatasetQueryCancellationRegistry.class);
|
||||
|
||||
private final DatacenterFederationQueryCancellationService cancellationService;
|
||||
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
|
||||
private final ConcurrentHashMap<String, ConcurrentHashMap<String, BigInteger>>
|
||||
localActive = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, Long> localCancelledUntil =
|
||||
new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 创建工作流查询取消登记表。
|
||||
*
|
||||
* @param cancellationService 数据中枢查询取消服务
|
||||
* @param redisTemplateProvider 可选 Redis 模板
|
||||
*/
|
||||
public WorkflowDatasetQueryCancellationRegistry(
|
||||
DatacenterFederationQueryCancellationService cancellationService,
|
||||
ObjectProvider<StringRedisTemplate> redisTemplateProvider) {
|
||||
this.cancellationService = cancellationService;
|
||||
this.redisTemplateProvider = redisTemplateProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在数据库查询开始前登记工作流与 QueryId。
|
||||
*
|
||||
* @param stateInstanceId 工作流实例 ID
|
||||
* @param account 执行账号
|
||||
* @param queryId 查询 UUID
|
||||
* @return 必须关闭的登记句柄
|
||||
* @throws BusinessException 参数缺失时抛出
|
||||
*/
|
||||
public Registration register(
|
||||
String stateInstanceId,
|
||||
LoginAccount account,
|
||||
String queryId) {
|
||||
String instanceId = requireText(stateInstanceId, "工作流实例 ID 不能为空");
|
||||
String normalizedQueryId = requireText(queryId, "queryId 不能为空");
|
||||
BigInteger tenantId = requireTenantId(account);
|
||||
localActive.computeIfAbsent(
|
||||
instanceId,
|
||||
ignored -> new ConcurrentHashMap<>())
|
||||
.put(normalizedQueryId, tenantId);
|
||||
persistActive(instanceId, normalizedQueryId, tenantId);
|
||||
Registration registration = new Registration(
|
||||
instanceId, normalizedQueryId, tenantId);
|
||||
if (isExecutionCancelled(instanceId)) {
|
||||
cancelQuery(normalizedQueryId, tenantId);
|
||||
}
|
||||
return registration;
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消工作流当前登记的全部数据集查询。
|
||||
*
|
||||
* @param stateInstanceId 工作流实例 ID
|
||||
* @return 是否发现至少一个活动查询
|
||||
*/
|
||||
public boolean cancelExecution(String stateInstanceId) {
|
||||
String instanceId = requireText(stateInstanceId, "工作流实例 ID 不能为空");
|
||||
long expiresAt = System.currentTimeMillis() + STATE_TTL.toMillis();
|
||||
localCancelledUntil.put(instanceId, expiresAt);
|
||||
persistCancellationMarker(instanceId);
|
||||
|
||||
Map<String, BigInteger> active = new LinkedHashMap<>();
|
||||
ConcurrentHashMap<String, BigInteger> local = localActive.get(instanceId);
|
||||
if (local != null) {
|
||||
active.putAll(local);
|
||||
}
|
||||
loadPersistedActive(instanceId).forEach(active::putIfAbsent);
|
||||
active.forEach((queryId, tenantId) -> {
|
||||
try {
|
||||
cancelQuery(queryId, tenantId);
|
||||
} catch (RuntimeException exception) {
|
||||
// 单个驱动取消失败不能阻断同一工作流的其他活动查询。
|
||||
log.error("Failed to cancel workflow dataset query {} for {}",
|
||||
queryId, instanceId, exception);
|
||||
}
|
||||
});
|
||||
return !active.isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理本机已过期的取消墓碑,避免不可达工作流持续占用内存。
|
||||
*/
|
||||
@Scheduled(fixedDelayString =
|
||||
"${easyflow.workflow.dataset-query-cancel-cleanup-ms:60000}")
|
||||
public void cleanupExpiredLocalMarkers() {
|
||||
long now = System.currentTimeMillis();
|
||||
localCancelledUntil.entrySet().removeIf(
|
||||
entry -> entry.getValue() <= now);
|
||||
}
|
||||
|
||||
private void cancelQuery(String queryId, BigInteger tenantId) {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setTenantId(tenantId);
|
||||
cancellationService.cancel(queryId, account);
|
||||
}
|
||||
|
||||
private boolean isExecutionCancelled(String stateInstanceId) {
|
||||
Long localDeadline = localCancelledUntil.get(stateInstanceId);
|
||||
if (localDeadline != null && localDeadline > System.currentTimeMillis()) {
|
||||
return true;
|
||||
}
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return Boolean.TRUE.equals(redisTemplate.hasKey(
|
||||
cancelledKey(stateInstanceId)));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to read workflow dataset cancellation marker for {}",
|
||||
stateInstanceId, exception);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void persistActive(
|
||||
String stateInstanceId,
|
||||
String queryId,
|
||||
BigInteger tenantId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String key = activeKey(stateInstanceId);
|
||||
redisTemplate.opsForHash().put(key, queryId, tenantId.toString());
|
||||
redisTemplate.expire(key, STATE_TTL);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to persist workflow dataset query mapping for {}",
|
||||
stateInstanceId, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, BigInteger> loadPersistedActive(String stateInstanceId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
Map<Object, Object> entries = redisTemplate.opsForHash()
|
||||
.entries(activeKey(stateInstanceId));
|
||||
Map<String, BigInteger> active = new LinkedHashMap<>();
|
||||
entries.forEach((queryId, tenantId) -> {
|
||||
try {
|
||||
active.put(String.valueOf(queryId),
|
||||
new BigInteger(String.valueOf(tenantId)));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Ignored invalid workflow dataset query mapping for {}",
|
||||
stateInstanceId, exception);
|
||||
}
|
||||
});
|
||||
return active;
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to load workflow dataset query mappings for {}",
|
||||
stateInstanceId, exception);
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
|
||||
private void persistCancellationMarker(String stateInstanceId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
redisTemplate.opsForValue().set(
|
||||
cancelledKey(stateInstanceId), "1", STATE_TTL);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to persist workflow dataset cancellation marker for {}",
|
||||
stateInstanceId, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private void unregister(
|
||||
String stateInstanceId,
|
||||
String queryId,
|
||||
BigInteger tenantId) {
|
||||
localActive.computeIfPresent(stateInstanceId, (ignored, queries) -> {
|
||||
queries.remove(queryId, tenantId);
|
||||
return queries.isEmpty() ? null : queries;
|
||||
});
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
redisTemplate.opsForHash().delete(
|
||||
activeKey(stateInstanceId), queryId);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to remove workflow dataset query mapping for {}",
|
||||
stateInstanceId, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private String activeKey(String stateInstanceId) {
|
||||
return ACTIVE_KEY_PREFIX + stateInstanceId;
|
||||
}
|
||||
|
||||
private String cancelledKey(String stateInstanceId) {
|
||||
return CANCELLED_KEY_PREFIX + stateInstanceId;
|
||||
}
|
||||
|
||||
private String requireText(String value, String message) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new BusinessException(message);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private BigInteger requireTenantId(LoginAccount account) {
|
||||
if (account == null || account.getTenantId() == null) {
|
||||
throw new BusinessException("工作流数据集查询缺少执行租户");
|
||||
}
|
||||
return account.getTenantId();
|
||||
}
|
||||
|
||||
/**
|
||||
* 单次工作流查询登记句柄。
|
||||
*/
|
||||
public final class Registration implements AutoCloseable {
|
||||
|
||||
private final String stateInstanceId;
|
||||
private final String queryId;
|
||||
private final BigInteger tenantId;
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
private Registration(
|
||||
String stateInstanceId,
|
||||
String queryId,
|
||||
BigInteger tenantId) {
|
||||
this.stateInstanceId = stateInstanceId;
|
||||
this.queryId = queryId;
|
||||
this.tenantId = tenantId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 幂等注销当前工作流查询。
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
unregister(stateInstanceId, queryId, tenantId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import org.springframework.context.annotation.Configuration;
|
||||
import tech.easyflow.ai.easyagentsflow.listener.ChainErrorListenerForSave;
|
||||
import tech.easyflow.ai.easyagentsflow.listener.ChainEventListenerForSave;
|
||||
import tech.easyflow.ai.easyagentsflow.listener.NodeErrorListenerForSave;
|
||||
import tech.easyflow.ai.easyagentsflow.cancellation.WorkflowDatasetQueryCancellationListener;
|
||||
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadCleanupListener;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
@@ -40,6 +41,9 @@ public class ChainExecutorConfig {
|
||||
@Resource
|
||||
private WorkflowApiUploadCleanupListener workflowApiUploadCleanupListener;
|
||||
@Resource
|
||||
private WorkflowDatasetQueryCancellationListener
|
||||
workflowDatasetQueryCancellationListener;
|
||||
@Resource
|
||||
private WorkflowExecutionBudgetProperties workflowExecutionBudgetProperties;
|
||||
@Resource
|
||||
private WorkflowRuntimeProperties workflowRuntimeProperties;
|
||||
@@ -91,6 +95,9 @@ public class ChainExecutorConfig {
|
||||
chainExecutor.addEventListener(
|
||||
ChainStatusChangeEvent.class,
|
||||
workflowApiUploadCleanupListener);
|
||||
chainExecutor.addEventListener(
|
||||
ChainStatusChangeEvent.class,
|
||||
workflowDatasetQueryCancellationListener);
|
||||
chainExecutor.addErrorListener(new ChainErrorListenerForSave());
|
||||
chainExecutor.addNodeErrorListener(new NodeErrorListenerForSave());
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ public class ChainEventListenerForSave implements ChainEventListener {
|
||||
record.setStartTime(new Date());
|
||||
record.setStatus(state.getStatus().getValue());
|
||||
record.setCreatedKey(WorkFlowUtil.getCreatedKey(chain));
|
||||
record.setCreatedBy(WorkFlowUtil.getOperator(chain).getId().toString());
|
||||
record.setCreatedBy(WorkFlowUtil.getCreatedBy(chain));
|
||||
// 启动记录保留同步确认,避免执行接口返回后立即查询时记录尚不可见。
|
||||
try {
|
||||
workflowExecResultService.save(record);
|
||||
|
||||
@@ -59,6 +59,8 @@ public class WorkflowCheckService {
|
||||
private static final String SYSTEM_START_PARAM_NAME = "user_input";
|
||||
private static final int MIN_LOOP_COUNT = 1;
|
||||
private static final int MAX_LOOP_COUNT = 300;
|
||||
private static final String JOIN_MODE_ANY = "any";
|
||||
private static final String JOIN_MODE_ALL = "all";
|
||||
|
||||
@Resource
|
||||
private WorkflowService workflowService;
|
||||
@@ -196,6 +198,10 @@ public class WorkflowCheckService {
|
||||
edge.id = trimToNull(edgeJson.getString("id"));
|
||||
edge.source = trimToNull(edgeJson.getString("source"));
|
||||
edge.target = trimToNull(edgeJson.getString("target"));
|
||||
JSONObject edgeData = edgeJson.getJSONObject("data");
|
||||
edge.condition = edgeData == null
|
||||
? null
|
||||
: trimToNull(edgeData.getString("condition"));
|
||||
|
||||
if (!StringUtils.hasText(edge.id)) {
|
||||
addIssue(issues, issueKeys, "EDGE_ID_EMPTY", "存在连线缺少 id", null, null, null);
|
||||
@@ -228,10 +234,162 @@ public class WorkflowCheckService {
|
||||
parsedWorkflow.nodes = nodes;
|
||||
parsedWorkflow.edges = edges;
|
||||
parsedWorkflow.nodeMap = nodeMap;
|
||||
checkJoinModes(parsedWorkflow, issues, issueKeys);
|
||||
checkDatacenterNodes(parsedWorkflow, issues, issueKeys);
|
||||
return parsedWorkflow;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验节点汇聚模式及其静态可证明的到达安全性。
|
||||
*
|
||||
* @param parsed 工作流视图
|
||||
* @param issues 问题列表
|
||||
* @param issueKeys 问题去重键
|
||||
*/
|
||||
private void checkJoinModes(
|
||||
ParsedWorkflow parsed,
|
||||
List<WorkflowCheckIssue> issues,
|
||||
Set<String> issueKeys) {
|
||||
Map<String, List<EdgeView>> inwardEdges = new LinkedHashMap<>();
|
||||
for (EdgeView edge : parsed.edges) {
|
||||
if (edge == null || !StringUtils.hasText(edge.target)) {
|
||||
continue;
|
||||
}
|
||||
inwardEdges.computeIfAbsent(
|
||||
edge.target, ignored -> new ArrayList<>()).add(edge);
|
||||
}
|
||||
|
||||
for (NodeView node : parsed.nodes) {
|
||||
String joinMode = resolveJoinMode(node);
|
||||
if (joinMode == null) {
|
||||
addIssue(
|
||||
issues,
|
||||
issueKeys,
|
||||
"JOIN_MODE_INVALID",
|
||||
"执行时机配置无效,joinMode 仅支持 any 或 all",
|
||||
node.id,
|
||||
null,
|
||||
node.name);
|
||||
continue;
|
||||
}
|
||||
if (JOIN_MODE_ALL.equals(joinMode)
|
||||
&& StringUtils.hasText(node.parentId)) {
|
||||
addIssue(
|
||||
issues,
|
||||
issueKeys,
|
||||
"JOIN_MODE_LOOP_CHILD_UNSUPPORTED",
|
||||
"显式循环子图暂不支持“全部上游完成”,请改为“任一上游完成”",
|
||||
node.id,
|
||||
null,
|
||||
node.name);
|
||||
}
|
||||
}
|
||||
|
||||
Set<String> guaranteedNodes = findGuaranteedNodes(
|
||||
parsed, inwardEdges);
|
||||
for (NodeView node : parsed.nodes) {
|
||||
if (!JOIN_MODE_ALL.equals(resolveJoinMode(node))
|
||||
|| StringUtils.hasText(node.parentId)) {
|
||||
continue;
|
||||
}
|
||||
List<EdgeView> directInward = inwardEdges.getOrDefault(
|
||||
node.id, Collections.emptyList());
|
||||
if (directInward.size() <= 1) {
|
||||
continue;
|
||||
}
|
||||
boolean allGuaranteed = directInward.stream().allMatch(edge ->
|
||||
!edge.hasCondition()
|
||||
&& guaranteedNodes.contains(edge.source));
|
||||
if (!allGuaranteed) {
|
||||
addIssue(
|
||||
issues,
|
||||
issueKeys,
|
||||
"JOIN_MODE_CONDITIONAL_PATH_UNSUPPORTED",
|
||||
"“全部上游完成”可能永久等待:存在条件、互斥或无法证明必达的上游路径。"
|
||||
+ "请改为“任一上游完成”或调整连线,确保所有直接入边都会到达",
|
||||
node.id,
|
||||
null,
|
||||
node.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用保守固定点传播计算能够保证执行的根级节点。
|
||||
*
|
||||
* @param parsed 工作流视图
|
||||
* @param inwardEdges 直接入边索引
|
||||
* @return 保证执行的节点 ID
|
||||
*/
|
||||
private Set<String> findGuaranteedNodes(
|
||||
ParsedWorkflow parsed,
|
||||
Map<String, List<EdgeView>> inwardEdges) {
|
||||
Set<String> guaranteed = parsed.nodes.stream()
|
||||
.filter(NodeView::isRootLevel)
|
||||
.filter(node -> TYPE_START.equals(node.type))
|
||||
.map(node -> node.id)
|
||||
.filter(StringUtils::hasText)
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
|
||||
boolean changed;
|
||||
do {
|
||||
changed = false;
|
||||
for (NodeView node : parsed.nodes) {
|
||||
if (!node.isRootLevel()
|
||||
|| guaranteed.contains(node.id)
|
||||
|| hasAdvancedCondition(node)) {
|
||||
continue;
|
||||
}
|
||||
String joinMode = resolveJoinMode(node);
|
||||
if (joinMode == null) {
|
||||
continue;
|
||||
}
|
||||
List<EdgeView> directInward = inwardEdges.getOrDefault(
|
||||
node.id, Collections.emptyList());
|
||||
boolean isGuaranteed;
|
||||
if (JOIN_MODE_ALL.equals(joinMode)) {
|
||||
isGuaranteed = !directInward.isEmpty()
|
||||
&& directInward.stream().allMatch(edge ->
|
||||
!edge.hasCondition()
|
||||
&& guaranteed.contains(edge.source));
|
||||
} else {
|
||||
isGuaranteed = directInward.stream().anyMatch(edge ->
|
||||
!edge.hasCondition()
|
||||
&& guaranteed.contains(edge.source));
|
||||
}
|
||||
if (isGuaranteed && guaranteed.add(node.id)) {
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
} while (changed);
|
||||
return guaranteed;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取节点汇聚模式。字段缺失时兼容为 any,显式非法值返回 null。
|
||||
*/
|
||||
private String resolveJoinMode(NodeView node) {
|
||||
if (node == null || node.data == null
|
||||
|| !node.data.containsKey("joinMode")) {
|
||||
return JOIN_MODE_ANY;
|
||||
}
|
||||
String value = trimToNull(node.data.getString("joinMode"));
|
||||
if (JOIN_MODE_ANY.equalsIgnoreCase(value)) {
|
||||
return JOIN_MODE_ANY;
|
||||
}
|
||||
if (JOIN_MODE_ALL.equalsIgnoreCase(value)) {
|
||||
return JOIN_MODE_ALL;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean hasAdvancedCondition(NodeView node) {
|
||||
return node != null
|
||||
&& node.data != null
|
||||
&& StringUtils.hasText(
|
||||
trimToNull(node.data.getString("condition")));
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验普通循环、显式循环和循环父子层级。
|
||||
*
|
||||
@@ -1610,5 +1768,10 @@ public class WorkflowCheckService {
|
||||
private String id;
|
||||
private String source;
|
||||
private String target;
|
||||
private String condition;
|
||||
|
||||
private boolean hasCondition() {
|
||||
return StringUtils.hasText(condition);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,12 +14,13 @@ import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
@@ -34,6 +35,7 @@ public class WorkflowDatacenterContentService {
|
||||
public static final String LLM_NODE_TYPE = "llmNode";
|
||||
public static final String QUERY_DATA_CONTEXT = "queryDataContext";
|
||||
public static final String SEARCH_SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
|
||||
public static final String SEARCH_TABLE_MISSING_MESSAGE = "查询数据节点未选择已接入表";
|
||||
public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
|
||||
public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
|
||||
public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效,请重新选择查询数据节点";
|
||||
@@ -135,11 +137,16 @@ public class WorkflowDatacenterContentService {
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
if (datasetRef.getTableId() == null) {
|
||||
throw new BusinessException(SEARCH_TABLE_MISSING_MESSAGE);
|
||||
}
|
||||
String querySql = data == null ? null : trimToNull(data.getString("querySql"));
|
||||
if (!StringUtils.hasText(querySql)) {
|
||||
throw new BusinessException(SEARCH_SQL_MISSING_MESSAGE);
|
||||
}
|
||||
return datasetRef;
|
||||
DatasetRef boundRef = bindAuthoritativeTenant(datasetRef);
|
||||
data.put("datasetRef", boundRef);
|
||||
return boundRef;
|
||||
}
|
||||
|
||||
public DatasetRef requireSaveDatasetRef(JSONObject data) {
|
||||
@@ -150,6 +157,31 @@ public class WorkflowDatacenterContentService {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
|
||||
}
|
||||
DatasetRef boundRef = bindAuthoritativeTenant(datasetRef);
|
||||
data.put("datasetRef", boundRef);
|
||||
return boundRef;
|
||||
}
|
||||
|
||||
/**
|
||||
* 依据当前租户可见的权威 Source/Table 覆盖调用方声明的租户字段。
|
||||
*
|
||||
* @param datasetRef 工作流数据集引用
|
||||
* @return 已绑定权威租户的数据集引用
|
||||
*/
|
||||
private DatasetRef bindAuthoritativeTenant(DatasetRef datasetRef) {
|
||||
DatacenterTable table = datasetRef.getTableId() == null
|
||||
? null : registryService.getTableWithFields(datasetRef.getTableId());
|
||||
BigInteger sourceId = table == null ? datasetRef.getSourceId() : table.getSourceId();
|
||||
if (sourceId == null) {
|
||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
||||
if (table != null && (!sourceId.equals(table.getSourceId())
|
||||
|| !java.util.Objects.equals(source.getTenantId(), table.getTenantId()))) {
|
||||
throw new BusinessException("数据集引用与当前租户不一致");
|
||||
}
|
||||
datasetRef.setTenantId(source.getTenantId());
|
||||
datasetRef.setSourceId(sourceId);
|
||||
return datasetRef;
|
||||
}
|
||||
|
||||
@@ -172,41 +204,68 @@ public class WorkflowDatacenterContentService {
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
if (datasetRef.getTableId() == null) {
|
||||
throw new BusinessException(SEARCH_TABLE_MISSING_MESSAGE);
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), datasetRef.getCatalogId());
|
||||
managedTables.sort(Comparator.comparing(table -> table.getTableName() == null ? "" : table.getTableName()));
|
||||
DatacenterTable fullTable = registryService.getTableWithFields(
|
||||
datasetRef.getTableId());
|
||||
if (fullTable == null
|
||||
|| !datasetRef.getSourceId().equals(fullTable.getSourceId())
|
||||
|| !Integer.valueOf(1).equals(fullTable.getQueryable())
|
||||
|| !DatacenterMetadataStatus.ACTIVE.name().equals(
|
||||
fullTable.getMetadataStatus())) {
|
||||
throw new BusinessException("查询数据节点绑定的表不可用");
|
||||
}
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(
|
||||
fullTable.getCatalogId());
|
||||
if (StringUtils.hasText(datasetRef.getCatalogName())
|
||||
&& (catalog == null || !datasetRef.getCatalogName().equals(
|
||||
catalog.getCatalogName()))) {
|
||||
throw new BusinessException("查询数据节点绑定的命名空间已变化");
|
||||
}
|
||||
JSONObject sourceSummary = new JSONObject();
|
||||
sourceSummary.put("sourceName", source.getSourceName());
|
||||
sourceSummary.put("sourceType", source.getSourceType());
|
||||
JSONArray tables = new JSONArray();
|
||||
for (DatacenterTable table : managedTables) {
|
||||
DatacenterTable fullTable = registryService.getTableWithFields(table.getId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(fullTable.getCatalogId());
|
||||
if (StringUtils.hasText(datasetRef.getCatalogName())
|
||||
&& (catalog == null || !datasetRef.getCatalogName().equals(catalog.getCatalogName()))) {
|
||||
continue;
|
||||
}
|
||||
JSONObject tableSummary = new JSONObject();
|
||||
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
|
||||
tableSummary.put("tableName", fullTable.getTableName());
|
||||
tableSummary.put("tableDesc", fullTable.getTableDesc());
|
||||
JSONArray fields = new JSONArray();
|
||||
if (fullTable.getFields() != null) {
|
||||
for (DatacenterTableField field : fullTable.getFields()) {
|
||||
JSONObject fieldSummary = new JSONObject();
|
||||
fieldSummary.put("fieldName", field.getFieldName());
|
||||
fieldSummary.put("fieldDesc", field.getFieldDesc());
|
||||
fieldSummary.put("fieldType", resolveFieldType(field));
|
||||
fields.add(fieldSummary);
|
||||
JSONObject tableSummary = new JSONObject();
|
||||
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
|
||||
tableSummary.put("tableName", fullTable.getTableName());
|
||||
tableSummary.put("tableDesc", fullTable.getTableDesc());
|
||||
JSONArray fields = new JSONArray();
|
||||
if (fullTable.getFields() != null) {
|
||||
for (DatacenterTableField field : fullTable.getFields()) {
|
||||
if (!isQueryablePublicField(field)) {
|
||||
continue;
|
||||
}
|
||||
JSONObject fieldSummary = new JSONObject();
|
||||
fieldSummary.put("fieldName", field.getFieldName());
|
||||
fieldSummary.put("fieldDesc", field.getFieldDesc());
|
||||
fieldSummary.put("fieldType", resolveFieldType(field));
|
||||
fields.add(fieldSummary);
|
||||
}
|
||||
tableSummary.put("fields", fields);
|
||||
tables.add(tableSummary);
|
||||
}
|
||||
tableSummary.put("fields", fields);
|
||||
tables.add(tableSummary);
|
||||
sourceSummary.put("tables", tables);
|
||||
return sourceSummary;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断字段是否允许暴露给 SQL 生成上下文。
|
||||
*
|
||||
* @param field 字段元数据
|
||||
* @return 是否为当前可查询的公开字段
|
||||
*/
|
||||
private boolean isQueryablePublicField(DatacenterTableField field) {
|
||||
return field != null
|
||||
&& Integer.valueOf(1).equals(field.getQueryable())
|
||||
&& DatacenterMetadataStatus.ACTIVE.name().equals(
|
||||
field.getMetadataStatus())
|
||||
&& DatacenterSensitivityLevel.PUBLIC.name().equals(
|
||||
field.getSensitivityLevel());
|
||||
}
|
||||
|
||||
private void injectQueryDataContext(JSONObject data, Map<String, JSONObject> nodeMap) {
|
||||
if (data == null) {
|
||||
return;
|
||||
@@ -217,7 +276,7 @@ public class WorkflowDatacenterContentService {
|
||||
removeQueryDataContextPlaceholder(data);
|
||||
return;
|
||||
}
|
||||
Map<BigInteger, JSONObject> sourceSummaries = new LinkedHashMap<>();
|
||||
Map<String, JSONObject> sourceSummaries = new LinkedHashMap<>();
|
||||
Set<String> visitedNodeIds = new LinkedHashSet<>();
|
||||
for (int i = 0; i < nodeIds.size(); i++) {
|
||||
String nodeId = trimToNull(nodeIds.getString(i));
|
||||
@@ -229,7 +288,10 @@ public class WorkflowDatacenterContentService {
|
||||
throw new BusinessException(INVALID_QUERY_CONTEXT_MESSAGE);
|
||||
}
|
||||
DatasetRef datasetRef = requireSearchDatasetRef(targetNode.getJSONObject("data"));
|
||||
sourceSummaries.putIfAbsent(datasetRef.getSourceId(), buildSourceSummary(datasetRef));
|
||||
String summaryKey = datasetRef.getSourceId() + ":" + datasetRef.getTableId();
|
||||
if (!sourceSummaries.containsKey(summaryKey)) {
|
||||
sourceSummaries.put(summaryKey, buildSourceSummary(datasetRef));
|
||||
}
|
||||
}
|
||||
String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values()));
|
||||
upsertQueryDataContextParameter(data, contextValue);
|
||||
@@ -343,6 +405,7 @@ public class WorkflowDatacenterContentService {
|
||||
|
||||
private DatasetRef copyDatasetRef(DatasetRef datasetRef) {
|
||||
DatasetRef copy = new DatasetRef();
|
||||
copy.setTenantId(datasetRef.getTenantId());
|
||||
copy.setSourceId(datasetRef.getSourceId());
|
||||
copy.setCatalogId(datasetRef.getCatalogId());
|
||||
copy.setCatalogName(datasetRef.getCatalogName());
|
||||
|
||||
@@ -139,7 +139,7 @@ public class WorkflowRunningParameterResolver {
|
||||
}
|
||||
|
||||
/**
|
||||
* 归一化工作流运行时变量,确保文件参数统一为文件对象数组。
|
||||
* 归一化工作流运行时变量,统一文件结构并移除空图片值。
|
||||
*
|
||||
* @param content 工作流内容
|
||||
* @param variables 原始运行变量
|
||||
@@ -162,7 +162,13 @@ public class WorkflowRunningParameterResolver {
|
||||
if (isFileParameter(parameter)) {
|
||||
normalized.put(name, normalizeFileVariableValue(normalized.get(name), name));
|
||||
} else if (isImageParameter(parameter)) {
|
||||
normalized.put(name, normalizeImageVariableValue(normalized.get(name), name));
|
||||
Object imageValue = normalizeImageVariableValue(
|
||||
normalized.get(name), name);
|
||||
if (imageValue == null) {
|
||||
normalized.remove(name);
|
||||
} else {
|
||||
normalized.put(name, imageValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
@@ -683,6 +689,10 @@ public class WorkflowRunningParameterResolver {
|
||||
if (value == null) {
|
||||
return;
|
||||
}
|
||||
if (value instanceof String stringValue
|
||||
&& !StringUtils.hasText(stringValue)) {
|
||||
return;
|
||||
}
|
||||
if (value instanceof Collection<?> collection) {
|
||||
for (Object item : collection) {
|
||||
collectFileValues(item, result);
|
||||
|
||||
@@ -15,6 +15,7 @@ import tech.easyflow.ai.document.support.DocumentInputStreamSupport;
|
||||
import tech.easyflow.ai.document.support.DocumentParseSourceType;
|
||||
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadedFileReader;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.filestorage.FileStorageWriteHandle;
|
||||
import tech.easyflow.common.util.StringUtil;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
@@ -326,20 +327,29 @@ public class DocNodeFileContentExtractor {
|
||||
private void copySourceToTemporaryFile(
|
||||
DocumentSourceRef sourceRef, Path target) throws IOException {
|
||||
String filePath = sourceRef.getFilePath();
|
||||
boolean managedUpload = StringUtil.hasText(filePath)
|
||||
&& uploadedFileReader != null
|
||||
&& uploadedFileReader.isManagedPathCandidate(filePath);
|
||||
Optional<FileStorageWriteHandle> trustedFile = Optional.empty();
|
||||
if (StringUtil.hasText(filePath)
|
||||
&& isRemoteUrl(filePath)
|
||||
&& !managedUpload) {
|
||||
trustedFile = fileStorageService.resolveTrustedFile(filePath);
|
||||
}
|
||||
boolean localStorage = StringUtil.hasText(filePath)
|
||||
&& (!isRemoteUrl(filePath)
|
||||
|| (uploadedFileReader != null
|
||||
&& uploadedFileReader.isManagedPathCandidate(filePath)));
|
||||
|| managedUpload
|
||||
|| trustedFile.isPresent());
|
||||
if (localStorage) {
|
||||
try (IoBulkhead.Permit ignored =
|
||||
IoBulkhead.storage().acquire("storage:document-read");
|
||||
InputStream inputStream = openInputStream(sourceRef);
|
||||
InputStream inputStream = openInputStream(sourceRef, trustedFile);
|
||||
OutputStream outputStream = Files.newOutputStream(target)) {
|
||||
copy(inputStream, outputStream);
|
||||
}
|
||||
return;
|
||||
}
|
||||
try (InputStream inputStream = openInputStream(sourceRef);
|
||||
try (InputStream inputStream = openInputStream(sourceRef, trustedFile);
|
||||
OutputStream outputStream = Files.newOutputStream(target)) {
|
||||
copy(inputStream, outputStream);
|
||||
}
|
||||
@@ -362,7 +372,17 @@ public class DocNodeFileContentExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
private InputStream openInputStream(DocumentSourceRef sourceRef) throws IOException {
|
||||
/**
|
||||
* 按可信受管上传、服务端文件记录、本地路径和普通公网 URL 的顺序打开源流。
|
||||
*
|
||||
* @param sourceRef 文档源
|
||||
* @param trustedFile 已通过服务端记录或存储配置确认的物理读取句柄
|
||||
* @return 受实际字节数限制的输入流
|
||||
* @throws IOException 文件无法安全读取时抛出
|
||||
*/
|
||||
private InputStream openInputStream(
|
||||
DocumentSourceRef sourceRef,
|
||||
Optional<FileStorageWriteHandle> trustedFile) throws IOException {
|
||||
String filePath = sourceRef.getFilePath();
|
||||
if (uploadedFileReader != null && StringUtil.hasText(filePath)) {
|
||||
Optional<InputStream> managed = uploadedFileReader.openVerified(filePath);
|
||||
@@ -372,6 +392,11 @@ public class DocNodeFileContentExtractor {
|
||||
FILE_MAX_SINGLE_SIZE);
|
||||
}
|
||||
}
|
||||
if (trustedFile.isPresent()) {
|
||||
return DocumentInputStreamSupport.limit(
|
||||
fileStorageService.readRecoverable(trustedFile.get()),
|
||||
FILE_MAX_SINGLE_SIZE);
|
||||
}
|
||||
if (StringUtil.hasText(filePath) && isRemoteUrl(filePath)) {
|
||||
return DocumentInputStreamSupport.openRemote(filePath, FILE_MAX_SINGLE_SIZE);
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.chain.runtime.RetryableTriggerException;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.util.IoBulkhead;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tech.easyflow.ai.utils.WorkFlowUtil;
|
||||
@@ -55,7 +54,6 @@ public class SaveDatasetNode extends BaseNode {
|
||||
rows.add(item instanceof JSONObject json ? json : JSONObject.from(item));
|
||||
}
|
||||
try {
|
||||
TenantManager.ignoreTenantCondition();
|
||||
try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) {
|
||||
writeService.saveRowsIdempotently(
|
||||
datasetRef,
|
||||
@@ -77,8 +75,6 @@ public class SaveDatasetNode extends BaseNode {
|
||||
} catch (Exception ex) {
|
||||
log.error("工作流保存数据到统一数据集失败,datasetRef={}", datasetRef, ex);
|
||||
throw ex;
|
||||
} finally {
|
||||
TenantManager.restoreTenantCondition();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,12 +7,13 @@ import com.easyagents.flow.core.chain.repository.LoopInputReference;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.util.IoBulkhead;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.ai.easyagentsflow.cancellation.WorkflowDatasetQueryCancellationRegistry;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
@@ -55,32 +56,38 @@ public class SearchDatasetNode extends BaseNode {
|
||||
Map<String, Object> params =
|
||||
chain.getExecutionState().resolveParameters(this);
|
||||
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
|
||||
WorkflowDatasetQueryCancellationRegistry cancellationRegistry =
|
||||
SpringContextUtil.getBean(
|
||||
WorkflowDatasetQueryCancellationRegistry.class);
|
||||
LoginAccount account = tech.easyflow.ai.utils.WorkFlowUtil.getOperator(chain);
|
||||
DatacenterSqlQueryRequest request = buildRuntimeRequest(params);
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
try {
|
||||
TenantManager.ignoreTenantCondition();
|
||||
try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) {
|
||||
String resultId = chain.getStateInstanceId()
|
||||
+ ":dataset:"
|
||||
+ UUID.randomUUID();
|
||||
int rowCount =
|
||||
chain.storeProducedLoopInputOutsideLock(
|
||||
resultId,
|
||||
sink -> queryService.consumeBySql(
|
||||
request,
|
||||
QUERY_PAGE_SIZE,
|
||||
sink::accept),
|
||||
0L,
|
||||
chain.currentFencingClaimId(),
|
||||
chain.currentClaimGeneration());
|
||||
result.put(
|
||||
resolveOutputKey("data"),
|
||||
new LoopInputReference(
|
||||
resultId, rowCount));
|
||||
return result;
|
||||
}
|
||||
} finally {
|
||||
TenantManager.restoreTenantCondition();
|
||||
String queryId = UUID.randomUUID().toString();
|
||||
try (WorkflowDatasetQueryCancellationRegistry.Registration registration =
|
||||
cancellationRegistry.register(
|
||||
chain.getStateInstanceId(), account, queryId);
|
||||
IoBulkhead.Permit ignored = IoBulkhead.dataset()
|
||||
.acquire(resolveIoTarget())) {
|
||||
String resultId = chain.getStateInstanceId()
|
||||
+ ":dataset:"
|
||||
+ queryId;
|
||||
int rowCount =
|
||||
chain.storeProducedLoopInputOutsideLock(
|
||||
resultId,
|
||||
sink -> queryService.consumeBySql(
|
||||
request,
|
||||
QUERY_PAGE_SIZE,
|
||||
account,
|
||||
queryId,
|
||||
sink::accept),
|
||||
0L,
|
||||
chain.currentFencingClaimId(),
|
||||
chain.currentClaimGeneration());
|
||||
result.put(
|
||||
resolveOutputKey("data"),
|
||||
new LoopInputReference(
|
||||
resultId, rowCount));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +104,9 @@ public class SearchDatasetNode extends BaseNode {
|
||||
}
|
||||
|
||||
private DatacenterSqlQueryRequest buildRuntimeRequest(Map<String, Object> params) {
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException("数据集绑定缺少连接信息,请重新选择数据集");
|
||||
}
|
||||
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
|
||||
request.setDatasetRef(copyDatasetRef());
|
||||
request.setSql(resolveQuerySql(params));
|
||||
@@ -128,12 +138,13 @@ public class SearchDatasetNode extends BaseNode {
|
||||
|
||||
private DatasetRef copyDatasetRef() {
|
||||
DatasetRef copy = new DatasetRef();
|
||||
copy.setTenantId(datasetRef == null ? null : datasetRef.getTenantId());
|
||||
copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId());
|
||||
copy.setCatalogId(datasetRef == null ? null : datasetRef.getCatalogId());
|
||||
copy.setCatalogName(datasetRef == null ? null : datasetRef.getCatalogName());
|
||||
copy.setTableId(null);
|
||||
copy.setTableName(null);
|
||||
copy.setVersionId(null);
|
||||
copy.setTableId(datasetRef == null ? null : datasetRef.getTableId());
|
||||
copy.setTableName(datasetRef == null ? null : datasetRef.getTableName());
|
||||
copy.setVersionId(datasetRef == null ? null : datasetRef.getVersionId());
|
||||
return copy;
|
||||
}
|
||||
|
||||
|
||||
@@ -92,4 +92,22 @@ public interface WorkflowShareService extends IService<WorkflowShare> {
|
||||
* @return 有效对话分享记录
|
||||
*/
|
||||
WorkflowShare resolveChatShare(String shareKey, BigInteger tenantId);
|
||||
|
||||
/**
|
||||
* 跨租户解析当前有效的匿名对话分享。
|
||||
*
|
||||
* @param shareKey 原始分享密钥
|
||||
* @return 有效且指向严格发布工作流的分享记录
|
||||
*/
|
||||
WorkflowShare resolvePublicChatShare(String shareKey);
|
||||
|
||||
/**
|
||||
* 跨租户解析匿名对话分享的历史记录。
|
||||
*
|
||||
* <p>仅用于详情和取消已发起执行,不校验分享状态、有效期与当前发布态。</p>
|
||||
*
|
||||
* @param shareKey 原始分享密钥
|
||||
* @return 对话分享记录
|
||||
*/
|
||||
WorkflowShare resolveHistoricalChatShare(String shareKey);
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ package tech.easyflow.ai.service;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.ai.enums.PublishStatus;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.system.enums.CategoryResourceType;
|
||||
@@ -15,7 +15,7 @@ import java.util.Objects;
|
||||
/**
|
||||
* 工作流使用权限校验服务。
|
||||
*
|
||||
* <p>统一封装工作流存在性、租户、启用状态和资源使用权限校验,供页面能力和后台任务复用。</p>
|
||||
* <p>统一封装工作流存在性、租户、发布快照和资源使用权限校验,供页面能力和后台任务复用。</p>
|
||||
*/
|
||||
@Service
|
||||
public class WorkflowUsageAuthorizationService {
|
||||
@@ -40,20 +40,20 @@ public class WorkflowUsageAuthorizationService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前账号可使用的启用工作流。
|
||||
* 获取当前账号可使用的已发布工作流视图。
|
||||
*
|
||||
* @param workflowId 工作流 ID
|
||||
* @param account 使用工作流的账号
|
||||
* @param denyMessage 校验失败提示
|
||||
* @return 可使用的工作流
|
||||
* @throws BusinessException 工作流不存在、未启用、跨租户或无使用权限时抛出
|
||||
* @return 可使用的工作流发布视图
|
||||
* @throws BusinessException 工作流不存在、未发布、缺少发布快照、跨租户或无使用权限时抛出
|
||||
*/
|
||||
public Workflow requireUsableWorkflow(
|
||||
BigInteger workflowId,
|
||||
LoginAccount account,
|
||||
String denyMessage) {
|
||||
String message = denyMessage == null || denyMessage.isBlank()
|
||||
? "工作流不存在、已禁用或无权使用"
|
||||
? "工作流不存在、未发布或无权使用"
|
||||
: denyMessage;
|
||||
if (workflowId == null || account == null || account.getId() == null
|
||||
|| account.getTenantId() == null) {
|
||||
@@ -62,7 +62,9 @@ public class WorkflowUsageAuthorizationService {
|
||||
Workflow workflow = workflowService.getById(workflowId);
|
||||
boolean usable = workflow != null
|
||||
&& Objects.equals(workflow.getTenantId(), account.getTenantId())
|
||||
&& EnumDataStatus.AVAILABLE.getCode().equals(workflow.getStatus())
|
||||
&& PublishStatus.PUBLISHED.getCode().equals(workflow.getPublishStatus())
|
||||
&& workflow.getPublishedSnapshotJson() != null
|
||||
&& !workflow.getPublishedSnapshotJson().isEmpty()
|
||||
&& resourceAccessService.canAccess(
|
||||
account,
|
||||
CategoryResourceType.WORKFLOW,
|
||||
@@ -71,6 +73,10 @@ public class WorkflowUsageAuthorizationService {
|
||||
if (!usable) {
|
||||
throw new BusinessException(403, 403, message);
|
||||
}
|
||||
return workflow;
|
||||
Workflow published = workflowService.toPublishedView(workflow);
|
||||
if (published == null) {
|
||||
throw new BusinessException(403, 403, message);
|
||||
}
|
||||
return published;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,7 +187,7 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
||||
RagScoreNormalizer.normalize(searchDocuments, retrievalMode, reranked);
|
||||
List<Document> formattedDocuments = formatDocuments(
|
||||
searchDocuments,
|
||||
shouldApplyMinSimilarityFilter(retrievalMode, reranked),
|
||||
true,
|
||||
minSimilarity,
|
||||
docRecallMaxNum
|
||||
);
|
||||
@@ -396,10 +396,6 @@ public class DocumentCollectionServiceImpl extends ServiceImpl<DocumentCollectio
|
||||
return modelRerank.toRerankModel();
|
||||
}
|
||||
|
||||
private boolean shouldApplyMinSimilarityFilter(RetrievalMode retrievalMode, boolean reranked) {
|
||||
return !reranked && retrievalMode == RetrievalMode.VECTOR;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析本次查询使用的召回上限,优先采用请求参数,其次回退到知识库默认配置。
|
||||
*
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package tech.easyflow.ai.service.impl;
|
||||
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
@@ -23,6 +24,7 @@ import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
@@ -184,6 +186,61 @@ public class WorkflowShareServiceImpl extends ServiceImpl<WorkflowShareMapper, W
|
||||
return resolveShare(shareKey, tenantId, WorkflowSharePurpose.CHAT);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public WorkflowShare resolvePublicChatShare(String shareKey) {
|
||||
if (shareKey == null || shareKey.isBlank()) {
|
||||
throw invalidShare();
|
||||
}
|
||||
return TenantManager.withoutTenantCondition(() -> {
|
||||
WorkflowShare share = findShare(
|
||||
shareKey,
|
||||
WorkflowSharePurpose.CHAT,
|
||||
true
|
||||
);
|
||||
if (share == null) {
|
||||
throw invalidShare();
|
||||
}
|
||||
if (share.getExpiresAt() == null
|
||||
|| !share.getExpiresAt().after(new Date())) {
|
||||
throw new BusinessException(403, 403, "工作流分享链接已过期");
|
||||
}
|
||||
Workflow workflow = workflowService.getPublishedById(
|
||||
share.getWorkflowId());
|
||||
if (workflow == null
|
||||
|| !Objects.equals(
|
||||
share.getTenantId(), workflow.getTenantId())) {
|
||||
throw invalidShare();
|
||||
}
|
||||
if (!isStrictlyPublished(workflow)) {
|
||||
throw new BusinessException(409, 409, "工作流尚未发布或已下线");
|
||||
}
|
||||
return share;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public WorkflowShare resolveHistoricalChatShare(String shareKey) {
|
||||
if (shareKey == null || shareKey.isBlank()) {
|
||||
throw invalidShare();
|
||||
}
|
||||
WorkflowShare share = TenantManager.withoutTenantCondition(
|
||||
() -> findShare(
|
||||
shareKey,
|
||||
WorkflowSharePurpose.CHAT,
|
||||
false
|
||||
));
|
||||
if (share == null) {
|
||||
throw invalidShare();
|
||||
}
|
||||
return share;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按用途校验并解析分享。
|
||||
*
|
||||
@@ -200,10 +257,7 @@ public class WorkflowShareServiceImpl extends ServiceImpl<WorkflowShareMapper, W
|
||||
if (shareKey == null || shareKey.isBlank() || tenantId == null) {
|
||||
throw invalidShare();
|
||||
}
|
||||
WorkflowShare share = getOne(QueryWrapper.create()
|
||||
.eq(WorkflowShare::getShareKeyHash, WorkflowSharePolicy.hashShareKey(shareKey))
|
||||
.eq(WorkflowShare::getSharePurpose, purpose.name())
|
||||
.eq(WorkflowShare::getStatus, KnowledgeShareStatus.ENABLED.name()));
|
||||
WorkflowShare share = findShare(shareKey, purpose, true);
|
||||
if (share == null || !tenantId.equals(share.getTenantId())) {
|
||||
throw invalidShare();
|
||||
}
|
||||
@@ -220,6 +274,34 @@ public class WorkflowShareServiceImpl extends ServiceImpl<WorkflowShareMapper, W
|
||||
return share;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按密钥与用途查询分享记录。
|
||||
*
|
||||
* @param shareKey 原始分享密钥
|
||||
* @param purpose 分享用途
|
||||
* @param activeOnly 是否仅查询启用记录
|
||||
* @return 分享记录
|
||||
*/
|
||||
private WorkflowShare findShare(
|
||||
String shareKey,
|
||||
WorkflowSharePurpose purpose,
|
||||
boolean activeOnly
|
||||
) {
|
||||
QueryWrapper query = QueryWrapper.create()
|
||||
.eq(
|
||||
WorkflowShare::getShareKeyHash,
|
||||
WorkflowSharePolicy.hashShareKey(shareKey)
|
||||
)
|
||||
.eq(WorkflowShare::getSharePurpose, purpose.name());
|
||||
if (activeOnly) {
|
||||
query.eq(
|
||||
WorkflowShare::getStatus,
|
||||
KnowledgeShareStatus.ENABLED.name()
|
||||
);
|
||||
}
|
||||
return getOne(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在锁保护下创建或替换工作流的唯一分享记录。
|
||||
*
|
||||
|
||||
@@ -9,6 +9,8 @@ import java.time.Duration;
|
||||
import java.util.Date;
|
||||
import java.util.HexFormat;
|
||||
import java.util.Set;
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
/**
|
||||
* 工作流协作分享的密钥、时效与接口授权策略。
|
||||
@@ -25,6 +27,11 @@ public final class WorkflowSharePolicy {
|
||||
*/
|
||||
public static final String CHAT_SHARE_KEY_HEADER = "X-Workflow-Chat-Share-Key";
|
||||
|
||||
/**
|
||||
* 工作流对话分享访客标识请求头。
|
||||
*/
|
||||
public static final String CHAT_VISITOR_HEADER = "X-Workflow-Chat-Visitor";
|
||||
|
||||
private static final Duration DEFAULT_EXPIRE_DURATION = Duration.ofMinutes(30);
|
||||
private static final Duration DEFAULT_CHAT_EXPIRE_DURATION = Duration.ofDays(7);
|
||||
private static final Set<String> ALLOWED_REQUESTS = Set.of(
|
||||
@@ -67,6 +74,28 @@ public final class WorkflowSharePolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算匿名访客的不可逆执行归属摘要。
|
||||
*
|
||||
* @param shareKey 原始分享密钥
|
||||
* @param visitorId 当前标签页访客标识
|
||||
* @return HMAC-SHA256 前 16 字节的小写十六进制摘要
|
||||
*/
|
||||
public static String hashChatVisitor(String shareKey, String visitorId) {
|
||||
try {
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(new SecretKeySpec(
|
||||
shareKey.getBytes(StandardCharsets.UTF_8),
|
||||
"HmacSHA256"
|
||||
));
|
||||
byte[] digest = mac.doFinal(
|
||||
visitorId.getBytes(StandardCharsets.UTF_8));
|
||||
return HexFormat.of().formatHex(digest, 0, 16);
|
||||
} catch (Exception exception) {
|
||||
throw new IllegalStateException("HmacSHA256 unavailable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算默认过期时间。
|
||||
*
|
||||
|
||||
@@ -19,6 +19,7 @@ public class WorkFlowUtil {
|
||||
public final static String WORKFLOW_CHAT_SHARE = "WORKFLOW_CHAT_SHARE";
|
||||
public final static String WORKFLOW_KEY = "workflow";
|
||||
public final static String CREATED_KEY_MEMORY_KEY = "workflowCreatedKey";
|
||||
public final static String CREATED_BY_MEMORY_KEY = "workflowCreatedBy";
|
||||
|
||||
public static String removeSensitiveInfo(String originJson) {
|
||||
JSONObject workflowInfo = JSON.parseObject(originJson);
|
||||
@@ -56,6 +57,35 @@ public class WorkFlowUtil {
|
||||
return value == null ? USER_KEY : String.valueOf(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取工作流执行记录的归属主体。
|
||||
*
|
||||
* <p>匿名分享可覆盖为访客摘要;其他入口继续使用权限主体账号 ID。</p>
|
||||
*
|
||||
* @param chain 当前工作流执行链
|
||||
* @return 执行归属主体
|
||||
*/
|
||||
public static String getCreatedBy(Chain chain) {
|
||||
Object value = chain.getExecutionState()
|
||||
.getMemory()
|
||||
.get(CREATED_BY_MEMORY_KEY);
|
||||
if (value != null) {
|
||||
return String.valueOf(value);
|
||||
}
|
||||
LoginAccount operator = getOperator(chain);
|
||||
return operator.getId() == null ? "0" : operator.getId().toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建工作流匿名分享的执行来源标识。
|
||||
*
|
||||
* @param shareId 分享记录 ID
|
||||
* @return 执行来源标识
|
||||
*/
|
||||
public static String publicChatShareCreatedKey(BigInteger shareId) {
|
||||
return WORKFLOW_CHAT_SHARE + ":" + shareId;
|
||||
}
|
||||
|
||||
public static LoginAccount defaultAccount() {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(new BigInteger("0"));
|
||||
|
||||
@@ -7,6 +7,7 @@ import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadedFileReader;
|
||||
import tech.easyflow.ai.document.exception.DocumentParseBridgeException;
|
||||
import tech.easyflow.ai.document.model.DocumentSourceRef;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.filestorage.FileStorageWriteHandle;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
@@ -73,6 +74,24 @@ public class DocumentSourceLoaderTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证服务端已登记的普通附件 URL 在公网地址校验前通过物理句柄直读。
|
||||
*/
|
||||
@Test
|
||||
public void shouldLoadRecordedInternalStorageUrlBeforeRemoteAddressGuard() {
|
||||
String fileUrl = "http://127.0.0.1:39000/easyflow/attachment/1/2026/8/26/demo.pdf";
|
||||
byte[] body = "recorded-pdf".getBytes(StandardCharsets.UTF_8);
|
||||
DocumentSourceLoader loader = new DocumentSourceLoader(
|
||||
new RecordedFileStorageService(fileUrl, body));
|
||||
DocumentSourceRef sourceRef = new DocumentSourceRef();
|
||||
sourceRef.setFileName("demo.pdf");
|
||||
sourceRef.setFilePath(fileUrl);
|
||||
|
||||
LoadedDocumentSource loadedSource = loader.load(sourceRef);
|
||||
|
||||
Assert.assertArrayEquals(body, loadedSource.getContentBytes());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证已通过上传记录校验的内网存储 URL 会走恢复句柄读取。
|
||||
*
|
||||
@@ -217,4 +236,42 @@ public class DocumentSourceLoaderTest {
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 仅允许通过服务端记录句柄读取内容的存储测试替身。
|
||||
*/
|
||||
private static class RecordedFileStorageService
|
||||
extends FailingFileStorageService {
|
||||
/** 允许解析的精确 URL。 */
|
||||
private final String recordedUrl;
|
||||
/** 固定文件内容。 */
|
||||
private final byte[] content;
|
||||
/** 固定可信读取句柄。 */
|
||||
private final FileStorageWriteHandle handle = new FileStorageWriteHandle(
|
||||
"recorded", "", "/storage", "attachment", "demo.pdf");
|
||||
|
||||
/**
|
||||
* 创建服务端记录存储替身。
|
||||
*
|
||||
* @param recordedUrl 允许解析的精确 URL
|
||||
* @param content 固定文件内容
|
||||
*/
|
||||
private RecordedFileStorageService(String recordedUrl, byte[] content) {
|
||||
this.recordedUrl = recordedUrl;
|
||||
this.content = content.clone();
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public Optional<FileStorageWriteHandle> resolveTrustedFile(String reference) {
|
||||
return recordedUrl.equals(reference) ? Optional.of(handle) : Optional.empty();
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public InputStream readRecoverable(FileStorageWriteHandle requestedHandle) {
|
||||
Assert.assertSame(handle, requestedHandle);
|
||||
return new ByteArrayInputStream(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import tech.easyflow.ai.enums.DocumentImportBatchItemStage;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchItemStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportBatchStatus;
|
||||
import tech.easyflow.ai.enums.DocumentImportMode;
|
||||
import tech.easyflow.ai.enums.DocumentProcessStatus;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchItemMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentImportBatchMapper;
|
||||
import tech.easyflow.ai.mapper.DocumentMapper;
|
||||
@@ -47,6 +48,89 @@ import java.util.function.BooleanSupplier;
|
||||
*/
|
||||
public class DocumentImportBatchAppServiceTest {
|
||||
|
||||
/**
|
||||
* 验证历史失败批次关联文档均已完成时返回真实完成标记。
|
||||
*/
|
||||
@Test
|
||||
public void latestAutoBatchShouldDetectActuallyCompletedDocuments() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setImportMode(DocumentImportMode.AUTO.name());
|
||||
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
|
||||
batch.setTotalCount(2);
|
||||
|
||||
DocumentImportBatchItem completed = uploadedItem(
|
||||
BigInteger.valueOf(11), batch.getId()
|
||||
);
|
||||
completed.setStatus(DocumentImportBatchItemStatus.COMPLETED.name());
|
||||
completed.setDocumentId(BigInteger.valueOf(101));
|
||||
DocumentImportBatchItem staleFailed = uploadedItem(
|
||||
BigInteger.valueOf(12), batch.getId()
|
||||
);
|
||||
staleFailed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||
staleFailed.setDocumentId(BigInteger.valueOf(102));
|
||||
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
||||
.thenReturn(List.of(completed, staleFailed));
|
||||
|
||||
tech.easyflow.ai.entity.Document recovered =
|
||||
new tech.easyflow.ai.entity.Document();
|
||||
recovered.setId(staleFailed.getDocumentId());
|
||||
recovered.setCollectionId(batch.getKnowledgeId());
|
||||
recovered.setProcessStatus(DocumentProcessStatus.COMPLETED.name());
|
||||
Mockito.when(context.documentMapper.selectListByQuery(Mockito.any()))
|
||||
.thenReturn(List.of(recovered));
|
||||
|
||||
DocumentImportBatchDtos.StatusResponse batchResponse =
|
||||
new DocumentImportBatchDtos.StatusResponse();
|
||||
batchResponse.setStatus(batch.getStatus());
|
||||
Mockito.when(context.batchTracker.toStatusResponse(batch))
|
||||
.thenReturn(batchResponse);
|
||||
|
||||
DocumentImportBatchDtos.StatusResponse response =
|
||||
context.service.getLatestAutoBatch(batch.getKnowledgeId());
|
||||
|
||||
Assert.assertTrue(response.getActualCompleted());
|
||||
Mockito.verify(context.documentMapper)
|
||||
.selectListByQuery(Mockito.any(QueryWrapper.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证失败项关联文档仍未完成时继续保留批次提示。
|
||||
*/
|
||||
@Test
|
||||
public void latestAutoBatchShouldKeepIncompleteFailureVisible() {
|
||||
TestContext context = createContext();
|
||||
DocumentImportBatch batch = context.batchService.getOne(
|
||||
QueryWrapper.create()
|
||||
);
|
||||
batch.setImportMode(DocumentImportMode.AUTO.name());
|
||||
batch.setStatus(DocumentImportBatchStatus.PARTIAL_SUCCEEDED.name());
|
||||
batch.setTotalCount(1);
|
||||
|
||||
DocumentImportBatchItem failed = uploadedItem(
|
||||
BigInteger.valueOf(13), batch.getId()
|
||||
);
|
||||
failed.setStatus(DocumentImportBatchItemStatus.FAILED.name());
|
||||
failed.setDocumentId(BigInteger.valueOf(103));
|
||||
Mockito.when(context.itemService.list(Mockito.any(QueryWrapper.class)))
|
||||
.thenReturn(List.of(failed));
|
||||
Mockito.when(context.documentMapper.selectListByQuery(Mockito.any()))
|
||||
.thenReturn(List.of());
|
||||
|
||||
DocumentImportBatchDtos.StatusResponse batchResponse =
|
||||
new DocumentImportBatchDtos.StatusResponse();
|
||||
batchResponse.setStatus(batch.getStatus());
|
||||
Mockito.when(context.batchTracker.toStatusResponse(batch))
|
||||
.thenReturn(batchResponse);
|
||||
|
||||
DocumentImportBatchDtos.StatusResponse response =
|
||||
context.service.getLatestAutoBatch(batch.getKnowledgeId());
|
||||
|
||||
Assert.assertFalse(response.getActualCompleted());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证上传领取通过单条多表更新同步刷新批次进度时间。
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
||||
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.chain.ChainStatus;
|
||||
import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent;
|
||||
import java.math.BigInteger;
|
||||
import java.util.UUID;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
||||
|
||||
/**
|
||||
* {@link WorkflowDatasetQueryCancellationRegistry} 取消竞态回归测试。
|
||||
*/
|
||||
public class WorkflowDatasetQueryCancellationRegistryTest {
|
||||
|
||||
/**
|
||||
* 验证工作流取消会使用登记租户取消活动 QueryId。
|
||||
*/
|
||||
@Test
|
||||
public void shouldCancelRegisteredQuery() {
|
||||
Fixture fixture = fixture();
|
||||
String queryId = UUID.randomUUID().toString();
|
||||
LoginAccount account = account(BigInteger.valueOf(7L));
|
||||
|
||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignored =
|
||||
fixture.registry.register("instance-a", account, queryId)) {
|
||||
Assert.assertTrue(fixture.registry.cancelExecution("instance-a"));
|
||||
}
|
||||
|
||||
ArgumentCaptor<LoginAccount> accountCaptor =
|
||||
ArgumentCaptor.forClass(LoginAccount.class);
|
||||
Mockito.verify(fixture.cancellationService).cancel(
|
||||
Mockito.eq(queryId), accountCaptor.capture());
|
||||
Assert.assertEquals(BigInteger.valueOf(7L),
|
||||
accountCaptor.getValue().getTenantId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证单个查询取消失败不会阻断同一工作流的其他活动查询。
|
||||
*/
|
||||
@Test
|
||||
public void shouldIsolateCancellationFailuresBetweenQueries() {
|
||||
Fixture fixture = fixture();
|
||||
String failedQueryId = UUID.randomUUID().toString();
|
||||
String succeedingQueryId = UUID.randomUUID().toString();
|
||||
LoginAccount account = account(BigInteger.valueOf(7L));
|
||||
Mockito.doThrow(new IllegalStateException("simulated cancellation failure"))
|
||||
.when(fixture.cancellationService)
|
||||
.cancel(Mockito.eq(failedQueryId), Mockito.any(LoginAccount.class));
|
||||
|
||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignoredA =
|
||||
fixture.registry.register("instance-failure", account, failedQueryId);
|
||||
WorkflowDatasetQueryCancellationRegistry.Registration ignoredB =
|
||||
fixture.registry.register("instance-failure", account, succeedingQueryId)) {
|
||||
Assert.assertTrue(fixture.registry.cancelExecution("instance-failure"));
|
||||
}
|
||||
|
||||
Mockito.verify(fixture.cancellationService).cancel(
|
||||
Mockito.eq(failedQueryId), Mockito.any(LoginAccount.class));
|
||||
Mockito.verify(fixture.cancellationService).cancel(
|
||||
Mockito.eq(succeedingQueryId), Mockito.any(LoginAccount.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证取消先于节点登记发生时,后续查询仍会立即收到取消。
|
||||
*/
|
||||
@Test
|
||||
public void shouldCancelQueryRegisteredAfterWorkflowCancellation() {
|
||||
Fixture fixture = fixture();
|
||||
String queryId = UUID.randomUUID().toString();
|
||||
|
||||
Assert.assertFalse(fixture.registry.cancelExecution("instance-b"));
|
||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignored =
|
||||
fixture.registry.register(
|
||||
"instance-b", account(BigInteger.ONE), queryId)) {
|
||||
Mockito.verify(fixture.cancellationService).cancel(
|
||||
Mockito.eq(queryId), Mockito.any(LoginAccount.class));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证监听器只桥接取消终态。
|
||||
*/
|
||||
@Test
|
||||
public void listenerShouldBridgeOnlyCancelledStatus() {
|
||||
WorkflowDatasetQueryCancellationRegistry registry = Mockito.mock(
|
||||
WorkflowDatasetQueryCancellationRegistry.class);
|
||||
WorkflowDatasetQueryCancellationListener listener =
|
||||
new WorkflowDatasetQueryCancellationListener(registry);
|
||||
Chain chain = Mockito.mock(Chain.class);
|
||||
Mockito.when(chain.getStateInstanceId()).thenReturn("instance-c");
|
||||
|
||||
listener.onEvent(new ChainStatusChangeEvent(
|
||||
chain, ChainStatus.SUCCEEDED, ChainStatus.RUNNING), chain);
|
||||
listener.onEvent(new ChainStatusChangeEvent(
|
||||
chain, ChainStatus.CANCELLED, ChainStatus.RUNNING), chain);
|
||||
|
||||
Mockito.verify(registry).cancelExecution("instance-c");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private Fixture fixture() {
|
||||
DatacenterFederationQueryCancellationService cancellationService =
|
||||
Mockito.mock(
|
||||
DatacenterFederationQueryCancellationService.class);
|
||||
ObjectProvider<StringRedisTemplate> redisProvider =
|
||||
Mockito.mock(ObjectProvider.class);
|
||||
Mockito.when(redisProvider.getIfAvailable()).thenReturn(null);
|
||||
return new Fixture(
|
||||
new WorkflowDatasetQueryCancellationRegistry(
|
||||
cancellationService, redisProvider),
|
||||
cancellationService);
|
||||
}
|
||||
|
||||
private LoginAccount account(BigInteger tenantId) {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setTenantId(tenantId);
|
||||
return account;
|
||||
}
|
||||
|
||||
private record Fixture(
|
||||
WorkflowDatasetQueryCancellationRegistry registry,
|
||||
DatacenterFederationQueryCancellationService cancellationService) {
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,163 @@ import java.util.Map;
|
||||
|
||||
public class WorkflowCheckServiceTest {
|
||||
|
||||
@Test
|
||||
public void testSaveAndPreExecuteShouldPassGuaranteedAllJoin() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
JSONObject joinData = data("汇聚");
|
||||
joinData.put("joinMode", "all");
|
||||
String content = workflowJson(
|
||||
array(
|
||||
node("start", "startNode", null, data("开始")),
|
||||
node("a", "codeNode", null, data("分支 A")),
|
||||
node("b", "codeNode", null, data("分支 B")),
|
||||
node("join", "codeNode", null, joinData),
|
||||
node("end", "endNode", null, data("结束"))),
|
||||
array(
|
||||
edge("start-a", "start", "a"),
|
||||
edge("start-b", "start", "b"),
|
||||
edge("a-join", "a", "join"),
|
||||
edge("b-join", "b", "join"),
|
||||
edge("join-end", "join", "end")));
|
||||
|
||||
Assert.assertTrue(service.checkContent(
|
||||
content, WorkflowCheckStage.SAVE, null).isPassed());
|
||||
Assert.assertTrue(service.checkContent(
|
||||
content, WorkflowCheckStage.PRE_EXECUTE, null).isPassed());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSaveAndPreExecuteShouldBlockConditionalAllJoin() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
JSONObject joinData = data("汇聚");
|
||||
joinData.put("joinMode", "all");
|
||||
String content = workflowJson(
|
||||
array(
|
||||
node("start", "startNode", null, data("开始")),
|
||||
node("a", "codeNode", null, data("条件来源")),
|
||||
node("b", "codeNode", null, data("普通来源")),
|
||||
node("join", "codeNode", null, joinData),
|
||||
node("end", "endNode", null, data("结束"))),
|
||||
array(
|
||||
conditionalEdge("start-a", "start", "a", "enabled === true"),
|
||||
edge("start-b", "start", "b"),
|
||||
edge("a-join", "a", "join"),
|
||||
edge("b-join", "b", "join"),
|
||||
edge("join-end", "join", "end")));
|
||||
|
||||
WorkflowCheckResult save = service.checkContent(
|
||||
content, WorkflowCheckStage.SAVE, null);
|
||||
WorkflowCheckResult preExecute = service.checkContent(
|
||||
content, WorkflowCheckStage.PRE_EXECUTE, null);
|
||||
|
||||
Assert.assertFalse(save.isPassed());
|
||||
Assert.assertFalse(preExecute.isPassed());
|
||||
assertHasCode(save, "JOIN_MODE_CONDITIONAL_PATH_UNSUPPORTED");
|
||||
assertHasCode(preExecute, "JOIN_MODE_CONDITIONAL_PATH_UNSUPPORTED");
|
||||
Assert.assertTrue(save.getIssues().stream().anyMatch(issue ->
|
||||
"join".equals(issue.getNodeId())
|
||||
&& issue.getMessage().contains("永久等待")
|
||||
&& issue.getMessage().contains("任一上游完成")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSaveShouldBlockAllJoinWithDirectConditionalEdge() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
JSONObject joinData = data("汇聚");
|
||||
joinData.put("joinMode", "all");
|
||||
String content = workflowJson(
|
||||
array(
|
||||
node("start", "startNode", null, data("开始")),
|
||||
node("a", "codeNode", null, data("分支 A")),
|
||||
node("b", "codeNode", null, data("分支 B")),
|
||||
node("join", "codeNode", null, joinData)),
|
||||
array(
|
||||
edge("start-a", "start", "a"),
|
||||
edge("start-b", "start", "b"),
|
||||
conditionalEdge("a-join", "a", "join", "matched === true"),
|
||||
edge("b-join", "b", "join")));
|
||||
|
||||
WorkflowCheckResult result = service.checkContent(
|
||||
content, WorkflowCheckStage.SAVE, null);
|
||||
|
||||
Assert.assertFalse(result.isPassed());
|
||||
assertHasCode(result, "JOIN_MODE_CONDITIONAL_PATH_UNSUPPORTED");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSaveShouldBlockAllJoinFromCustomConditionSource() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
JSONObject conditionalSource = data("高级条件来源");
|
||||
conditionalSource.put("condition", "score > 0");
|
||||
JSONObject joinData = data("汇聚");
|
||||
joinData.put("joinMode", "all");
|
||||
String content = workflowJson(
|
||||
array(
|
||||
node("start", "startNode", null, data("开始")),
|
||||
node("a", "codeNode", null, conditionalSource),
|
||||
node("b", "codeNode", null, data("普通来源")),
|
||||
node("join", "codeNode", null, joinData)),
|
||||
array(
|
||||
edge("start-a", "start", "a"),
|
||||
edge("start-b", "start", "b"),
|
||||
edge("a-join", "a", "join"),
|
||||
edge("b-join", "b", "join")));
|
||||
|
||||
WorkflowCheckResult result = service.checkContent(
|
||||
content, WorkflowCheckStage.SAVE, null);
|
||||
|
||||
Assert.assertFalse(result.isPassed());
|
||||
assertHasCode(result, "JOIN_MODE_CONDITIONAL_PATH_UNSUPPORTED");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSaveShouldBlockInvalidAndLoopChildJoinModes() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
JSONObject invalidData = data("非法汇聚");
|
||||
invalidData.put("joinMode", "first");
|
||||
JSONObject loopData = loopData(
|
||||
fixedParameter("count", "2", "Number"), null);
|
||||
JSONObject childData = data("循环子节点");
|
||||
childData.put("joinMode", "all");
|
||||
String content = workflowJson(
|
||||
array(
|
||||
node("invalid", "codeNode", null, invalidData),
|
||||
node("loop", "loopNode", null, loopData),
|
||||
node("child", "codeNode", "loop", childData)),
|
||||
new JSONArray());
|
||||
|
||||
WorkflowCheckResult result = service.checkContent(
|
||||
content, WorkflowCheckStage.SAVE, null);
|
||||
|
||||
Assert.assertFalse(result.isPassed());
|
||||
assertHasCode(result, "JOIN_MODE_INVALID");
|
||||
assertHasCode(result, "JOIN_MODE_LOOP_CHILD_UNSUPPORTED");
|
||||
Assert.assertTrue(result.getIssues().stream().anyMatch(issue ->
|
||||
"invalid".equals(issue.getNodeId())
|
||||
&& "JOIN_MODE_INVALID".equals(issue.getCode())));
|
||||
Assert.assertTrue(result.getIssues().stream().anyMatch(issue ->
|
||||
"child".equals(issue.getNodeId())
|
||||
&& "JOIN_MODE_LOOP_CHILD_UNSUPPORTED".equals(issue.getCode())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSaveShouldAllowSingleConditionalInboundAllJoin() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
JSONObject joinData = data("单入边汇聚");
|
||||
joinData.put("joinMode", "all");
|
||||
String content = workflowJson(
|
||||
array(
|
||||
node("start", "startNode", null, data("开始")),
|
||||
node("join", "codeNode", null, joinData)),
|
||||
array(conditionalEdge(
|
||||
"start-join", "start", "join", "enabled === true")));
|
||||
|
||||
WorkflowCheckResult result = service.checkContent(
|
||||
content, WorkflowCheckStage.SAVE, null);
|
||||
|
||||
Assert.assertTrue(result.isPassed());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证保存阶段接受合法的正则条件规则。
|
||||
*/
|
||||
@@ -992,4 +1149,13 @@ public class WorkflowCheckServiceTest {
|
||||
edge.put("target", target);
|
||||
return edge;
|
||||
}
|
||||
|
||||
private static JSONObject conditionalEdge(
|
||||
String id, String source, String target, String condition) {
|
||||
JSONObject edge = edge(id, source, target);
|
||||
JSONObject data = new JSONObject();
|
||||
data.put("condition", condition);
|
||||
edge.put("data", data);
|
||||
return edge;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ import org.mockito.Mockito;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
@@ -41,13 +43,22 @@ public class WorkflowDatacenterContentServiceTest {
|
||||
DatacenterSource source = Mockito.mock(DatacenterSource.class);
|
||||
Mockito.when(source.getSourceName()).thenReturn("ama 实验基线模型预算");
|
||||
Mockito.when(source.getSourceType()).thenReturn("EXCEL");
|
||||
Mockito.when(source.getTenantId()).thenReturn(BigInteger.ONE);
|
||||
|
||||
DatacenterTableField modelId = mockField("col_id", "模型ID", "VARCHAR");
|
||||
DatacenterTableField inputPrice = mockField("token", "输入价格", "DECIMAL");
|
||||
DatacenterTableField hidden = mockField("hidden_col", "受限字段", "VARCHAR");
|
||||
Mockito.when(hidden.getQueryable()).thenReturn(0);
|
||||
DatacenterTable table = Mockito.mock(DatacenterTable.class);
|
||||
Mockito.when(table.getId()).thenReturn(TABLE_ID);
|
||||
Mockito.when(table.getSourceId()).thenReturn(SOURCE_ID);
|
||||
Mockito.when(table.getTenantId()).thenReturn(BigInteger.ONE);
|
||||
Mockito.when(table.getTableName()).thenReturn("Sheet1");
|
||||
Mockito.when(table.getFields()).thenReturn(List.of(modelId, inputPrice));
|
||||
Mockito.when(table.getQueryable()).thenReturn(1);
|
||||
Mockito.when(table.getMetadataStatus()).thenReturn(
|
||||
DatacenterMetadataStatus.ACTIVE.name());
|
||||
Mockito.when(table.getFields()).thenReturn(List.of(
|
||||
modelId, inputPrice, hidden));
|
||||
|
||||
Mockito.when(registryService.getSourceRequired(SOURCE_ID)).thenReturn(source);
|
||||
Mockito.when(registryService.listManagedTables(SOURCE_ID, null))
|
||||
@@ -77,6 +88,7 @@ public class WorkflowDatacenterContentServiceTest {
|
||||
Assert.assertTrue(contextValue.contains("col_id"));
|
||||
Assert.assertTrue(contextValue.contains("模型ID"));
|
||||
Assert.assertTrue(contextValue.contains("token AS input_price"));
|
||||
Assert.assertFalse(contextValue.contains("hidden_col"));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -103,6 +115,7 @@ public class WorkflowDatacenterContentServiceTest {
|
||||
private JSONObject buildWorkflowRoot() {
|
||||
JSONObject datasetRef = new JSONObject();
|
||||
datasetRef.put("sourceId", SOURCE_ID);
|
||||
datasetRef.put("tableId", TABLE_ID);
|
||||
|
||||
JSONObject queryData = new JSONObject();
|
||||
queryData.put("datasetRef", datasetRef);
|
||||
@@ -160,6 +173,11 @@ public class WorkflowDatacenterContentServiceTest {
|
||||
Mockito.when(field.getFieldName()).thenReturn(fieldName);
|
||||
Mockito.when(field.getFieldDesc()).thenReturn(fieldDesc);
|
||||
Mockito.when(field.getJdbcType()).thenReturn(jdbcType);
|
||||
Mockito.when(field.getQueryable()).thenReturn(1);
|
||||
Mockito.when(field.getMetadataStatus()).thenReturn(
|
||||
DatacenterMetadataStatus.ACTIVE.name());
|
||||
Mockito.when(field.getSensitivityLevel()).thenReturn(
|
||||
DatacenterSensitivityLevel.PUBLIC.name());
|
||||
return field;
|
||||
}
|
||||
|
||||
|
||||
@@ -219,6 +219,38 @@ public class WorkflowRunningParameterResolverTest {
|
||||
Assert.assertTrue(((List<?>) attachments).get(0) instanceof Map<?, ?>);
|
||||
}
|
||||
|
||||
/**
|
||||
* 空文件参数应统一归一化为空数组,避免旧客户端空字符串触发格式错误。
|
||||
*
|
||||
* @throws Exception 反射注入失败
|
||||
*/
|
||||
@Test
|
||||
public void testNormalizeRuntimeVariablesShouldTreatBlankFileValuesAsEmptyList()
|
||||
throws Exception {
|
||||
WorkflowRunningParameterResolver resolver = newResolver();
|
||||
Object[] emptyValues = {null, "", " ", List.of()};
|
||||
|
||||
for (Object emptyValue : emptyValues) {
|
||||
Map<String, Object> variables = new LinkedHashMap<>();
|
||||
variables.put("attachments", emptyValue);
|
||||
|
||||
Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
|
||||
workflowContentWithStartParameters(),
|
||||
variables);
|
||||
|
||||
Assert.assertEquals(List.of(), normalized.get("attachments"));
|
||||
}
|
||||
|
||||
Map<String, Object> variables = new LinkedHashMap<>();
|
||||
variables.put("attachments", List.of(
|
||||
" ",
|
||||
"https://files.example.com/contracts/contract.docx"));
|
||||
List<?> normalizedFiles = (List<?>) resolver.normalizeRuntimeVariables(
|
||||
workflowContentWithStartParameters(),
|
||||
variables).get("attachments");
|
||||
Assert.assertEquals(1, normalizedFiles.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件参数应接受远程 URL 字符串数组并自动提取文件名。
|
||||
*
|
||||
@@ -446,6 +478,30 @@ public class WorkflowRunningParameterResolverTest {
|
||||
Assert.assertEquals("https://example.com/image.png", image.get("url"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 空图片参数应从运行变量中移除,避免向执行引擎的并发 Map 写入 null。
|
||||
*
|
||||
* @throws Exception 反射注入失败
|
||||
*/
|
||||
@Test
|
||||
public void testNormalizeRuntimeVariablesShouldRemoveBlankImageValues()
|
||||
throws Exception {
|
||||
WorkflowRunningParameterResolver resolver = newResolver();
|
||||
Object[] emptyValues = {null, "", " "};
|
||||
|
||||
for (Object emptyValue : emptyValues) {
|
||||
Map<String, Object> variables = new LinkedHashMap<>();
|
||||
variables.put("image_input", emptyValue);
|
||||
|
||||
Map<String, Object> normalized = resolver.normalizeRuntimeVariables(
|
||||
workflowContentWithImageStartParameter(),
|
||||
variables);
|
||||
|
||||
Assert.assertFalse(normalized.containsKey("image_input"));
|
||||
Assert.assertFalse(normalized.containsValue(null));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 运行入口不应接收 Data URI,避免 Base64 写入工作流状态和审计参数。
|
||||
*
|
||||
|
||||
@@ -10,6 +10,7 @@ import tech.easyflow.ai.document.model.DocumentParsedResult;
|
||||
import tech.easyflow.ai.document.model.DocumentSourceRef;
|
||||
import tech.easyflow.ai.document.service.DocumentParseBridgeService;
|
||||
import tech.easyflow.common.filestorage.FileStorageService;
|
||||
import tech.easyflow.common.filestorage.FileStorageWriteHandle;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
@@ -203,6 +204,27 @@ public class DocNodeFileContentExtractorTest {
|
||||
Assert.assertNull(bridgeService.lastSource);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证默认读取器也能通过服务端文件记录读取内部附件 URL。
|
||||
*/
|
||||
@Test
|
||||
public void shouldReadRecordedInternalUrlForUnsupportedType() {
|
||||
RecordingDocumentParseBridgeService bridgeService = new RecordingDocumentParseBridgeService();
|
||||
String fileUrl = "http://127.0.0.1:39000/easyflow/attachment/1/2026/8/26/note.txt";
|
||||
DocNodeFileContentExtractor extractor = new DocNodeFileContentExtractor(
|
||||
bridgeService,
|
||||
new RecordedFileStorageService(fileUrl, "recorded text"),
|
||||
new ReadingReaderManager());
|
||||
|
||||
String content = extractor.extract(buildFileValue(
|
||||
"note.txt",
|
||||
fileUrl,
|
||||
"text/plain"));
|
||||
|
||||
Assert.assertEquals("recorded text", content);
|
||||
Assert.assertNull(bridgeService.lastSource);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证受管上传 URL 的非桥接文件通过记录校验后走内部存储读取。
|
||||
*
|
||||
@@ -574,4 +596,42 @@ public class DocNodeFileContentExtractorTest {
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 仅允许通过服务端记录句柄读取内容的存储测试替身。
|
||||
*/
|
||||
private static class RecordedFileStorageService
|
||||
extends FailingFileStorageService {
|
||||
/** 允许解析的精确 URL。 */
|
||||
private final String recordedUrl;
|
||||
/** 固定内容。 */
|
||||
private final byte[] content;
|
||||
/** 固定可信读取句柄。 */
|
||||
private final FileStorageWriteHandle handle = new FileStorageWriteHandle(
|
||||
"recorded", "", "/storage", "attachment", "note.txt");
|
||||
|
||||
/**
|
||||
* 创建服务端记录存储替身。
|
||||
*
|
||||
* @param recordedUrl 允许解析的精确 URL
|
||||
* @param content 固定文本内容
|
||||
*/
|
||||
private RecordedFileStorageService(String recordedUrl, String content) {
|
||||
this.recordedUrl = recordedUrl;
|
||||
this.content = content.getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public Optional<FileStorageWriteHandle> resolveTrustedFile(String reference) {
|
||||
return recordedUrl.equals(reference) ? Optional.of(handle) : Optional.empty();
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public InputStream readRecoverable(FileStorageWriteHandle requestedHandle) {
|
||||
Assert.assertSame(handle, requestedHandle);
|
||||
return new ByteArrayInputStream(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ package tech.easyflow.ai.service;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.ai.enums.PublishStatus;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.system.enums.CategoryResourceType;
|
||||
@@ -11,6 +11,7 @@ import tech.easyflow.system.enums.ResourceAction;
|
||||
import tech.easyflow.system.service.ResourceAccessService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -21,14 +22,14 @@ import static org.mockito.Mockito.when;
|
||||
public class WorkflowUsageAuthorizationServiceTest {
|
||||
|
||||
/**
|
||||
* 验证禁用工作流即使资源权限允许也不能被使用。
|
||||
* 验证未发布工作流即使资源权限允许也不能被使用。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRejectDisabledWorkflow() {
|
||||
public void shouldRejectUnpublishedWorkflow() {
|
||||
BigInteger workflowId = BigInteger.valueOf(101);
|
||||
WorkflowService workflowService = mock(WorkflowService.class);
|
||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||
Workflow workflow = workflow(workflowId, BigInteger.TEN, EnumDataStatus.UNAVAILABLE.getCode());
|
||||
Workflow workflow = workflow(workflowId, BigInteger.TEN, PublishStatus.DRAFT, Map.of());
|
||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||
when(resourceAccessService.canAccess(
|
||||
@@ -58,7 +59,8 @@ public class WorkflowUsageAuthorizationServiceTest {
|
||||
Workflow workflow = workflow(
|
||||
workflowId,
|
||||
BigInteger.valueOf(20),
|
||||
EnumDataStatus.AVAILABLE.getCode());
|
||||
PublishStatus.PUBLISHED,
|
||||
Map.of("title", "published"));
|
||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||
WorkflowUsageAuthorizationService service =
|
||||
@@ -71,17 +73,50 @@ public class WorkflowUsageAuthorizationServiceTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证启用、同租户且具有使用权限的工作流可以返回。
|
||||
* 验证已发布、同租户且具有使用权限的工作流返回发布视图。
|
||||
*/
|
||||
@Test
|
||||
public void shouldReturnUsableWorkflow() {
|
||||
public void shouldReturnPublishedWorkflowView() {
|
||||
BigInteger workflowId = BigInteger.valueOf(103);
|
||||
WorkflowService workflowService = mock(WorkflowService.class);
|
||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||
Workflow workflow = workflow(
|
||||
workflowId,
|
||||
BigInteger.TEN,
|
||||
EnumDataStatus.AVAILABLE.getCode());
|
||||
PublishStatus.PUBLISHED,
|
||||
Map.of("title", "published"));
|
||||
Workflow published = new Workflow();
|
||||
published.setId(workflowId);
|
||||
published.setTitle("发布版");
|
||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||
when(resourceAccessService.canAccess(
|
||||
account,
|
||||
CategoryResourceType.WORKFLOW,
|
||||
workflow,
|
||||
ResourceAction.USE)).thenReturn(true);
|
||||
when(workflowService.toPublishedView(workflow)).thenReturn(published);
|
||||
WorkflowUsageAuthorizationService service =
|
||||
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
|
||||
|
||||
Workflow result = service.requireUsableWorkflow(workflowId, account, "工作流不可用");
|
||||
|
||||
Assert.assertSame(result, published);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证发布状态异常但缺少快照的工作流不可被后台任务使用。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRejectPublishedWorkflowWithoutSnapshot() {
|
||||
BigInteger workflowId = BigInteger.valueOf(104);
|
||||
WorkflowService workflowService = mock(WorkflowService.class);
|
||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||
Workflow workflow = workflow(
|
||||
workflowId,
|
||||
BigInteger.TEN,
|
||||
PublishStatus.PUBLISHED,
|
||||
Map.of());
|
||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||
when(resourceAccessService.canAccess(
|
||||
@@ -92,9 +127,10 @@ public class WorkflowUsageAuthorizationServiceTest {
|
||||
WorkflowUsageAuthorizationService service =
|
||||
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
|
||||
|
||||
Workflow result = service.requireUsableWorkflow(workflowId, account, "工作流不可用");
|
||||
|
||||
Assert.assertSame(result, workflow);
|
||||
Assert.assertThrows(
|
||||
BusinessException.class,
|
||||
() -> service.requireUsableWorkflow(workflowId, account, "工作流不可用")
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -102,14 +138,18 @@ public class WorkflowUsageAuthorizationServiceTest {
|
||||
*
|
||||
* @param id 工作流 ID
|
||||
* @param tenantId 租户 ID
|
||||
* @param status 工作流状态
|
||||
* @param publishStatus 工作流发布状态
|
||||
* @param snapshot 工作流发布快照
|
||||
* @return 工作流
|
||||
*/
|
||||
private Workflow workflow(BigInteger id, BigInteger tenantId, Integer status) {
|
||||
private Workflow workflow(BigInteger id, BigInteger tenantId,
|
||||
PublishStatus publishStatus,
|
||||
Map<String, Object> snapshot) {
|
||||
Workflow workflow = new Workflow();
|
||||
workflow.setId(id);
|
||||
workflow.setTenantId(tenantId);
|
||||
workflow.setStatus(status);
|
||||
workflow.setPublishStatus(publishStatus.getCode());
|
||||
workflow.setPublishedSnapshotJson(snapshot);
|
||||
return workflow;
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,28 @@ import static tech.easyflow.ai.entity.DocumentCollection.KEY_SIMILARITY_THRESHOL
|
||||
*/
|
||||
public class DocumentCollectionServiceImplTest {
|
||||
|
||||
/**
|
||||
* 验证最终相关度阈值会过滤所有已统一到零到一范围的检索结果。
|
||||
*/
|
||||
@Test
|
||||
public void formatDocumentsShouldApplyFinalScoreThreshold() {
|
||||
Document lowScore = buildHit(BigInteger.ONE, 0.49D);
|
||||
Document thresholdScore = buildHit(BigInteger.TWO, 0.5D);
|
||||
Document highScore = buildHit(BigInteger.valueOf(3), 0.9D);
|
||||
DocumentCollectionServiceImpl service = new DocumentCollectionServiceImpl();
|
||||
|
||||
List<Document> result = service.formatDocuments(
|
||||
List.of(lowScore, thresholdScore, highScore),
|
||||
true,
|
||||
0.5F,
|
||||
5
|
||||
);
|
||||
|
||||
Assert.assertEquals(2, result.size());
|
||||
Assert.assertEquals(highScore.getId(), result.get(0).getId());
|
||||
Assert.assertEquals(thresholdScore.getId(), result.get(1).getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证检索结果会在重排前过滤掉未完成文档,避免高分进行中文档挤占最终名额。
|
||||
*
|
||||
|
||||
@@ -24,6 +24,26 @@ public class WorkflowSharePolicyTest {
|
||||
Assert.assertNotEquals("share-key", first);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证匿名访客归属摘要稳定、定长且按分享密钥隔离。
|
||||
*/
|
||||
@Test
|
||||
public void shouldHashChatVisitorPerShareWithoutLeakingIdentity() {
|
||||
String visitorId = "00112233445566778899aabbccddeeff";
|
||||
|
||||
String first = WorkflowSharePolicy.hashChatVisitor(
|
||||
"share-key-a", visitorId);
|
||||
String second = WorkflowSharePolicy.hashChatVisitor(
|
||||
"share-key-a", visitorId);
|
||||
String otherShare = WorkflowSharePolicy.hashChatVisitor(
|
||||
"share-key-b", visitorId);
|
||||
|
||||
Assert.assertEquals(first, second);
|
||||
Assert.assertEquals(32, first.length());
|
||||
Assert.assertNotEquals(first, otherShare);
|
||||
Assert.assertFalse(first.contains(visitorId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证默认过期时间为创建时间后 30 分钟。
|
||||
*/
|
||||
|
||||
@@ -5,6 +5,7 @@ tech.easyflow.approval.config.ApprovalModuleConfig
|
||||
tech.easyflow.auth.config.AuthModuleConfig
|
||||
tech.easyflow.chatlog.config.ChatlogModuleConfig
|
||||
tech.easyflow.datacenter.config.DatacenterModuleConfig
|
||||
tech.easyflow.dataspace.config.DataspaceModuleConfig
|
||||
tech.easyflow.job.config.JobModuleConfig
|
||||
tech.easyflow.log.config.LogModuleConfig
|
||||
tech.easyflow.skill.config.SkillModuleConfig
|
||||
|
||||
@@ -20,6 +20,10 @@
|
||||
<groupId>com.zaxxer</groupId>
|
||||
<artifactId>HikariCP</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-federation-sql-adapter-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
package tech.easyflow.datacenter.audit;
|
||||
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据中枢统一只读查询审计记录。
|
||||
*/
|
||||
@Table(value = "tb_datacenter_query_audit", comment = "数据中枢统一查询审计")
|
||||
public class DatacenterQueryAudit implements Serializable {
|
||||
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||
private BigInteger id;
|
||||
@Column(comment = "查询标识")
|
||||
private String queryId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(comment = "执行账号ID")
|
||||
private BigInteger executorAccountId;
|
||||
@Column(comment = "调用方类型")
|
||||
private String callerType;
|
||||
@Column(comment = "调用方标识")
|
||||
private String callerId;
|
||||
@Column(comment = "数据源ID")
|
||||
private BigInteger sourceId;
|
||||
@Column(comment = "数据源版本")
|
||||
private Long sourceRevision;
|
||||
@Column(comment = "纳管范围版本")
|
||||
private Long scopeRevision;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "引用表摘要")
|
||||
private List<String> referencedTablesJson;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "引用字段摘要")
|
||||
private List<String> referencedFieldsJson;
|
||||
@Column(comment = "参数化或脱敏SQL")
|
||||
private String parameterizedSql;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "脱敏参数摘要")
|
||||
private Map<String, Object> maskedParametersJson;
|
||||
@Column(comment = "执行状态")
|
||||
private String status;
|
||||
@Column(comment = "稳定错误码")
|
||||
private String errorCode;
|
||||
@Column(comment = "有界错误说明")
|
||||
private String errorMessage;
|
||||
@Column(comment = "开始时间")
|
||||
private Date startedAt;
|
||||
@Column(comment = "结束时间")
|
||||
private Date finishedAt;
|
||||
@Column(comment = "总耗时毫秒")
|
||||
private Long durationMs;
|
||||
@Column(comment = "数据库耗时毫秒")
|
||||
private Long databaseDurationMs;
|
||||
@Column(comment = "返回行数")
|
||||
private Long returnedRows;
|
||||
@Column(comment = "是否截断")
|
||||
private Integer truncated;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
|
||||
/** @return 主键 */
|
||||
@JsonSerialize(using = ToStringSerializer.class)
|
||||
public BigInteger getId() { return id; }
|
||||
/** @param id 主键 */
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
/** @return 查询标识 */
|
||||
public String getQueryId() { return queryId; }
|
||||
/** @param queryId 查询标识 */
|
||||
public void setQueryId(String queryId) { this.queryId = queryId; }
|
||||
/** @return 租户 ID */
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
/** @param tenantId 租户 ID */
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
/** @return 部门 ID */
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
/** @param deptId 部门 ID */
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
/** @return 执行账号 ID;系统任务为空 */
|
||||
public BigInteger getExecutorAccountId() { return executorAccountId; }
|
||||
/** @param executorAccountId 执行账号 ID */
|
||||
public void setExecutorAccountId(BigInteger executorAccountId) { this.executorAccountId = executorAccountId; }
|
||||
/** @return 调用方类型 */
|
||||
public String getCallerType() { return callerType; }
|
||||
/** @param callerType 调用方类型 */
|
||||
public void setCallerType(String callerType) { this.callerType = callerType; }
|
||||
/** @return 调用方标识 */
|
||||
public String getCallerId() { return callerId; }
|
||||
/** @param callerId 调用方标识 */
|
||||
public void setCallerId(String callerId) { this.callerId = callerId; }
|
||||
/** @return 数据源 ID */
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
/** @param sourceId 数据源 ID */
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
/** @return 数据源版本 */
|
||||
public Long getSourceRevision() { return sourceRevision; }
|
||||
/** @param sourceRevision 数据源版本 */
|
||||
public void setSourceRevision(Long sourceRevision) { this.sourceRevision = sourceRevision; }
|
||||
/** @return 纳管范围版本 */
|
||||
public Long getScopeRevision() { return scopeRevision; }
|
||||
/** @param scopeRevision 纳管范围版本 */
|
||||
public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; }
|
||||
/** @return 引用表 */
|
||||
public List<String> getReferencedTablesJson() { return referencedTablesJson; }
|
||||
/** @param referencedTablesJson 引用表 */
|
||||
public void setReferencedTablesJson(List<String> referencedTablesJson) { this.referencedTablesJson = referencedTablesJson; }
|
||||
/** @return 引用字段 */
|
||||
public List<String> getReferencedFieldsJson() { return referencedFieldsJson; }
|
||||
/** @param referencedFieldsJson 引用字段 */
|
||||
public void setReferencedFieldsJson(List<String> referencedFieldsJson) { this.referencedFieldsJson = referencedFieldsJson; }
|
||||
/** @return 参数化 SQL */
|
||||
public String getParameterizedSql() { return parameterizedSql; }
|
||||
/** @param parameterizedSql 参数化 SQL */
|
||||
public void setParameterizedSql(String parameterizedSql) { this.parameterizedSql = parameterizedSql; }
|
||||
/** @return 脱敏参数 */
|
||||
public Map<String, Object> getMaskedParametersJson() { return maskedParametersJson; }
|
||||
/** @param maskedParametersJson 脱敏参数 */
|
||||
public void setMaskedParametersJson(Map<String, Object> maskedParametersJson) { this.maskedParametersJson = maskedParametersJson; }
|
||||
/** @return 状态 */
|
||||
public String getStatus() { return status; }
|
||||
/** @param status 状态 */
|
||||
public void setStatus(String status) { this.status = status; }
|
||||
/** @return 错误码 */
|
||||
public String getErrorCode() { return errorCode; }
|
||||
/** @param errorCode 错误码 */
|
||||
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
|
||||
/** @return 错误说明 */
|
||||
public String getErrorMessage() { return errorMessage; }
|
||||
/** @param errorMessage 错误说明 */
|
||||
public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
|
||||
/** @return 开始时间 */
|
||||
public Date getStartedAt() { return startedAt; }
|
||||
/** @param startedAt 开始时间 */
|
||||
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
|
||||
/** @return 结束时间 */
|
||||
public Date getFinishedAt() { return finishedAt; }
|
||||
/** @param finishedAt 结束时间 */
|
||||
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
|
||||
/** @return 总耗时 */
|
||||
public Long getDurationMs() { return durationMs; }
|
||||
/** @param durationMs 总耗时 */
|
||||
public void setDurationMs(Long durationMs) { this.durationMs = durationMs; }
|
||||
/** @return 数据库耗时 */
|
||||
public Long getDatabaseDurationMs() { return databaseDurationMs; }
|
||||
/** @param databaseDurationMs 数据库耗时 */
|
||||
public void setDatabaseDurationMs(Long databaseDurationMs) { this.databaseDurationMs = databaseDurationMs; }
|
||||
/** @return 返回行数 */
|
||||
public Long getReturnedRows() { return returnedRows; }
|
||||
/** @param returnedRows 返回行数 */
|
||||
public void setReturnedRows(Long returnedRows) { this.returnedRows = returnedRows; }
|
||||
/** @return 是否截断 */
|
||||
public Integer getTruncated() { return truncated; }
|
||||
/** @param truncated 是否截断 */
|
||||
public void setTruncated(Integer truncated) { this.truncated = truncated; }
|
||||
/** @return 创建时间 */
|
||||
public Date getCreated() { return created; }
|
||||
/** @param created 创建时间 */
|
||||
public void setCreated(Date created) { this.created = created; }
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package tech.easyflow.datacenter.audit;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterQueryAuditMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
/**
|
||||
* 在查询前落账并在资源释放后收口状态的审计服务。
|
||||
*/
|
||||
@Service
|
||||
public class DatacenterQueryAuditService {
|
||||
|
||||
private final DatacenterQueryAuditMapper auditMapper;
|
||||
|
||||
/**
|
||||
* 创建审计服务。
|
||||
*
|
||||
* @param auditMapper 审计 Mapper
|
||||
*/
|
||||
public DatacenterQueryAuditService(DatacenterQueryAuditMapper auditMapper) {
|
||||
this.auditMapper = auditMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在数据库查询开始前写入 RUNNING 记录。
|
||||
*
|
||||
* @param queryId 查询标识
|
||||
* @param source 数据源快照
|
||||
* @param sql 参数化或无参数只读 SQL
|
||||
* @param parameters 脱敏参数摘要
|
||||
* @param account 执行账号
|
||||
* @param callerType 调用方类型
|
||||
* @param callerId 调用方标识
|
||||
* @return 已持久化审计记录
|
||||
*/
|
||||
public DatacenterQueryAudit start(
|
||||
String queryId,
|
||||
DatacenterSource source,
|
||||
String sql,
|
||||
Map<String, Object> parameters,
|
||||
LoginAccount account,
|
||||
String callerType,
|
||||
String callerId) {
|
||||
Date now = new Date();
|
||||
DatacenterQueryAudit audit = new DatacenterQueryAudit();
|
||||
audit.setQueryId(queryId);
|
||||
audit.setTenantId(account == null || account.getTenantId() == null
|
||||
? value(source.getTenantId()) : account.getTenantId());
|
||||
audit.setDeptId(account == null || account.getDeptId() == null
|
||||
? value(source.getDeptId()) : account.getDeptId());
|
||||
audit.setExecutorAccountId(account == null ? null : account.getId());
|
||||
audit.setCallerType(callerType == null ? "SYSTEM" : callerType);
|
||||
audit.setCallerId(callerId);
|
||||
audit.setSourceId(source.getId());
|
||||
audit.setSourceRevision(value(source.getDefinitionRevision()));
|
||||
audit.setScopeRevision(value(source.getScopeRevision()));
|
||||
audit.setReferencedTablesJson(List.of());
|
||||
audit.setReferencedFieldsJson(List.of());
|
||||
audit.setParameterizedSql(redactSql(sql));
|
||||
audit.setMaskedParametersJson(parameters == null ? Map.of() : Map.copyOf(parameters));
|
||||
audit.setStatus("RUNNING");
|
||||
audit.setStartedAt(now);
|
||||
audit.setReturnedRows(0L);
|
||||
audit.setTruncated(0);
|
||||
audit.setCreated(now);
|
||||
auditMapper.insert(audit);
|
||||
return audit;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记查询成功。
|
||||
*
|
||||
* @param audit 审计记录
|
||||
* @param referencedTables Calcite 解析出的引用表
|
||||
* @param referencedFields Calcite 解析出的引用字段
|
||||
* @param returnedRows 返回行数
|
||||
* @param truncated 是否截断
|
||||
* @param durationMs 总耗时
|
||||
* @param databaseDurationMs 数据库耗时
|
||||
*/
|
||||
public void succeed(
|
||||
DatacenterQueryAudit audit,
|
||||
List<String> referencedTables,
|
||||
List<String> referencedFields,
|
||||
long returnedRows,
|
||||
boolean truncated,
|
||||
long durationMs,
|
||||
long databaseDurationMs) {
|
||||
finish(audit, "SUCCESS", null, null, referencedTables, referencedFields,
|
||||
returnedRows, truncated, durationMs, databaseDurationMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记查询失败。
|
||||
*
|
||||
* @param audit 审计记录
|
||||
* @param errorCode 稳定错误码
|
||||
* @param message 有界错误说明
|
||||
* @param referencedTables Calcite 已解析出的引用表
|
||||
* @param referencedFields Calcite 已解析出的引用字段
|
||||
* @param durationMs 总耗时
|
||||
*/
|
||||
public void fail(
|
||||
DatacenterQueryAudit audit,
|
||||
String errorCode,
|
||||
String message,
|
||||
List<String> referencedTables,
|
||||
List<String> referencedFields,
|
||||
long durationMs) {
|
||||
finish(audit, "FAILED", errorCode, bounded(message), referencedTables, referencedFields,
|
||||
0L, false, durationMs, null);
|
||||
}
|
||||
|
||||
private void finish(
|
||||
DatacenterQueryAudit audit,
|
||||
String status,
|
||||
String errorCode,
|
||||
String errorMessage,
|
||||
List<String> referencedTables,
|
||||
List<String> referencedFields,
|
||||
long returnedRows,
|
||||
boolean truncated,
|
||||
long durationMs,
|
||||
Long databaseDurationMs) {
|
||||
audit.setStatus(status);
|
||||
audit.setErrorCode(errorCode);
|
||||
audit.setErrorMessage(errorMessage);
|
||||
audit.setReferencedTablesJson(List.copyOf(
|
||||
referencedTables == null ? List.of() : referencedTables));
|
||||
audit.setReferencedFieldsJson(List.copyOf(
|
||||
referencedFields == null ? List.of() : referencedFields));
|
||||
audit.setReturnedRows(returnedRows);
|
||||
audit.setTruncated(truncated ? 1 : 0);
|
||||
audit.setDurationMs(durationMs);
|
||||
audit.setDatabaseDurationMs(databaseDurationMs);
|
||||
audit.setFinishedAt(new Date());
|
||||
auditMapper.update(audit);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将超过十分钟仍为 RUNNING 的遗留审计收口为失败,覆盖进程异常退出场景。
|
||||
*/
|
||||
@Scheduled(fixedDelayString = "${easyflow.datacenter.audit-reconcile-ms:60000}")
|
||||
public void reconcileAbandonedQueries() {
|
||||
Date now = new Date();
|
||||
DatacenterQueryAudit patch = new DatacenterQueryAudit();
|
||||
patch.setStatus("FAILED");
|
||||
patch.setErrorCode("QUERY_ABANDONED");
|
||||
patch.setErrorMessage("查询进程异常结束或超出审计存活时间");
|
||||
patch.setFinishedAt(now);
|
||||
QueryWrapper guard = QueryWrapper.create()
|
||||
.eq(DatacenterQueryAudit::getStatus, "RUNNING")
|
||||
.lt(DatacenterQueryAudit::getStartedAt, new Date(now.getTime() - 600_000L));
|
||||
auditMapper.updateByQuery(patch, guard);
|
||||
}
|
||||
|
||||
private long value(Long value) {
|
||||
return value == null ? 0L : value;
|
||||
}
|
||||
|
||||
private BigInteger value(BigInteger value) {
|
||||
return value == null ? BigInteger.ZERO : value;
|
||||
}
|
||||
|
||||
private String redactSql(String sql) {
|
||||
if (sql == null || sql.isBlank()) {
|
||||
return sql;
|
||||
}
|
||||
StringBuilder result = new StringBuilder(Math.min(sql.length(), 100_000));
|
||||
boolean quoted = false;
|
||||
for (int index = 0; index < sql.length(); index++) {
|
||||
char current = sql.charAt(index);
|
||||
if (quoted) {
|
||||
if (current == '\'' && index + 1 < sql.length() && sql.charAt(index + 1) == '\'') {
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
if (current == '\'') {
|
||||
quoted = false;
|
||||
result.append('?');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (current == '\'') {
|
||||
quoted = true;
|
||||
continue;
|
||||
}
|
||||
if (current == '-' && index + 1 < sql.length() && sql.charAt(index + 1) == '-') {
|
||||
result.append("--?");
|
||||
index += 2;
|
||||
while (index < sql.length() && sql.charAt(index) != '\n'
|
||||
&& sql.charAt(index) != '\r') {
|
||||
index++;
|
||||
}
|
||||
if (index < sql.length()) {
|
||||
result.append(sql.charAt(index));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (current == '/' && index + 1 < sql.length() && sql.charAt(index + 1) == '*') {
|
||||
result.append("/*?*/");
|
||||
index = skipBlockComment(sql, index + 2);
|
||||
continue;
|
||||
}
|
||||
String dollarDelimiter = current == '$' ? dollarDelimiter(sql, index) : null;
|
||||
if (dollarDelimiter != null) {
|
||||
int end = sql.indexOf(dollarDelimiter, index + dollarDelimiter.length());
|
||||
result.append('?');
|
||||
index = end < 0 ? sql.length() : end + dollarDelimiter.length() - 1;
|
||||
continue;
|
||||
}
|
||||
if (Character.isDigit(current)
|
||||
&& (index == 0 || !Character.isJavaIdentifierPart(sql.charAt(index - 1)))) {
|
||||
result.append('?');
|
||||
while (index + 1 < sql.length()) {
|
||||
char next = sql.charAt(index + 1);
|
||||
if (!Character.isDigit(next) && next != '.' && next != 'e' && next != 'E'
|
||||
&& next != '+' && next != '-') {
|
||||
break;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
result.append(current);
|
||||
if (result.length() >= 100_000) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (quoted) {
|
||||
result.append('?');
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 跳过支持嵌套的块注释内容。
|
||||
*
|
||||
* @param sql 原始 SQL
|
||||
* @param contentStart 首个注释内容字符位置
|
||||
* @return 外层注释结束标记的末尾位置;未闭合时返回末尾
|
||||
*/
|
||||
private int skipBlockComment(String sql, int contentStart) {
|
||||
int depth = 1;
|
||||
for (int index = contentStart; index < sql.length() - 1; index++) {
|
||||
if (sql.charAt(index) == '/' && sql.charAt(index + 1) == '*') {
|
||||
depth++;
|
||||
index++;
|
||||
} else if (sql.charAt(index) == '*' && sql.charAt(index + 1) == '/') {
|
||||
depth--;
|
||||
if (depth == 0) {
|
||||
return index + 1;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
}
|
||||
return sql.length();
|
||||
}
|
||||
|
||||
/**
|
||||
* 识别 PostgreSQL dollar-quoted 字面量起始分隔符。
|
||||
*
|
||||
* @param sql 原始 SQL
|
||||
* @param start 美元符号位置
|
||||
* @return 完整分隔符;当前位置不是合法起始符时为空
|
||||
*/
|
||||
private String dollarDelimiter(String sql, int start) {
|
||||
int end = sql.indexOf('$', start + 1);
|
||||
if (end < 0) {
|
||||
return null;
|
||||
}
|
||||
String tag = sql.substring(start + 1, end);
|
||||
if (tag.isEmpty()) {
|
||||
return "$$";
|
||||
}
|
||||
if (!Character.isLetter(tag.charAt(0)) && tag.charAt(0) != '_') {
|
||||
return null;
|
||||
}
|
||||
for (int index = 1; index < tag.length(); index++) {
|
||||
char character = tag.charAt(index);
|
||||
if (!Character.isLetterOrDigit(character) && character != '_') {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return '$' + tag + '$';
|
||||
}
|
||||
|
||||
private String bounded(String message) {
|
||||
if (message == null) {
|
||||
return null;
|
||||
}
|
||||
return message.length() <= 500 ? message : message.substring(0, 500);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package tech.easyflow.datacenter.config;
|
||||
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
||||
import org.springframework.data.redis.listener.ChannelTopic;
|
||||
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationSourceStateProvider;
|
||||
|
||||
/**
|
||||
* 数据中枢无凭据 Definition 变更提示订阅配置。
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
public class DatacenterFederationRedisConfig {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationRedisConfig.class);
|
||||
|
||||
/**
|
||||
* 创建独立 Redis 监听容器,收到提示后始终回源数据库。
|
||||
*
|
||||
* @param connectionFactory Redis 连接工厂
|
||||
* @param stateProvider 数据库权威状态 Provider
|
||||
* @param queryCancellationService 查询取消服务
|
||||
* @return 监听容器
|
||||
*/
|
||||
@Bean(name = "datacenterFederationRedisListenerContainer")
|
||||
@ConditionalOnBean(RedisConnectionFactory.class)
|
||||
public RedisMessageListenerContainer datacenterFederationRedisListenerContainer(
|
||||
RedisConnectionFactory connectionFactory,
|
||||
DatacenterFederationSourceStateProvider stateProvider,
|
||||
DatacenterFederationQueryCancellationService queryCancellationService) {
|
||||
RedisMessageListenerContainer container = new RedisMessageListenerContainer();
|
||||
container.setConnectionFactory(connectionFactory);
|
||||
container.addMessageListener((message, pattern) -> {
|
||||
String value = new String(message.getBody(), StandardCharsets.UTF_8);
|
||||
try {
|
||||
stateProvider.refresh(new SourceId(value));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to refresh datacenter Federation source {}", value, exception);
|
||||
}
|
||||
}, new ChannelTopic(DatacenterFederationChangeNotifier.CHANNEL));
|
||||
container.addMessageListener((message, pattern) ->
|
||||
queryCancellationService.acceptCancellationHint(
|
||||
new String(message.getBody(), StandardCharsets.UTF_8)),
|
||||
new ChannelTopic(DatacenterFederationQueryCancellationService.CHANNEL));
|
||||
return container;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package tech.easyflow.datacenter.connector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.util.List;
|
||||
@@ -10,7 +12,97 @@ import java.util.List;
|
||||
public interface MetadataExplorer {
|
||||
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
|
||||
|
||||
/**
|
||||
* 分页浏览 Catalog 或 Schema。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param keyword 名称搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
default DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
|
||||
DatacenterSource source,
|
||||
String keyword,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
String normalizedKeyword = keyword == null
|
||||
? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
List<DatacenterCatalogMeta> filtered = listCatalogs(source).stream()
|
||||
.filter(catalog -> normalizedKeyword.isEmpty()
|
||||
|| String.valueOf(catalog.getCatalogName())
|
||||
.toLowerCase(java.util.Locale.ROOT)
|
||||
.contains(normalizedKeyword))
|
||||
.toList();
|
||||
return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize);
|
||||
}
|
||||
|
||||
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
|
||||
|
||||
/**
|
||||
* 分页浏览表或视图。
|
||||
*
|
||||
* <p>非 JDBC Connector 默认复用已有列表能力;外部 JDBC Connector 应覆盖此方法,
|
||||
* 直接在元数据 ResultSet 上做有界读取。</p>
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param keyword 表名搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界表列表
|
||||
*/
|
||||
default DatacenterMetadataPage<DatacenterTable> listTablesPage(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
String normalizedKeyword = keyword == null ? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
List<DatacenterTable> filtered = listTables(source, catalogName).stream()
|
||||
.filter(table -> normalizedKeyword.isEmpty()
|
||||
|| String.valueOf(table.getTableName())
|
||||
.toLowerCase(java.util.Locale.ROOT)
|
||||
.contains(normalizedKeyword))
|
||||
.toList();
|
||||
return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize);
|
||||
}
|
||||
|
||||
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
|
||||
|
||||
/**
|
||||
* 批量读取表详情。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param tableNames 表名集合
|
||||
* @return 表详情集合
|
||||
*/
|
||||
default List<DatacenterTableDetailMeta> getTableDetails(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
List<String> tableNames) {
|
||||
return tableNames.stream()
|
||||
.map(tableName -> getTableDetail(source, catalogName, tableName))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取已纳管表的刷新快照。
|
||||
*
|
||||
* <p>默认实现保留 Connector 的既有逐表语义;JDBC Connector 应覆盖此方法以复用
|
||||
* 单个连接,并明确区分物理表缺失与连接故障。</p>
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param tableNames 已纳管表名集合
|
||||
* @return 存在与缺失对象的快照
|
||||
*/
|
||||
default DatacenterManagedMetadataSnapshot inspectManagedTables(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
List<String> tableNames) {
|
||||
return new DatacenterManagedMetadataSnapshot(
|
||||
getTableDetails(source, catalogName, tableNames), java.util.Set.of());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,13 +28,12 @@ public class MysqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
// MySQL 查询 Runtime 已由 Federation SQL 独占管理;此 Connector 只保留短生命周期元数据访问。
|
||||
HikariDataSource dataSource = datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,13 +28,12 @@ public class PostgresqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
// PostgreSQL 查询 Runtime 已由 Federation SQL 独占管理;此 Connector 只保留短生命周期元数据访问。
|
||||
HikariDataSource dataSource = datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package tech.easyflow.datacenter.connector.support;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryColumn;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Db;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
@@ -15,6 +16,7 @@ import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
@@ -85,35 +87,180 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||
if (request == null || request.getPageSize() == null
|
||||
|| request.getPageSize() < 1L
|
||||
|| request.getPageSize() > 500L) {
|
||||
throw new BusinessException("pageSize 必须在 1 到 500 之间");
|
||||
}
|
||||
String actualTable = resolveTableName(table);
|
||||
long count = Db.selectCountByQuery(
|
||||
actualTable, createQueryWrapper(request.getWhere()));
|
||||
actualTable, createQueryWrapper(table, request, false));
|
||||
if (count == 0) {
|
||||
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
|
||||
}
|
||||
// selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*),分页查询必须使用独立实例。
|
||||
QueryWrapper pageQuery = createQueryWrapper(table, request, true);
|
||||
pageQuery.select(resolveSelectedColumns(table, request).stream()
|
||||
.map(this::quoteInternalIdentifier)
|
||||
.toArray(String[]::new));
|
||||
Page<Row> page = Db.paginate(
|
||||
actualTable,
|
||||
new Page<>(request.getPageNumber(), request.getPageSize(), count),
|
||||
createQueryWrapper(request.getWhere()));
|
||||
pageQuery);
|
||||
normalizeRows(page.getRecords());
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将已在服务层校验的逻辑字段映射为内部物理列。
|
||||
*
|
||||
* @param table 绑定表及字段
|
||||
* @param request 查询请求
|
||||
* @return 按请求顺序排列的物理列名
|
||||
* @throws BusinessException 字段元数据缺失时抛出
|
||||
*/
|
||||
private List<String> resolveSelectedColumns(
|
||||
DatacenterTable table,
|
||||
DatacenterQueryRequest request) {
|
||||
Map<String, DatacenterTableField> fields = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
fields.put(field.getFieldName(), field);
|
||||
}
|
||||
List<String> columns = new ArrayList<>();
|
||||
for (String selected : request.getSelectedColumns()) {
|
||||
DatacenterTableField field = fields.get(selected);
|
||||
if (field == null) {
|
||||
throw new BusinessException("字段不存在: " + selected);
|
||||
}
|
||||
String sourceColumn = StrUtil.blankToDefault(
|
||||
field.getSourceColumnName(), field.getFieldName());
|
||||
columns.add(sourceColumn);
|
||||
}
|
||||
if (columns.isEmpty()) {
|
||||
throw new BusinessException("当前数据集没有可查询字段");
|
||||
}
|
||||
return columns;
|
||||
}
|
||||
|
||||
/**
|
||||
* 引用可信元数据列名,避免保留字或特殊字符改变 SQL 结构。
|
||||
*
|
||||
* @param identifier 物理列名
|
||||
* @return MySQL 反引号标识符
|
||||
*/
|
||||
private String quoteInternalIdentifier(String identifier) {
|
||||
return "`" + identifier.replace("`", "``") + "`";
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建用于动态表查询的独立条件包装器。
|
||||
*
|
||||
* @param where 已校验的筛选表达式
|
||||
* @param where 调用方传入的旧原始筛选表达式;非空时拒绝
|
||||
* @return 新建的查询条件包装器
|
||||
*/
|
||||
static QueryWrapper createQueryWrapper(String where) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
if (StrUtil.isNotBlank(where)) {
|
||||
wrapper.where(where);
|
||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
||||
}
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
private QueryWrapper createQueryWrapper(
|
||||
DatacenterTable table,
|
||||
DatacenterQueryRequest request,
|
||||
boolean includeSorts) {
|
||||
QueryWrapper wrapper = createQueryWrapper(request.getWhere());
|
||||
Map<String, DatacenterTableField> fields = table.getFields().stream()
|
||||
.collect(java.util.stream.Collectors.toMap(
|
||||
DatacenterTableField::getFieldName,
|
||||
field -> field,
|
||||
(first, ignored) -> first,
|
||||
LinkedHashMap::new));
|
||||
List<Object> parameters = new ArrayList<>();
|
||||
StringBuilder condition = new StringBuilder();
|
||||
if (request.getFilters() != null) {
|
||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||
DatacenterTableField field = fields.get(filter.getColumn());
|
||||
if (field == null) {
|
||||
throw new BusinessException("字段不存在: " + filter.getColumn());
|
||||
}
|
||||
appendFilter(condition, parameters, field, filter);
|
||||
}
|
||||
}
|
||||
if (!condition.isEmpty()) {
|
||||
wrapper.where(condition.toString(), parameters.toArray());
|
||||
}
|
||||
if (includeSorts && request.getSorts() != null && !request.getSorts().isEmpty()) {
|
||||
request.getSorts().forEach(sort -> {
|
||||
DatacenterTableField field = fields.get(sort.getColumn());
|
||||
if (field == null) {
|
||||
throw new BusinessException("字段不存在: " + sort.getColumn());
|
||||
}
|
||||
wrapper.orderBy(
|
||||
new QueryColumn(sourceColumnName(field)),
|
||||
!"DESC".equalsIgnoreCase(sort.getDirection()));
|
||||
});
|
||||
}
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
private void appendFilter(
|
||||
StringBuilder condition,
|
||||
List<Object> parameters,
|
||||
DatacenterTableField field,
|
||||
DatacenterQueryFilter filter) {
|
||||
if (!condition.isEmpty()) {
|
||||
condition.append(" AND ");
|
||||
}
|
||||
String column = quoteInternalIdentifier(sourceColumnName(field));
|
||||
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ")
|
||||
.toUpperCase(Locale.ROOT);
|
||||
switch (operator) {
|
||||
case "EQ" -> appendComparison(condition, parameters, column, "=", filter.getValue());
|
||||
case "LIKE" -> appendComparison(
|
||||
condition, parameters, column, "LIKE",
|
||||
"%" + String.valueOf(filter.getValue()) + "%");
|
||||
case "GT" -> appendComparison(condition, parameters, column, ">", filter.getValue());
|
||||
case "GTE" -> appendComparison(condition, parameters, column, ">=", filter.getValue());
|
||||
case "LT" -> appendComparison(condition, parameters, column, "<", filter.getValue());
|
||||
case "LTE" -> appendComparison(condition, parameters, column, "<=", filter.getValue());
|
||||
case "IS_NULL" -> condition.append(column).append(" IS NULL");
|
||||
case "IN" -> appendInFilter(condition, parameters, column, filter.getValues());
|
||||
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
||||
}
|
||||
}
|
||||
|
||||
private void appendComparison(
|
||||
StringBuilder condition,
|
||||
List<Object> parameters,
|
||||
String column,
|
||||
String operator,
|
||||
Object value) {
|
||||
condition.append(column).append(' ').append(operator).append(" ?");
|
||||
parameters.add(value);
|
||||
}
|
||||
|
||||
private void appendInFilter(
|
||||
StringBuilder condition,
|
||||
List<Object> parameters,
|
||||
String column,
|
||||
List<Object> values) {
|
||||
if (values == null || values.isEmpty()) {
|
||||
condition.append("1 = 0");
|
||||
return;
|
||||
}
|
||||
condition.append(column).append(" IN (")
|
||||
.append(Collections.nCopies(values.size(), "?").stream()
|
||||
.collect(java.util.stream.Collectors.joining(", ")))
|
||||
.append(')');
|
||||
parameters.addAll(values);
|
||||
}
|
||||
|
||||
private String sourceColumnName(DatacenterTableField field) {
|
||||
return StrUtil.blankToDefault(field.getSourceColumnName(), field.getFieldName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||
List<Row> rows = Db.selectListBySql(sql);
|
||||
|
||||
@@ -16,12 +16,17 @@ import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||
import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
@@ -105,30 +110,39 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
return withConnection(source, true, connection -> {
|
||||
List<DatacenterCatalogMeta> result = new ArrayList<>();
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
try (ResultSet catalogs = metaData.getCatalogs()) {
|
||||
while (catalogs.next()) {
|
||||
String name = catalogs.getString("TABLE_CAT");
|
||||
if (StrUtil.isBlank(name)) {
|
||||
continue;
|
||||
if (usesCatalogNamespace()) {
|
||||
try (ResultSet catalogs = metaData.getCatalogs()) {
|
||||
while (catalogs.next()) {
|
||||
String name = catalogs.getString("TABLE_CAT");
|
||||
if (StrUtil.isBlank(name)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("CATALOG");
|
||||
meta.setLogicalSchemaName(name);
|
||||
meta.setPhysicalCatalogName(name);
|
||||
result.add(meta);
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("CATALOG");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
try (ResultSet schemas = metaData.getSchemas()) {
|
||||
while (schemas.next()) {
|
||||
String name = schemas.getString("TABLE_SCHEM");
|
||||
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
|
||||
continue;
|
||||
if (!usesCatalogNamespace()) {
|
||||
try (ResultSet schemas = metaData.getSchemas()) {
|
||||
while (schemas.next()) {
|
||||
String name = schemas.getString("TABLE_SCHEM");
|
||||
if (StrUtil.isBlank(name)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("SCHEMA");
|
||||
meta.setLogicalSchemaName(name);
|
||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
||||
meta.setPhysicalSchemaName(name);
|
||||
result.add(meta);
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("SCHEMA");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
result = filterConfiguredCatalogs(source, result);
|
||||
@@ -139,6 +153,13 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(fallback);
|
||||
meta.setCatalogType("DEFAULT");
|
||||
meta.setLogicalSchemaName(fallback);
|
||||
if (usesCatalogNamespace()) {
|
||||
meta.setPhysicalCatalogName(fallback);
|
||||
} else {
|
||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
||||
meta.setPhysicalSchemaName(fallback);
|
||||
}
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
@@ -149,24 +170,98 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接在 JDBC Catalog/Schema 元数据游标上执行有界分页。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param keyword 名称搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
@Override
|
||||
public DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
|
||||
DatacenterSource source,
|
||||
String keyword,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
String normalizedKeyword = keyword == null
|
||||
? "" : keyword.trim().toLowerCase(Locale.ROOT);
|
||||
String configuredName = usesCatalogNamespace()
|
||||
? StrUtil.trimToNull(source.getDatabaseName())
|
||||
: StrUtil.trimToNull(source.getSchemaName());
|
||||
long offset = (pageNumber - 1L) * pageSize;
|
||||
long matched = 0L;
|
||||
boolean hasMore = false;
|
||||
List<DatacenterCatalogMeta> records = new ArrayList<>((int) pageSize);
|
||||
try (ResultSet names = usesCatalogNamespace()
|
||||
? metadata.getCatalogs() : metadata.getSchemas()) {
|
||||
String column = usesCatalogNamespace() ? "TABLE_CAT" : "TABLE_SCHEM";
|
||||
while (names.next()) {
|
||||
String name = names.getString(column);
|
||||
if (StrUtil.isBlank(name)
|
||||
|| (configuredName != null && !configuredName.equals(name))
|
||||
|| (!normalizedKeyword.isEmpty()
|
||||
&& !name.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) {
|
||||
continue;
|
||||
}
|
||||
if (matched++ < offset) {
|
||||
continue;
|
||||
}
|
||||
if (records.size() >= pageSize) {
|
||||
hasMore = true;
|
||||
break;
|
||||
}
|
||||
records.add(catalogMeta(source, name));
|
||||
}
|
||||
}
|
||||
if (records.isEmpty() && offset == 0L && normalizedKeyword.isEmpty()) {
|
||||
String fallback = resolveCatalogName(source, null);
|
||||
if (StrUtil.isNotBlank(fallback)) {
|
||||
records.add(catalogMeta(source, fallback));
|
||||
}
|
||||
}
|
||||
return new DatacenterMetadataPage<>(
|
||||
records, pageNumber, pageSize, hasMore);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterCatalogMeta catalogMeta(
|
||||
DatacenterSource source,
|
||||
String name) {
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType(usesCatalogNamespace() ? "CATALOG" : "SCHEMA");
|
||||
meta.setLogicalSchemaName(name);
|
||||
if (usesCatalogNamespace()) {
|
||||
meta.setPhysicalCatalogName(name);
|
||||
} else {
|
||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
||||
meta.setPhysicalSchemaName(name);
|
||||
}
|
||||
return meta;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
List<DatacenterTable> tables = new ArrayList<>();
|
||||
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
|
||||
try (ResultSet resultSet = metaData.getTables(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
||||
"%",
|
||||
new String[]{"TABLE", "VIEW"})) {
|
||||
while (resultSet.next()) {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setSourceId(source.getId());
|
||||
table.setTableName(resultSet.getString("TABLE_NAME"));
|
||||
table.setTableDesc(resultSet.getString("REMARKS"));
|
||||
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
tables.add(table);
|
||||
tables.add(tableFromMetadata(source, resultSet));
|
||||
}
|
||||
}
|
||||
return tables;
|
||||
@@ -176,10 +271,166 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接在 JDBC 元数据游标上完成表名筛选和有界分页。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param keyword 表名搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界表列表
|
||||
*/
|
||||
@Override
|
||||
public DatacenterMetadataPage<DatacenterTable> listTablesPage(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
String normalizedKeyword = keyword == null
|
||||
? "" : keyword.trim().toLowerCase(Locale.ROOT);
|
||||
long offset = (pageNumber - 1L) * pageSize;
|
||||
long matched = 0L;
|
||||
List<DatacenterTable> tables = new ArrayList<>((int) pageSize);
|
||||
boolean hasMore = false;
|
||||
try (ResultSet resultSet = metaData.getTables(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
||||
"%",
|
||||
new String[]{"TABLE", "VIEW"})) {
|
||||
while (resultSet.next()) {
|
||||
String tableName = resultSet.getString("TABLE_NAME");
|
||||
if (!normalizedKeyword.isEmpty()
|
||||
&& (tableName == null
|
||||
|| !tableName.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) {
|
||||
continue;
|
||||
}
|
||||
if (matched++ < offset) {
|
||||
continue;
|
||||
}
|
||||
if (tables.size() >= pageSize) {
|
||||
hasMore = true;
|
||||
break;
|
||||
}
|
||||
tables.add(tableFromMetadata(source, resultSet));
|
||||
}
|
||||
}
|
||||
return new DatacenterMetadataPage<>(
|
||||
tables, pageNumber, pageSize, hasMore);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterTable tableFromMetadata(
|
||||
DatacenterSource source,
|
||||
ResultSet resultSet) throws SQLException {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setSourceId(source.getId());
|
||||
table.setTableName(resultSet.getString("TABLE_NAME"));
|
||||
table.setTableDesc(resultSet.getString("REMARKS"));
|
||||
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION)
|
||||
? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
||||
table.setQueryable(1);
|
||||
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
table.setLastSeenAt(new java.util.Date());
|
||||
table.setTimeSemantics("NONE");
|
||||
table.setMetadataRevision(1L);
|
||||
table.setCapabilitiesJson(Map.of(
|
||||
"capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
return table;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||
try {
|
||||
return withConnection(source, true,
|
||||
connection -> readTableDetail(connection, source, catalogName, tableName));
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用单个元数据连接批量读取表详情,避免每张表重复创建连接池。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param tableNames 表名集合
|
||||
* @return 表详情集合
|
||||
*/
|
||||
@Override
|
||||
public List<DatacenterTableDetailMeta> getTableDetails(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
List<String> tableNames) {
|
||||
if (tableNames == null || tableNames.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
List<DatacenterTableDetailMeta> details = new ArrayList<>(tableNames.size());
|
||||
for (String tableName : tableNames) {
|
||||
details.add(readTableDetail(connection, source, catalogName, tableName));
|
||||
}
|
||||
return List.copyOf(details);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("批量读取表详情失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用单个连接刷新已纳管表,并把物理缺失作为快照状态返回。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param tableNames 已纳管表名集合
|
||||
* @return 已发现详情与缺失表名
|
||||
*/
|
||||
@Override
|
||||
public DatacenterManagedMetadataSnapshot inspectManagedTables(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
List<String> tableNames) {
|
||||
if (tableNames == null || tableNames.isEmpty()) {
|
||||
return new DatacenterManagedMetadataSnapshot(List.of(), Set.of());
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
List<DatacenterTableDetailMeta> details = new ArrayList<>(tableNames.size());
|
||||
Set<String> missing = new java.util.LinkedHashSet<>();
|
||||
for (String tableName : tableNames) {
|
||||
try {
|
||||
details.add(readTableDetail(connection, source, catalogName, tableName));
|
||||
} catch (BusinessException exception) {
|
||||
if (exception.getMessage() == null
|
||||
|| !exception.getMessage().startsWith("所选数据表已变化")) {
|
||||
throw exception;
|
||||
}
|
||||
missing.add(tableName);
|
||||
}
|
||||
}
|
||||
return new DatacenterManagedMetadataSnapshot(details, missing);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("刷新已纳管表失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterTableDetailMeta readTableDetail(
|
||||
Connection connection,
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String tableName) throws SQLException {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
@@ -189,24 +440,35 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
table.setMaterializedTable(tableName);
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
|
||||
table.setQueryable(1);
|
||||
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
table.setLastSeenAt(new java.util.Date());
|
||||
table.setTimeSemantics("NONE");
|
||||
table.setMetadataRevision(1L);
|
||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
detail.setTable(table);
|
||||
|
||||
boolean tableFound = false;
|
||||
try (ResultSet tableSet = metaData.getTables(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
resolveSchemaArgument(source, catalogName),
|
||||
tableName,
|
||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
||||
metadataPattern(metaData, tableName),
|
||||
new String[]{"TABLE", "VIEW"})) {
|
||||
while (tableSet.next()) {
|
||||
String currentTableName = tableSet.getString("TABLE_NAME");
|
||||
if (!matchesTableName(currentTableName, tableName)) {
|
||||
if (!matchesMetadataTable(
|
||||
tableSet, source, catalogName, tableName)) {
|
||||
continue;
|
||||
}
|
||||
table.setTableDesc(tableSet.getString("REMARKS"));
|
||||
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
|
||||
tableFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!tableFound) {
|
||||
throw new BusinessException(
|
||||
"所选数据表已变化,请刷新后重试: " + tableName);
|
||||
}
|
||||
|
||||
Set<String> primaryKeys = new HashSet<>();
|
||||
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
|
||||
@@ -216,13 +478,24 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
}
|
||||
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
|
||||
try (ResultSet columns = metaData.getColumns(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
||||
metadataPattern(metaData, tableName),
|
||||
"%")) {
|
||||
while (columns.next()) {
|
||||
if (!matchesMetadataTable(
|
||||
columns, source, catalogName, tableName)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(columns.getString("COLUMN_NAME"));
|
||||
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
|
||||
field.setFieldDesc(columns.getString("REMARKS"));
|
||||
field.setJdbcType(columns.getString("TYPE_NAME"));
|
||||
field.setJdbcTypeCode(columns.getInt("DATA_TYPE"));
|
||||
field.setNativeTypeName(columns.getString("TYPE_NAME"));
|
||||
field.setOrdinalPosition(columns.getInt("ORDINAL_POSITION"));
|
||||
field.setPrecision(columns.getInt("COLUMN_SIZE"));
|
||||
field.setScale(columns.getInt("DECIMAL_DIGITS"));
|
||||
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
|
||||
@@ -231,15 +504,21 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
|
||||
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
|
||||
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
|
||||
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
field.setLastSeenAt(new java.util.Date());
|
||||
field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name());
|
||||
field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field));
|
||||
fields.add(field);
|
||||
}
|
||||
}
|
||||
if (fields.isEmpty()) {
|
||||
throw new BusinessException(
|
||||
"无法读取数据表字段,请检查权限: " + tableName);
|
||||
}
|
||||
table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
|
||||
table.getTableName(), table.getTableKind(), fields));
|
||||
detail.setFields(fields);
|
||||
return detail;
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -247,6 +526,11 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||
throw new BusinessException("当前数据源暂不支持查询");
|
||||
}
|
||||
if (request == null || request.getPageSize() == null
|
||||
|| request.getPageSize() < 1L
|
||||
|| request.getPageSize() > 500L) {
|
||||
throw new BusinessException("pageSize 必须在 1 到 500 之间");
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
|
||||
} catch (Exception ex) {
|
||||
@@ -325,7 +609,7 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
|
||||
StringBuilder whereClause = new StringBuilder();
|
||||
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||
whereClause.append(" WHERE ").append(request.getWhere());
|
||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
||||
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
whereClause.append(" WHERE 1=1 ");
|
||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||
@@ -622,13 +906,9 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
return items;
|
||||
}
|
||||
List<DatacenterCatalogMeta> matched = items.stream()
|
||||
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
|
||||
.filter(item -> configuredName.equals(item.getCatalogName()))
|
||||
.collect(Collectors.toList());
|
||||
return matched.isEmpty() ? items : matched;
|
||||
}
|
||||
|
||||
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
|
||||
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
|
||||
return matched;
|
||||
}
|
||||
|
||||
private DatacenterTableKind resolveTableKind(String tableType) {
|
||||
@@ -639,8 +919,58 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
if (currentTableName == null || targetTableName == null) {
|
||||
return false;
|
||||
}
|
||||
return currentTableName.equals(targetTableName)
|
||||
|| currentTableName.equalsIgnoreCase(targetTableName);
|
||||
return currentTableName.equals(targetTableName);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将精确标识符转换为 JDBC 元数据 Pattern,转义驱动声明的通配符字符。
|
||||
*
|
||||
* @param metaData JDBC 元数据
|
||||
* @param value 精确 Schema 或表名
|
||||
* @return 可安全用于元数据 Pattern 参数的文本
|
||||
* @throws SQLException 驱动无法返回转义字符时抛出
|
||||
*/
|
||||
private String metadataPattern(DatabaseMetaData metaData, String value)
|
||||
throws SQLException {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
String escape = metaData.getSearchStringEscape();
|
||||
if (escape == null || escape.isEmpty()) {
|
||||
return value;
|
||||
}
|
||||
return value.replace(escape, escape + escape)
|
||||
.replace("%", escape + "%")
|
||||
.replace("_", escape + "_");
|
||||
}
|
||||
|
||||
/**
|
||||
* 对元数据结果再次执行精确 Catalog、Schema 与表名校验,防御驱动忽略 Pattern 转义。
|
||||
*
|
||||
* @param resultSet 元数据结果集
|
||||
* @param source 数据源
|
||||
* @param catalogName 用户选择的物理命名空间
|
||||
* @param tableName 目标表名
|
||||
* @return 当前结果行是否属于精确目标
|
||||
* @throws SQLException 读取元数据列失败时抛出
|
||||
*/
|
||||
private boolean matchesMetadataTable(
|
||||
ResultSet resultSet,
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String tableName) throws SQLException {
|
||||
if (!matchesTableName(resultSet.getString("TABLE_NAME"), tableName)) {
|
||||
return false;
|
||||
}
|
||||
String expectedCatalog = resolveCatalogArgument(source, catalogName);
|
||||
String expectedSchema = resolveSchemaArgument(source, catalogName);
|
||||
String actualCatalog = resultSet.getString("TABLE_CAT");
|
||||
String actualSchema = resultSet.getString("TABLE_SCHEM");
|
||||
// Schema 型数据库已由当前连接锁定 Database;部分 PostgreSQL 驱动不会回填 TABLE_CAT。
|
||||
return (!usesCatalogNamespace()
|
||||
|| expectedCatalog == null
|
||||
|| expectedCatalog.equals(actualCatalog))
|
||||
&& (expectedSchema == null || expectedSchema.equals(actualSchema));
|
||||
}
|
||||
|
||||
private Integer mapFieldType(int jdbcType) {
|
||||
|
||||
@@ -65,6 +65,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
@Column(comment = "物理表名")
|
||||
private String actualTable;
|
||||
|
||||
/** 物理对象稳定摘要。 */
|
||||
@Column(comment = "物理对象稳定摘要")
|
||||
private String physicalIdentityKey;
|
||||
|
||||
/** 表结构摘要。 */
|
||||
@Column(comment = "表结构摘要")
|
||||
private String metadataFingerprint;
|
||||
|
||||
/** 元数据状态。 */
|
||||
@Column(comment = "元数据状态")
|
||||
private String metadataStatus;
|
||||
|
||||
/** 最近发现时间。 */
|
||||
@Column(comment = "最近发现时间")
|
||||
private Date lastSeenAt;
|
||||
|
||||
/** 是否允许统一只读查询。 */
|
||||
@Column(comment = "是否允许统一只读查询")
|
||||
private Integer queryable;
|
||||
|
||||
/** 时间语义。 */
|
||||
@Column(comment = "时间语义")
|
||||
private String timeSemantics;
|
||||
|
||||
/** 默认时间字段 ID。 */
|
||||
@Column(comment = "默认时间字段ID")
|
||||
private BigInteger defaultTimeFieldId;
|
||||
|
||||
/** 表元数据版本。 */
|
||||
@Column(comment = "表元数据版本")
|
||||
private Long metadataRevision;
|
||||
|
||||
/**
|
||||
* 表类型
|
||||
*/
|
||||
@@ -198,6 +230,39 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
this.actualTable = actualTable;
|
||||
}
|
||||
|
||||
public String getPhysicalIdentityKey() { return physicalIdentityKey; }
|
||||
|
||||
public void setPhysicalIdentityKey(String physicalIdentityKey) { this.physicalIdentityKey = physicalIdentityKey; }
|
||||
|
||||
public String getMetadataFingerprint() { return metadataFingerprint; }
|
||||
|
||||
public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; }
|
||||
|
||||
public String getMetadataStatus() { return metadataStatus; }
|
||||
|
||||
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
|
||||
|
||||
public Date getLastSeenAt() { return lastSeenAt; }
|
||||
|
||||
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
|
||||
|
||||
public Integer getQueryable() { return queryable; }
|
||||
|
||||
public void setQueryable(Integer queryable) { this.queryable = queryable; }
|
||||
|
||||
public String getTimeSemantics() { return timeSemantics; }
|
||||
|
||||
public void setTimeSemantics(String timeSemantics) { this.timeSemantics = timeSemantics; }
|
||||
|
||||
@JsonSerialize(using = ToStringSerializer.class)
|
||||
public BigInteger getDefaultTimeFieldId() { return defaultTimeFieldId; }
|
||||
|
||||
public void setDefaultTimeFieldId(BigInteger defaultTimeFieldId) { this.defaultTimeFieldId = defaultTimeFieldId; }
|
||||
|
||||
public Long getMetadataRevision() { return metadataRevision; }
|
||||
|
||||
public void setMetadataRevision(Long metadataRevision) { this.metadataRevision = metadataRevision; }
|
||||
|
||||
public String getTableKind() {
|
||||
return tableKind;
|
||||
}
|
||||
|
||||
@@ -41,6 +41,10 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "源字段名")
|
||||
private String sourceColumnName;
|
||||
|
||||
/** JDBC 字段顺序。 */
|
||||
@Column(comment = "JDBC字段顺序")
|
||||
private Integer ordinalPosition;
|
||||
|
||||
/**
|
||||
* 字段描述
|
||||
*/
|
||||
@@ -59,6 +63,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "JDBC类型")
|
||||
private String jdbcType;
|
||||
|
||||
/** java.sql.Types 数值。 */
|
||||
@Column(comment = "JDBC类型编码")
|
||||
private Integer jdbcTypeCode;
|
||||
|
||||
/** 数据库原生类型名。 */
|
||||
@Column(comment = "数据库原生类型名")
|
||||
private String nativeTypeName;
|
||||
|
||||
/**
|
||||
* 精度
|
||||
*/
|
||||
@@ -71,6 +83,26 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "小数位")
|
||||
private Integer scale;
|
||||
|
||||
/** 字段元数据摘要。 */
|
||||
@Column(comment = "字段元数据摘要")
|
||||
private String metadataFingerprint;
|
||||
|
||||
/** 元数据状态。 */
|
||||
@Column(comment = "元数据状态")
|
||||
private String metadataStatus;
|
||||
|
||||
/** 最近发现时间。 */
|
||||
@Column(comment = "最近发现时间")
|
||||
private Date lastSeenAt;
|
||||
|
||||
/** 敏感级别。 */
|
||||
@Column(comment = "敏感级别")
|
||||
private String sensitivityLevel;
|
||||
|
||||
/** 脱敏策略。 */
|
||||
@Column(comment = "脱敏策略")
|
||||
private String maskingStrategy;
|
||||
|
||||
/**
|
||||
* 是否必填
|
||||
*/
|
||||
@@ -165,6 +197,10 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.sourceColumnName = sourceColumnName;
|
||||
}
|
||||
|
||||
public Integer getOrdinalPosition() { return ordinalPosition; }
|
||||
|
||||
public void setOrdinalPosition(Integer ordinalPosition) { this.ordinalPosition = ordinalPosition; }
|
||||
|
||||
public String getFieldDesc() {
|
||||
return fieldDesc;
|
||||
}
|
||||
@@ -189,6 +225,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.jdbcType = jdbcType;
|
||||
}
|
||||
|
||||
public Integer getJdbcTypeCode() { return jdbcTypeCode; }
|
||||
|
||||
public void setJdbcTypeCode(Integer jdbcTypeCode) { this.jdbcTypeCode = jdbcTypeCode; }
|
||||
|
||||
public String getNativeTypeName() { return nativeTypeName; }
|
||||
|
||||
public void setNativeTypeName(String nativeTypeName) { this.nativeTypeName = nativeTypeName; }
|
||||
|
||||
public Integer getPrecision() {
|
||||
return precision;
|
||||
}
|
||||
@@ -205,6 +249,26 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.scale = scale;
|
||||
}
|
||||
|
||||
public String getMetadataFingerprint() { return metadataFingerprint; }
|
||||
|
||||
public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; }
|
||||
|
||||
public String getMetadataStatus() { return metadataStatus; }
|
||||
|
||||
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
|
||||
|
||||
public Date getLastSeenAt() { return lastSeenAt; }
|
||||
|
||||
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
|
||||
|
||||
public String getSensitivityLevel() { return sensitivityLevel; }
|
||||
|
||||
public void setSensitivityLevel(String sensitivityLevel) { this.sensitivityLevel = sensitivityLevel; }
|
||||
|
||||
public String getMaskingStrategy() { return maskingStrategy; }
|
||||
|
||||
public void setMaskingStrategy(String maskingStrategy) { this.maskingStrategy = maskingStrategy; }
|
||||
|
||||
public Integer getRequired() {
|
||||
return required;
|
||||
}
|
||||
|
||||
@@ -346,7 +346,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||
final int[] rowIndex = {1};
|
||||
totalRows += iterateRows(queryRequest, row -> {
|
||||
totalRows += iterateRows(queryRequest, account, row -> {
|
||||
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
|
||||
for (int i = 0; i < table.getFields().size(); i++) {
|
||||
Cell cell = excelRow.createCell(i);
|
||||
@@ -434,7 +434,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
final Holder holder = new Holder();
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> {
|
||||
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
|
||||
holder.batchNo++;
|
||||
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||
@@ -466,7 +466,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> {
|
||||
String fieldValue = stringify(row.get(splitField.getFieldName()));
|
||||
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
|
||||
DatacenterTable targetTable = targets.get(bucket);
|
||||
@@ -543,7 +543,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
Map<String, String> mapping = fieldMappings.get(table.getId());
|
||||
iterateRows(buildFullQuery(table), row -> {
|
||||
iterateRows(buildFullQuery(table), account, row -> {
|
||||
String joinValue = stringify(row.get(request.getJoinKey()));
|
||||
if (joinValue == null || joinValue.isBlank()) {
|
||||
return;
|
||||
@@ -729,16 +729,22 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
}
|
||||
|
||||
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
|
||||
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
|
||||
return iterateRows(
|
||||
queryRequest,
|
||||
account,
|
||||
row -> saveToTable(targetTable, mapper.map(row), account));
|
||||
}
|
||||
|
||||
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
|
||||
private long iterateRows(
|
||||
DatacenterQueryRequest queryRequest,
|
||||
LoginAccount account,
|
||||
RowConsumer consumer) {
|
||||
long total = 0L;
|
||||
long pageNumber = 1L;
|
||||
while (true) {
|
||||
queryRequest.setPageNumber(pageNumber);
|
||||
queryRequest.setPageSize(QUERY_BATCH_SIZE);
|
||||
Page<Row> page = queryService.queryPage(queryRequest);
|
||||
Page<Row> page = queryService.queryPage(queryRequest, account);
|
||||
if (page.getRecords() == null || page.getRecords().isEmpty()) {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -5,31 +5,46 @@ import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterSchemaResponse {
|
||||
private DatasetRef datasetRef;
|
||||
private DatacenterSource source;
|
||||
private DatacenterSourceView source;
|
||||
private DatacenterCatalog catalog;
|
||||
private DatacenterTable table;
|
||||
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||
private long fieldPageNumber = 1L;
|
||||
private long fieldPageSize;
|
||||
private boolean hasMoreFields;
|
||||
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
|
||||
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
|
||||
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
|
||||
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public DatacenterSource getSource() { return source; }
|
||||
public void setSource(DatacenterSource source) { this.source = source; }
|
||||
public DatacenterSourceView getSource() { return source; }
|
||||
public void setSource(DatacenterSourceView source) { this.source = source; }
|
||||
public DatacenterCatalog getCatalog() { return catalog; }
|
||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||
public DatacenterTable getTable() { return table; }
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
public List<DatacenterTableField> getFields() { return fields; }
|
||||
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||
/** @return 当前字段页码 */
|
||||
public long getFieldPageNumber() { return fieldPageNumber; }
|
||||
/** @param fieldPageNumber 当前字段页码 */
|
||||
public void setFieldPageNumber(long fieldPageNumber) { this.fieldPageNumber = fieldPageNumber; }
|
||||
/** @return 当前字段页大小 */
|
||||
public long getFieldPageSize() { return fieldPageSize; }
|
||||
/** @param fieldPageSize 当前字段页大小 */
|
||||
public void setFieldPageSize(long fieldPageSize) { this.fieldPageSize = fieldPageSize; }
|
||||
/** @return 是否还有下一页字段 */
|
||||
public boolean isHasMoreFields() { return hasMoreFields; }
|
||||
/** @param hasMoreFields 是否还有下一页字段 */
|
||||
public void setHasMoreFields(boolean hasMoreFields) { this.hasMoreFields = hasMoreFields; }
|
||||
public List<DatacenterDatasetVersion> getVersions() { return versions; }
|
||||
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
|
||||
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
/**
|
||||
* 管理端取消只读 SQL 查询请求。
|
||||
*
|
||||
* @param queryId 客户端在执行前生成的查询 UUID
|
||||
*/
|
||||
public record DatacenterSqlCancelRequest(String queryId) {
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
/**
|
||||
* SQL 控制台结果列。
|
||||
*
|
||||
* @param key 前端稳定列键
|
||||
* @param label JDBC 列标签
|
||||
* @param jdbcType JDBC 类型编码
|
||||
* @param typeName 数据库类型名
|
||||
* @param nullable 是否允许空值
|
||||
*/
|
||||
public record DatacenterSqlColumnView(
|
||||
String key,
|
||||
String label,
|
||||
int jdbcType,
|
||||
String typeName,
|
||||
boolean nullable) {
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* 管理端只读 SQL 控制台请求。
|
||||
*
|
||||
* @param sourceId 已激活数据源 ID
|
||||
* @param queryId 客户端在执行前生成的查询 UUID
|
||||
* @param sql 单条只读 SQL
|
||||
* @param maxRows 最大返回行数
|
||||
*/
|
||||
public record DatacenterSqlConsoleRequest(
|
||||
BigInteger sourceId,
|
||||
String queryId,
|
||||
String sql,
|
||||
Integer maxRows) {
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 有界 SQL 控制台结果。
|
||||
*
|
||||
* @param queryId 查询标识
|
||||
* @param columns 结果列
|
||||
* @param rows 结果行
|
||||
* @param returnedRows 返回行数
|
||||
* @param truncated 是否因上限截断
|
||||
* @param durationMs 总耗时毫秒
|
||||
*/
|
||||
public record DatacenterSqlConsoleResult(
|
||||
String queryId,
|
||||
List<DatacenterSqlColumnView> columns,
|
||||
List<Map<String, Object>> rows,
|
||||
long returnedRows,
|
||||
boolean truncated,
|
||||
long durationMs) {
|
||||
|
||||
/**
|
||||
* 防御性复制结果集合。
|
||||
*/
|
||||
public DatacenterSqlConsoleResult {
|
||||
columns = List.copyOf(columns == null ? List.of() : columns);
|
||||
rows = List.copyOf(rows == null ? List.of() : rows);
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import java.math.BigInteger;
|
||||
public class DatasetRef implements java.io.Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private BigInteger tenantId;
|
||||
private BigInteger sourceId;
|
||||
private BigInteger catalogId;
|
||||
private String catalogName;
|
||||
@@ -12,6 +13,8 @@ public class DatasetRef implements java.io.Serializable {
|
||||
private String tableName;
|
||||
private BigInteger versionId;
|
||||
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
|
||||
@@ -2,6 +2,7 @@ package tech.easyflow.datacenter.execution.service;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
@@ -20,6 +21,19 @@ public interface DatacenterDatasetQueryService {
|
||||
*/
|
||||
Page<Row> queryPage(DatacenterQueryRequest request);
|
||||
|
||||
/**
|
||||
* 使用明确执行账号分页查询结构化数据集。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @param account 执行账号
|
||||
* @return 分页结果
|
||||
*/
|
||||
default Page<Row> queryPage(
|
||||
DatacenterQueryRequest request,
|
||||
LoginAccount account) {
|
||||
return queryPage(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行原生 SQL 并返回完整结果。
|
||||
*
|
||||
@@ -35,9 +49,43 @@ public interface DatacenterDatasetQueryService {
|
||||
* @param fetchSize JDBC 建议拉取行数
|
||||
* @param consumer 单行消费者
|
||||
*/
|
||||
default void consumeBySql(
|
||||
DatacenterSqlQueryRequest request,
|
||||
int fetchSize,
|
||||
Consumer<Row> consumer) {
|
||||
consumeBySql(request, fetchSize, null, consumer);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用明确执行账号流式消费 SQL,后台工作流据此绑定权威租户。
|
||||
*
|
||||
* @param request SQL 查询请求
|
||||
* @param fetchSize JDBC 建议拉取行数
|
||||
* @param account 执行账号
|
||||
* @param consumer 单行消费者
|
||||
*/
|
||||
default void consumeBySql(
|
||||
DatacenterSqlQueryRequest request,
|
||||
int fetchSize,
|
||||
LoginAccount account,
|
||||
Consumer<Row> consumer) {
|
||||
consumeBySql(request, fetchSize, account, null, consumer);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用明确执行账号和调用方 QueryId 流式消费 SQL。
|
||||
*
|
||||
* @param request SQL 查询请求
|
||||
* @param fetchSize JDBC 建议拉取行数
|
||||
* @param account 执行账号
|
||||
* @param requestedQueryId 调用方预生成的查询 UUID;为空时服务端生成
|
||||
* @param consumer 单行消费者
|
||||
*/
|
||||
void consumeBySql(
|
||||
DatacenterSqlQueryRequest request,
|
||||
int fetchSize,
|
||||
LoginAccount account,
|
||||
String requestedQueryId,
|
||||
Consumer<Row> consumer);
|
||||
|
||||
/**
|
||||
@@ -48,6 +96,19 @@ public interface DatacenterDatasetQueryService {
|
||||
*/
|
||||
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
|
||||
|
||||
/**
|
||||
* 获取有界字段页的数据集结构。
|
||||
*
|
||||
* @param datasetRef 数据集引用
|
||||
* @param fieldPageNumber 字段页码
|
||||
* @param fieldPageSize 字段页大小
|
||||
* @return 数据集结构和当前字段页
|
||||
*/
|
||||
DatacenterSchemaResponse getSchema(
|
||||
DatasetRef datasetRef,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize);
|
||||
|
||||
/**
|
||||
* 仅解析数据集定位信息,不加载版本和血缘。
|
||||
*
|
||||
|
||||
@@ -1,22 +1,43 @@
|
||||
package tech.easyflow.datacenter.execution.service.impl;
|
||||
|
||||
import com.easyagents.federation.sql.execute.SqlParameter;
|
||||
import com.easyagents.federation.sql.execute.QueryId;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.Types;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
@@ -25,19 +46,14 @@ import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceViews;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.utils.SqlSupportUtils;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
|
||||
@@ -54,18 +70,34 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||
@Resource
|
||||
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||
@Resource
|
||||
private DatacenterFederationQueryService federationQueryService;
|
||||
@Resource
|
||||
private DatacenterQueryAuditService queryAuditService;
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterQueryRequest request) {
|
||||
throw new BusinessException("结构化查询必须提供执行账号");
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public Page<Row> queryPage(
|
||||
DatacenterQueryRequest request,
|
||||
LoginAccount account) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("datasetRef 不能为空");
|
||||
}
|
||||
normalizePage(request);
|
||||
DatacenterTable table = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
validateStructuredQueryTenant(
|
||||
request.getDatasetRef(), table, source, account);
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
|
||||
validateRequest(queryTable, request, source);
|
||||
validateRequest(queryTable, request);
|
||||
request.getDatasetRef().setSourceId(table.getSourceId());
|
||||
request.getDatasetRef().setCatalogId(table.getCatalogId());
|
||||
request.getDatasetRef().setTableId(table.getId());
|
||||
@@ -73,15 +105,199 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
if (catalog != null) {
|
||||
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
|
||||
}
|
||||
if (isFederated(source)) {
|
||||
return queryFederatedPage(
|
||||
source, catalog, queryTable, request, account);
|
||||
}
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
return connector.queryPage(source, queryTable, request);
|
||||
QueryId queryId = QueryId.create();
|
||||
long startedNanos = System.nanoTime();
|
||||
DatacenterQueryAudit audit = queryAuditService.start(
|
||||
queryId.value(), source, structuredQueryDescription(queryTable, request),
|
||||
structuredParameterSummary(request), account,
|
||||
"DATASET_PAGE", table.getId().toString());
|
||||
try {
|
||||
Page<Row> result = connector.queryPage(source, queryTable, request);
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
queryAuditService.succeed(
|
||||
audit,
|
||||
List.of(table.getTableName()),
|
||||
structuredReferencedFields(request),
|
||||
result.getRecords().size(),
|
||||
false,
|
||||
durationMs,
|
||||
durationMs);
|
||||
return result;
|
||||
} catch (RuntimeException exception) {
|
||||
queryAuditService.fail(
|
||||
audit,
|
||||
"DATASET_PAGE_QUERY_FAILED",
|
||||
safeMessage(exception),
|
||||
List.of(table.getTableName()),
|
||||
structuredReferencedFields(request),
|
||||
elapsedMillis(startedNanos));
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用执行账号绑定结构化查询的权威租户。
|
||||
*
|
||||
* @param datasetRef 数据集引用
|
||||
* @param table 已解析表
|
||||
* @param source 已解析数据源
|
||||
* @param account 执行账号
|
||||
* @throws BusinessException 账号或租户边界不合法时抛出
|
||||
*/
|
||||
private void validateStructuredQueryTenant(
|
||||
DatasetRef datasetRef,
|
||||
DatacenterTable table,
|
||||
DatacenterSource source,
|
||||
LoginAccount account) {
|
||||
if (account == null || account.getTenantId() == null) {
|
||||
throw new BusinessException("结构化查询缺少执行账号租户");
|
||||
}
|
||||
BigInteger tenantId = account.getTenantId();
|
||||
if ((datasetRef.getTenantId() != null
|
||||
&& !tenantId.equals(datasetRef.getTenantId()))
|
||||
|| !tenantId.equals(table.getTenantId())
|
||||
|| !tenantId.equals(source.getTenantId())) {
|
||||
throw new BusinessException("数据集不属于当前租户");
|
||||
}
|
||||
datasetRef.setTenantId(tenantId);
|
||||
}
|
||||
|
||||
private String structuredQueryDescription(
|
||||
DatacenterTable table,
|
||||
DatacenterQueryRequest request) {
|
||||
StringBuilder sql = new StringBuilder("SELECT ")
|
||||
.append(String.join(", ", request.getSelectedColumns()))
|
||||
.append(" FROM ").append(table.getTableName());
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
sql.append(" WHERE ");
|
||||
for (int index = 0; index < request.getFilters().size(); index++) {
|
||||
DatacenterQueryFilter filter = request.getFilters().get(index);
|
||||
if (index > 0) {
|
||||
sql.append(" AND ");
|
||||
}
|
||||
String operator = normalizeFilterOperator(filter.getOperator());
|
||||
sql.append(filter.getColumn()).append(' ');
|
||||
switch (operator) {
|
||||
case "IS_NULL" -> sql.append("IS NULL");
|
||||
case "IN" -> sql.append("IN (?)");
|
||||
default -> sql.append(operator).append(" ?");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||
sql.append(" ORDER BY ");
|
||||
for (int index = 0; index < request.getSorts().size(); index++) {
|
||||
DatacenterQuerySort sort = request.getSorts().get(index);
|
||||
if (index > 0) {
|
||||
sql.append(", ");
|
||||
}
|
||||
sql.append(sort.getColumn()).append(' ')
|
||||
.append("DESC".equalsIgnoreCase(sort.getDirection())
|
||||
? "DESC" : "ASC");
|
||||
}
|
||||
}
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建不包含实际值的结构化查询参数摘要。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @return 脱敏参数摘要
|
||||
*/
|
||||
private Map<String, Object> structuredParameterSummary(
|
||||
DatacenterQueryRequest request) {
|
||||
List<Map<String, Object>> filters = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||
String operator = normalizeFilterOperator(filter.getOperator());
|
||||
int valueCount = "IN".equals(operator)
|
||||
? (filter.getValues() == null ? 0 : filter.getValues().size())
|
||||
: "IS_NULL".equals(operator) ? 0 : 1;
|
||||
filters.add(Map.of(
|
||||
"column", filter.getColumn(),
|
||||
"operator", operator,
|
||||
"valueCount", valueCount));
|
||||
}
|
||||
}
|
||||
return Map.of(
|
||||
"filters", List.copyOf(filters),
|
||||
"pageNumber", request.getPageNumber(),
|
||||
"pageSize", request.getPageSize());
|
||||
}
|
||||
|
||||
/**
|
||||
* 收集结构化查询实际引用的投影、筛选和排序字段。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @return 去重后的字段列表
|
||||
*/
|
||||
private List<String> structuredReferencedFields(
|
||||
DatacenterQueryRequest request) {
|
||||
Set<String> fields = new LinkedHashSet<>(request.getSelectedColumns());
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
request.getFilters().stream()
|
||||
.map(DatacenterQueryFilter::getColumn)
|
||||
.forEach(fields::add);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||
request.getSorts().stream()
|
||||
.map(DatacenterQuerySort::getColumn)
|
||||
.forEach(fields::add);
|
||||
}
|
||||
return List.copyOf(fields);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将结构化筛选操作符归一化为审计中的稳定名称。
|
||||
*
|
||||
* @param operator 原始操作符
|
||||
* @return 大写稳定操作符
|
||||
*/
|
||||
private String normalizeFilterOperator(String operator) {
|
||||
return trimToNull(operator) == null
|
||||
? "EQ" : operator.trim().toUpperCase(java.util.Locale.ROOT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
|
||||
ResolvedSqlQuery query = resolveSqlQuery(request);
|
||||
return query.connector.queryBySql(
|
||||
query.source, query.sql);
|
||||
if (isFederated(query.source)) {
|
||||
DatacenterSqlConsoleResult result = federationQueryService.execute(
|
||||
query.source,
|
||||
query.sql,
|
||||
List.of(),
|
||||
1_000,
|
||||
null,
|
||||
"DATASET",
|
||||
request.getDatasetRef().getTableId() == null
|
||||
? null : request.getDatasetRef().getTableId().toString());
|
||||
if (result.truncated()) {
|
||||
throw new BusinessException("数据集查询结果超过返回上限,请使用流式消费");
|
||||
}
|
||||
return toRows(result);
|
||||
}
|
||||
QueryId queryId = QueryId.create();
|
||||
long startedNanos = System.nanoTime();
|
||||
DatacenterQueryAudit audit = queryAuditService.start(
|
||||
queryId.value(), query.source, query.sql, Map.of(), null,
|
||||
"DATASET", request.getDatasetRef().getTableId().toString());
|
||||
try {
|
||||
List<Row> rows = query.connector.queryBySql(query.source, query.sql);
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
queryAuditService.succeed(audit, query.referencedTables, List.of(),
|
||||
rows.size(), false, durationMs, durationMs);
|
||||
return rows;
|
||||
} catch (RuntimeException exception) {
|
||||
queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception),
|
||||
query.referencedTables, List.of(), elapsedMillis(startedNanos));
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -91,12 +307,38 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
public void consumeBySql(
|
||||
DatacenterSqlQueryRequest request,
|
||||
int fetchSize,
|
||||
LoginAccount account,
|
||||
String requestedQueryId,
|
||||
Consumer<Row> consumer) {
|
||||
if (fetchSize <= 0 || consumer == null) {
|
||||
throw new IllegalArgumentException(
|
||||
"fetchSize and consumer must be valid");
|
||||
}
|
||||
ResolvedSqlQuery query = resolveSqlQuery(request);
|
||||
ResolvedSqlQuery query = resolveSqlQuery(request, account);
|
||||
if (isFederated(query.source)) {
|
||||
int maxRows = Integer.getInteger(
|
||||
"easyflow.datacenter.query.max-rows", 1_000_000);
|
||||
long maxBytes = Long.getLong(
|
||||
"easyflow.datacenter.query.max-bytes", 512L * 1024L * 1024L);
|
||||
federationQueryService.consume(
|
||||
query.source,
|
||||
query.sql,
|
||||
List.of(),
|
||||
fetchSize,
|
||||
maxRows,
|
||||
maxBytes,
|
||||
account,
|
||||
"DATASET",
|
||||
request.getDatasetRef().getTableId() == null
|
||||
? null : request.getDatasetRef().getTableId().toString(),
|
||||
requestedQueryId,
|
||||
sourceRow -> {
|
||||
Row row = new Row();
|
||||
sourceRow.forEach(row::put);
|
||||
consumer.accept(row);
|
||||
});
|
||||
return;
|
||||
}
|
||||
int maxRows = Integer.getInteger(
|
||||
"easyflow.datacenter.query.max-rows",
|
||||
1_000_000);
|
||||
@@ -105,34 +347,48 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
512L * 1024L * 1024L);
|
||||
long[] accumulatedRows = {0L};
|
||||
long[] accumulatedBytes = {0L};
|
||||
query.connector.consumeBySql(
|
||||
query.source,
|
||||
query.sql,
|
||||
fetchSize,
|
||||
row -> {
|
||||
accumulatedRows[0]++;
|
||||
if (maxRows > 0
|
||||
&& accumulatedRows[0] > maxRows) {
|
||||
throw new BusinessException(
|
||||
"数据集查询结果超过行数上限: "
|
||||
+ maxRows);
|
||||
}
|
||||
for (Map.Entry<String, Object> entry
|
||||
: row.entrySet()) {
|
||||
accumulatedBytes[0] +=
|
||||
estimateQueryValueBytes(
|
||||
entry.getKey(),
|
||||
entry.getValue());
|
||||
if (maxBytes > 0L
|
||||
&& accumulatedBytes[0] > maxBytes) {
|
||||
QueryId queryId = resolveQueryId(requestedQueryId);
|
||||
long startedNanos = System.nanoTime();
|
||||
DatacenterQueryAudit audit = queryAuditService.start(
|
||||
queryId.value(), query.source, query.sql, Map.of(), account,
|
||||
"DATASET", request.getDatasetRef().getTableId().toString());
|
||||
try {
|
||||
query.connector.consumeBySql(
|
||||
query.source,
|
||||
query.sql,
|
||||
fetchSize,
|
||||
row -> {
|
||||
accumulatedRows[0]++;
|
||||
if (maxRows > 0
|
||||
&& accumulatedRows[0] > maxRows) {
|
||||
throw new BusinessException(
|
||||
"数据集查询结果超过字节上限: "
|
||||
+ maxBytes);
|
||||
"数据集查询结果超过行数上限: "
|
||||
+ maxRows);
|
||||
}
|
||||
for (Map.Entry<String, Object> entry
|
||||
: row.entrySet()) {
|
||||
accumulatedBytes[0] +=
|
||||
estimateQueryValueBytes(
|
||||
entry.getKey(),
|
||||
entry.getValue());
|
||||
if (maxBytes > 0L
|
||||
&& accumulatedBytes[0] > maxBytes) {
|
||||
throw new BusinessException(
|
||||
"数据集查询结果超过字节上限: "
|
||||
+ maxBytes);
|
||||
}
|
||||
}
|
||||
consumer.accept(row);
|
||||
}
|
||||
consumer.accept(row);
|
||||
}
|
||||
);
|
||||
);
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
queryAuditService.succeed(audit, query.referencedTables, List.of(),
|
||||
accumulatedRows[0], false, durationMs, durationMs);
|
||||
} catch (RuntimeException exception) {
|
||||
queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception),
|
||||
query.referencedTables, List.of(), elapsedMillis(startedNanos));
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -171,6 +427,19 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
*/
|
||||
private ResolvedSqlQuery resolveSqlQuery(
|
||||
DatacenterSqlQueryRequest request) {
|
||||
return resolveSqlQuery(request, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验请求、执行账号租户并解析实际连接器与可执行 SQL。
|
||||
*
|
||||
* @param request SQL 查询请求
|
||||
* @param account 后台执行账号;普通同步调用可为空
|
||||
* @return 已解析查询
|
||||
*/
|
||||
private ResolvedSqlQuery resolveSqlQuery(
|
||||
DatacenterSqlQueryRequest request,
|
||||
LoginAccount account) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("datasetRef 不能为空");
|
||||
}
|
||||
@@ -183,8 +452,28 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
throw new BusinessException("缺少连接服务配置");
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||
BigInteger authoritativeTenantId = account != null && account.getTenantId() != null
|
||||
? account.getTenantId() : datasetRef.getTenantId();
|
||||
if (authoritativeTenantId == null) {
|
||||
throw new BusinessException("数据集绑定缺少租户信息");
|
||||
}
|
||||
if ((datasetRef.getTenantId() != null
|
||||
&& !datasetRef.getTenantId().equals(authoritativeTenantId))
|
||||
|| !authoritativeTenantId.equals(source.getTenantId())) {
|
||||
throw new BusinessException("数据集不属于当前租户");
|
||||
}
|
||||
datasetRef.setTenantId(authoritativeTenantId);
|
||||
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
|
||||
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
|
||||
if (datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("数据集 SQL 查询必须绑定具体表");
|
||||
}
|
||||
DatacenterTable boundTable = registryService.getTableWithFields(datasetRef.getTableId());
|
||||
if (!datasetRef.getSourceId().equals(boundTable.getSourceId())
|
||||
|| (catalogId != null && !catalogId.equals(boundTable.getCatalogId()))
|
||||
|| !authoritativeTenantId.equals(boundTable.getTenantId())) {
|
||||
throw new BusinessException("数据集引用与绑定表不一致");
|
||||
}
|
||||
List<DatacenterTable> managedTables = List.of(boundTable);
|
||||
if (CollectionUtils.isEmpty(managedTables)) {
|
||||
throw new BusinessException("当前连接下没有已接入表");
|
||||
}
|
||||
@@ -198,12 +487,15 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
// 内部连接的 catalog 是逻辑命名空间,底层项目 MySQL 只执行物理表名。
|
||||
SqlSupportUtils.ResolvedSql resolvedSql =
|
||||
connector instanceof AbstractInternalTableConnector
|
||||
|| DatacenterSourceType.PROJECT_MYSQL.name()
|
||||
.equals(source.getSourceType())
|
||||
? SqlSupportUtils.resolveInternalMysql(sql, sqlTables)
|
||||
: SqlSupportUtils.resolve(sql, sqlTables);
|
||||
return new ResolvedSqlQuery(
|
||||
source,
|
||||
connector,
|
||||
resolvedSql.getExecutableSql());
|
||||
isFederated(source) ? sql : resolvedSql.getExecutableSql(),
|
||||
resolvedSql.getLogicalTables());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -212,30 +504,123 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
* @param source 数据源
|
||||
* @param connector 数据连接器
|
||||
* @param sql 可执行 SQL
|
||||
* @param referencedTables 已校验引用表
|
||||
*/
|
||||
private record ResolvedSqlQuery(
|
||||
DatacenterSource source,
|
||||
DatacenterConnector connector,
|
||||
String sql) {
|
||||
String sql,
|
||||
List<String> referencedTables) {
|
||||
}
|
||||
|
||||
private long elapsedMillis(long startedNanos) {
|
||||
return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验调用方查询 UUID,内部连接也使用同一标识完成审计关联。
|
||||
*
|
||||
* @param requestedQueryId 调用方标识;为空时生成
|
||||
* @return 查询标识
|
||||
*/
|
||||
private QueryId resolveQueryId(String requestedQueryId) {
|
||||
if (!StringUtils.hasText(requestedQueryId)) {
|
||||
return QueryId.create();
|
||||
}
|
||||
try {
|
||||
return new QueryId(UUID.fromString(
|
||||
requestedQueryId.trim()).toString());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new BusinessException("queryId 必须是 UUID");
|
||||
}
|
||||
}
|
||||
|
||||
private String safeMessage(RuntimeException exception) {
|
||||
String message = exception.getMessage();
|
||||
if (message == null || message.isBlank()) {
|
||||
return "数据集查询失败";
|
||||
}
|
||||
return message.length() <= 500 ? message : message.substring(0, 500);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
|
||||
return getSchema(datasetRef, null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public DatacenterSchemaResponse getSchema(
|
||||
DatasetRef datasetRef,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
List<DatacenterTableField> allFields = List.copyOf(table.getFields());
|
||||
long actualPage = fieldPageNumber == null ? 1L : Math.max(1L, fieldPageNumber);
|
||||
long actualSize = fieldPageSize == null
|
||||
? Math.max(1L, allFields.size())
|
||||
: Math.min(200L, Math.max(1L, fieldPageSize));
|
||||
List<DatacenterTableField> pageFields = sliceFields(
|
||||
allFields, actualPage, actualSize);
|
||||
long offset = safePageOffset(actualPage, actualSize);
|
||||
boolean hasMoreFields = offset < allFields.size()
|
||||
&& offset + pageFields.size() < allFields.size();
|
||||
// 字段只通过顶层分页字段返回,避免在 table 内重复序列化完整列表。
|
||||
table.setFields(List.of());
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
|
||||
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
response.setSource(source);
|
||||
response.setSource(DatacenterSourceViews.from(source));
|
||||
response.setCatalog(catalog);
|
||||
response.setTable(table);
|
||||
response.setFields(table.getFields());
|
||||
response.setFields(pageFields);
|
||||
response.setFieldPageNumber(actualPage);
|
||||
response.setFieldPageSize(actualSize);
|
||||
response.setHasMoreFields(hasMoreFields);
|
||||
response.setVersions(listVersions(table.getId()));
|
||||
response.setUpstreamLineage(listUpstream(table.getId()));
|
||||
response.setDownstreamLineage(listDownstream(table.getId()));
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* 截取字段页并避免向响应暴露可变子列表。
|
||||
*
|
||||
* @param fields 完整字段列表
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @return 当前页字段
|
||||
*/
|
||||
private List<DatacenterTableField> sliceFields(
|
||||
List<DatacenterTableField> fields,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
long offset = safePageOffset(pageNumber, pageSize);
|
||||
if (offset >= fields.size()) {
|
||||
return List.of();
|
||||
}
|
||||
int fromIndex = Math.toIntExact(offset);
|
||||
int toIndex = (int) Math.min((long) fields.size(), offset + pageSize);
|
||||
return List.copyOf(fields.subList(fromIndex, toIndex));
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算字段页偏移并防止长整型乘法溢出。
|
||||
*
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @return 安全偏移;溢出时返回最大长整型
|
||||
*/
|
||||
private long safePageOffset(long pageNumber, long pageSize) {
|
||||
if (pageNumber - 1L > Long.MAX_VALUE / pageSize) {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
return (pageNumber - 1L) * pageSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@@ -245,8 +630,8 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
DatacenterSchemaResponse response =
|
||||
new DatacenterSchemaResponse();
|
||||
response.setDatasetRef(datasetRef);
|
||||
response.setSource(
|
||||
registryService.getSourceRequired(table.getSourceId()));
|
||||
response.setSource(DatacenterSourceViews.from(
|
||||
registryService.getSourceRequired(table.getSourceId())));
|
||||
response.setCatalog(
|
||||
registryService.getCatalogById(table.getCatalogId()));
|
||||
response.setTable(table);
|
||||
@@ -319,13 +704,29 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
DatacenterCatalog catalog = catalogId == null
|
||||
? null
|
||||
: catalogsById.get(catalogId);
|
||||
List<String> knownColumns = table.getFields().stream()
|
||||
.filter(field -> DatacenterMetadataStatus.ACTIVE.name()
|
||||
.equals(field.getMetadataStatus()))
|
||||
.map(this::sourceColumnName)
|
||||
.toList();
|
||||
List<String> queryableColumns = table.getFields().stream()
|
||||
.filter(this::isQueryable)
|
||||
.map(this::sourceColumnName)
|
||||
.toList();
|
||||
return new SqlSupportUtils.ManagedTable(
|
||||
catalog == null ? null : catalog.getCatalogName(),
|
||||
table.getTableName(),
|
||||
resolvePhysicalTableName(table)
|
||||
resolvePhysicalTableName(table),
|
||||
knownColumns,
|
||||
queryableColumns
|
||||
);
|
||||
}
|
||||
|
||||
private String sourceColumnName(DatacenterTableField field) {
|
||||
String sourceColumnName = trimToNull(field.getSourceColumnName());
|
||||
return sourceColumnName == null ? field.getFieldName() : sourceColumnName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 一次批量加载 SQL 白名单表关联的目录,避免逐表查询。
|
||||
*
|
||||
@@ -367,6 +768,153 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
return actualTable != null ? actualTable : table.getTableName();
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用逻辑 Schema 构建参数化 ANSI SQL,再由 Calcite 转换为目标方言。
|
||||
*/
|
||||
private Page<Row> queryFederatedPage(
|
||||
DatacenterSource source,
|
||||
DatacenterCatalog catalog,
|
||||
DatacenterTable table,
|
||||
DatacenterQueryRequest request,
|
||||
LoginAccount account) {
|
||||
if (catalog == null || catalog.getLogicalSchemaName() == null) {
|
||||
throw new BusinessException("数据集缺少 Federation Schema 绑定");
|
||||
}
|
||||
if (request.getPageSize() > 500L) {
|
||||
throw new BusinessException("外部数据源单页最多返回 500 行");
|
||||
}
|
||||
Map<String, DatacenterTableField> fields = table.getFields().stream()
|
||||
.collect(Collectors.toMap(
|
||||
DatacenterTableField::getFieldName,
|
||||
Function.identity(),
|
||||
(first, ignored) -> first,
|
||||
LinkedHashMap::new));
|
||||
FederatedWhere where = buildFederatedWhere(request.getFilters(), fields);
|
||||
String from = quote(catalog.getLogicalSchemaName()) + "." + quote(table.getTableName());
|
||||
String countSql = "SELECT COUNT(*) AS total FROM " + from + where.sql();
|
||||
DatacenterSqlConsoleResult countResult = federationQueryService.execute(
|
||||
source, countSql, where.parameters(), 1, account,
|
||||
"DATASET", table.getId().toString());
|
||||
long total = countResult.rows().isEmpty()
|
||||
? 0L : Long.parseLong(String.valueOf(countResult.rows().get(0).get("c1")));
|
||||
|
||||
if (total == 0L) {
|
||||
return new Page<>(List.of(), request.getPageNumber(), request.getPageSize(), 0L);
|
||||
}
|
||||
String columns = request.getSelectedColumns().stream()
|
||||
.map(name -> quote(requireField(fields, name).getSourceColumnName()))
|
||||
.collect(Collectors.joining(", "));
|
||||
String orderBy = buildFederatedOrderBy(request.getSorts(), fields);
|
||||
long offset = (request.getPageNumber() - 1L) * request.getPageSize();
|
||||
String pageSql = "SELECT " + columns + " FROM " + from + where.sql() + orderBy
|
||||
+ " OFFSET " + offset + " ROWS FETCH NEXT " + request.getPageSize() + " ROWS ONLY";
|
||||
DatacenterSqlConsoleResult pageResult = federationQueryService.execute(
|
||||
source, pageSql, where.parameters(), request.getPageSize().intValue(),
|
||||
account, "DATASET", table.getId().toString());
|
||||
return new Page<>(toRows(pageResult), request.getPageNumber(), request.getPageSize(), total);
|
||||
}
|
||||
|
||||
private FederatedWhere buildFederatedWhere(
|
||||
List<DatacenterQueryFilter> filters,
|
||||
Map<String, DatacenterTableField> fields) {
|
||||
if (CollectionUtils.isEmpty(filters)) {
|
||||
return new FederatedWhere("", List.of());
|
||||
}
|
||||
StringBuilder sql = new StringBuilder(" WHERE 1 = 1");
|
||||
List<SqlParameter> parameters = new ArrayList<>();
|
||||
for (DatacenterQueryFilter filter : filters) {
|
||||
DatacenterTableField field = requireField(fields, filter.getColumn());
|
||||
String column = quote(field.getSourceColumnName());
|
||||
String operator = trimToNull(filter.getOperator()) == null
|
||||
? "EQ" : filter.getOperator().trim().toUpperCase(java.util.Locale.ROOT);
|
||||
switch (operator) {
|
||||
case "EQ", "GT", "GTE", "LT", "LTE", "LIKE" -> {
|
||||
String sqlOperator = switch (operator) {
|
||||
case "GT" -> ">";
|
||||
case "GTE" -> ">=";
|
||||
case "LT" -> "<";
|
||||
case "LTE" -> "<=";
|
||||
case "LIKE" -> "LIKE";
|
||||
default -> "=";
|
||||
};
|
||||
sql.append(" AND ").append(column).append(' ').append(sqlOperator).append(" ?");
|
||||
Object value = "LIKE".equals(operator)
|
||||
? "%" + filter.getValue() + "%" : filter.getValue();
|
||||
parameters.add(parameter(field, value));
|
||||
}
|
||||
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
|
||||
case "IN" -> {
|
||||
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
|
||||
if (values.isEmpty()) {
|
||||
sql.append(" AND 1 = 0");
|
||||
} else {
|
||||
sql.append(" AND ").append(column).append(" IN (")
|
||||
.append(values.stream().map(ignored -> "?").collect(Collectors.joining(",")))
|
||||
.append(')');
|
||||
values.forEach(value -> parameters.add(parameter(field, value)));
|
||||
}
|
||||
}
|
||||
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
||||
}
|
||||
}
|
||||
return new FederatedWhere(sql.toString(), List.copyOf(parameters));
|
||||
}
|
||||
|
||||
private String buildFederatedOrderBy(
|
||||
List<DatacenterQuerySort> sorts,
|
||||
Map<String, DatacenterTableField> fields) {
|
||||
if (CollectionUtils.isEmpty(sorts)) {
|
||||
return "";
|
||||
}
|
||||
return " ORDER BY " + sorts.stream()
|
||||
.map(sort -> quote(requireField(fields, sort.getColumn()).getSourceColumnName())
|
||||
+ ("DESC".equalsIgnoreCase(sort.getDirection()) ? " DESC" : " ASC"))
|
||||
.collect(Collectors.joining(", "));
|
||||
}
|
||||
|
||||
private DatacenterTableField requireField(
|
||||
Map<String, DatacenterTableField> fields,
|
||||
String name) {
|
||||
DatacenterTableField field = fields.get(name);
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
throw new BusinessException("字段不可查询: " + name);
|
||||
}
|
||||
return field;
|
||||
}
|
||||
|
||||
private SqlParameter parameter(DatacenterTableField field, Object value) {
|
||||
int jdbcType = field.getJdbcTypeCode() == null ? Types.VARCHAR : field.getJdbcTypeCode();
|
||||
return value == null ? new SqlParameter(jdbcType, null) : new SqlParameter(jdbcType, value);
|
||||
}
|
||||
|
||||
private String quote(String identifier) {
|
||||
if (!StringUtils.hasText(identifier)) {
|
||||
throw new BusinessException("数据集物理名称为空");
|
||||
}
|
||||
return '"' + identifier.replace("\"", "\"\"") + '"';
|
||||
}
|
||||
|
||||
private List<Row> toRows(DatacenterSqlConsoleResult result) {
|
||||
List<Row> rows = new ArrayList<>(result.rows().size());
|
||||
for (Map<String, Object> sourceRow : result.rows()) {
|
||||
Row row = new Row();
|
||||
for (int index = 0; index < result.columns().size(); index++) {
|
||||
var column = result.columns().get(index);
|
||||
row.put(column.label(), sourceRow.get(column.key()));
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
private boolean isFederated(DatacenterSource source) {
|
||||
return source != null && (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|
||||
|| DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType()));
|
||||
}
|
||||
|
||||
private record FederatedWhere(String sql, List<SqlParameter> parameters) {
|
||||
}
|
||||
|
||||
private void normalizePage(DatacenterQueryRequest request) {
|
||||
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
|
||||
request.setPageNumber(1L);
|
||||
@@ -374,6 +922,9 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
if (request.getPageSize() == null || request.getPageSize() < 1L) {
|
||||
throw new BusinessException("pageSize 必须大于 0");
|
||||
}
|
||||
if (request.getPageSize() > 500L) {
|
||||
throw new BusinessException("单页最多返回 500 行");
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
|
||||
@@ -400,7 +951,7 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
return queryTable;
|
||||
}
|
||||
|
||||
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
|
||||
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request) {
|
||||
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
fieldMap.put(field.getFieldName(), field);
|
||||
@@ -408,22 +959,25 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
||||
for (String column : request.getSelectedColumns()) {
|
||||
DatacenterTableField field = fieldMap.get(column);
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
if (field == null || !isQueryable(field)) {
|
||||
throw new BusinessException("字段不可查询: " + column);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
request.setSelectedColumns(
|
||||
table.getFields().stream()
|
||||
.filter(field -> isEnabled(field.getQueryable()))
|
||||
.filter(this::isQueryable)
|
||||
.map(DatacenterTableField::getFieldName)
|
||||
.toList()
|
||||
);
|
||||
}
|
||||
if (CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
||||
throw new BusinessException("当前数据集没有可查询字段");
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
request.getFilters().forEach(filter -> {
|
||||
DatacenterTableField field = fieldMap.get(filter.getColumn());
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
if (field == null || !isQueryable(field)) {
|
||||
throw new BusinessException("字段不可过滤: " + filter.getColumn());
|
||||
}
|
||||
});
|
||||
@@ -431,18 +985,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||
request.getSorts().forEach(sort -> {
|
||||
DatacenterTableField field = fieldMap.get(sort.getColumn());
|
||||
if (field == null || !isEnabled(field.getSortable())) {
|
||||
if (field == null || !isQueryable(field) || !isEnabled(field.getSortable())) {
|
||||
throw new BusinessException("字段不可排序: " + sort.getColumn());
|
||||
}
|
||||
});
|
||||
}
|
||||
if (request.getWhere() != null && !request.getWhere().isBlank()) {
|
||||
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|
||||
|| "MYSQL".equals(source.getSourceType())
|
||||
|| "POSTGRESQL".equals(source.getSourceType());
|
||||
if (!allowLegacyWhere) {
|
||||
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
|
||||
}
|
||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -450,6 +999,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
return value == null || value == 1;
|
||||
}
|
||||
|
||||
private boolean isQueryable(DatacenterTableField field) {
|
||||
return field != null
|
||||
&& isEnabled(field.getQueryable())
|
||||
&& DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus())
|
||||
&& DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel());
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (!StringUtils.hasText(value)) {
|
||||
return null;
|
||||
|
||||
@@ -38,7 +38,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
||||
*/
|
||||
@Override
|
||||
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterTable table = resolveTable(datasetRef, account);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.saveRow(source, table, data, account);
|
||||
@@ -52,7 +52,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterTable table = resolveTable(datasetRef, account);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.saveRows(source, table, rows, account, Math.max(1, batchSize));
|
||||
@@ -70,7 +70,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterTable table = resolveTable(datasetRef, account);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
String payloadHash = sha256Rows(rows);
|
||||
@@ -95,17 +95,27 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
||||
*/
|
||||
@Override
|
||||
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterTable table = resolveTable(datasetRef, account);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.deleteRow(source, table, id, account);
|
||||
}
|
||||
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef, LoginAccount account) {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("缺少 tableId");
|
||||
}
|
||||
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||
if (account == null || account.getTenantId() == null) {
|
||||
throw new BusinessException("数据集写入缺少执行租户");
|
||||
}
|
||||
DatacenterTable table = registryService.getTableWithFields(datasetRef.getTableId());
|
||||
if (!account.getTenantId().equals(table.getTenantId())
|
||||
|| (datasetRef.getTenantId() != null
|
||||
&& !account.getTenantId().equals(datasetRef.getTenantId()))) {
|
||||
throw new BusinessException("数据集不属于当前租户");
|
||||
}
|
||||
datasetRef.setTenantId(account.getTenantId());
|
||||
return table;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
|
||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
||||
import com.easyagents.federation.sql.execute.FederationQueryAdmissionController;
|
||||
import com.easyagents.federation.sql.execute.FederationQueryPermit;
|
||||
import com.easyagents.federation.sql.execute.LocalFederationQueryAdmissionController;
|
||||
import com.easyagents.federation.sql.execute.QueryId;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.BooleanSupplier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 对查询执行租户、数据源与节点三级本地并发准入。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationAdmissionController
|
||||
implements FederationQueryAdmissionController {
|
||||
|
||||
private final LocalFederationQueryAdmissionController globalAdmission;
|
||||
private final int maximumConcurrentQueriesPerTenant;
|
||||
private final int maximumConcurrentQueriesPerSource;
|
||||
private final Map<String, AdmissionEntry>
|
||||
tenantAdmissions = new ConcurrentHashMap<>();
|
||||
private final Map<SourceId, AdmissionEntry>
|
||||
sourceAdmissions = new ConcurrentHashMap<>();
|
||||
private final Object lifecycleLock = new Object();
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
/**
|
||||
* 创建三级准入控制器。
|
||||
*
|
||||
* @param maximumConcurrentQueries 节点查询上限
|
||||
* @param maximumConcurrentQueriesPerTenant 单租户查询上限
|
||||
* @param maximumConcurrentQueriesPerSource 单数据源查询上限
|
||||
*/
|
||||
public DatacenterFederationAdmissionController(
|
||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries:64}")
|
||||
int maximumConcurrentQueries,
|
||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-tenant:20}")
|
||||
int maximumConcurrentQueriesPerTenant,
|
||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-source:10}")
|
||||
int maximumConcurrentQueriesPerSource) {
|
||||
if (maximumConcurrentQueries <= 0
|
||||
|| maximumConcurrentQueriesPerTenant <= 0
|
||||
|| maximumConcurrentQueriesPerSource <= 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"query admission limits must be positive");
|
||||
}
|
||||
this.globalAdmission = new LocalFederationQueryAdmissionController(
|
||||
maximumConcurrentQueries);
|
||||
this.maximumConcurrentQueriesPerTenant =
|
||||
maximumConcurrentQueriesPerTenant;
|
||||
this.maximumConcurrentQueriesPerSource =
|
||||
maximumConcurrentQueriesPerSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取三级查询许可。
|
||||
*
|
||||
* @param sourceId 数据源标识
|
||||
* @param queryId 查询标识
|
||||
* @param timeout 最大等待时间
|
||||
* @return 查询许可
|
||||
*/
|
||||
@Override
|
||||
public FederationQueryPermit acquire(
|
||||
SourceId sourceId,
|
||||
QueryId queryId,
|
||||
Duration timeout) {
|
||||
return acquire(sourceId, queryId, timeout, () -> false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按数据源、租户、节点顺序获取许可,避免热点连接池等待占满全局配额。
|
||||
*
|
||||
* @param sourceId 数据源标识
|
||||
* @param queryId 查询标识
|
||||
* @param timeout 最大等待时间
|
||||
* @param cancellationRequested 查询取消状态
|
||||
* @return 查询许可
|
||||
*/
|
||||
@Override
|
||||
public FederationQueryPermit acquire(
|
||||
SourceId sourceId,
|
||||
QueryId queryId,
|
||||
Duration timeout,
|
||||
BooleanSupplier cancellationRequested) {
|
||||
long started = System.nanoTime();
|
||||
String tenantKey = tenantKey(sourceId);
|
||||
AdmissionEntry sourceEntry;
|
||||
AdmissionEntry tenantEntry;
|
||||
synchronized (lifecycleLock) {
|
||||
ensureOpen();
|
||||
sourceEntry = retain(sourceAdmissions, sourceId,
|
||||
maximumConcurrentQueriesPerSource);
|
||||
tenantEntry = retain(tenantAdmissions, tenantKey,
|
||||
maximumConcurrentQueriesPerTenant);
|
||||
}
|
||||
FederationQueryPermit sourcePermit = null;
|
||||
FederationQueryPermit tenantPermit = null;
|
||||
FederationQueryPermit globalPermit = null;
|
||||
try {
|
||||
sourcePermit = sourceEntry.controller.acquire(
|
||||
sourceId, queryId, remaining(timeout, started),
|
||||
cancellationRequested);
|
||||
tenantPermit = tenantEntry.controller.acquire(
|
||||
sourceId, queryId, remaining(timeout, started),
|
||||
cancellationRequested);
|
||||
globalPermit = globalAdmission.acquire(
|
||||
sourceId, queryId, remaining(timeout, started),
|
||||
cancellationRequested);
|
||||
FederationQueryPermit acquiredSource = sourcePermit;
|
||||
FederationQueryPermit acquiredTenant = tenantPermit;
|
||||
FederationQueryPermit acquiredGlobal = globalPermit;
|
||||
AtomicBoolean released = new AtomicBoolean();
|
||||
return () -> {
|
||||
if (released.compareAndSet(false, true)) {
|
||||
try {
|
||||
acquiredGlobal.close();
|
||||
} finally {
|
||||
try {
|
||||
acquiredTenant.close();
|
||||
} finally {
|
||||
try {
|
||||
acquiredSource.close();
|
||||
} finally {
|
||||
releaseEntries(
|
||||
sourceId, sourceEntry,
|
||||
tenantKey, tenantEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
} catch (RuntimeException exception) {
|
||||
closeQuietly(globalPermit);
|
||||
closeQuietly(tenantPermit);
|
||||
closeQuietly(sourcePermit);
|
||||
releaseEntries(sourceId, sourceEntry, tenantKey, tenantEntry);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭全部准入控制器并唤醒等待线程。
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
synchronized (lifecycleLock) {
|
||||
if (!closed.compareAndSet(false, true)) {
|
||||
return;
|
||||
}
|
||||
globalAdmission.close();
|
||||
tenantAdmissions.values().forEach(
|
||||
entry -> entry.controller.close());
|
||||
sourceAdmissions.values().forEach(
|
||||
entry -> entry.controller.close());
|
||||
tenantAdmissions.clear();
|
||||
sourceAdmissions.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private Duration remaining(Duration timeout, long started) {
|
||||
if (timeout == null || timeout.isZero() || timeout.isNegative()) {
|
||||
return Duration.ZERO;
|
||||
}
|
||||
long remaining = timeout.toNanos() - (System.nanoTime() - started);
|
||||
return remaining <= 0L ? Duration.ZERO : Duration.ofNanos(remaining);
|
||||
}
|
||||
|
||||
private String tenantKey(SourceId sourceId) {
|
||||
String value = sourceId == null ? "" : sourceId.value();
|
||||
int marker = value.lastIndexOf("-source-");
|
||||
return marker <= 0 ? value : value.substring(0, marker);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取或创建分层准入项并登记一个活动查询或等待者。
|
||||
*
|
||||
* @param admissions 准入项映射
|
||||
* @param key 租户或数据源键
|
||||
* @param maximumConcurrentQueries 并发上限
|
||||
* @param <K> 键类型
|
||||
* @return 已登记引用的准入项
|
||||
*/
|
||||
private <K> AdmissionEntry retain(
|
||||
Map<K, AdmissionEntry> admissions,
|
||||
K key,
|
||||
int maximumConcurrentQueries) {
|
||||
AdmissionEntry entry = admissions.computeIfAbsent(
|
||||
key,
|
||||
ignored -> new AdmissionEntry(
|
||||
new LocalFederationQueryAdmissionController(
|
||||
maximumConcurrentQueries)));
|
||||
entry.references++;
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步释放数据源与租户准入项引用。
|
||||
*
|
||||
* @param sourceId 数据源标识
|
||||
* @param sourceEntry 数据源准入项
|
||||
* @param tenantKey 租户键
|
||||
* @param tenantEntry 租户准入项
|
||||
*/
|
||||
private void releaseEntries(
|
||||
SourceId sourceId,
|
||||
AdmissionEntry sourceEntry,
|
||||
String tenantKey,
|
||||
AdmissionEntry tenantEntry) {
|
||||
synchronized (lifecycleLock) {
|
||||
release(sourceAdmissions, sourceId, sourceEntry);
|
||||
release(tenantAdmissions, tenantKey, tenantEntry);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 释放一个准入项引用,并在无查询或等待者时移除并关闭该项。
|
||||
*
|
||||
* @param admissions 准入项映射
|
||||
* @param key 租户或数据源键
|
||||
* @param entry 待释放准入项
|
||||
* @param <K> 键类型
|
||||
*/
|
||||
private <K> void release(
|
||||
Map<K, AdmissionEntry> admissions,
|
||||
K key,
|
||||
AdmissionEntry entry) {
|
||||
entry.references--;
|
||||
if (entry.references < 0) {
|
||||
throw new IllegalStateException(
|
||||
"query admission reference count became negative");
|
||||
}
|
||||
if (entry.references == 0 && admissions.remove(key, entry)) {
|
||||
entry.controller.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回当前仍有查询或等待者的数据源准入项数量。
|
||||
*
|
||||
* @return 活动数据源准入项数量
|
||||
*/
|
||||
int trackedSourceAdmissionCount() {
|
||||
return sourceAdmissions.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回当前仍有查询或等待者的租户准入项数量。
|
||||
*
|
||||
* @return 活动租户准入项数量
|
||||
*/
|
||||
int trackedTenantAdmissionCount() {
|
||||
return tenantAdmissions.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回指定数据源当前活动查询与等待者的引用总数。
|
||||
*
|
||||
* @param sourceId 数据源标识
|
||||
* @return 当前引用数
|
||||
*/
|
||||
int trackedSourceReferenceCount(SourceId sourceId) {
|
||||
synchronized (lifecycleLock) {
|
||||
AdmissionEntry entry = sourceAdmissions.get(sourceId);
|
||||
return entry == null ? 0 : entry.references;
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureOpen() {
|
||||
if (closed.get()) {
|
||||
throw new FederationSqlException(
|
||||
FederationSqlErrorCode.ENGINE_CLOSED,
|
||||
"datacenter query admission controller is closed");
|
||||
}
|
||||
}
|
||||
|
||||
private void closeQuietly(FederationQueryPermit permit) {
|
||||
if (permit != null) {
|
||||
permit.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 共享公平信号量及其活动查询、等待者引用计数。
|
||||
*/
|
||||
private static final class AdmissionEntry {
|
||||
|
||||
private final LocalFederationQueryAdmissionController controller;
|
||||
private int references;
|
||||
|
||||
/**
|
||||
* 创建准入项。
|
||||
*
|
||||
* @param controller 本地公平准入控制器
|
||||
*/
|
||||
private AdmissionEntry(LocalFederationQueryAdmissionController controller) {
|
||||
this.controller = controller;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
/**
|
||||
* 发布不含凭据的 Federation Definition 变更提示。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationChangeNotifier {
|
||||
|
||||
/** Redis Pub/Sub 频道。 */
|
||||
public static final String CHANNEL = "easyflow:datacenter:federation-source-changed";
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationChangeNotifier.class);
|
||||
|
||||
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
|
||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
||||
|
||||
/**
|
||||
* 创建通知器。
|
||||
*
|
||||
* @param redisTemplateProvider 可选 Redis 模板
|
||||
* @param definitionFactory Definition 工厂
|
||||
*/
|
||||
public DatacenterFederationChangeNotifier(
|
||||
ObjectProvider<StringRedisTemplate> redisTemplateProvider,
|
||||
DatacenterFederationDefinitionFactory definitionFactory) {
|
||||
this.redisTemplateProvider = redisTemplateProvider;
|
||||
this.definitionFactory = definitionFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在当前事务提交后发布 SourceId;无 Redis 时不影响数据库权威状态。
|
||||
*
|
||||
* @param source 已提交或即将提交的数据源
|
||||
*/
|
||||
public void publishAfterCommit(DatacenterSource source) {
|
||||
String sourceId = definitionFactory.sourceId(source).value();
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override
|
||||
public void afterCommit() {
|
||||
publish(sourceId);
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
publish(sourceId);
|
||||
}
|
||||
|
||||
private void publish(String sourceId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
redisTemplate.convertAndSend(CHANNEL, sourceId);
|
||||
} catch (RuntimeException exception) {
|
||||
// Redis 仅缩短传播时间;数据库按需读取继续保证 minimumRevision 正确性。
|
||||
log.warn("Failed to publish datacenter source change hint for {}", sourceId, exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user