Compare commits
28 Commits
develop
...
9722bea701
| Author | SHA1 | Date | |
|---|---|---|---|
| 9722bea701 | |||
| 386cebf342 | |||
| c7cac61ce8 | |||
| b99e275371 | |||
| ebade41e40 | |||
| cc7f0c1a43 | |||
| c4799760cf | |||
| 6cbc330f55 | |||
| d7b0d442eb | |||
| 198d592dd6 | |||
| 7257f41eb8 | |||
| e9fc0bd810 | |||
| 9ef20119a9 | |||
| 1bd9810518 | |||
| fa174f6c16 | |||
| e40bd9dc82 | |||
| 38078741f2 | |||
| 1d3147e7bf | |||
| a771affc5d | |||
| be10eabb64 | |||
| 7e1490d5f8 | |||
| 7aed4bcc37 | |||
| 6248e2c7b8 | |||
| 4de8cc5bd0 | |||
| 71b3d3d620 | |||
| 1870ac4028 | |||
| 2a383ef3f2 | |||
| 34ff62d317 |
@@ -40,10 +40,6 @@
|
|||||||
<groupId>tech.easyflow</groupId>
|
<groupId>tech.easyflow</groupId>
|
||||||
<artifactId>easyflow-module-job</artifactId>
|
<artifactId>easyflow-module-job</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<groupId>tech.easyflow</groupId>
|
|
||||||
<artifactId>easyflow-module-dataspace</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>tech.easyflow</groupId>
|
<groupId>tech.easyflow</groupId>
|
||||||
<artifactId>easyflow-common-captcha</artifactId>
|
<artifactId>easyflow-common-captcha</artifactId>
|
||||||
|
|||||||
@@ -224,7 +224,6 @@ public class WorkflowChatController {
|
|||||||
Map<String, Object> detail = new LinkedHashMap<>();
|
Map<String, Object> detail = new LinkedHashMap<>();
|
||||||
detail.put("record", recordView);
|
detail.put("record", recordView);
|
||||||
detail.put("steps", stepViews);
|
detail.put("steps", stepViews);
|
||||||
detail.put("runtime", eventStream.runtimeView(executeId));
|
|
||||||
return Result.ok(detail);
|
return Result.ok(detail);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
package tech.easyflow.admin.controller.ai;
|
package tech.easyflow.admin.controller.ai;
|
||||||
|
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowExecutionErrorMapper;
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
import cn.dev33.satoken.stp.StpUtil;
|
import cn.dev33.satoken.stp.StpUtil;
|
||||||
import cn.hutool.core.io.IoUtil;
|
import cn.hutool.core.io.IoUtil;
|
||||||
@@ -262,7 +261,6 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
|
|||||||
if (StpUtil.isLogin()) {
|
if (StpUtil.isLogin()) {
|
||||||
variables.put(Constants.LOGIN_USER_KEY, SaTokenUtil.getLoginAccount());
|
variables.put(Constants.LOGIN_USER_KEY, SaTokenUtil.getLoginAccount());
|
||||||
}
|
}
|
||||||
WorkflowExecutionErrorMapper.installRequestProfile();
|
|
||||||
Map<String, Object> res = chainExecutor.executeNode(workflowId.toString(), nodeId, variables);
|
Map<String, Object> res = chainExecutor.executeNode(workflowId.toString(), nodeId, variables);
|
||||||
return Result.ok(res);
|
return Result.ok(res);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,11 +24,6 @@ import java.util.List;
|
|||||||
@RequestMapping("/api/v1/datacenterDataset")
|
@RequestMapping("/api/v1/datacenterDataset")
|
||||||
public class DatacenterDatasetController {
|
public class DatacenterDatasetController {
|
||||||
|
|
||||||
/** 对外 Schema 接口的默认字段页码。 */
|
|
||||||
private static final long DEFAULT_FIELD_PAGE_NUMBER = 1L;
|
|
||||||
/** 对外 Schema 接口的默认字段页大小。 */
|
|
||||||
private static final long DEFAULT_FIELD_PAGE_SIZE = 200L;
|
|
||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
private DatacenterDatasetQueryService queryService;
|
private DatacenterDatasetQueryService queryService;
|
||||||
@Resource
|
@Resource
|
||||||
@@ -37,18 +32,13 @@ public class DatacenterDatasetController {
|
|||||||
@PostMapping("/queryPage")
|
@PostMapping("/queryPage")
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
public Result<Page<Row>> queryPage(@RequestBody DatacenterQueryRequest request) {
|
public Result<Page<Row>> queryPage(@RequestBody DatacenterQueryRequest request) {
|
||||||
return Result.ok(queryService.queryPage(
|
return Result.ok(queryService.queryPage(request));
|
||||||
request, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/schema")
|
@GetMapping("/schema")
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
public Result<DatacenterSchemaResponse> schema(
|
public Result<DatacenterSchemaResponse> schema(DatasetRef datasetRef) {
|
||||||
DatasetRef datasetRef,
|
return Result.ok(queryService.getSchema(datasetRef));
|
||||||
@RequestParam(defaultValue = "1") Long fieldPageNumber,
|
|
||||||
@RequestParam(defaultValue = "200") Long fieldPageSize) {
|
|
||||||
return Result.ok(queryService.getSchema(
|
|
||||||
datasetRef, fieldPageNumber, fieldPageSize));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/managedTables")
|
@GetMapping("/managedTables")
|
||||||
@@ -73,13 +63,6 @@ public class DatacenterDatasetController {
|
|||||||
request == null ? List.of() : request.getFields(),
|
request == null ? List.of() : request.getFields(),
|
||||||
account
|
account
|
||||||
);
|
);
|
||||||
return Result.ok(queryService.getSchema(
|
return Result.ok(queryService.getSchema(registryService.resolveDatasetRef(table.getId())));
|
||||||
registryService.resolveDatasetRef(table.getId()),
|
|
||||||
request == null || request.getFieldPageNumber() == null
|
|
||||||
? DEFAULT_FIELD_PAGE_NUMBER
|
|
||||||
: request.getFieldPageNumber(),
|
|
||||||
request == null || request.getFieldPageSize() == null
|
|
||||||
? DEFAULT_FIELD_PAGE_SIZE
|
|
||||||
: request.getFieldPageSize()));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,84 +0,0 @@
|
|||||||
package tech.easyflow.admin.controller.datacenter;
|
|
||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestBody;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
import tech.easyflow.common.domain.Result;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleRequest;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlCancelRequest;
|
|
||||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
|
|
||||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据中枢管理端只读 SQL 控制台。
|
|
||||||
*/
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/api/v1/datacenterQuery")
|
|
||||||
public class DatacenterQueryController {
|
|
||||||
|
|
||||||
private final DatacenterDatasetRegistryService registryService;
|
|
||||||
private final DatacenterFederationQueryService queryService;
|
|
||||||
private final DatacenterFederationQueryCancellationService cancellationService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建查询 Controller。
|
|
||||||
*
|
|
||||||
* @param registryService 数据集注册服务
|
|
||||||
* @param queryService Federation 查询服务
|
|
||||||
* @param cancellationService 跨节点查询取消服务
|
|
||||||
*/
|
|
||||||
public DatacenterQueryController(
|
|
||||||
DatacenterDatasetRegistryService registryService,
|
|
||||||
DatacenterFederationQueryService queryService,
|
|
||||||
DatacenterFederationQueryCancellationService cancellationService) {
|
|
||||||
this.registryService = registryService;
|
|
||||||
this.queryService = queryService;
|
|
||||||
this.cancellationService = cancellationService;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 执行一条受 Calcite 与业务 Policy 校验的只读 SQL。
|
|
||||||
*
|
|
||||||
* @param request 查询请求
|
|
||||||
* @return 有界查询结果
|
|
||||||
*/
|
|
||||||
@PostMapping("/execute")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
|
||||||
public Result<DatacenterSqlConsoleResult> execute(
|
|
||||||
@RequestBody DatacenterSqlConsoleRequest request) {
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
|
||||||
DatacenterSource source = registryService.getSourceRequired(
|
|
||||||
request == null ? null : request.sourceId());
|
|
||||||
return Result.ok(queryService.execute(
|
|
||||||
source,
|
|
||||||
request == null ? null : request.sql(),
|
|
||||||
java.util.List.of(),
|
|
||||||
request == null ? null : request.maxRows(),
|
|
||||||
account,
|
|
||||||
"MANUAL",
|
|
||||||
account == null || account.getId() == null
|
|
||||||
? null : account.getId().toString(),
|
|
||||||
request == null ? null : request.queryId()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 取消当前租户在任一节点执行的 SQL 查询。
|
|
||||||
*
|
|
||||||
* @param request 取消请求
|
|
||||||
* @return 是否在本地或集群中接受取消提示
|
|
||||||
*/
|
|
||||||
@PostMapping("/cancel")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
|
||||||
public Result<Boolean> cancel(@RequestBody DatacenterSqlCancelRequest request) {
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
|
||||||
return Result.ok(cancellationService.cancel(
|
|
||||||
request == null ? null : request.queryId(), account));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,16 +8,10 @@ import tech.easyflow.common.entity.LoginAccount;
|
|||||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest;
|
import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateMetadataRequest;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateCatalogRequest;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||||
|
|
||||||
@@ -25,9 +19,6 @@ import javax.annotation.Resource;
|
|||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据源绑定、生命周期与元数据浏览接口。
|
|
||||||
*/
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/api/v1/datacenterSource")
|
@RequestMapping("/api/v1/datacenterSource")
|
||||||
public class DatacenterSourceController {
|
public class DatacenterSourceController {
|
||||||
@@ -35,100 +26,23 @@ public class DatacenterSourceController {
|
|||||||
@Resource
|
@Resource
|
||||||
private DatacenterSourceService sourceService;
|
private DatacenterSourceService sourceService;
|
||||||
|
|
||||||
@PostMapping("/draft")
|
@PostMapping("/testConnection")
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<DatacenterSourceView> saveDraft(@RequestBody DatacenterSourceDraftRequest request) {
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
|
||||||
return Result.ok(sourceService.saveDraft(request, account));
|
|
||||||
}
|
|
||||||
|
|
||||||
@PostMapping("/{sourceId}/probe")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
public Result<DatacenterConnectionTestResult> probe(@PathVariable BigInteger sourceId) {
|
public Result<DatacenterConnectionTestResult> testConnection(@RequestBody DatacenterSource source) {
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
return Result.ok(sourceService.probe(sourceId, account));
|
return Result.ok(sourceService.testConnection(source, account));
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/activate")
|
@PostMapping("/save")
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
public Result<DatacenterSourceView> activate(@RequestBody DatacenterSourceActivateRequest request) {
|
public Result<DatacenterSource> save(@RequestBody DatacenterSource source) {
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
return Result.ok(sourceService.activate(request, account));
|
return Result.ok(sourceService.saveSource(source, account));
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 探测活动数据源的未发布候选配置。
|
|
||||||
*
|
|
||||||
* @param request 候选连接配置
|
|
||||||
* @return 连接探测结果
|
|
||||||
*/
|
|
||||||
@PostMapping("/candidate/probe")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<DatacenterConnectionTestResult> probeCandidate(
|
|
||||||
@RequestBody DatacenterSourceDraftRequest request) {
|
|
||||||
return Result.ok(sourceService.probeCandidate(
|
|
||||||
request, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 浏览活动数据源候选配置可访问的命名空间。
|
|
||||||
*
|
|
||||||
* @param request 候选连接配置
|
|
||||||
* @return 命名空间列表
|
|
||||||
*/
|
|
||||||
@PostMapping("/candidate/catalogs")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<List<DatacenterCatalogMeta>> candidateCatalogs(
|
|
||||||
@RequestBody DatacenterSourceDraftRequest request) {
|
|
||||||
return Result.ok(sourceService.listCandidateCatalogs(
|
|
||||||
request, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分页浏览候选配置可访问的命名空间。
|
|
||||||
*
|
|
||||||
* @param request 候选配置和分页条件
|
|
||||||
* @return 有界命名空间列表
|
|
||||||
*/
|
|
||||||
@PostMapping("/candidate/catalogs/page")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<DatacenterMetadataPage<DatacenterCatalogMeta>> candidateCatalogsPage(
|
|
||||||
@RequestBody DatacenterSourceCandidateCatalogRequest request) {
|
|
||||||
return Result.ok(sourceService.listCandidateCatalogsPage(
|
|
||||||
request, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分页浏览活动数据源候选配置可访问的表。
|
|
||||||
*
|
|
||||||
* @param request 候选配置与分页条件
|
|
||||||
* @return 有界表列表
|
|
||||||
*/
|
|
||||||
@PostMapping("/candidate/tables")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<DatacenterMetadataPage<DatacenterTable>> candidateTables(
|
|
||||||
@RequestBody DatacenterSourceCandidateMetadataRequest request) {
|
|
||||||
return Result.ok(sourceService.listCandidateTables(
|
|
||||||
request, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 原地发布活动数据源的新连接配置和纳管范围。
|
|
||||||
*
|
|
||||||
* @param request 重配置请求
|
|
||||||
* @return 发布后的数据源视图
|
|
||||||
*/
|
|
||||||
@PostMapping("/reconfigure")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<DatacenterSourceView> reconfigure(
|
|
||||||
@RequestBody DatacenterSourceReconfigureRequest request) {
|
|
||||||
return Result.ok(sourceService.reconfigure(
|
|
||||||
request, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/page")
|
@GetMapping("/page")
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
public Result<Page<DatacenterSourceView>> page(Long pageNumber, Long pageSize) {
|
public Result<Page<DatacenterSource>> page(Long pageNumber, Long pageSize) {
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
return Result.ok(sourceService.pageSources(pageNumber, pageSize, account));
|
return Result.ok(sourceService.pageSources(pageNumber, pageSize, account));
|
||||||
}
|
}
|
||||||
@@ -140,53 +54,19 @@ public class DatacenterSourceController {
|
|||||||
return Result.ok(sourceService.listCatalogs(sourceId, account));
|
return Result.ok(sourceService.listCatalogs(sourceId, account));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 分页浏览当前数据源的命名空间。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源 ID
|
|
||||||
* @param keyword 名称搜索词
|
|
||||||
* @param pageNumber 页码
|
|
||||||
* @param pageSize 每页大小
|
|
||||||
* @return 有界命名空间列表
|
|
||||||
*/
|
|
||||||
@GetMapping("/catalogs/page")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
|
||||||
public Result<DatacenterMetadataPage<DatacenterCatalogMeta>> catalogsPage(
|
|
||||||
BigInteger sourceId,
|
|
||||||
String keyword,
|
|
||||||
Long pageNumber,
|
|
||||||
Long pageSize) {
|
|
||||||
return Result.ok(sourceService.listCatalogsPage(
|
|
||||||
sourceId, keyword, pageNumber, pageSize,
|
|
||||||
SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/tables")
|
@GetMapping("/tables")
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
public Result<DatacenterMetadataPage<DatacenterTable>> tables(
|
public Result<List<DatacenterTable>> tables(BigInteger sourceId, String catalogName) {
|
||||||
BigInteger sourceId,
|
|
||||||
String catalogName,
|
|
||||||
String keyword,
|
|
||||||
Long pageNumber,
|
|
||||||
Long pageSize) {
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
return Result.ok(sourceService.listTables(
|
return Result.ok(sourceService.listTables(sourceId, catalogName, account));
|
||||||
sourceId, catalogName, keyword, pageNumber, pageSize, account));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/tableDetail")
|
@GetMapping("/tableDetail")
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
public Result<DatacenterTableDetailMeta> tableDetail(
|
public Result<DatacenterTableDetailMeta> tableDetail(BigInteger sourceId, String catalogName, String tableName,
|
||||||
BigInteger sourceId,
|
@RequestParam(defaultValue = "false") boolean register) {
|
||||||
String catalogName,
|
|
||||||
String tableName,
|
|
||||||
@RequestParam(defaultValue = "false") boolean register,
|
|
||||||
Long fieldPageNumber,
|
|
||||||
Long fieldPageSize) {
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
return Result.ok(sourceService.getTableDetail(
|
return Result.ok(sourceService.getTableDetail(sourceId, catalogName, tableName, register, account));
|
||||||
sourceId, catalogName, tableName, register,
|
|
||||||
fieldPageNumber, fieldPageSize, account));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/registerBatch")
|
@PostMapping("/registerBatch")
|
||||||
@@ -203,44 +83,4 @@ public class DatacenterSourceController {
|
|||||||
sourceService.removeSource(request == null ? null : request.getSourceId(), account);
|
sourceService.removeSource(request == null ? null : request.getSourceId(), account);
|
||||||
return Result.ok();
|
return Result.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 停用活动数据源。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源 ID
|
|
||||||
* @return 停用后的数据源视图
|
|
||||||
*/
|
|
||||||
@PostMapping("/{sourceId}/disable")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<DatacenterSourceView> disable(@PathVariable BigInteger sourceId) {
|
|
||||||
return Result.ok(sourceService.disable(
|
|
||||||
sourceId, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 重新启用已停用数据源。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源 ID
|
|
||||||
* @return 启用后的数据源视图
|
|
||||||
*/
|
|
||||||
@PostMapping("/{sourceId}/enable")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<DatacenterSourceView> enable(@PathVariable BigInteger sourceId) {
|
|
||||||
return Result.ok(sourceService.enable(
|
|
||||||
sourceId, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 刷新已纳管对象的 JDBC 元数据观测状态。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源 ID
|
|
||||||
* @return 刷新后的数据源视图
|
|
||||||
*/
|
|
||||||
@PostMapping("/{sourceId}/metadata/refresh")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
|
||||||
public Result<DatacenterSourceView> refreshMetadata(
|
|
||||||
@PathVariable BigInteger sourceId) {
|
|
||||||
return Result.ok(sourceService.refreshMetadata(
|
|
||||||
sourceId, SaTokenUtil.getLoginAccount()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,161 +0,0 @@
|
|||||||
package tech.easyflow.admin.controller.dataspace;
|
|
||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.List;
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
import tech.easyflow.common.domain.Result;
|
|
||||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
import tech.easyflow.dataspace.model.ConnectionDefinition;
|
|
||||||
import tech.easyflow.dataspace.model.ConnectionView;
|
|
||||||
import tech.easyflow.dataspace.model.ObjectView;
|
|
||||||
import tech.easyflow.dataspace.provider.DataspaceProbe;
|
|
||||||
import tech.easyflow.dataspace.service.DataspaceConnectionService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据空间物理连接管理端 API。
|
|
||||||
*/
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/api/v1/dataspaceConnection")
|
|
||||||
public class DataspaceConnectionController {
|
|
||||||
|
|
||||||
private final DataspaceConnectionService connectionService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建连接控制器。
|
|
||||||
*
|
|
||||||
* @param connectionService 连接服务
|
|
||||||
*/
|
|
||||||
public DataspaceConnectionController(DataspaceConnectionService connectionService) {
|
|
||||||
this.connectionService = connectionService;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 查询当前租户连接列表。
|
|
||||||
*
|
|
||||||
* @param keyword 搜索关键词
|
|
||||||
* @return 连接列表
|
|
||||||
*/
|
|
||||||
@GetMapping("/list")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceConnection/query")
|
|
||||||
public Result<List<ConnectionView>> list(String keyword) {
|
|
||||||
return Result.ok(connectionService.list(keyword));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取连接详情。
|
|
||||||
*
|
|
||||||
* @param id 连接 ID
|
|
||||||
* @return 连接详情
|
|
||||||
*/
|
|
||||||
@GetMapping("/detail")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceConnection/query")
|
|
||||||
public Result<ConnectionView> detail(BigInteger id) {
|
|
||||||
return Result.ok(connectionService.detail(id));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 测试候选或已保存连接。
|
|
||||||
*
|
|
||||||
* @param definition 连接定义
|
|
||||||
* @return 测试结果
|
|
||||||
*/
|
|
||||||
@PostMapping("/test")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceConnection/test")
|
|
||||||
public Result<DataspaceProbe> test(
|
|
||||||
@JsonBody(required = true, skipConvertError = false) ConnectionDefinition definition) {
|
|
||||||
return Result.ok(connectionService.test(definition));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建或更新连接。
|
|
||||||
*
|
|
||||||
* @param definition 连接定义
|
|
||||||
* @return 保存后的连接
|
|
||||||
*/
|
|
||||||
@PostMapping("/save")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceConnection/save")
|
|
||||||
public Result<ConnectionView> save(
|
|
||||||
@JsonBody(required = true, skipConvertError = false) ConnectionDefinition definition) {
|
|
||||||
return Result.ok(connectionService.save(definition));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 启用或禁用连接。
|
|
||||||
*
|
|
||||||
* @param request 状态变更请求
|
|
||||||
* @return 变更后的连接
|
|
||||||
*/
|
|
||||||
@PostMapping("/status")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceConnection/save")
|
|
||||||
public Result<ConnectionView> status(
|
|
||||||
@JsonBody(required = true, skipConvertError = false) StatusRequest request) {
|
|
||||||
if (request == null || request.enabled() == null) {
|
|
||||||
throw new BusinessException("连接状态不能为空");
|
|
||||||
}
|
|
||||||
return Result.ok(connectionService.setEnabled(request.id(), request.enabled()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 查询当前连接的对象树数据。
|
|
||||||
*
|
|
||||||
* @param connectionId 连接 ID
|
|
||||||
* @param keyword Schema 或表名关键词
|
|
||||||
* @return 对象列表
|
|
||||||
*/
|
|
||||||
@GetMapping("/objects")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceConnection/query")
|
|
||||||
public Result<List<ObjectView>> objects(BigInteger connectionId, String keyword) {
|
|
||||||
return Result.ok(connectionService.objects(connectionId, keyword));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 刷新连接元数据。
|
|
||||||
*
|
|
||||||
* @param request 刷新请求
|
|
||||||
* @return 新 revision 对象列表
|
|
||||||
*/
|
|
||||||
@PostMapping("/refreshMetadata")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceConnection/metadata")
|
|
||||||
public Result<List<ObjectView>> refreshMetadata(
|
|
||||||
@JsonBody(required = true, skipConvertError = false) RefreshRequest request) {
|
|
||||||
return Result.ok(connectionService.refreshMetadata(
|
|
||||||
request.connectionId(), request.expectedRevision()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 删除未被引用的连接。
|
|
||||||
*
|
|
||||||
* @param id 连接 ID
|
|
||||||
* @return 成功结果
|
|
||||||
*/
|
|
||||||
@PostMapping("/remove")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceConnection/remove")
|
|
||||||
public Result<Void> remove(
|
|
||||||
@JsonBody(value = "id", required = true, skipConvertError = false) BigInteger id) {
|
|
||||||
connectionService.remove(id);
|
|
||||||
return Result.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 元数据刷新请求。
|
|
||||||
*
|
|
||||||
* @param connectionId 连接 ID
|
|
||||||
* @param expectedRevision 期望 revision
|
|
||||||
*/
|
|
||||||
public record RefreshRequest(BigInteger connectionId, long expectedRevision) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 连接状态变更请求。
|
|
||||||
*
|
|
||||||
* @param id 连接 ID
|
|
||||||
* @param enabled 是否启用
|
|
||||||
*/
|
|
||||||
public record StatusRequest(BigInteger id, Boolean enabled) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
package tech.easyflow.admin.controller.dataspace;
|
|
||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.List;
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestBody;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
import tech.easyflow.common.domain.Result;
|
|
||||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceDefinition;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceSummary;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceView;
|
|
||||||
import tech.easyflow.dataspace.service.DataspaceService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 虚拟数据空间管理端 API。
|
|
||||||
*/
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/api/v1/dataspace")
|
|
||||||
public class DataspaceController {
|
|
||||||
|
|
||||||
private final DataspaceService dataspaceService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建数据空间控制器。
|
|
||||||
*
|
|
||||||
* @param dataspaceService 数据空间服务
|
|
||||||
*/
|
|
||||||
public DataspaceController(DataspaceService dataspaceService) {
|
|
||||||
this.dataspaceService = dataspaceService;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 查询数据空间列表。
|
|
||||||
*
|
|
||||||
* @param keyword 搜索关键词
|
|
||||||
* @return 数据空间摘要
|
|
||||||
*/
|
|
||||||
@GetMapping("/list")
|
|
||||||
@SaCheckPermission("/api/v1/dataspace/query")
|
|
||||||
public Result<List<DataspaceSummary>> list(String keyword) {
|
|
||||||
return Result.ok(dataspaceService.list(keyword));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取数据空间当前 revision 详情。
|
|
||||||
*
|
|
||||||
* @param id 数据空间 ID
|
|
||||||
* @return 数据空间详情
|
|
||||||
*/
|
|
||||||
@GetMapping("/detail")
|
|
||||||
@SaCheckPermission("/api/v1/dataspace/detail")
|
|
||||||
public Result<DataspaceView> detail(BigInteger id) {
|
|
||||||
return Result.ok(dataspaceService.detail(id));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 保存数据空间并生成新 revision。
|
|
||||||
*
|
|
||||||
* @param definition 数据空间定义
|
|
||||||
* @return 保存后的详情
|
|
||||||
*/
|
|
||||||
@PostMapping("/save")
|
|
||||||
@SaCheckPermission("/api/v1/dataspace/save")
|
|
||||||
public Result<DataspaceView> save(
|
|
||||||
@RequestBody DataspaceDefinition definition) {
|
|
||||||
return Result.ok(dataspaceService.save(definition));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 启用或禁用数据空间。
|
|
||||||
*
|
|
||||||
* @param request 状态变更请求
|
|
||||||
* @return 成功结果
|
|
||||||
*/
|
|
||||||
@PostMapping("/status")
|
|
||||||
@SaCheckPermission("/api/v1/dataspace/save")
|
|
||||||
public Result<Void> status(
|
|
||||||
@JsonBody(required = true, skipConvertError = false) StatusRequest request) {
|
|
||||||
if (request == null || request.enabled() == null) {
|
|
||||||
throw new BusinessException("数据空间状态不能为空");
|
|
||||||
}
|
|
||||||
dataspaceService.setEnabled(request.id(), request.enabled());
|
|
||||||
return Result.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 逻辑删除数据空间。
|
|
||||||
*
|
|
||||||
* @param id 数据空间 ID
|
|
||||||
* @return 成功结果
|
|
||||||
*/
|
|
||||||
@PostMapping("/remove")
|
|
||||||
@SaCheckPermission("/api/v1/dataspace/remove")
|
|
||||||
public Result<Void> remove(
|
|
||||||
@JsonBody(value = "id", required = true, skipConvertError = false) BigInteger id) {
|
|
||||||
dataspaceService.remove(id);
|
|
||||||
return Result.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据空间状态变更请求。
|
|
||||||
*
|
|
||||||
* @param id 数据空间 ID
|
|
||||||
* @param enabled 是否启用
|
|
||||||
*/
|
|
||||||
public record StatusRequest(BigInteger id, Boolean enabled) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
package tech.easyflow.admin.controller.dataspace;
|
|
||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
import tech.easyflow.common.domain.Result;
|
|
||||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceExplainResult;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceQueryRequest;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceQueryResult;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceSqlCompletionRequest;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceSqlCompletionResult;
|
|
||||||
import tech.easyflow.dataspace.service.DataspaceQueryService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据空间 SQL 工作台 Query、Explain、Complete 与 Cancel API。
|
|
||||||
*/
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/api/v1/dataspaceSql")
|
|
||||||
public class DataspaceSqlController {
|
|
||||||
|
|
||||||
private final DataspaceQueryService queryService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建 SQL 控制器。
|
|
||||||
*
|
|
||||||
* @param queryService 查询服务
|
|
||||||
*/
|
|
||||||
public DataspaceSqlController(DataspaceQueryService queryService) {
|
|
||||||
this.queryService = queryService;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 执行只读 SQL。
|
|
||||||
*
|
|
||||||
* @param request 查询请求
|
|
||||||
* @return 查询结果与指标
|
|
||||||
*/
|
|
||||||
@PostMapping("/query")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceSql/query")
|
|
||||||
public Result<DataspaceQueryResult> query(
|
|
||||||
@JsonBody(required = true, skipConvertError = false) DataspaceQueryRequest request) {
|
|
||||||
return Result.ok(queryService.query(request));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 显式执行非 ANALYZE Explain。
|
|
||||||
*
|
|
||||||
* @param request Explain 请求
|
|
||||||
* @return Explain 与索引信息
|
|
||||||
*/
|
|
||||||
@PostMapping("/explain")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceSql/explain")
|
|
||||||
public Result<DataspaceExplainResult> explain(
|
|
||||||
@JsonBody(required = true, skipConvertError = false) DataspaceQueryRequest request) {
|
|
||||||
return Result.ok(queryService.explain(request));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回当前数据空间内的 Calcite SQL 补全候选。
|
|
||||||
*
|
|
||||||
* @param request 补全请求
|
|
||||||
* @return 补全替换区间与候选
|
|
||||||
*/
|
|
||||||
@PostMapping("/complete")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceSql/query")
|
|
||||||
public Result<DataspaceSqlCompletionResult> complete(
|
|
||||||
@JsonBody(required = true, skipConvertError = false)
|
|
||||||
DataspaceSqlCompletionRequest request) {
|
|
||||||
return Result.ok(queryService.complete(request));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 尝试取消当前节点查询。
|
|
||||||
*
|
|
||||||
* @param queryId 查询 ID
|
|
||||||
* @return 是否找到并发起取消
|
|
||||||
*/
|
|
||||||
@PostMapping("/cancel")
|
|
||||||
@SaCheckPermission("/api/v1/dataspaceSql/query")
|
|
||||||
public Result<Boolean> cancel(
|
|
||||||
@JsonBody(value = "queryId", required = true, skipConvertError = false) String queryId) {
|
|
||||||
return Result.ok(queryService.cancel(queryId));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,31 +1,26 @@
|
|||||||
package tech.easyflow.admin.controller.job;
|
package tech.easyflow.admin.controller.job;
|
||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
|
import cn.hutool.core.date.DateUtil;
|
||||||
import com.easyagents.flow.core.chain.Parameter;
|
import com.easyagents.flow.core.chain.Parameter;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.quartz.CronExpression;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
import org.springframework.transaction.annotation.Propagation;
|
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.enums.PublishStatus;
|
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
|
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
|
||||||
import tech.easyflow.admin.model.SysJobWorkflowOptionView;
|
import tech.easyflow.admin.model.SysJobWorkflowOptionView;
|
||||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||||
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
|
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
import tech.easyflow.common.web.controller.BaseCurdController;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
|
||||||
import tech.easyflow.job.entity.SysJob;
|
import tech.easyflow.job.entity.SysJob;
|
||||||
import tech.easyflow.job.job.JobConstant;
|
import tech.easyflow.job.job.JobConstant;
|
||||||
import tech.easyflow.job.service.SysJobService;
|
import tech.easyflow.job.service.SysJobService;
|
||||||
@@ -37,8 +32,7 @@ import tech.easyflow.system.service.ResourceAccessService;
|
|||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.time.ZoneId;
|
import java.util.ArrayList;
|
||||||
import java.time.format.DateTimeFormatter;
|
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -67,9 +61,6 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
/** 工作流运行参数解析器。 */
|
/** 工作流运行参数解析器。 */
|
||||||
private final WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
private final WorkflowRunningParameterResolver workflowRunningParameterResolver;
|
||||||
|
|
||||||
/** 与调度计算一致的 Cron 预览格式化器。 */
|
|
||||||
private final DateTimeFormatter jobTimeFormatter;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建定时任务控制器。
|
* 创建定时任务控制器。
|
||||||
*
|
*
|
||||||
@@ -78,21 +69,17 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
* @param workflowUsageAuthorizationService 工作流使用权限校验服务
|
* @param workflowUsageAuthorizationService 工作流使用权限校验服务
|
||||||
* @param resourceAccessService 资源访问控制服务
|
* @param resourceAccessService 资源访问控制服务
|
||||||
* @param workflowRunningParameterResolver 工作流运行参数解析器
|
* @param workflowRunningParameterResolver 工作流运行参数解析器
|
||||||
* @param jobTimezone 定时任务业务时区
|
|
||||||
*/
|
*/
|
||||||
public SysJobController(SysJobService service,
|
public SysJobController(SysJobService service,
|
||||||
WorkflowService workflowService,
|
WorkflowService workflowService,
|
||||||
WorkflowUsageAuthorizationService workflowUsageAuthorizationService,
|
WorkflowUsageAuthorizationService workflowUsageAuthorizationService,
|
||||||
ResourceAccessService resourceAccessService,
|
ResourceAccessService resourceAccessService,
|
||||||
WorkflowRunningParameterResolver workflowRunningParameterResolver,
|
WorkflowRunningParameterResolver workflowRunningParameterResolver) {
|
||||||
@Value("${easyflow.job.timezone:Asia/Shanghai}") String jobTimezone) {
|
|
||||||
super(service);
|
super(service);
|
||||||
this.workflowService = workflowService;
|
this.workflowService = workflowService;
|
||||||
this.workflowUsageAuthorizationService = workflowUsageAuthorizationService;
|
this.workflowUsageAuthorizationService = workflowUsageAuthorizationService;
|
||||||
this.resourceAccessService = resourceAccessService;
|
this.resourceAccessService = resourceAccessService;
|
||||||
this.workflowRunningParameterResolver = workflowRunningParameterResolver;
|
this.workflowRunningParameterResolver = workflowRunningParameterResolver;
|
||||||
this.jobTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
|
|
||||||
.withZone(ZoneId.of(jobTimezone));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -124,43 +111,18 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
return Result.ok();
|
return Result.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/trigger")
|
|
||||||
@SaCheckPermission("/api/v1/sysJob/save")
|
|
||||||
@LogRecord("立即执行定时任务")
|
|
||||||
public Result<String> trigger(BigInteger id) {
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
|
||||||
SysJob job = requireExistingJob(id);
|
|
||||||
validateWorkflowReference(job, account);
|
|
||||||
return Result.ok(service.triggerNow(id));
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/getNextTimes")
|
@GetMapping("/getNextTimes")
|
||||||
@SaCheckPermission("/api/v1/sysJob/save")
|
@SaCheckPermission("/api/v1/sysJob/save")
|
||||||
public Result<List<String>> getNextTimes(String cronExpression) {
|
public Result<List<String>> getNextTimes(String cronExpression) throws Exception{
|
||||||
return Result.ok(service.nextFireTimes(cronExpression, 5).stream()
|
CronExpression ex = new CronExpression(cronExpression);
|
||||||
.map(Date::toInstant)
|
List<String> times = new ArrayList<>();
|
||||||
.map(jobTimeFormatter::format)
|
Date date = new Date();
|
||||||
.toList());
|
for (int i = 0; i < 5; i++) {
|
||||||
}
|
Date next = ex.getNextValidTimeAfter(date);
|
||||||
|
times.add(DateUtil.formatDateTime(next));
|
||||||
@Override
|
date = next;
|
||||||
@PostMapping("remove")
|
|
||||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
|
||||||
public Result<?> remove(@JsonBody(value = "id", required = true) Serializable id) {
|
|
||||||
service.deleteJob(List.of(id));
|
|
||||||
return Result.ok(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
@PostMapping("removeBatch")
|
|
||||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
|
||||||
public Result<?> removeBatch(
|
|
||||||
@JsonBody(value = "ids", required = true) Collection<Serializable> ids) {
|
|
||||||
if (ids == null || ids.isEmpty()) {
|
|
||||||
return Result.fail("id不能为空");
|
|
||||||
}
|
}
|
||||||
service.deleteJob(ids);
|
return Result.ok(times);
|
||||||
return Result.ok(true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -174,19 +136,15 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
List<SysJobWorkflowOptionView> options = workflowService.list(QueryWrapper.create()
|
List<SysJobWorkflowOptionView> options = workflowService.list(QueryWrapper.create()
|
||||||
.eq(Workflow::getTenantId, account.getTenantId())
|
.eq(Workflow::getTenantId, account.getTenantId())
|
||||||
.eq(Workflow::getPublishStatus, PublishStatus.PUBLISHED.getCode())
|
.eq(Workflow::getStatus, EnumDataStatus.AVAILABLE.getCode())
|
||||||
.orderBy(Workflow::getModified, false))
|
.orderBy(Workflow::getModified, false))
|
||||||
.stream()
|
.stream()
|
||||||
.filter(workflow -> Objects.equals(workflow.getTenantId(), account.getTenantId()))
|
.filter(workflow -> Objects.equals(workflow.getTenantId(), account.getTenantId()))
|
||||||
.filter(workflow -> workflow.getPublishedSnapshotJson() != null
|
|
||||||
&& !workflow.getPublishedSnapshotJson().isEmpty())
|
|
||||||
.filter(workflow -> resourceAccessService.canAccess(
|
.filter(workflow -> resourceAccessService.canAccess(
|
||||||
account,
|
account,
|
||||||
CategoryResourceType.WORKFLOW,
|
CategoryResourceType.WORKFLOW,
|
||||||
workflow,
|
workflow,
|
||||||
ResourceAction.USE))
|
ResourceAction.USE))
|
||||||
.map(workflowService::toPublishedView)
|
|
||||||
.filter(Objects::nonNull)
|
|
||||||
.map(workflow -> new SysJobWorkflowOptionView(
|
.map(workflow -> new SysJobWorkflowOptionView(
|
||||||
workflow.getId(),
|
workflow.getId(),
|
||||||
workflow.getTitle(),
|
workflow.getTitle(),
|
||||||
@@ -208,7 +166,7 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||||
id,
|
id,
|
||||||
SaTokenUtil.getLoginAccount(),
|
SaTokenUtil.getLoginAccount(),
|
||||||
"工作流不存在、未发布或无权运行");
|
"工作流不存在、已禁用或无权运行");
|
||||||
Map<String, Object> result = workflowRunningParameterResolver.buildRunningParametersView(workflow);
|
Map<String, Object> result = workflowRunningParameterResolver.buildRunningParametersView(workflow);
|
||||||
if (result == null) {
|
if (result == null) {
|
||||||
throw new BusinessException("工作流参数配置无效,请检查工作流后重试");
|
throw new BusinessException("工作流参数配置无效,请检查工作流后重试");
|
||||||
@@ -224,9 +182,6 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
||||||
SysJob effectiveEntity = entity;
|
SysJob effectiveEntity = entity;
|
||||||
if (isSave) {
|
if (isSave) {
|
||||||
// 新任务固定从 STOP 和第 0 代开始,禁止请求绕过启动协议。
|
|
||||||
entity.setStatus(EnumJobStatus.STOP.getCode());
|
|
||||||
entity.setScheduleGeneration(0L);
|
|
||||||
commonFiled(entity,loginUser.getId(),loginUser.getTenantId(), loginUser.getDeptId());
|
commonFiled(entity,loginUser.getId(),loginUser.getTenantId(), loginUser.getDeptId());
|
||||||
} else {
|
} else {
|
||||||
SysJob existing = requireExistingJob(entity.getId());
|
SysJob existing = requireExistingJob(entity.getId());
|
||||||
@@ -236,25 +191,9 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
entity.setModifiedBy(loginUser.getId());
|
entity.setModifiedBy(loginUser.getId());
|
||||||
}
|
}
|
||||||
validateWorkflowReference(effectiveEntity, loginUser);
|
validateWorkflowReference(effectiveEntity, loginUser);
|
||||||
validateCronExpression(effectiveEntity.getCronExpression());
|
|
||||||
validateMisfirePolicy(effectiveEntity.getMisfirePolicy());
|
|
||||||
return super.onSaveOrUpdateBefore(entity, isSave);
|
return super.onSaveOrUpdateBefore(entity, isSave);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void onSaveOrUpdateAfter(SysJob entity, boolean isSave) {
|
|
||||||
service.syncJob(entity.getId());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
@PostMapping("update")
|
|
||||||
public Result<?> update(@JsonBody SysJob entity) {
|
|
||||||
Result<?> result = onSaveOrUpdateBefore(entity, false);
|
|
||||||
if (result != null) return result;
|
|
||||||
service.updateJobDefinition(entity);
|
|
||||||
return Result.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 校验工作流类型任务引用的工作流可被当前用户运行。
|
* 校验工作流类型任务引用的工作流可被当前用户运行。
|
||||||
*
|
*
|
||||||
@@ -271,7 +210,7 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||||
workflowId,
|
workflowId,
|
||||||
account,
|
account,
|
||||||
"工作流不存在、未发布或无权运行");
|
"工作流不存在、已禁用或无权运行");
|
||||||
validateRequiredWorkflowParams(entity, workflow);
|
validateRequiredWorkflowParams(entity, workflow);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -304,8 +243,6 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
entity.setDeptId(existing.getDeptId());
|
entity.setDeptId(existing.getDeptId());
|
||||||
entity.setCreated(existing.getCreated());
|
entity.setCreated(existing.getCreated());
|
||||||
entity.setCreatedBy(existing.getCreatedBy());
|
entity.setCreatedBy(existing.getCreatedBy());
|
||||||
entity.setStatus(existing.getStatus());
|
|
||||||
entity.setScheduleGeneration(existing.getScheduleGeneration());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -323,30 +260,9 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
effective.setJobParams(entity.getJobParams() == null
|
effective.setJobParams(entity.getJobParams() == null
|
||||||
? existing.getJobParams()
|
? existing.getJobParams()
|
||||||
: entity.getJobParams());
|
: entity.getJobParams());
|
||||||
effective.setCronExpression(entity.getCronExpression() == null
|
|
||||||
? existing.getCronExpression()
|
|
||||||
: entity.getCronExpression());
|
|
||||||
effective.setMisfirePolicy(entity.getMisfirePolicy() == null
|
|
||||||
? existing.getMisfirePolicy()
|
|
||||||
: entity.getMisfirePolicy());
|
|
||||||
return effective;
|
return effective;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void validateMisfirePolicy(Integer misfirePolicy) {
|
|
||||||
if (!Integer.valueOf(EnumMisfirePolicy.FIRE_ONCE_NOW.getCode()).equals(misfirePolicy)
|
|
||||||
&& !Integer.valueOf(EnumMisfirePolicy.SKIP.getCode()).equals(misfirePolicy)) {
|
|
||||||
throw new BusinessException("错过策略只支持恢复后补执行一次或跳过本次");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void validateCronExpression(String cronExpression) {
|
|
||||||
try {
|
|
||||||
service.nextFireTimes(cronExpression, 1);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
throw new BusinessException(400, 1, "Cron 表达式无效", exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 校验定时任务已填写工作流的全部必填运行参数。
|
* 校验定时任务已填写工作流的全部必填运行参数。
|
||||||
*
|
*
|
||||||
@@ -406,4 +322,9 @@ public class SysJobController extends BaseCurdController<SysJobService, SysJob>
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Result onRemoveBefore(Collection<Serializable> ids) {
|
||||||
|
service.deleteJob(ids);
|
||||||
|
return super.onRemoveBefore(ids);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,29 +1,15 @@
|
|||||||
package tech.easyflow.admin.controller.job;
|
package tech.easyflow.admin.controller.job;
|
||||||
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
import tech.easyflow.common.annotation.UsePermission;
|
import tech.easyflow.common.annotation.UsePermission;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
import tech.easyflow.common.util.StringUtil;
|
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
import tech.easyflow.common.web.controller.BaseCurdController;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
import tech.easyflow.job.entity.SysJobLog;
|
import tech.easyflow.job.entity.SysJobLog;
|
||||||
import tech.easyflow.job.service.SysJobLogService;
|
import tech.easyflow.job.service.SysJobLogService;
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.time.LocalDateTime;
|
|
||||||
import java.time.ZoneId;
|
|
||||||
import java.time.format.DateTimeFormatter;
|
|
||||||
import java.time.format.DateTimeParseException;
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.Date;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 系统任务日志 控制层。
|
* 系统任务日志 控制层。
|
||||||
*
|
*
|
||||||
@@ -34,112 +20,16 @@ import java.util.List;
|
|||||||
@RequestMapping("/api/v1/sysJobLog")
|
@RequestMapping("/api/v1/sysJobLog")
|
||||||
@UsePermission(moduleName = "/api/v1/sysJob")
|
@UsePermission(moduleName = "/api/v1/sysJob")
|
||||||
public class SysJobLogController extends BaseCurdController<SysJobLogService, SysJobLog> {
|
public class SysJobLogController extends BaseCurdController<SysJobLogService, SysJobLog> {
|
||||||
private static final long DEFAULT_PAGE_SIZE = 10L;
|
public SysJobLogController(SysJobLogService service) {
|
||||||
private static final long MAX_PAGE_SIZE = 100L;
|
|
||||||
private static final DateTimeFormatter QUERY_TIME_FORMATTER =
|
|
||||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
|
||||||
|
|
||||||
private final ZoneId jobZoneId;
|
|
||||||
|
|
||||||
public SysJobLogController(
|
|
||||||
SysJobLogService service,
|
|
||||||
@Value("${easyflow.job.timezone:Asia/Shanghai}") String jobTimezone) {
|
|
||||||
super(service);
|
super(service);
|
||||||
this.jobZoneId = ZoneId.of(jobTimezone);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 构造日志筛选条件,并追加计划触发时间和实际触发时间范围。
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
protected QueryWrapper buildQueryWrapper(HttpServletRequest request) {
|
|
||||||
QueryWrapper queryWrapper = super.buildQueryWrapper(request);
|
|
||||||
Date scheduledStart = parseQueryTime(
|
|
||||||
request.getParameter("scheduledStart"), "计划触发开始时间");
|
|
||||||
Date scheduledEnd = parseQueryTime(
|
|
||||||
request.getParameter("scheduledEnd"), "计划触发结束时间");
|
|
||||||
Date actualStart = parseQueryTime(
|
|
||||||
request.getParameter("actualStart"), "实际触发开始时间");
|
|
||||||
Date actualEnd = parseQueryTime(
|
|
||||||
request.getParameter("actualEnd"), "实际触发结束时间");
|
|
||||||
|
|
||||||
validateTimeRange(scheduledStart, scheduledEnd, "计划触发时间");
|
|
||||||
validateTimeRange(actualStart, actualEnd, "实际触发时间");
|
|
||||||
if (scheduledStart != null) {
|
|
||||||
queryWrapper.ge(SysJobLog::getScheduledFireTime, scheduledStart);
|
|
||||||
}
|
|
||||||
if (scheduledEnd != null) {
|
|
||||||
queryWrapper.le(SysJobLog::getScheduledFireTime, scheduledEnd);
|
|
||||||
}
|
|
||||||
if (actualStart != null) {
|
|
||||||
queryWrapper.ge(SysJobLog::getActualFireTime, actualStart);
|
|
||||||
}
|
|
||||||
if (actualEnd != null) {
|
|
||||||
queryWrapper.le(SysJobLog::getActualFireTime, actualEnd);
|
|
||||||
}
|
|
||||||
return queryWrapper;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 自动刷新只读取当前第一页,不执行分页总数统计。
|
|
||||||
*/
|
|
||||||
@GetMapping("refresh")
|
|
||||||
public Result<List<SysJobLog>> refresh(HttpServletRequest request, Long pageSize) {
|
|
||||||
QueryWrapper queryWrapper = buildQueryWrapper(request);
|
|
||||||
queryWrapper.orderBy(buildOrderBy(null, null, getDefaultOrderBy()));
|
|
||||||
queryWrapper.limit(resolvePageSize(pageSize));
|
|
||||||
return Result.ok(service.list(queryWrapper));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 最新计划触发记录优先,并用主键保证毫秒时间相同时顺序稳定。
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
protected String getDefaultOrderBy() {
|
|
||||||
return "scheduled_fire_time desc, id desc";
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected Page<SysJobLog> queryPage(
|
|
||||||
Page<SysJobLog> page, QueryWrapper queryWrapper) {
|
|
||||||
page.setPageSize(resolvePageSize(page.getPageSize()));
|
|
||||||
return super.queryPage(page, queryWrapper);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Result onSaveOrUpdateBefore(SysJobLog entity, boolean isSave) {
|
protected Result onSaveOrUpdateBefore(SysJobLog entity, boolean isSave) {
|
||||||
throw new IllegalStateException("定时任务执行记录由系统维护,禁止外部写入");
|
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
||||||
}
|
if (isSave) {
|
||||||
|
commonFiled(entity,loginUser.getId(),loginUser.getTenantId(), loginUser.getDeptId());
|
||||||
@Override
|
|
||||||
protected Result onRemoveBefore(Collection<Serializable> ids) {
|
|
||||||
service.requireTerminal(ids);
|
|
||||||
return super.onRemoveBefore(ids);
|
|
||||||
}
|
|
||||||
|
|
||||||
private long resolvePageSize(Long pageSize) {
|
|
||||||
if (pageSize == null || pageSize < 1) {
|
|
||||||
return DEFAULT_PAGE_SIZE;
|
|
||||||
}
|
|
||||||
return Math.min(pageSize, MAX_PAGE_SIZE);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Date parseQueryTime(String value, String fieldName) {
|
|
||||||
if (!StringUtil.hasText(value)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
LocalDateTime dateTime = LocalDateTime.parse(value, QUERY_TIME_FORMATTER);
|
|
||||||
return Date.from(dateTime.atZone(jobZoneId).toInstant());
|
|
||||||
} catch (DateTimeParseException exception) {
|
|
||||||
throw new BusinessException(
|
|
||||||
400, 400, fieldName + "格式不正确", exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void validateTimeRange(Date start, Date end, String fieldName) {
|
|
||||||
if (start != null && end != null && start.after(end)) {
|
|
||||||
throw new BusinessException(400, 400, fieldName + "范围不正确");
|
|
||||||
}
|
}
|
||||||
|
return super.onSaveOrUpdateBefore(entity, isSave);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -132,7 +132,6 @@ public record WorkflowDesignerOptionsView(
|
|||||||
* 已接入数据集安全选项。
|
* 已接入数据集安全选项。
|
||||||
*
|
*
|
||||||
* @param id 数据集 ID
|
* @param id 数据集 ID
|
||||||
* @param tenantId 租户 ID
|
|
||||||
* @param sourceId 数据源 ID
|
* @param sourceId 数据源 ID
|
||||||
* @param catalogId 目录 ID
|
* @param catalogId 目录 ID
|
||||||
* @param tableName 数据表名称
|
* @param tableName 数据表名称
|
||||||
@@ -140,7 +139,6 @@ public record WorkflowDesignerOptionsView(
|
|||||||
*/
|
*/
|
||||||
public record DatasetOption(
|
public record DatasetOption(
|
||||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger id,
|
@JsonSerialize(using = ToStringSerializer.class) BigInteger id,
|
||||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger tenantId,
|
|
||||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger sourceId,
|
@JsonSerialize(using = ToStringSerializer.class) BigInteger sourceId,
|
||||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger catalogId,
|
@JsonSerialize(using = ToStringSerializer.class) BigInteger catalogId,
|
||||||
String tableName,
|
String tableName,
|
||||||
|
|||||||
@@ -4,14 +4,9 @@ import com.alibaba.fastjson.JSON;
|
|||||||
import com.easyagents.flow.core.chain.Chain;
|
import com.easyagents.flow.core.chain.Chain;
|
||||||
import com.easyagents.flow.core.chain.ChainConsts;
|
import com.easyagents.flow.core.chain.ChainConsts;
|
||||||
import com.easyagents.flow.core.chain.ChainStatus;
|
import com.easyagents.flow.core.chain.ChainStatus;
|
||||||
import com.easyagents.flow.core.chain.ChainState;
|
|
||||||
import com.easyagents.flow.core.chain.Edge;
|
import com.easyagents.flow.core.chain.Edge;
|
||||||
import com.easyagents.flow.core.chain.Event;
|
import com.easyagents.flow.core.chain.Event;
|
||||||
import com.easyagents.flow.core.chain.Node;
|
import com.easyagents.flow.core.chain.Node;
|
||||||
import com.easyagents.flow.core.chain.NodeStatus;
|
|
||||||
import com.easyagents.flow.core.chain.ExceptionSummary;
|
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowExecutionError;
|
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowExecutionErrorMapper;
|
|
||||||
import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent;
|
import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent;
|
||||||
import com.easyagents.flow.core.chain.event.EdgeConditionCheckFailedEvent;
|
import com.easyagents.flow.core.chain.event.EdgeConditionCheckFailedEvent;
|
||||||
import com.easyagents.flow.core.chain.event.EdgeTriggerEvent;
|
import com.easyagents.flow.core.chain.event.EdgeTriggerEvent;
|
||||||
@@ -44,41 +39,6 @@ import java.util.concurrent.atomic.AtomicLong;
|
|||||||
@Service
|
@Service
|
||||||
public class WorkflowChatEventStream {
|
public class WorkflowChatEventStream {
|
||||||
|
|
||||||
public Map<String, Object> runtimeView(String executeId) {
|
|
||||||
try {
|
|
||||||
ChainState state = chainExecutor.getChainStateRepository()
|
|
||||||
.load(executeId);
|
|
||||||
if (state == null || state.getStatus() == null) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
Map<String, Object> view = new LinkedHashMap<>();
|
|
||||||
view.put("status", state.getStatus().name());
|
|
||||||
view.put("statusValue", state.getStatus().getValue());
|
|
||||||
WorkflowExecutionError error = WorkflowExecutionErrorMapper.chain(state.getError(), state.getStatus());
|
|
||||||
view.put("error", error);
|
|
||||||
view.put("message", state.getStatus() == ChainStatus.SUSPEND ? state.getMessage()
|
|
||||||
: WorkflowExecutionErrorMapper.summary(error));
|
|
||||||
if (state.getStatus() == ChainStatus.SUSPEND) {
|
|
||||||
view.put("parameters", state.getSuspendForParameters());
|
|
||||||
}
|
|
||||||
if (state.getStatus() == ChainStatus.SUCCEEDED) {
|
|
||||||
view.put(
|
|
||||||
"output",
|
|
||||||
WorkflowChatEventStream.visibleFinalOutput(
|
|
||||||
state.getExecuteResult())
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return view;
|
|
||||||
} catch (RuntimeException error) {
|
|
||||||
log.warn(
|
|
||||||
"failed to load public workflow runtime state, executeId={}",
|
|
||||||
executeId,
|
|
||||||
error
|
|
||||||
);
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static final Logger log =
|
private static final Logger log =
|
||||||
LoggerFactory.getLogger(WorkflowChatEventStream.class);
|
LoggerFactory.getLogger(WorkflowChatEventStream.class);
|
||||||
private static final long SSE_TIMEOUT_MILLIS = 30L * 60L * 1000L;
|
private static final long SSE_TIMEOUT_MILLIS = 30L * 60L * 1000L;
|
||||||
@@ -251,9 +211,9 @@ public class WorkflowChatEventStream {
|
|||||||
StreamSession session = findSession(chain);
|
StreamSession session = findSession(chain);
|
||||||
if (session != null
|
if (session != null
|
||||||
&& Objects.equals(chain.getStateInstanceId(), session.executeId)) {
|
&& Objects.equals(chain.getStateInstanceId(), session.executeId)) {
|
||||||
WorkflowExecutionError detail = WorkflowExecutionErrorMapper.map(
|
session.send("execution_error", Map.of(
|
||||||
chain.getState().getError(), true, null, null, false);
|
"message", safeErrorMessage(error)
|
||||||
session.send("execution_error", Map.of("message", detail.getMessage(), "error", detail));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -309,6 +269,20 @@ public class WorkflowChatEventStream {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 读取适合返回给用户的异常信息。
|
||||||
|
*
|
||||||
|
* @param error 异常
|
||||||
|
* @return 非空异常信息
|
||||||
|
*/
|
||||||
|
private String safeErrorMessage(Throwable error) {
|
||||||
|
if (error == null || error.getMessage() == null
|
||||||
|
|| error.getMessage().isBlank()) {
|
||||||
|
return "工作流执行失败";
|
||||||
|
}
|
||||||
|
return error.getMessage();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 去掉顶级工作流结果中的内部状态控制字段。
|
* 去掉顶级工作流结果中的内部状态控制字段。
|
||||||
*
|
*
|
||||||
@@ -457,13 +431,8 @@ public class WorkflowChatEventStream {
|
|||||||
data.put("output", event.getResult() == null
|
data.put("output", event.getResult() == null
|
||||||
? Map.of()
|
? Map.of()
|
||||||
: event.getResult());
|
: event.getResult());
|
||||||
if (event.getErrorSummary() != null) {
|
if (event.getError() != null) {
|
||||||
WorkflowExecutionError detail = WorkflowExecutionErrorMapper.node(event.getErrorSummary(),
|
data.put("error", safeErrorMessage(event.getError()));
|
||||||
event.getStatus() == null ? NodeStatus.FAILED : event.getStatus(), node.getId(), node.getName());
|
|
||||||
if (detail != null) {
|
|
||||||
data.put("error", detail.getMessage());
|
|
||||||
data.put("errorDetail", detail);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
send("node_finished", nodePayload(node, data));
|
send("node_finished", nodePayload(node, data));
|
||||||
}
|
}
|
||||||
@@ -592,11 +561,6 @@ public class WorkflowChatEventStream {
|
|||||||
Map<String, Object> data = new LinkedHashMap<>();
|
Map<String, Object> data = new LinkedHashMap<>();
|
||||||
data.put("status", status.name());
|
data.put("status", status.name());
|
||||||
data.put("message", chain.getState().getMessage());
|
data.put("message", chain.getState().getMessage());
|
||||||
WorkflowExecutionError detail = WorkflowExecutionErrorMapper.chain(chain.getState().getError(), status);
|
|
||||||
if (detail != null) {
|
|
||||||
data.put("error", detail);
|
|
||||||
data.put("message", WorkflowExecutionErrorMapper.summary(detail));
|
|
||||||
}
|
|
||||||
if (status == ChainStatus.SUCCEEDED) {
|
if (status == ChainStatus.SUCCEEDED) {
|
||||||
data.put(
|
data.put(
|
||||||
"output",
|
"output",
|
||||||
@@ -650,12 +614,8 @@ public class WorkflowChatEventStream {
|
|||||||
*/
|
*/
|
||||||
private void fail(Throwable error) {
|
private void fail(Throwable error) {
|
||||||
if (terminal.compareAndSet(false, true)) {
|
if (terminal.compareAndSet(false, true)) {
|
||||||
WorkflowExecutionError detail = WorkflowExecutionErrorMapper.map(
|
|
||||||
error == null ? null : new ExceptionSummary(error), true, null, null, false);
|
|
||||||
send("execution_failed", Map.of(
|
send("execution_failed", Map.of(
|
||||||
"status", ChainStatus.FAILED.name(),
|
"message", safeErrorMessage(error)
|
||||||
"message", detail.getMessage(),
|
|
||||||
"error", detail
|
|
||||||
));
|
));
|
||||||
removeSession(this);
|
removeSession(this);
|
||||||
if (connected.compareAndSet(true, false)) {
|
if (connected.compareAndSet(true, false)) {
|
||||||
|
|||||||
@@ -334,7 +334,7 @@ public class WorkflowDesignerOptionService {
|
|||||||
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
Workflow workflow = workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||||
childWorkflowId,
|
childWorkflowId,
|
||||||
account,
|
account,
|
||||||
"子流程不存在、未发布或无权使用");
|
"子流程不存在、已禁用或无权使用");
|
||||||
assertContentReferences(workflow.getContent());
|
assertContentReferences(workflow.getContent());
|
||||||
|
|
||||||
ChainDefinition definition = chainParser.parse(
|
ChainDefinition definition = chainParser.parse(
|
||||||
@@ -572,7 +572,7 @@ public class WorkflowDesignerOptionService {
|
|||||||
workflowUsageAuthorizationService.requireUsableWorkflow(
|
workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||||
workflowId,
|
workflowId,
|
||||||
account,
|
account,
|
||||||
"子流程不存在、未发布或无权使用");
|
"子流程不存在、已禁用或无权使用");
|
||||||
}
|
}
|
||||||
|
|
||||||
private void assertDatasetReference(
|
private void assertDatasetReference(
|
||||||
@@ -707,7 +707,6 @@ public class WorkflowDesignerOptionService {
|
|||||||
private WorkflowDesignerOptionsView.DatasetOption toDatasetOption(DatacenterTable table) {
|
private WorkflowDesignerOptionsView.DatasetOption toDatasetOption(DatacenterTable table) {
|
||||||
return new WorkflowDesignerOptionsView.DatasetOption(
|
return new WorkflowDesignerOptionsView.DatasetOption(
|
||||||
table.getId(),
|
table.getId(),
|
||||||
table.getTenantId(),
|
|
||||||
table.getSourceId(),
|
table.getSourceId(),
|
||||||
table.getCatalogId(),
|
table.getCatalogId(),
|
||||||
table.getTableName(),
|
table.getTableName(),
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ public class WorkflowPublicChatService {
|
|||||||
.eq(WorkflowExecStep::getRecordId, record.getId())
|
.eq(WorkflowExecStep::getRecordId, record.getId())
|
||||||
.orderBy(WorkflowExecStep::getStartTime, true)
|
.orderBy(WorkflowExecStep::getStartTime, true)
|
||||||
));
|
));
|
||||||
return buildExecutionDetail(record, steps, eventStream.runtimeView(executeId));
|
return buildExecutionDetail(record, steps, runtimeView(executeId));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -292,4 +292,35 @@ public class WorkflowPublicChatService {
|
|||||||
/**
|
/**
|
||||||
* 构建刷新恢复所需的最小 Runtime 视图。
|
* 构建刷新恢复所需的最小 Runtime 视图。
|
||||||
*/
|
*/
|
||||||
|
private Map<String, Object> runtimeView(String executeId) {
|
||||||
|
try {
|
||||||
|
ChainState state = chainExecutor.getChainStateRepository()
|
||||||
|
.load(executeId);
|
||||||
|
if (state == null || state.getStatus() == null) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
Map<String, Object> view = new LinkedHashMap<>();
|
||||||
|
view.put("status", state.getStatus().name());
|
||||||
|
view.put("statusValue", state.getStatus().getValue());
|
||||||
|
view.put("message", state.getMessage());
|
||||||
|
if (state.getStatus() == ChainStatus.SUSPEND) {
|
||||||
|
view.put("parameters", state.getSuspendForParameters());
|
||||||
|
}
|
||||||
|
if (state.getStatus() == ChainStatus.SUCCEEDED) {
|
||||||
|
view.put(
|
||||||
|
"output",
|
||||||
|
WorkflowChatEventStream.visibleFinalOutput(
|
||||||
|
state.getExecuteResult())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return view;
|
||||||
|
} catch (RuntimeException error) {
|
||||||
|
log.warn(
|
||||||
|
"failed to load public workflow runtime state, executeId={}",
|
||||||
|
executeId,
|
||||||
|
error
|
||||||
|
);
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
package tech.easyflow.admin.controller.dataspace;
|
|
||||||
|
|
||||||
import static org.testng.Assert.assertEquals;
|
|
||||||
import static org.testng.Assert.assertNotNull;
|
|
||||||
import static org.testng.Assert.assertNull;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import java.lang.reflect.Method;
|
|
||||||
import java.lang.reflect.Parameter;
|
|
||||||
import org.springframework.web.bind.annotation.RequestBody;
|
|
||||||
import org.testng.annotations.Test;
|
|
||||||
import tech.easyflow.common.web.jsonbody.JsonBody;
|
|
||||||
import tech.easyflow.dataspace.model.DataspaceDefinition;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据空间管理接口请求绑定契约测试。
|
|
||||||
*/
|
|
||||||
public class DataspaceControllerContractTest {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 验证保存接口使用 Jackson 请求体绑定,避免嵌套定义残留为 JSONObject。
|
|
||||||
*
|
|
||||||
* @throws Exception 反射或 JSON 转换失败时抛出
|
|
||||||
*/
|
|
||||||
@Test
|
|
||||||
public void shouldBindNestedDataspaceDefinitionWithJackson() throws Exception {
|
|
||||||
Method method = DataspaceController.class.getMethod(
|
|
||||||
"save", DataspaceDefinition.class);
|
|
||||||
Parameter parameter = method.getParameters()[0];
|
|
||||||
assertNotNull(parameter.getAnnotation(RequestBody.class));
|
|
||||||
assertNull(parameter.getAnnotation(JsonBody.class));
|
|
||||||
|
|
||||||
String request = """
|
|
||||||
{
|
|
||||||
"name": "网点经营分析",
|
|
||||||
"tables": [
|
|
||||||
{
|
|
||||||
"clientKey": "table:outlet",
|
|
||||||
"objectId": "1001",
|
|
||||||
"sourceAlias": "MYSQL_1",
|
|
||||||
"schemaAlias": "MAIN",
|
|
||||||
"tableAlias": "outlet",
|
|
||||||
"positionX": 80,
|
|
||||||
"positionY": 120
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"clientKey": "table:region",
|
|
||||||
"objectId": "1002",
|
|
||||||
"sourceAlias": "PG_1",
|
|
||||||
"schemaAlias": "PUBLIC",
|
|
||||||
"tableAlias": "outlet_region",
|
|
||||||
"positionX": 420,
|
|
||||||
"positionY": 120
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"relations": [
|
|
||||||
{
|
|
||||||
"leftClientKey": "table:outlet",
|
|
||||||
"rightClientKey": "table:region",
|
|
||||||
"joinType": "INNER",
|
|
||||||
"leftColumn": "institution_id",
|
|
||||||
"rightColumn": "institution_id"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""";
|
|
||||||
|
|
||||||
DataspaceDefinition definition = new ObjectMapper().readValue(
|
|
||||||
request, DataspaceDefinition.class);
|
|
||||||
|
|
||||||
assertEquals(2, definition.tables().size());
|
|
||||||
assertEquals("outlet", definition.tables().get(0).tableAlias());
|
|
||||||
assertEquals("outlet_region", definition.tables().get(1).tableAlias());
|
|
||||||
assertEquals(1, definition.relations().size());
|
|
||||||
assertEquals("institution_id", definition.relations().get(0).leftColumn());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,39 +1,26 @@
|
|||||||
package tech.easyflow.admin.controller.job;
|
package tech.easyflow.admin.controller.job;
|
||||||
|
|
||||||
import com.easyagents.flow.core.chain.Parameter;
|
import com.easyagents.flow.core.chain.Parameter;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
|
||||||
import org.mockito.ArgumentCaptor;
|
|
||||||
import org.mockito.MockedStatic;
|
import org.mockito.MockedStatic;
|
||||||
import org.testng.Assert;
|
import org.testng.Assert;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowRunningParameterResolver;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.enums.PublishStatus;
|
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
|
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
|
||||||
import tech.easyflow.admin.model.SysJobWorkflowOptionView;
|
|
||||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
|
||||||
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
import tech.easyflow.job.entity.SysJob;
|
import tech.easyflow.job.entity.SysJob;
|
||||||
import tech.easyflow.job.job.JobConstant;
|
import tech.easyflow.job.job.JobConstant;
|
||||||
import tech.easyflow.job.service.SysJobService;
|
import tech.easyflow.job.service.SysJobService;
|
||||||
import tech.easyflow.system.enums.CategoryResourceType;
|
|
||||||
import tech.easyflow.system.enums.ResourceAction;
|
|
||||||
import tech.easyflow.system.service.ResourceAccessService;
|
import tech.easyflow.system.service.ResourceAccessService;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.time.Instant;
|
|
||||||
import java.util.Date;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.TimeZone;
|
|
||||||
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.mockStatic;
|
import static org.mockito.Mockito.mockStatic;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
@@ -44,114 +31,6 @@ import static org.mockito.Mockito.when;
|
|||||||
*/
|
*/
|
||||||
public class SysJobControllerTest {
|
public class SysJobControllerTest {
|
||||||
|
|
||||||
@Test
|
|
||||||
public void shouldQueryPublishedWorkflowOptionsAndReturnPublishedMetadata() {
|
|
||||||
BigInteger workflowId = BigInteger.valueOf(501);
|
|
||||||
LoginAccount account = account();
|
|
||||||
Workflow raw = new Workflow();
|
|
||||||
raw.setId(workflowId);
|
|
||||||
raw.setTenantId(account.getTenantId());
|
|
||||||
raw.setPublishStatus(PublishStatus.PUBLISHED.getCode());
|
|
||||||
raw.setPublishedSnapshotJson(Map.of("title", "发布标题"));
|
|
||||||
raw.setTitle("草稿标题");
|
|
||||||
Workflow withoutSnapshot = new Workflow();
|
|
||||||
withoutSnapshot.setId(BigInteger.valueOf(502));
|
|
||||||
withoutSnapshot.setTenantId(account.getTenantId());
|
|
||||||
withoutSnapshot.setPublishStatus(PublishStatus.PUBLISHED.getCode());
|
|
||||||
Workflow published = new Workflow();
|
|
||||||
published.setId(workflowId);
|
|
||||||
published.setTitle("发布标题");
|
|
||||||
published.setDescription("发布描述");
|
|
||||||
WorkflowService workflowService = mock(WorkflowService.class);
|
|
||||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
|
||||||
when(workflowService.list(any(QueryWrapper.class)))
|
|
||||||
.thenReturn(List.of(raw, withoutSnapshot));
|
|
||||||
when(resourceAccessService.canAccess(
|
|
||||||
account,
|
|
||||||
CategoryResourceType.WORKFLOW,
|
|
||||||
raw,
|
|
||||||
ResourceAction.USE)).thenReturn(true);
|
|
||||||
when(workflowService.toPublishedView(raw)).thenReturn(published);
|
|
||||||
SysJobController controller = new SysJobController(
|
|
||||||
mock(SysJobService.class),
|
|
||||||
workflowService,
|
|
||||||
mock(WorkflowUsageAuthorizationService.class),
|
|
||||||
resourceAccessService,
|
|
||||||
mock(WorkflowRunningParameterResolver.class),
|
|
||||||
"Asia/Shanghai");
|
|
||||||
|
|
||||||
List<SysJobWorkflowOptionView> options;
|
|
||||||
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
|
||||||
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account);
|
|
||||||
options = controller.workflowOptions().getData();
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.assertEquals(options.size(), 1);
|
|
||||||
Assert.assertEquals(options.get(0).title(), "发布标题");
|
|
||||||
ArgumentCaptor<QueryWrapper> queryCaptor = ArgumentCaptor.forClass(QueryWrapper.class);
|
|
||||||
verify(workflowService).list(queryCaptor.capture());
|
|
||||||
String sql = queryCaptor.getValue().toSQL().toLowerCase(Locale.ROOT);
|
|
||||||
Assert.assertTrue(sql.contains("publish_status"));
|
|
||||||
Assert.assertFalse(sql.replace("publish_status", "").matches("(?s).*\\bstatus\\b.*"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void shouldFormatCronPreviewWithConfiguredTimezone() {
|
|
||||||
SysJobService service = mock(SysJobService.class);
|
|
||||||
when(service.nextFireTimes("0 0 9 * * ?", 5))
|
|
||||||
.thenReturn(List.of(Date.from(Instant.parse("2026-01-01T01:00:00Z"))));
|
|
||||||
TimeZone previous = TimeZone.getDefault();
|
|
||||||
try {
|
|
||||||
TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
|
|
||||||
Assert.assertEquals(
|
|
||||||
controller(service).getNextTimes("0 0 9 * * ?").getData().get(0),
|
|
||||||
"2026-01-01 09:00:00");
|
|
||||||
} finally {
|
|
||||||
TimeZone.setDefault(previous);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void shouldForceNewJobToStoppedGenerationZero() {
|
|
||||||
SysJobController controller = controller(mock(SysJobService.class));
|
|
||||||
SysJob job = validJavaJob();
|
|
||||||
job.setStatus(EnumJobStatus.RUNNING.getCode());
|
|
||||||
job.setScheduleGeneration(99L);
|
|
||||||
LoginAccount account = account();
|
|
||||||
|
|
||||||
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
|
||||||
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account);
|
|
||||||
controller.onSaveOrUpdateBefore(job, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.assertEquals(job.getStatus(), Integer.valueOf(EnumJobStatus.STOP.getCode()));
|
|
||||||
Assert.assertEquals(job.getScheduleGeneration(), Long.valueOf(0L));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void shouldRejectStatusAndGenerationMutationThroughOrdinaryUpdate() {
|
|
||||||
BigInteger id = BigInteger.valueOf(401);
|
|
||||||
SysJobService service = mock(SysJobService.class);
|
|
||||||
SysJob existing = validJavaJob();
|
|
||||||
existing.setId(id);
|
|
||||||
existing.setStatus(EnumJobStatus.STOP.getCode());
|
|
||||||
existing.setScheduleGeneration(8L);
|
|
||||||
when(service.getById(id)).thenReturn(existing);
|
|
||||||
|
|
||||||
SysJob update = validJavaJob();
|
|
||||||
update.setId(id);
|
|
||||||
update.setStatus(EnumJobStatus.RUNNING.getCode());
|
|
||||||
update.setScheduleGeneration(100L);
|
|
||||||
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
|
||||||
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account());
|
|
||||||
controller(service).update(update);
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.assertEquals(update.getStatus(), Integer.valueOf(EnumJobStatus.STOP.getCode()));
|
|
||||||
Assert.assertEquals(update.getScheduleGeneration(), Long.valueOf(8L));
|
|
||||||
verify(service).updateJobDefinition(update);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证缺少工作流必填参数时拒绝保存定时任务。
|
* 验证缺少工作流必填参数时拒绝保存定时任务。
|
||||||
*/
|
*/
|
||||||
@@ -184,8 +63,7 @@ public class SysJobControllerTest {
|
|||||||
workflowService,
|
workflowService,
|
||||||
workflowAuthorizationService,
|
workflowAuthorizationService,
|
||||||
resourceAccessService,
|
resourceAccessService,
|
||||||
parameterResolver,
|
parameterResolver
|
||||||
"Asia/Shanghai"
|
|
||||||
);
|
);
|
||||||
SysJob job = new SysJob();
|
SysJob job = new SysJob();
|
||||||
job.setJobType(EnumJobType.TINY_FLOW.getCode());
|
job.setJobType(EnumJobType.TINY_FLOW.getCode());
|
||||||
@@ -252,8 +130,7 @@ public class SysJobControllerTest {
|
|||||||
workflowService,
|
workflowService,
|
||||||
workflowAuthorizationService,
|
workflowAuthorizationService,
|
||||||
resourceAccessService,
|
resourceAccessService,
|
||||||
parameterResolver,
|
parameterResolver
|
||||||
"Asia/Shanghai"
|
|
||||||
);
|
);
|
||||||
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
try (MockedStatic<SaTokenUtil> saToken = mockStatic(SaTokenUtil.class)) {
|
||||||
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account);
|
saToken.when(SaTokenUtil::getLoginAccount).thenReturn(account);
|
||||||
@@ -270,34 +147,4 @@ public class SysJobControllerTest {
|
|||||||
org.mockito.ArgumentMatchers.anyString());
|
org.mockito.ArgumentMatchers.anyString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static SysJobController controller(SysJobService service) {
|
|
||||||
return new SysJobController(
|
|
||||||
service,
|
|
||||||
mock(WorkflowService.class),
|
|
||||||
mock(WorkflowUsageAuthorizationService.class),
|
|
||||||
mock(ResourceAccessService.class),
|
|
||||||
mock(WorkflowRunningParameterResolver.class),
|
|
||||||
"Asia/Shanghai");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SysJob validJavaJob() {
|
|
||||||
SysJob job = new SysJob();
|
|
||||||
job.setJobName("generation-test");
|
|
||||||
job.setJobType(EnumJobType.JAVA_CLASS.getCode());
|
|
||||||
job.setCronExpression("0 0 0 1 1 ? 2099");
|
|
||||||
job.setMisfirePolicy(EnumMisfirePolicy.SKIP.getCode());
|
|
||||||
job.setAllowConcurrent(0);
|
|
||||||
job.setJobParams(Map.of(JobConstant.JAVA_METHOD_KEY,
|
|
||||||
"tech.easyflow.job.util.JobUtil.test()"));
|
|
||||||
return job;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static LoginAccount account() {
|
|
||||||
LoginAccount account = new LoginAccount();
|
|
||||||
account.setId(BigInteger.ONE);
|
|
||||||
account.setTenantId(BigInteger.ONE);
|
|
||||||
account.setDeptId(BigInteger.ONE);
|
|
||||||
return account;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,103 +0,0 @@
|
|||||||
package tech.easyflow.admin.controller.job;
|
|
||||||
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
|
||||||
import org.mockito.ArgumentCaptor;
|
|
||||||
import org.testng.Assert;
|
|
||||||
import org.testng.annotations.Test;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
import tech.easyflow.job.entity.SysJobLog;
|
|
||||||
import tech.easyflow.job.service.SysJobLogService;
|
|
||||||
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Locale;
|
|
||||||
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* {@link SysJobLogController} 查询与轻量刷新边界测试。
|
|
||||||
*/
|
|
||||||
public class SysJobLogControllerTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void shouldBuildBothFireTimeRanges() {
|
|
||||||
SysJobLogController controller = controller(mock(SysJobLogService.class));
|
|
||||||
HttpServletRequest request = emptyRequest();
|
|
||||||
when(request.getParameter("scheduledStart")).thenReturn("2026-08-31 10:00:00");
|
|
||||||
when(request.getParameter("scheduledEnd")).thenReturn("2026-08-31 11:00:00");
|
|
||||||
when(request.getParameter("actualStart")).thenReturn("2026-08-31 10:00:01");
|
|
||||||
when(request.getParameter("actualEnd")).thenReturn("2026-08-31 11:00:01");
|
|
||||||
|
|
||||||
String sql = controller.buildQueryWrapper(request).toSQL().toLowerCase(Locale.ROOT);
|
|
||||||
|
|
||||||
Assert.assertEquals(countOccurrences(sql, "scheduled_fire_time"), 2);
|
|
||||||
Assert.assertEquals(countOccurrences(sql, "actual_fire_time"), 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test(expectedExceptions = BusinessException.class)
|
|
||||||
public void shouldRejectInvalidFireTime() {
|
|
||||||
SysJobLogController controller = controller(mock(SysJobLogService.class));
|
|
||||||
HttpServletRequest request = emptyRequest();
|
|
||||||
when(request.getParameter("scheduledStart")).thenReturn("2026/08/31 10:00:00");
|
|
||||||
|
|
||||||
controller.buildQueryWrapper(request);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test(expectedExceptions = BusinessException.class)
|
|
||||||
public void shouldRejectReversedActualFireTimeRange() {
|
|
||||||
SysJobLogController controller = controller(mock(SysJobLogService.class));
|
|
||||||
HttpServletRequest request = emptyRequest();
|
|
||||||
when(request.getParameter("actualStart")).thenReturn("2026-08-31 11:00:00");
|
|
||||||
when(request.getParameter("actualEnd")).thenReturn("2026-08-31 10:00:00");
|
|
||||||
|
|
||||||
controller.buildQueryWrapper(request);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void shouldClampRefreshAndPageSize() {
|
|
||||||
SysJobLogService service = mock(SysJobLogService.class);
|
|
||||||
when(service.list(any(QueryWrapper.class))).thenReturn(List.of());
|
|
||||||
when(service.page(any(Page.class), any(QueryWrapper.class)))
|
|
||||||
.thenAnswer(invocation -> invocation.getArgument(0));
|
|
||||||
SysJobLogController controller = controller(service);
|
|
||||||
HttpServletRequest request = emptyRequest();
|
|
||||||
|
|
||||||
controller.refresh(request, 500L);
|
|
||||||
Page<SysJobLog> page = controller.queryPage(
|
|
||||||
new Page<>(1, 500), QueryWrapper.create());
|
|
||||||
|
|
||||||
ArgumentCaptor<QueryWrapper> queryCaptor =
|
|
||||||
ArgumentCaptor.forClass(QueryWrapper.class);
|
|
||||||
verify(service).list(queryCaptor.capture());
|
|
||||||
String refreshSql = queryCaptor.getValue().toSQL().toLowerCase(Locale.ROOT);
|
|
||||||
Assert.assertTrue(refreshSql.contains("limit 100"));
|
|
||||||
Assert.assertEquals(page.getPageSize(), 100L);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void shouldUseStableScheduledFireTimeOrder() {
|
|
||||||
Assert.assertEquals(
|
|
||||||
controller(mock(SysJobLogService.class)).getDefaultOrderBy(),
|
|
||||||
"scheduled_fire_time desc, id desc");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SysJobLogController controller(SysJobLogService service) {
|
|
||||||
return new SysJobLogController(service, "Asia/Shanghai");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static HttpServletRequest emptyRequest() {
|
|
||||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
|
||||||
when(request.getParameterMap()).thenReturn(Collections.emptyMap());
|
|
||||||
return request;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static int countOccurrences(String source, String expected) {
|
|
||||||
return (source.length() - source.replace(expected, "").length())
|
|
||||||
/ expected.length();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +1,6 @@
|
|||||||
package tech.easyflow.admin.service.ai;
|
package tech.easyflow.admin.service.ai;
|
||||||
|
|
||||||
import com.easyagents.flow.core.chain.*;
|
import com.easyagents.flow.core.chain.ChainConsts;
|
||||||
import com.easyagents.flow.core.chain.repository.*;
|
|
||||||
import com.easyagents.flow.core.chain.runtime.InMemoryTriggerStore;
|
|
||||||
import com.easyagents.flow.core.chain.runtime.TriggerScheduler;
|
|
||||||
import com.easyagents.flow.core.node.StartNode;
|
|
||||||
import com.alibaba.fastjson.JSON;
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
|
||||||
import java.util.concurrent.*;
|
|
||||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||||
import org.testng.Assert;
|
import org.testng.Assert;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
@@ -145,50 +138,6 @@ public class WorkflowChatEventStreamTest {
|
|||||||
Assert.assertEquals(cleanupCount.get(), 1);
|
Assert.assertEquals(cleanupCount.get(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void terminalMustCarryReasonAfterFailedNode() throws Exception {
|
|
||||||
ChainDefinition definition = new ChainDefinition(); definition.setId("sse-test");
|
|
||||||
StartNode start = new StartNode(); start.setId("start"); definition.addNode(start);
|
|
||||||
Node failed = new Node() {
|
|
||||||
@Override public Map<String, Object> execute(Chain chain) {
|
|
||||||
throw new WorkflowExecutionException(WorkflowErrorReason.MODEL_UNAVAILABLE, "PRIVATE_PROVIDER_BODY");
|
|
||||||
}
|
|
||||||
};
|
|
||||||
failed.setId("llm"); failed.setName("模型分析"); definition.addNode(failed);
|
|
||||||
Edge edge = new Edge(); edge.setId("edge"); edge.setSource("start"); edge.setTarget("llm"); definition.addEdge(edge);
|
|
||||||
TriggerScheduler scheduler = new TriggerScheduler(new InMemoryTriggerStore(), Executors.newSingleThreadScheduledExecutor(), Executors.newFixedThreadPool(2), 1000);
|
|
||||||
ChainExecutor executor = new ChainExecutor(id -> definition, new InMemoryChainStateRepository(), new InMemoryNodeStateRepository(), scheduler);
|
|
||||||
List<JSONObject> events = new CopyOnWriteArrayList<>();
|
|
||||||
CountDownLatch complete = new CountDownLatch(1);
|
|
||||||
SseEmitter emitter = new SseEmitter() {
|
|
||||||
@Override public void send(SseEventBuilder event) {
|
|
||||||
event.build().forEach(data -> {
|
|
||||||
String value = String.valueOf(data.getData());
|
|
||||||
if (value.startsWith("{")) events.add(JSON.parseObject(value));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
@Override public void complete() { complete.countDown(); }
|
|
||||||
};
|
|
||||||
WorkflowChatEventStream stream = new WorkflowChatEventStream(executor) {
|
|
||||||
@Override SseEmitter createEmitter() { return emitter; }
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
stream.registerListeners(); stream.start("sse-test", Map.of());
|
|
||||||
Assert.assertTrue(complete.await(5, TimeUnit.SECONDS));
|
|
||||||
List<JSONObject> terminals = events.stream().filter(e -> "execution_failed".equals(e.getString("type"))).toList();
|
|
||||||
Assert.assertEquals(terminals.size(), 1);
|
|
||||||
JSONObject detail = terminals.get(0).getJSONObject("data").getJSONObject("error");
|
|
||||||
Assert.assertEquals(detail.getString("reasonCode"), "MODEL_UNAVAILABLE");
|
|
||||||
Assert.assertEquals(detail.getString("nodeId"), "llm");
|
|
||||||
JSONObject ended = events.stream().filter(e -> "node_finished".equals(e.getString("type")) && "llm".equals(e.getJSONObject("data").getString("nodeId"))).findFirst().orElseThrow();
|
|
||||||
Assert.assertEquals(ended.getJSONObject("data").getString("status"), "FAILED");
|
|
||||||
Assert.assertTrue(events.indexOf(ended) < events.indexOf(terminals.get(0)));
|
|
||||||
Assert.assertTrue(ended.getJSONObject("data").get("error") instanceof String);
|
|
||||||
Assert.assertEquals(ended.getJSONObject("data").getJSONObject("errorDetail").getString("reasonCode"), "MODEL_UNAVAILABLE");
|
|
||||||
Assert.assertFalse(JSON.toJSONString(events).contains("PRIVATE_PROVIDER_BODY"));
|
|
||||||
} finally { stream.shutdown(); scheduler.shutdown(); }
|
|
||||||
}
|
|
||||||
|
|
||||||
private static final class CapturingSseEmitter extends SseEmitter {
|
private static final class CapturingSseEmitter extends SseEmitter {
|
||||||
|
|
||||||
private Runnable completion;
|
private Runnable completion;
|
||||||
|
|||||||
@@ -123,8 +123,6 @@ public class WorkflowPublicChatServiceTest {
|
|||||||
when(fixture.chainExecutor.getChainStateRepository())
|
when(fixture.chainExecutor.getChainStateRepository())
|
||||||
.thenReturn(repository);
|
.thenReturn(repository);
|
||||||
when(repository.load("execution-1")).thenReturn(state);
|
when(repository.load("execution-1")).thenReturn(state);
|
||||||
Map<String, Object> runtimeSnapshot = new WorkflowChatEventStream(fixture.chainExecutor).runtimeView("execution-1");
|
|
||||||
when(fixture.eventStream.runtimeView("execution-1")).thenReturn(runtimeSnapshot);
|
|
||||||
|
|
||||||
Map<String, Object> detail = fixture.service.detail(
|
Map<String, Object> detail = fixture.service.detail(
|
||||||
"share-key", visitorId(), "execution-1");
|
"share-key", visitorId(), "execution-1");
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
package tech.easyflow.publicapi.controller;
|
package tech.easyflow.publicapi.controller;
|
||||||
|
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowExecutionErrorMapper;
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
import cn.dev33.satoken.stp.StpUtil;
|
import cn.dev33.satoken.stp.StpUtil;
|
||||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||||
@@ -126,7 +125,6 @@ public class PublicWorkflowController {
|
|||||||
if (StpUtil.isLogin()) {
|
if (StpUtil.isLogin()) {
|
||||||
variables.put(Constants.LOGIN_USER_KEY, SaTokenUtil.getLoginAccount());
|
variables.put(Constants.LOGIN_USER_KEY, SaTokenUtil.getLoginAccount());
|
||||||
}
|
}
|
||||||
WorkflowExecutionErrorMapper.installRequestProfile();
|
|
||||||
Map<String, Object> res = chainExecutor.executeNode(workflowId.toString(), nodeId, variables);
|
Map<String, Object> res = chainExecutor.executeNode(workflowId.toString(), nodeId, variables);
|
||||||
return Result.ok(res);
|
return Result.ok(res);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,10 +22,5 @@ public record PublicWorkflowNodeStatus(
|
|||||||
PublicWorkflowExecutionStatus status,
|
PublicWorkflowExecutionStatus status,
|
||||||
String message,
|
String message,
|
||||||
Map<String, Object> result,
|
Map<String, Object> result,
|
||||||
List<Parameter> suspendForParameters,
|
List<Parameter> suspendForParameters) implements Serializable {
|
||||||
PublicWorkflowStatusError error) implements Serializable {
|
|
||||||
public PublicWorkflowNodeStatus(String nodeId, String nodeName, PublicWorkflowExecutionStatus status,
|
|
||||||
String message, Map<String, Object> result, List<Parameter> suspendForParameters) {
|
|
||||||
this(nodeId, nodeName, status, message, result, suspendForParameters, null);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ public class PublicWorkflowStatusError implements Serializable {
|
|||||||
private static final long serialVersionUID = 1L;
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
private final String code;
|
private final String code;
|
||||||
private final String reasonCode;
|
|
||||||
private final String message;
|
private final String message;
|
||||||
private final String nodeId;
|
private final String nodeId;
|
||||||
private final String nodeName;
|
private final String nodeName;
|
||||||
@@ -31,13 +30,7 @@ public class PublicWorkflowStatusError implements Serializable {
|
|||||||
String nodeId,
|
String nodeId,
|
||||||
String nodeName,
|
String nodeName,
|
||||||
boolean retryable) {
|
boolean retryable) {
|
||||||
this(code, null, message, nodeId, nodeName, retryable);
|
|
||||||
}
|
|
||||||
|
|
||||||
public PublicWorkflowStatusError(String code, String reasonCode, String message,
|
|
||||||
String nodeId, String nodeName, boolean retryable) {
|
|
||||||
this.code = code;
|
this.code = code;
|
||||||
this.reasonCode = reasonCode;
|
|
||||||
this.message = message;
|
this.message = message;
|
||||||
this.nodeId = nodeId;
|
this.nodeId = nodeId;
|
||||||
this.nodeName = nodeName;
|
this.nodeName = nodeName;
|
||||||
@@ -53,8 +46,6 @@ public class PublicWorkflowStatusError implements Serializable {
|
|||||||
return code;
|
return code;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getReasonCode() { return reasonCode; }
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取安全消息。
|
* 获取安全消息。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
package tech.easyflow.publicapi.service;
|
package tech.easyflow.publicapi.service;
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.StringUtils;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowExecutionError;
|
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowExecutionErrorMapper;
|
|
||||||
import tech.easyflow.publicapi.dto.PublicWorkflowChainStatus;
|
import tech.easyflow.publicapi.dto.PublicWorkflowChainStatus;
|
||||||
import tech.easyflow.publicapi.dto.PublicWorkflowExecutionStatus;
|
import tech.easyflow.publicapi.dto.PublicWorkflowExecutionStatus;
|
||||||
import tech.easyflow.publicapi.dto.PublicWorkflowNodeStatus;
|
import tech.easyflow.publicapi.dto.PublicWorkflowNodeStatus;
|
||||||
@@ -13,44 +12,130 @@ import tech.easyflow.publicapi.dto.PublicWorkflowStatusError;
|
|||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
/** 复用公共错误规范,兼容没有结构化错误的旧状态。 */
|
/**
|
||||||
|
* 将内部工作流执行错误转换为 Public API 安全状态。
|
||||||
|
*/
|
||||||
@Service
|
@Service
|
||||||
public class PublicWorkflowStatusSanitizer {
|
public class PublicWorkflowStatusSanitizer {
|
||||||
|
|
||||||
|
private static final String CHAIN_FAILED_MESSAGE =
|
||||||
|
"工作流执行失败,请检查输入或稍后重试";
|
||||||
|
private static final String NODE_FAILED_MESSAGE =
|
||||||
|
"节点执行失败,请检查输入或稍后重试";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 复制执行状态并移除异常类名、底层地址和内部错误详情。
|
||||||
|
*
|
||||||
|
* @param source 内部执行状态
|
||||||
|
* @return 可公开状态
|
||||||
|
*/
|
||||||
public PublicWorkflowChainStatus sanitize(ChainInfo source) {
|
public PublicWorkflowChainStatus sanitize(ChainInfo source) {
|
||||||
if (source == null) throw new IllegalArgumentException("source must not be null");
|
if (source == null) {
|
||||||
PublicWorkflowExecutionStatus status = PublicWorkflowExecutionStatus.fromChainStatus(source.getStatus());
|
throw new IllegalArgumentException(
|
||||||
Map<String, PublicWorkflowNodeStatus> nodes = new LinkedHashMap<>();
|
"source must not be null");
|
||||||
|
}
|
||||||
|
PublicWorkflowExecutionStatus chainStatus =
|
||||||
|
PublicWorkflowExecutionStatus.fromChainStatus(
|
||||||
|
source.getStatus());
|
||||||
|
|
||||||
|
Map<String, PublicWorkflowNodeStatus> safeNodes =
|
||||||
|
new LinkedHashMap<>();
|
||||||
PublicWorkflowStatusError firstNodeError = null;
|
PublicWorkflowStatusError firstNodeError = null;
|
||||||
if (source.getNodes() != null) {
|
if (source.getNodes() != null) {
|
||||||
for (var entry : source.getNodes().entrySet()) {
|
for (Map.Entry<String, NodeInfo> entry
|
||||||
NodeInfo node = entry.getValue();
|
: source.getNodes().entrySet()) {
|
||||||
if (node == null) continue;
|
PublicWorkflowNodeStatus safeNode = copyNode(
|
||||||
PublicWorkflowExecutionStatus nodeStatus = PublicWorkflowExecutionStatus.fromNodeStatus(node.getStatus());
|
entry.getValue());
|
||||||
PublicWorkflowStatusError error = copyError(node.getError(), false, nodeStatus, node.getNodeId(), node.getNodeName(), status.isTerminal());
|
safeNodes.put(entry.getKey(), safeNode);
|
||||||
nodes.put(entry.getKey(), new PublicWorkflowNodeStatus(node.getNodeId(), node.getNodeName(), nodeStatus,
|
if (firstNodeError == null
|
||||||
error == null ? null : error.getMessage(), node.getResult(), node.getSuspendForParameters(), error));
|
&& StringUtils.hasText(safeNode.message())) {
|
||||||
if (firstNodeError == null && error != null) firstNodeError = error;
|
firstNodeError = new PublicWorkflowStatusError(
|
||||||
|
"NODE_EXECUTION_FAILED",
|
||||||
|
safeNode.message(),
|
||||||
|
safeNode.nodeId(),
|
||||||
|
safeNode.nodeName(),
|
||||||
|
isRetryable(safeNode.status()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
PublicWorkflowStatusError error = copyError(source.getError(), true, status,
|
|
||||||
firstNodeError == null ? null : firstNodeError.getNodeId(),
|
String message = null;
|
||||||
firstNodeError == null ? null : firstNodeError.getNodeName(), status.isTerminal());
|
PublicWorkflowStatusError error = null;
|
||||||
// 暂态节点错误仍可查询;成功、取消和挂起不携带旧错误。
|
if (StringUtils.hasText(source.getMessage())) {
|
||||||
if (error == null && status == PublicWorkflowExecutionStatus.RUNNING) error = firstNodeError;
|
message = chainMessage(chainStatus);
|
||||||
return new PublicWorkflowChainStatus(source.getExecuteId(), status, status.isTerminal(),
|
error = new PublicWorkflowStatusError(
|
||||||
error == null ? null : error.getMessage(), source.getResult(), nodes, error);
|
"WORKFLOW_EXECUTION_FAILED",
|
||||||
|
message,
|
||||||
|
firstNodeError == null
|
||||||
|
? null
|
||||||
|
: firstNodeError.getNodeId(),
|
||||||
|
firstNodeError == null
|
||||||
|
? null
|
||||||
|
: firstNodeError.getNodeName(),
|
||||||
|
isRetryable(chainStatus));
|
||||||
|
} else if (firstNodeError != null) {
|
||||||
|
error = firstNodeError;
|
||||||
|
}
|
||||||
|
return new PublicWorkflowChainStatus(
|
||||||
|
source.getExecuteId(),
|
||||||
|
chainStatus,
|
||||||
|
chainStatus.isTerminal(),
|
||||||
|
message,
|
||||||
|
source.getResult(),
|
||||||
|
safeNodes,
|
||||||
|
error);
|
||||||
}
|
}
|
||||||
|
|
||||||
private PublicWorkflowStatusError copyError(WorkflowExecutionError source, boolean workflow,
|
/**
|
||||||
PublicWorkflowExecutionStatus status, String nodeId, String nodeName, boolean executionTerminal) {
|
* 复制并脱敏单个节点状态。
|
||||||
if (status != PublicWorkflowExecutionStatus.FAILED && status != PublicWorkflowExecutionStatus.ERROR) return null;
|
*
|
||||||
if (source != null) {
|
* @param source 内部节点状态
|
||||||
nodeId = source.getNodeId();
|
* @return 安全节点状态
|
||||||
nodeName = source.getNodeName();
|
*/
|
||||||
|
private PublicWorkflowNodeStatus copyNode(NodeInfo source) {
|
||||||
|
if (source == null) {
|
||||||
|
return new PublicWorkflowNodeStatus(
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
PublicWorkflowExecutionStatus.UNKNOWN,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null);
|
||||||
}
|
}
|
||||||
WorkflowExecutionError safe = WorkflowExecutionErrorMapper.fromReason(source == null ? null : source.getReasonCode(), workflow, nodeId, nodeName,
|
return new PublicWorkflowNodeStatus(
|
||||||
status == PublicWorkflowExecutionStatus.ERROR && !executionTerminal);
|
source.getNodeId(),
|
||||||
return new PublicWorkflowStatusError(safe.getCode(), safe.getReasonCode(), safe.getMessage(),
|
source.getNodeName(),
|
||||||
safe.getNodeId(), safe.getNodeName(), safe.isRetryable());
|
PublicWorkflowExecutionStatus.fromNodeStatus(
|
||||||
|
source.getStatus()),
|
||||||
|
StringUtils.hasText(source.getMessage())
|
||||||
|
? NODE_FAILED_MESSAGE
|
||||||
|
: null,
|
||||||
|
source.getResult(),
|
||||||
|
source.getSuspendForParameters());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 根据工作流状态生成安全消息。
|
||||||
|
*
|
||||||
|
* @param status 可读状态
|
||||||
|
* @return 安全消息
|
||||||
|
*/
|
||||||
|
private String chainMessage(
|
||||||
|
PublicWorkflowExecutionStatus status) {
|
||||||
|
if (status == PublicWorkflowExecutionStatus.CANCELLED) {
|
||||||
|
return "工作流执行已取消";
|
||||||
|
}
|
||||||
|
return CHAIN_FAILED_MESSAGE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断执行状态是否仍可能由运行时继续处理。
|
||||||
|
*
|
||||||
|
* @param status 可读状态
|
||||||
|
* @return 是否可重试
|
||||||
|
*/
|
||||||
|
private boolean isRetryable(
|
||||||
|
PublicWorkflowExecutionStatus status) {
|
||||||
|
return status == PublicWorkflowExecutionStatus.ERROR;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
package tech.easyflow.publicapi.service;
|
package tech.easyflow.publicapi.service;
|
||||||
|
|
||||||
import com.easyagents.flow.core.chain.ChainStatus;
|
import com.easyagents.flow.core.chain.ChainStatus;
|
||||||
import com.easyagents.flow.core.chain.WorkflowErrorReason;
|
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowExecutionError;
|
|
||||||
import com.easyagents.flow.core.chain.NodeStatus;
|
import com.easyagents.flow.core.chain.NodeStatus;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
@@ -41,11 +39,11 @@ public class PublicWorkflowStatusSanitizerTest {
|
|||||||
PublicWorkflowChainStatus result = sanitizer.sanitize(source);
|
PublicWorkflowChainStatus result = sanitizer.sanitize(source);
|
||||||
|
|
||||||
Assert.assertEquals(
|
Assert.assertEquals(
|
||||||
WorkflowErrorReason.NODE_EXECUTION_FAILED.getDefaultMessage(),
|
"工作流执行失败,请检查输入或稍后重试",
|
||||||
result.message());
|
result.message());
|
||||||
Assert.assertFalse(result.message().contains("minio"));
|
Assert.assertFalse(result.message().contains("minio"));
|
||||||
Assert.assertEquals(
|
Assert.assertEquals(
|
||||||
WorkflowErrorReason.NODE_EXECUTION_FAILED.getDefaultMessage(),
|
"节点执行失败,请检查输入或稍后重试",
|
||||||
result.nodes().get("node-1").message());
|
result.nodes().get("node-1").message());
|
||||||
Assert.assertEquals("node-1", result.error().getNodeId());
|
Assert.assertEquals("node-1", result.error().getNodeId());
|
||||||
Assert.assertFalse(result.error().isRetryable());
|
Assert.assertFalse(result.error().isRetryable());
|
||||||
@@ -103,46 +101,4 @@ public class PublicWorkflowStatusSanitizerTest {
|
|||||||
PublicWorkflowExecutionStatus.RUNNING,
|
PublicWorkflowExecutionStatus.RUNNING,
|
||||||
result.nodes().get("node-1").status());
|
result.nodes().get("node-1").status());
|
||||||
}
|
}
|
||||||
@Test
|
|
||||||
public void shouldReturnStructuredReasonWithoutRequestedNodes() {
|
|
||||||
for (WorkflowErrorReason reason : WorkflowErrorReason.values()) {
|
|
||||||
ChainInfo source = new ChainInfo();
|
|
||||||
source.setStatus(ChainStatus.FAILED.getValue());
|
|
||||||
source.setError(new WorkflowExecutionError("WORKFLOW_EXECUTION_FAILED", reason.getCode(),
|
|
||||||
"raw provider body must not escape", "llm", "分析", false));
|
|
||||||
var result = sanitizer.sanitize(source);
|
|
||||||
Assert.assertEquals(reason.getCode(), result.error().getReasonCode());
|
|
||||||
Assert.assertEquals(reason.getDefaultMessage(), result.message());
|
|
||||||
Assert.assertEquals("llm", result.error().getNodeId());
|
|
||||||
Assert.assertTrue(result.nodes().isEmpty());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void successfulRetryMustNotExposeStaleErrors() {
|
|
||||||
ChainInfo source = new ChainInfo();
|
|
||||||
source.setStatus(ChainStatus.SUCCEEDED.getValue());
|
|
||||||
source.setMessage("stale error");
|
|
||||||
NodeInfo node = new NodeInfo();
|
|
||||||
node.setNodeId("llm"); node.setStatus(NodeStatus.SUCCEEDED.getValue()); node.setMessage("old failure");
|
|
||||||
source.setNodes(Map.of("llm", node));
|
|
||||||
Assert.assertNull(sanitizer.sanitize(source).error());
|
|
||||||
Assert.assertNull(sanitizer.sanitize(source).nodes().get("llm").message());
|
|
||||||
}
|
|
||||||
@Test
|
|
||||||
public void terminalWorkflowMustNotAdvertiseNodeRetry() {
|
|
||||||
for (ChainStatus status : new ChainStatus[]{ChainStatus.RUNNING, ChainStatus.FAILED, ChainStatus.CANCELLED}) {
|
|
||||||
ChainInfo source = new ChainInfo();
|
|
||||||
source.setStatus(status.getValue());
|
|
||||||
NodeInfo node = new NodeInfo();
|
|
||||||
node.setNodeId("llm");
|
|
||||||
node.setStatus(NodeStatus.ERROR.getValue());
|
|
||||||
node.setError(new WorkflowExecutionError("NODE_EXECUTION_FAILED", "MODEL_TIMEOUT",
|
|
||||||
"raw error", "llm", "模型分析", true));
|
|
||||||
source.setNodes(Map.of("llm", node));
|
|
||||||
var result = sanitizer.sanitize(source);
|
|
||||||
Assert.assertEquals(status == ChainStatus.RUNNING, result.nodes().get("llm").error().isRetryable());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,12 +5,9 @@ import tech.easyflow.common.annotation.DictDef;
|
|||||||
@DictDef(name = "任务执行结果", code = "jobResult", keyField = "code", labelField = "text")
|
@DictDef(name = "任务执行结果", code = "jobResult", keyField = "code", labelField = "text")
|
||||||
public enum EnumJobResult {
|
public enum EnumJobResult {
|
||||||
|
|
||||||
|
|
||||||
SUCCESS(1,"成功"),
|
SUCCESS(1,"成功"),
|
||||||
FAIL(0,"失败"),
|
FAIL(0,"失败"),
|
||||||
PENDING(2,"等待执行"),
|
|
||||||
RUNNING(3,"执行中"),
|
|
||||||
DEAD(4,"需人工处理"),
|
|
||||||
CANCELLED(5,"已取消"),
|
|
||||||
;
|
;
|
||||||
|
|
||||||
private final int code;
|
private final int code;
|
||||||
|
|||||||
@@ -5,8 +5,10 @@ import tech.easyflow.common.annotation.DictDef;
|
|||||||
@DictDef(name = "错过策略", code = "misfirePolicy", keyField = "code", labelField = "text")
|
@DictDef(name = "错过策略", code = "misfirePolicy", keyField = "code", labelField = "text")
|
||||||
public enum EnumMisfirePolicy {
|
public enum EnumMisfirePolicy {
|
||||||
|
|
||||||
FIRE_ONCE_NOW(2,"恢复后补执行一次"),
|
DEFAULT(0,"默认"),
|
||||||
SKIP(3,"跳过本次");
|
MISFIRE_IGNORE_MISFIRES(1,"立即触发"),
|
||||||
|
MISFIRE_FIRE_AND_PROCEED(2,"立即触发一次"),
|
||||||
|
MISFIRE_DO_NOTHING(3,"忽略");
|
||||||
;
|
;
|
||||||
|
|
||||||
private final int code;
|
private final int code;
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import com.easyagents.core.model.embedding.EmbeddingOptions;
|
|||||||
import com.easyagents.core.store.DocumentStore;
|
import com.easyagents.core.store.DocumentStore;
|
||||||
import com.easyagents.core.store.StoreOptions;
|
import com.easyagents.core.store.StoreOptions;
|
||||||
import com.easyagents.core.store.StoreResult;
|
import com.easyagents.core.store.StoreResult;
|
||||||
import com.easyagents.core.store.VectorData;
|
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
import com.easyagents.search.engine.service.DocumentSearcher;
|
||||||
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
|
import com.easyagents.search.engine.service.KeywordSearchMetadataKeys;
|
||||||
import com.easyagents.store.milvus.MilvusVectorStore;
|
import com.easyagents.store.milvus.MilvusVectorStore;
|
||||||
@@ -23,7 +22,6 @@ import tech.easyflow.ai.entity.DocumentChunk;
|
|||||||
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
import tech.easyflow.ai.entity.DocumentCollection;
|
||||||
import tech.easyflow.ai.entity.Model;
|
import tech.easyflow.ai.entity.Model;
|
||||||
import tech.easyflow.ai.dto.DocumentChunkSyncStatus;
|
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
@@ -35,12 +33,7 @@ import java.math.BigInteger;
|
|||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 持久化分块索引同步任务的投递、执行和恢复。
|
* 持久化分块索引同步任务的投递、执行和恢复。
|
||||||
@@ -123,31 +116,6 @@ public class DocumentChunkSyncTaskAppService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<DocumentChunkSyncStatus> listSyncStatuses(List<DocumentChunk> chunks) {
|
|
||||||
if (chunks.isEmpty()) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
Map<BigInteger, DocumentChunkSyncTask> tasks = taskMapper.selectCurrentForChunks(
|
|
||||||
chunks.stream().map(DocumentChunk::getId).toList()
|
|
||||||
).stream().collect(Collectors.toMap(DocumentChunkSyncTask::getChunkId, Function.identity()));
|
|
||||||
return chunks.stream().map(chunk -> {
|
|
||||||
DocumentChunkSyncTask task = tasks.get(chunk.getId());
|
|
||||||
boolean current = task != null
|
|
||||||
&& Objects.equals(chunk.getIndexSyncVersion(), task.getSyncVersion());
|
|
||||||
// 正文状态可能在两次查询间推进,只有同版本仍在重试的任务提供失败原因。
|
|
||||||
boolean retrying = current && DocumentChunkSyncState.PENDING.equals(chunk.getIndexSyncStatus())
|
|
||||||
&& (DocumentChunkSyncState.PENDING.equals(task.getStatus())
|
|
||||||
|| DocumentChunkSyncState.TASK_RUNNING.equals(task.getStatus()));
|
|
||||||
return new DocumentChunkSyncStatus(
|
|
||||||
chunk.getId(), chunk.getIndexSyncStatus(), chunk.getIndexSyncVersion(),
|
|
||||||
retrying ? task.getErrorCode() : chunk.getIndexSyncErrorCode(),
|
|
||||||
retrying ? task.getErrorMessage() : chunk.getIndexSyncErrorMessage(),
|
|
||||||
current ? task.getAttemptCount() : null,
|
|
||||||
MAX_ATTEMPTS
|
|
||||||
);
|
|
||||||
}).toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void dispatchPendingTasks() {
|
public void dispatchPendingTasks() {
|
||||||
Date now = new Date();
|
Date now = new Date();
|
||||||
taskMapper.recoverExpired(now);
|
taskMapper.recoverExpired(now);
|
||||||
@@ -260,7 +228,6 @@ public class DocumentChunkSyncTaskAppService {
|
|||||||
StoreContext context = prepareUpsertContext(collection, task.getVectorCollection());
|
StoreContext context = prepareUpsertContext(collection, task.getVectorCollection());
|
||||||
try {
|
try {
|
||||||
com.easyagents.core.document.Document document = toSearchDocument(chunk, task.getDocumentCollectionId());
|
com.easyagents.core.document.Document document = toSearchDocument(chunk, task.getDocumentCollectionId());
|
||||||
embedDocument(context, document);
|
|
||||||
StoreResult vectorResult = context.documentStore.update(
|
StoreResult vectorResult = context.documentStore.update(
|
||||||
Collections.singletonList(document),
|
Collections.singletonList(document),
|
||||||
context.storeOptions
|
context.storeOptions
|
||||||
@@ -281,21 +248,6 @@ public class DocumentChunkSyncTaskAppService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void embedDocument(StoreContext context, com.easyagents.core.document.Document document) {
|
|
||||||
try {
|
|
||||||
VectorData vectorData = context.documentStore.getEmbeddingModel().embed(
|
|
||||||
document, context.storeOptions.getEmbeddingOptions()
|
|
||||||
);
|
|
||||||
if (vectorData == null || vectorData.getVector() == null || vectorData.getVector().length == 0) {
|
|
||||||
throw new IllegalStateException("向量模型未返回有效向量");
|
|
||||||
}
|
|
||||||
// update 复用已生成的向量,不重复调用模型;分开捕获以区分模型与索引写入失败。
|
|
||||||
document.setVector(vectorData.getVector());
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
throw new IndexSyncException("EMBEDDING_REQUEST_FAILED", "向量模型服务调用失败", exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void synchronizeDelete(DocumentChunkSyncTask task) {
|
private void synchronizeDelete(DocumentChunkSyncTask task) {
|
||||||
MilvusVectorStoreConfig storeConfig = milvusConfig.copyForCollection(task.getVectorCollection());
|
MilvusVectorStoreConfig storeConfig = milvusConfig.copyForCollection(task.getVectorCollection());
|
||||||
AiMilvusClientManager clientManager = milvusClientManagerProvider.getObject();
|
AiMilvusClientManager clientManager = milvusClientManagerProvider.getObject();
|
||||||
|
|||||||
@@ -10,8 +10,6 @@ public record DocumentChunkSyncStatus(
|
|||||||
String indexSyncStatus,
|
String indexSyncStatus,
|
||||||
Long indexSyncVersion,
|
Long indexSyncVersion,
|
||||||
String indexSyncErrorCode,
|
String indexSyncErrorCode,
|
||||||
String indexSyncErrorMessage,
|
String indexSyncErrorMessage
|
||||||
Integer indexSyncAttemptCount,
|
|
||||||
int indexSyncMaxAttempts
|
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,42 +0,0 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
|
||||||
|
|
||||||
import com.easyagents.flow.core.chain.Chain;
|
|
||||||
import com.easyagents.flow.core.chain.ChainStatus;
|
|
||||||
import com.easyagents.flow.core.chain.Event;
|
|
||||||
import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent;
|
|
||||||
import com.easyagents.flow.core.chain.listener.ChainEventListener;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 将工作流取消终态桥接到活动数据集查询。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class WorkflowDatasetQueryCancellationListener
|
|
||||||
implements ChainEventListener {
|
|
||||||
|
|
||||||
private final WorkflowDatasetQueryCancellationRegistry registry;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建工作流查询取消监听器。
|
|
||||||
*
|
|
||||||
* @param registry 工作流查询取消登记表
|
|
||||||
*/
|
|
||||||
public WorkflowDatasetQueryCancellationListener(
|
|
||||||
WorkflowDatasetQueryCancellationRegistry registry) {
|
|
||||||
this.registry = registry;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在工作流进入取消终态后取消该实例的活动查询。
|
|
||||||
*
|
|
||||||
* @param event 工作流事件
|
|
||||||
* @param chain 工作流实例
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public void onEvent(Event event, Chain chain) {
|
|
||||||
if (event instanceof ChainStatusChangeEvent statusChangeEvent
|
|
||||||
&& statusChangeEvent.getStatus() == ChainStatus.CANCELLED) {
|
|
||||||
registry.cancelExecution(chain.getStateInstanceId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,279 +0,0 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.ObjectProvider;
|
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 维护工作流实例到活动数据集查询的跨节点取消映射。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class WorkflowDatasetQueryCancellationRegistry {
|
|
||||||
|
|
||||||
private static final String ACTIVE_KEY_PREFIX =
|
|
||||||
"easyflow:workflow:dataset-query:active:";
|
|
||||||
private static final String CANCELLED_KEY_PREFIX =
|
|
||||||
"easyflow:workflow:dataset-query:cancelled:";
|
|
||||||
private static final Duration STATE_TTL = Duration.ofHours(24);
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(
|
|
||||||
WorkflowDatasetQueryCancellationRegistry.class);
|
|
||||||
|
|
||||||
private final DatacenterFederationQueryCancellationService cancellationService;
|
|
||||||
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
|
|
||||||
private final ConcurrentHashMap<String, ConcurrentHashMap<String, BigInteger>>
|
|
||||||
localActive = new ConcurrentHashMap<>();
|
|
||||||
private final ConcurrentHashMap<String, Long> localCancelledUntil =
|
|
||||||
new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建工作流查询取消登记表。
|
|
||||||
*
|
|
||||||
* @param cancellationService 数据中枢查询取消服务
|
|
||||||
* @param redisTemplateProvider 可选 Redis 模板
|
|
||||||
*/
|
|
||||||
public WorkflowDatasetQueryCancellationRegistry(
|
|
||||||
DatacenterFederationQueryCancellationService cancellationService,
|
|
||||||
ObjectProvider<StringRedisTemplate> redisTemplateProvider) {
|
|
||||||
this.cancellationService = cancellationService;
|
|
||||||
this.redisTemplateProvider = redisTemplateProvider;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在数据库查询开始前登记工作流与 QueryId。
|
|
||||||
*
|
|
||||||
* @param stateInstanceId 工作流实例 ID
|
|
||||||
* @param account 执行账号
|
|
||||||
* @param queryId 查询 UUID
|
|
||||||
* @return 必须关闭的登记句柄
|
|
||||||
* @throws BusinessException 参数缺失时抛出
|
|
||||||
*/
|
|
||||||
public Registration register(
|
|
||||||
String stateInstanceId,
|
|
||||||
LoginAccount account,
|
|
||||||
String queryId) {
|
|
||||||
String instanceId = requireText(stateInstanceId, "工作流实例 ID 不能为空");
|
|
||||||
String normalizedQueryId = requireText(queryId, "queryId 不能为空");
|
|
||||||
BigInteger tenantId = requireTenantId(account);
|
|
||||||
localActive.computeIfAbsent(
|
|
||||||
instanceId,
|
|
||||||
ignored -> new ConcurrentHashMap<>())
|
|
||||||
.put(normalizedQueryId, tenantId);
|
|
||||||
persistActive(instanceId, normalizedQueryId, tenantId);
|
|
||||||
Registration registration = new Registration(
|
|
||||||
instanceId, normalizedQueryId, tenantId);
|
|
||||||
if (isExecutionCancelled(instanceId)) {
|
|
||||||
cancelQuery(normalizedQueryId, tenantId);
|
|
||||||
}
|
|
||||||
return registration;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 取消工作流当前登记的全部数据集查询。
|
|
||||||
*
|
|
||||||
* @param stateInstanceId 工作流实例 ID
|
|
||||||
* @return 是否发现至少一个活动查询
|
|
||||||
*/
|
|
||||||
public boolean cancelExecution(String stateInstanceId) {
|
|
||||||
String instanceId = requireText(stateInstanceId, "工作流实例 ID 不能为空");
|
|
||||||
long expiresAt = System.currentTimeMillis() + STATE_TTL.toMillis();
|
|
||||||
localCancelledUntil.put(instanceId, expiresAt);
|
|
||||||
persistCancellationMarker(instanceId);
|
|
||||||
|
|
||||||
Map<String, BigInteger> active = new LinkedHashMap<>();
|
|
||||||
ConcurrentHashMap<String, BigInteger> local = localActive.get(instanceId);
|
|
||||||
if (local != null) {
|
|
||||||
active.putAll(local);
|
|
||||||
}
|
|
||||||
loadPersistedActive(instanceId).forEach(active::putIfAbsent);
|
|
||||||
active.forEach((queryId, tenantId) -> {
|
|
||||||
try {
|
|
||||||
cancelQuery(queryId, tenantId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
// 单个驱动取消失败不能阻断同一工作流的其他活动查询。
|
|
||||||
log.error("Failed to cancel workflow dataset query {} for {}",
|
|
||||||
queryId, instanceId, exception);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return !active.isEmpty();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 清理本机已过期的取消墓碑,避免不可达工作流持续占用内存。
|
|
||||||
*/
|
|
||||||
@Scheduled(fixedDelayString =
|
|
||||||
"${easyflow.workflow.dataset-query-cancel-cleanup-ms:60000}")
|
|
||||||
public void cleanupExpiredLocalMarkers() {
|
|
||||||
long now = System.currentTimeMillis();
|
|
||||||
localCancelledUntil.entrySet().removeIf(
|
|
||||||
entry -> entry.getValue() <= now);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void cancelQuery(String queryId, BigInteger tenantId) {
|
|
||||||
LoginAccount account = new LoginAccount();
|
|
||||||
account.setTenantId(tenantId);
|
|
||||||
cancellationService.cancel(queryId, account);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isExecutionCancelled(String stateInstanceId) {
|
|
||||||
Long localDeadline = localCancelledUntil.get(stateInstanceId);
|
|
||||||
if (localDeadline != null && localDeadline > System.currentTimeMillis()) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
|
||||||
if (redisTemplate == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return Boolean.TRUE.equals(redisTemplate.hasKey(
|
|
||||||
cancelledKey(stateInstanceId)));
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to read workflow dataset cancellation marker for {}",
|
|
||||||
stateInstanceId, exception);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void persistActive(
|
|
||||||
String stateInstanceId,
|
|
||||||
String queryId,
|
|
||||||
BigInteger tenantId) {
|
|
||||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
|
||||||
if (redisTemplate == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
String key = activeKey(stateInstanceId);
|
|
||||||
redisTemplate.opsForHash().put(key, queryId, tenantId.toString());
|
|
||||||
redisTemplate.expire(key, STATE_TTL);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to persist workflow dataset query mapping for {}",
|
|
||||||
stateInstanceId, exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, BigInteger> loadPersistedActive(String stateInstanceId) {
|
|
||||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
|
||||||
if (redisTemplate == null) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
Map<Object, Object> entries = redisTemplate.opsForHash()
|
|
||||||
.entries(activeKey(stateInstanceId));
|
|
||||||
Map<String, BigInteger> active = new LinkedHashMap<>();
|
|
||||||
entries.forEach((queryId, tenantId) -> {
|
|
||||||
try {
|
|
||||||
active.put(String.valueOf(queryId),
|
|
||||||
new BigInteger(String.valueOf(tenantId)));
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Ignored invalid workflow dataset query mapping for {}",
|
|
||||||
stateInstanceId, exception);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return active;
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to load workflow dataset query mappings for {}",
|
|
||||||
stateInstanceId, exception);
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void persistCancellationMarker(String stateInstanceId) {
|
|
||||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
|
||||||
if (redisTemplate == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
redisTemplate.opsForValue().set(
|
|
||||||
cancelledKey(stateInstanceId), "1", STATE_TTL);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to persist workflow dataset cancellation marker for {}",
|
|
||||||
stateInstanceId, exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void unregister(
|
|
||||||
String stateInstanceId,
|
|
||||||
String queryId,
|
|
||||||
BigInteger tenantId) {
|
|
||||||
localActive.computeIfPresent(stateInstanceId, (ignored, queries) -> {
|
|
||||||
queries.remove(queryId, tenantId);
|
|
||||||
return queries.isEmpty() ? null : queries;
|
|
||||||
});
|
|
||||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
|
||||||
if (redisTemplate == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
redisTemplate.opsForHash().delete(
|
|
||||||
activeKey(stateInstanceId), queryId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to remove workflow dataset query mapping for {}",
|
|
||||||
stateInstanceId, exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String activeKey(String stateInstanceId) {
|
|
||||||
return ACTIVE_KEY_PREFIX + stateInstanceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String cancelledKey(String stateInstanceId) {
|
|
||||||
return CANCELLED_KEY_PREFIX + stateInstanceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String requireText(String value, String message) {
|
|
||||||
if (value == null || value.isBlank()) {
|
|
||||||
throw new BusinessException(message);
|
|
||||||
}
|
|
||||||
return value.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
private BigInteger requireTenantId(LoginAccount account) {
|
|
||||||
if (account == null || account.getTenantId() == null) {
|
|
||||||
throw new BusinessException("工作流数据集查询缺少执行租户");
|
|
||||||
}
|
|
||||||
return account.getTenantId();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 单次工作流查询登记句柄。
|
|
||||||
*/
|
|
||||||
public final class Registration implements AutoCloseable {
|
|
||||||
|
|
||||||
private final String stateInstanceId;
|
|
||||||
private final String queryId;
|
|
||||||
private final BigInteger tenantId;
|
|
||||||
private final AtomicBoolean closed = new AtomicBoolean();
|
|
||||||
|
|
||||||
private Registration(
|
|
||||||
String stateInstanceId,
|
|
||||||
String queryId,
|
|
||||||
BigInteger tenantId) {
|
|
||||||
this.stateInstanceId = stateInstanceId;
|
|
||||||
this.queryId = queryId;
|
|
||||||
this.tenantId = tenantId;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 幂等注销当前工作流查询。
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public void close() {
|
|
||||||
if (closed.compareAndSet(false, true)) {
|
|
||||||
unregister(stateInstanceId, queryId, tenantId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,7 +14,6 @@ import org.springframework.context.annotation.Configuration;
|
|||||||
import tech.easyflow.ai.easyagentsflow.listener.ChainErrorListenerForSave;
|
import tech.easyflow.ai.easyagentsflow.listener.ChainErrorListenerForSave;
|
||||||
import tech.easyflow.ai.easyagentsflow.listener.ChainEventListenerForSave;
|
import tech.easyflow.ai.easyagentsflow.listener.ChainEventListenerForSave;
|
||||||
import tech.easyflow.ai.easyagentsflow.listener.NodeErrorListenerForSave;
|
import tech.easyflow.ai.easyagentsflow.listener.NodeErrorListenerForSave;
|
||||||
import tech.easyflow.ai.easyagentsflow.cancellation.WorkflowDatasetQueryCancellationListener;
|
|
||||||
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadCleanupListener;
|
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadCleanupListener;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
@@ -41,9 +40,6 @@ public class ChainExecutorConfig {
|
|||||||
@Resource
|
@Resource
|
||||||
private WorkflowApiUploadCleanupListener workflowApiUploadCleanupListener;
|
private WorkflowApiUploadCleanupListener workflowApiUploadCleanupListener;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowDatasetQueryCancellationListener
|
|
||||||
workflowDatasetQueryCancellationListener;
|
|
||||||
@Resource
|
|
||||||
private WorkflowExecutionBudgetProperties workflowExecutionBudgetProperties;
|
private WorkflowExecutionBudgetProperties workflowExecutionBudgetProperties;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowRuntimeProperties workflowRuntimeProperties;
|
private WorkflowRuntimeProperties workflowRuntimeProperties;
|
||||||
@@ -95,9 +91,6 @@ public class ChainExecutorConfig {
|
|||||||
chainExecutor.addEventListener(
|
chainExecutor.addEventListener(
|
||||||
ChainStatusChangeEvent.class,
|
ChainStatusChangeEvent.class,
|
||||||
workflowApiUploadCleanupListener);
|
workflowApiUploadCleanupListener);
|
||||||
chainExecutor.addEventListener(
|
|
||||||
ChainStatusChangeEvent.class,
|
|
||||||
workflowDatasetQueryCancellationListener);
|
|
||||||
chainExecutor.addErrorListener(new ChainErrorListenerForSave());
|
chainExecutor.addErrorListener(new ChainErrorListenerForSave());
|
||||||
chainExecutor.addNodeErrorListener(new NodeErrorListenerForSave());
|
chainExecutor.addNodeErrorListener(new NodeErrorListenerForSave());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,10 +23,6 @@ public class ChainInfo implements Serializable {
|
|||||||
* 消息,错误时显示
|
* 消息,错误时显示
|
||||||
*/
|
*/
|
||||||
private String message;
|
private String message;
|
||||||
private WorkflowExecutionError error;
|
|
||||||
|
|
||||||
public WorkflowExecutionError getError() { return error; }
|
|
||||||
public void setError(WorkflowExecutionError error) { this.error = error; }
|
|
||||||
/**
|
/**
|
||||||
* 执行结果
|
* 执行结果
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -28,10 +28,6 @@ public class NodeInfo implements Serializable {
|
|||||||
* 消息,错误时显示
|
* 消息,错误时显示
|
||||||
*/
|
*/
|
||||||
private String message;
|
private String message;
|
||||||
private WorkflowExecutionError error;
|
|
||||||
|
|
||||||
public WorkflowExecutionError getError() { return error; }
|
|
||||||
public void setError(WorkflowExecutionError error) { this.error = error; }
|
|
||||||
/**
|
/**
|
||||||
* 执行结果
|
* 执行结果
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -1,32 +0,0 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.entity;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
|
|
||||||
/** 三个运行出口共用的安全错误信息。 */
|
|
||||||
public class WorkflowExecutionError implements Serializable {
|
|
||||||
private static final long serialVersionUID = 1L;
|
|
||||||
|
|
||||||
private final String code;
|
|
||||||
private final String reasonCode;
|
|
||||||
private final String message;
|
|
||||||
private final String nodeId;
|
|
||||||
private final String nodeName;
|
|
||||||
private final boolean retryable;
|
|
||||||
|
|
||||||
public WorkflowExecutionError(String code, String reasonCode, String message,
|
|
||||||
String nodeId, String nodeName, boolean retryable) {
|
|
||||||
this.code = code;
|
|
||||||
this.reasonCode = reasonCode;
|
|
||||||
this.message = message;
|
|
||||||
this.nodeId = nodeId;
|
|
||||||
this.nodeName = nodeName;
|
|
||||||
this.retryable = retryable;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getCode() { return code; }
|
|
||||||
public String getReasonCode() { return reasonCode; }
|
|
||||||
public String getMessage() { return message; }
|
|
||||||
public String getNodeId() { return nodeId; }
|
|
||||||
public String getNodeName() { return nodeName; }
|
|
||||||
public boolean isRetryable() { return retryable; }
|
|
||||||
}
|
|
||||||
@@ -10,7 +10,6 @@ import org.slf4j.LoggerFactory;
|
|||||||
import org.springframework.dao.DuplicateKeyException;
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import tech.easyflow.ai.easyagentsflow.event.WorkflowExecutionAuditEvent;
|
import tech.easyflow.ai.easyagentsflow.event.WorkflowExecutionAuditEvent;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowExecutionErrorMapper;
|
|
||||||
import tech.easyflow.ai.easyagentsflow.event.WorkflowExecutionAuditProducer;
|
import tech.easyflow.ai.easyagentsflow.event.WorkflowExecutionAuditProducer;
|
||||||
import tech.easyflow.ai.easyagentsflow.repository.FrozenWorkflowDefinitionRegistry;
|
import tech.easyflow.ai.easyagentsflow.repository.FrozenWorkflowDefinitionRegistry;
|
||||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||||
@@ -112,7 +111,7 @@ public class ChainEventListenerForSave implements ChainEventListener {
|
|||||||
state.getExecuteResult()));
|
state.getExecuteResult()));
|
||||||
ExceptionSummary error = state.getError();
|
ExceptionSummary error = state.getError();
|
||||||
if (error != null) {
|
if (error != null) {
|
||||||
record.setErrorInfo(WorkflowExecutionErrorMapper.summary(WorkflowExecutionErrorMapper.chain(error, state.getStatus())));
|
record.setErrorInfo(error.getRootCauseClass() + " --> " + error.getRootCauseMessage());
|
||||||
}
|
}
|
||||||
sendAuditEvent(
|
sendAuditEvent(
|
||||||
WorkflowExecutionAuditEvent.Type.CHAIN_ENDED,
|
WorkflowExecutionAuditEvent.Type.CHAIN_ENDED,
|
||||||
@@ -210,13 +209,14 @@ public class ChainEventListenerForSave implements ChainEventListener {
|
|||||||
step.setEndTime(new Date());
|
step.setEndTime(new Date());
|
||||||
step.setStatus(nodeStatus.getValue());
|
step.setStatus(nodeStatus.getValue());
|
||||||
ExceptionSummary error =
|
ExceptionSummary error =
|
||||||
event.getErrorSummary() == null
|
event.getError() == null
|
||||||
? (legacyNodeState == null
|
? (legacyNodeState == null
|
||||||
? null
|
? null
|
||||||
: legacyNodeState.getError())
|
: legacyNodeState.getError())
|
||||||
: event.getErrorSummary();
|
: new ExceptionSummary(
|
||||||
|
event.getError());
|
||||||
if (error != null) {
|
if (error != null) {
|
||||||
step.setErrorInfo(WorkflowExecutionErrorMapper.summary(WorkflowExecutionErrorMapper.node(error, nodeStatus, node.getId(), node.getName())));
|
step.setErrorInfo(error.getRootCauseClass() + " --> " + error.getRootCauseMessage());
|
||||||
}
|
}
|
||||||
sendAuditEvent(
|
sendAuditEvent(
|
||||||
WorkflowExecutionAuditEvent.Type.NODE_ENDED,
|
WorkflowExecutionAuditEvent.Type.NODE_ENDED,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.service;
|
package tech.easyflow.ai.easyagentsflow.service;
|
||||||
|
|
||||||
|
import com.easyagents.document.core.exception.DocumentParseException;
|
||||||
import com.easyagents.flow.core.chain.ChainState;
|
import com.easyagents.flow.core.chain.ChainState;
|
||||||
import com.easyagents.flow.core.chain.ExceptionSummary;
|
import com.easyagents.flow.core.chain.ExceptionSummary;
|
||||||
import com.easyagents.flow.core.chain.NodeState;
|
import com.easyagents.flow.core.chain.NodeState;
|
||||||
@@ -7,9 +8,11 @@ import com.easyagents.flow.core.chain.NodeStatus;
|
|||||||
import com.easyagents.flow.core.chain.repository.ChainStateRepository;
|
import com.easyagents.flow.core.chain.repository.ChainStateRepository;
|
||||||
import com.easyagents.flow.core.chain.repository.NodeStateRepository;
|
import com.easyagents.flow.core.chain.repository.NodeStateRepository;
|
||||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||||
|
import com.easyagents.flow.core.code.impl.JavascriptExecutionException;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
||||||
|
import tech.easyflow.common.util.StringUtil;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
@@ -56,8 +59,10 @@ public class TinyFlowService {
|
|||||||
? Map.of()
|
? Map.of()
|
||||||
: resolvedNodeNames;
|
: resolvedNodeNames;
|
||||||
for (NodeInfo node : nodes) {
|
for (NodeInfo node : nodes) {
|
||||||
if (node == null) continue;
|
if (node != null
|
||||||
node.setNodeName(nodeNames.get(node.getNodeId()));
|
&& StringUtil.noText(node.getNodeName())) {
|
||||||
|
node.setNodeName(nodeNames.get(node.getNodeId()));
|
||||||
|
}
|
||||||
processNodeState(executeId, node, chainState, nodeStateRepository);
|
processNodeState(executeId, node, chainState, nodeStateRepository);
|
||||||
res.getNodes().put(node.getNodeId(), node);
|
res.getNodes().put(node.getNodeId(), node);
|
||||||
}
|
}
|
||||||
@@ -95,8 +100,9 @@ public class TinyFlowService {
|
|||||||
res.setExecuteId(executeId);
|
res.setExecuteId(executeId);
|
||||||
res.setStatus(chainState.getStatus().getValue());
|
res.setStatus(chainState.getStatus().getValue());
|
||||||
ExceptionSummary chainError = chainState.getError();
|
ExceptionSummary chainError = chainState.getError();
|
||||||
res.setError(WorkflowExecutionErrorMapper.chain(chainError, chainState.getStatus()));
|
if (chainError != null) {
|
||||||
res.setMessage(WorkflowExecutionErrorMapper.summary(res.getError()));
|
res.setMessage(formatError(chainError));
|
||||||
|
}
|
||||||
Map<String, Object> executeResult = chainState.getExecuteResult();
|
Map<String, Object> executeResult = chainState.getExecuteResult();
|
||||||
if (executeResult != null && !executeResult.isEmpty()) {
|
if (executeResult != null && !executeResult.isEmpty()) {
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
@@ -121,9 +127,12 @@ public class TinyFlowService {
|
|||||||
? NodeStatus.READY.getValue()
|
? NodeStatus.READY.getValue()
|
||||||
: nodeState.getStatus().getValue());
|
: nodeState.getStatus().getValue());
|
||||||
|
|
||||||
node.setError(nodeState == null ? null : WorkflowExecutionErrorMapper.node(
|
if (nodeState != null) {
|
||||||
nodeState.getError(), nodeState.getStatus(), nodeId, node.getNodeName(), chainState.getStatus().isTerminal()));
|
ExceptionSummary error = nodeState.getError();
|
||||||
node.setMessage(WorkflowExecutionErrorMapper.summary(node.getError()));
|
if (error != null) {
|
||||||
|
node.setMessage(formatError(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Map<String, Object> nodeExecuteResult = chainState.getNodeExecuteResult(nodeId);
|
Map<String, Object> nodeExecuteResult = chainState.getNodeExecuteResult(nodeId);
|
||||||
if (nodeExecuteResult != null && !nodeExecuteResult.isEmpty()) {
|
if (nodeExecuteResult != null && !nodeExecuteResult.isEmpty()) {
|
||||||
@@ -142,4 +151,34 @@ public class TinyFlowService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将执行异常转换为试运行界面可读的错误信息。
|
||||||
|
*
|
||||||
|
* @param error 持久化的异常摘要
|
||||||
|
* @return 可展示的错误信息
|
||||||
|
*/
|
||||||
|
private String formatError(ExceptionSummary error) {
|
||||||
|
if (JavascriptExecutionException.class.getName()
|
||||||
|
.equals(error.getExceptionClass())
|
||||||
|
&& StringUtil.hasText(error.getMessage())) {
|
||||||
|
return error.getMessage();
|
||||||
|
}
|
||||||
|
String rootClass = StringUtil.hasText(error.getRootCauseClass())
|
||||||
|
? error.getRootCauseClass()
|
||||||
|
: error.getExceptionClass();
|
||||||
|
String rootMessage = StringUtil.hasText(error.getRootCauseMessage())
|
||||||
|
? error.getRootCauseMessage()
|
||||||
|
: error.getMessage();
|
||||||
|
if (DocumentParseException.class.getName().equals(rootClass)
|
||||||
|
&& StringUtil.hasText(rootMessage)) {
|
||||||
|
return rootMessage;
|
||||||
|
}
|
||||||
|
if (StringUtil.noText(rootClass)) {
|
||||||
|
return rootMessage;
|
||||||
|
}
|
||||||
|
if (StringUtil.noText(rootMessage)) {
|
||||||
|
return rootClass;
|
||||||
|
}
|
||||||
|
return rootClass + " --> " + rootMessage;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,13 +14,12 @@ import tech.easyflow.datacenter.execution.model.DatasetRef;
|
|||||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
|
||||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Comparator;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -35,7 +34,6 @@ public class WorkflowDatacenterContentService {
|
|||||||
public static final String LLM_NODE_TYPE = "llmNode";
|
public static final String LLM_NODE_TYPE = "llmNode";
|
||||||
public static final String QUERY_DATA_CONTEXT = "queryDataContext";
|
public static final String QUERY_DATA_CONTEXT = "queryDataContext";
|
||||||
public static final String SEARCH_SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
|
public static final String SEARCH_SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
|
||||||
public static final String SEARCH_TABLE_MISSING_MESSAGE = "查询数据节点未选择已接入表";
|
|
||||||
public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
|
public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
|
||||||
public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
|
public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
|
||||||
public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效,请重新选择查询数据节点";
|
public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效,请重新选择查询数据节点";
|
||||||
@@ -137,16 +135,11 @@ public class WorkflowDatacenterContentService {
|
|||||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||||
}
|
}
|
||||||
if (datasetRef.getTableId() == null) {
|
|
||||||
throw new BusinessException(SEARCH_TABLE_MISSING_MESSAGE);
|
|
||||||
}
|
|
||||||
String querySql = data == null ? null : trimToNull(data.getString("querySql"));
|
String querySql = data == null ? null : trimToNull(data.getString("querySql"));
|
||||||
if (!StringUtils.hasText(querySql)) {
|
if (!StringUtils.hasText(querySql)) {
|
||||||
throw new BusinessException(SEARCH_SQL_MISSING_MESSAGE);
|
throw new BusinessException(SEARCH_SQL_MISSING_MESSAGE);
|
||||||
}
|
}
|
||||||
DatasetRef boundRef = bindAuthoritativeTenant(datasetRef);
|
return datasetRef;
|
||||||
data.put("datasetRef", boundRef);
|
|
||||||
return boundRef;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public DatasetRef requireSaveDatasetRef(JSONObject data) {
|
public DatasetRef requireSaveDatasetRef(JSONObject data) {
|
||||||
@@ -157,31 +150,6 @@ public class WorkflowDatacenterContentService {
|
|||||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||||
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
|
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
|
||||||
}
|
}
|
||||||
DatasetRef boundRef = bindAuthoritativeTenant(datasetRef);
|
|
||||||
data.put("datasetRef", boundRef);
|
|
||||||
return boundRef;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 依据当前租户可见的权威 Source/Table 覆盖调用方声明的租户字段。
|
|
||||||
*
|
|
||||||
* @param datasetRef 工作流数据集引用
|
|
||||||
* @return 已绑定权威租户的数据集引用
|
|
||||||
*/
|
|
||||||
private DatasetRef bindAuthoritativeTenant(DatasetRef datasetRef) {
|
|
||||||
DatacenterTable table = datasetRef.getTableId() == null
|
|
||||||
? null : registryService.getTableWithFields(datasetRef.getTableId());
|
|
||||||
BigInteger sourceId = table == null ? datasetRef.getSourceId() : table.getSourceId();
|
|
||||||
if (sourceId == null) {
|
|
||||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
|
||||||
}
|
|
||||||
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
|
||||||
if (table != null && (!sourceId.equals(table.getSourceId())
|
|
||||||
|| !java.util.Objects.equals(source.getTenantId(), table.getTenantId()))) {
|
|
||||||
throw new BusinessException("数据集引用与当前租户不一致");
|
|
||||||
}
|
|
||||||
datasetRef.setTenantId(source.getTenantId());
|
|
||||||
datasetRef.setSourceId(sourceId);
|
|
||||||
return datasetRef;
|
return datasetRef;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -204,68 +172,41 @@ public class WorkflowDatacenterContentService {
|
|||||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||||
}
|
}
|
||||||
if (datasetRef.getTableId() == null) {
|
|
||||||
throw new BusinessException(SEARCH_TABLE_MISSING_MESSAGE);
|
|
||||||
}
|
|
||||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||||
DatacenterTable fullTable = registryService.getTableWithFields(
|
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), datasetRef.getCatalogId());
|
||||||
datasetRef.getTableId());
|
managedTables.sort(Comparator.comparing(table -> table.getTableName() == null ? "" : table.getTableName()));
|
||||||
if (fullTable == null
|
|
||||||
|| !datasetRef.getSourceId().equals(fullTable.getSourceId())
|
|
||||||
|| !Integer.valueOf(1).equals(fullTable.getQueryable())
|
|
||||||
|| !DatacenterMetadataStatus.ACTIVE.name().equals(
|
|
||||||
fullTable.getMetadataStatus())) {
|
|
||||||
throw new BusinessException("查询数据节点绑定的表不可用");
|
|
||||||
}
|
|
||||||
DatacenterCatalog catalog = registryService.getCatalogById(
|
|
||||||
fullTable.getCatalogId());
|
|
||||||
if (StringUtils.hasText(datasetRef.getCatalogName())
|
|
||||||
&& (catalog == null || !datasetRef.getCatalogName().equals(
|
|
||||||
catalog.getCatalogName()))) {
|
|
||||||
throw new BusinessException("查询数据节点绑定的命名空间已变化");
|
|
||||||
}
|
|
||||||
JSONObject sourceSummary = new JSONObject();
|
JSONObject sourceSummary = new JSONObject();
|
||||||
sourceSummary.put("sourceName", source.getSourceName());
|
sourceSummary.put("sourceName", source.getSourceName());
|
||||||
sourceSummary.put("sourceType", source.getSourceType());
|
sourceSummary.put("sourceType", source.getSourceType());
|
||||||
JSONArray tables = new JSONArray();
|
JSONArray tables = new JSONArray();
|
||||||
JSONObject tableSummary = new JSONObject();
|
for (DatacenterTable table : managedTables) {
|
||||||
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
|
DatacenterTable fullTable = registryService.getTableWithFields(table.getId());
|
||||||
tableSummary.put("tableName", fullTable.getTableName());
|
DatacenterCatalog catalog = registryService.getCatalogById(fullTable.getCatalogId());
|
||||||
tableSummary.put("tableDesc", fullTable.getTableDesc());
|
if (StringUtils.hasText(datasetRef.getCatalogName())
|
||||||
JSONArray fields = new JSONArray();
|
&& (catalog == null || !datasetRef.getCatalogName().equals(catalog.getCatalogName()))) {
|
||||||
if (fullTable.getFields() != null) {
|
continue;
|
||||||
for (DatacenterTableField field : fullTable.getFields()) {
|
|
||||||
if (!isQueryablePublicField(field)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
JSONObject fieldSummary = new JSONObject();
|
|
||||||
fieldSummary.put("fieldName", field.getFieldName());
|
|
||||||
fieldSummary.put("fieldDesc", field.getFieldDesc());
|
|
||||||
fieldSummary.put("fieldType", resolveFieldType(field));
|
|
||||||
fields.add(fieldSummary);
|
|
||||||
}
|
}
|
||||||
|
JSONObject tableSummary = new JSONObject();
|
||||||
|
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
|
||||||
|
tableSummary.put("tableName", fullTable.getTableName());
|
||||||
|
tableSummary.put("tableDesc", fullTable.getTableDesc());
|
||||||
|
JSONArray fields = new JSONArray();
|
||||||
|
if (fullTable.getFields() != null) {
|
||||||
|
for (DatacenterTableField field : fullTable.getFields()) {
|
||||||
|
JSONObject fieldSummary = new JSONObject();
|
||||||
|
fieldSummary.put("fieldName", field.getFieldName());
|
||||||
|
fieldSummary.put("fieldDesc", field.getFieldDesc());
|
||||||
|
fieldSummary.put("fieldType", resolveFieldType(field));
|
||||||
|
fields.add(fieldSummary);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tableSummary.put("fields", fields);
|
||||||
|
tables.add(tableSummary);
|
||||||
}
|
}
|
||||||
tableSummary.put("fields", fields);
|
|
||||||
tables.add(tableSummary);
|
|
||||||
sourceSummary.put("tables", tables);
|
sourceSummary.put("tables", tables);
|
||||||
return sourceSummary;
|
return sourceSummary;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断字段是否允许暴露给 SQL 生成上下文。
|
|
||||||
*
|
|
||||||
* @param field 字段元数据
|
|
||||||
* @return 是否为当前可查询的公开字段
|
|
||||||
*/
|
|
||||||
private boolean isQueryablePublicField(DatacenterTableField field) {
|
|
||||||
return field != null
|
|
||||||
&& Integer.valueOf(1).equals(field.getQueryable())
|
|
||||||
&& DatacenterMetadataStatus.ACTIVE.name().equals(
|
|
||||||
field.getMetadataStatus())
|
|
||||||
&& DatacenterSensitivityLevel.PUBLIC.name().equals(
|
|
||||||
field.getSensitivityLevel());
|
|
||||||
}
|
|
||||||
|
|
||||||
private void injectQueryDataContext(JSONObject data, Map<String, JSONObject> nodeMap) {
|
private void injectQueryDataContext(JSONObject data, Map<String, JSONObject> nodeMap) {
|
||||||
if (data == null) {
|
if (data == null) {
|
||||||
return;
|
return;
|
||||||
@@ -276,7 +217,7 @@ public class WorkflowDatacenterContentService {
|
|||||||
removeQueryDataContextPlaceholder(data);
|
removeQueryDataContextPlaceholder(data);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Map<String, JSONObject> sourceSummaries = new LinkedHashMap<>();
|
Map<BigInteger, JSONObject> sourceSummaries = new LinkedHashMap<>();
|
||||||
Set<String> visitedNodeIds = new LinkedHashSet<>();
|
Set<String> visitedNodeIds = new LinkedHashSet<>();
|
||||||
for (int i = 0; i < nodeIds.size(); i++) {
|
for (int i = 0; i < nodeIds.size(); i++) {
|
||||||
String nodeId = trimToNull(nodeIds.getString(i));
|
String nodeId = trimToNull(nodeIds.getString(i));
|
||||||
@@ -288,10 +229,7 @@ public class WorkflowDatacenterContentService {
|
|||||||
throw new BusinessException(INVALID_QUERY_CONTEXT_MESSAGE);
|
throw new BusinessException(INVALID_QUERY_CONTEXT_MESSAGE);
|
||||||
}
|
}
|
||||||
DatasetRef datasetRef = requireSearchDatasetRef(targetNode.getJSONObject("data"));
|
DatasetRef datasetRef = requireSearchDatasetRef(targetNode.getJSONObject("data"));
|
||||||
String summaryKey = datasetRef.getSourceId() + ":" + datasetRef.getTableId();
|
sourceSummaries.putIfAbsent(datasetRef.getSourceId(), buildSourceSummary(datasetRef));
|
||||||
if (!sourceSummaries.containsKey(summaryKey)) {
|
|
||||||
sourceSummaries.put(summaryKey, buildSourceSummary(datasetRef));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values()));
|
String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values()));
|
||||||
upsertQueryDataContextParameter(data, contextValue);
|
upsertQueryDataContextParameter(data, contextValue);
|
||||||
@@ -405,7 +343,6 @@ public class WorkflowDatacenterContentService {
|
|||||||
|
|
||||||
private DatasetRef copyDatasetRef(DatasetRef datasetRef) {
|
private DatasetRef copyDatasetRef(DatasetRef datasetRef) {
|
||||||
DatasetRef copy = new DatasetRef();
|
DatasetRef copy = new DatasetRef();
|
||||||
copy.setTenantId(datasetRef.getTenantId());
|
|
||||||
copy.setSourceId(datasetRef.getSourceId());
|
copy.setSourceId(datasetRef.getSourceId());
|
||||||
copy.setCatalogId(datasetRef.getCatalogId());
|
copy.setCatalogId(datasetRef.getCatalogId());
|
||||||
copy.setCatalogName(datasetRef.getCatalogName());
|
copy.setCatalogName(datasetRef.getCatalogName());
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.service;
|
|
||||||
|
|
||||||
import com.easyagents.flow.core.chain.ChainStatus;
|
|
||||||
import com.easyagents.flow.core.chain.ExceptionSummary;
|
|
||||||
import com.easyagents.flow.core.chain.NodeStatus;
|
|
||||||
import com.easyagents.flow.core.chain.WorkflowErrorReason;
|
|
||||||
import com.easyagents.flow.core.chain.WorkflowExecutionException;
|
|
||||||
import org.springframework.web.context.request.RequestContextHolder;
|
|
||||||
import org.springframework.web.context.request.ServletRequestAttributes;
|
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowExecutionError;
|
|
||||||
import tech.easyflow.common.web.error.RequestErrorProfile;
|
|
||||||
import tech.easyflow.common.web.error.WebErrorMapping;
|
|
||||||
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
|
|
||||||
/** 只通过稳定原因码生成外部信息,原始 cause 与响应正文不进入展示或审计摘要。 */
|
|
||||||
public final class WorkflowExecutionErrorMapper {
|
|
||||||
private WorkflowExecutionErrorMapper() { }
|
|
||||||
|
|
||||||
public static WorkflowExecutionError chain(ExceptionSummary error, ChainStatus status) {
|
|
||||||
if (status != ChainStatus.FAILED && status != ChainStatus.ERROR) return null;
|
|
||||||
return map(error, true, null, null, status == ChainStatus.ERROR);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static WorkflowExecutionError node(ExceptionSummary error, NodeStatus status, String nodeId, String nodeName) {
|
|
||||||
return node(error, status, nodeId, nodeName, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static WorkflowExecutionError node(ExceptionSummary error, NodeStatus status, String nodeId, String nodeName,
|
|
||||||
boolean executionTerminal) {
|
|
||||||
if (status != NodeStatus.FAILED && status != NodeStatus.ERROR) return null;
|
|
||||||
return map(error, false, nodeId, nodeName, status == NodeStatus.ERROR && !executionTerminal);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static WorkflowExecutionError map(ExceptionSummary error, boolean workflow,
|
|
||||||
String nodeId, String nodeName, boolean retryable) {
|
|
||||||
if (error != null && error.getNodeId() != null) {
|
|
||||||
nodeId = error.getNodeId();
|
|
||||||
nodeName = error.getNodeName() == null ? nodeName : error.getNodeName();
|
|
||||||
}
|
|
||||||
return fromReason(error == null ? null : error.getErrorCode(), workflow, nodeId, nodeName, retryable);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static WorkflowExecutionError fromReason(String reasonCode, boolean workflow,
|
|
||||||
String nodeId, String nodeName, boolean retryable) {
|
|
||||||
WorkflowErrorReason reason = WorkflowErrorReason.fromCode(reasonCode);
|
|
||||||
if (reason == null) {
|
|
||||||
reason = nodeId == null ? WorkflowErrorReason.WORKFLOW_INTERNAL_ERROR : WorkflowErrorReason.NODE_EXECUTION_FAILED;
|
|
||||||
}
|
|
||||||
return new WorkflowExecutionError(workflow ? "WORKFLOW_EXECUTION_FAILED" : "NODE_EXECUTION_FAILED",
|
|
||||||
reason.getCode(), reason.getDefaultMessage(), nodeId, nodeName, retryable);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String summary(WorkflowExecutionError error) {
|
|
||||||
if (error == null) return null;
|
|
||||||
String name = error.getNodeName();
|
|
||||||
return name == null || name.isBlank() ? error.getMessage() : "「" + name + "」:" + error.getMessage();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 单节点同步执行沿用全局 HTTP 错误处理,并保留原请求的其他错误契约。 */
|
|
||||||
public static void installRequestProfile() {
|
|
||||||
if (!(RequestContextHolder.getRequestAttributes() instanceof ServletRequestAttributes attributes)) return;
|
|
||||||
var request = attributes.getRequest();
|
|
||||||
Object previous = request.getAttribute(RequestErrorProfile.ATTRIBUTE_NAME);
|
|
||||||
request.setAttribute(RequestErrorProfile.ATTRIBUTE_NAME, (RequestErrorProfile) (req, exception) -> {
|
|
||||||
if (exception instanceof WorkflowExecutionException failure) {
|
|
||||||
WorkflowExecutionError error = map(new ExceptionSummary(failure), false, null, null, false);
|
|
||||||
Map<String, Object> data = new LinkedHashMap<>();
|
|
||||||
data.put("error", error);
|
|
||||||
if (failure.getChainId() != null) data.put("executeId", failure.getChainId());
|
|
||||||
return new WebErrorMapping(500, 500, error.getMessage(), data);
|
|
||||||
}
|
|
||||||
return previous instanceof RequestErrorProfile profile ? profile.map(req, exception) : null;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -485,7 +485,7 @@ public class WorkflowApiUploadLifecycleService {
|
|||||||
return new BusinessException(
|
return new BusinessException(
|
||||||
500,
|
500,
|
||||||
50001,
|
50001,
|
||||||
"文件存储处理失败",
|
"文件存储处理失败,请联系管理员并提供 requestId",
|
||||||
error);
|
error);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -44,13 +44,6 @@ public interface DocumentChunkSyncTaskMapper extends BaseMapper<DocumentChunkSyn
|
|||||||
+ "ORDER BY sync_version, id FOR UPDATE")
|
+ "ORDER BY sync_version, id FOR UPDATE")
|
||||||
List<BigInteger> lockChunkTasks(@Param("chunkId") BigInteger chunkId);
|
List<BigInteger> lockChunkTasks(@Param("chunkId") BigInteger chunkId);
|
||||||
|
|
||||||
@Select("<script>SELECT " + SELECT_COLUMNS + " FROM tb_document_chunk_sync_task "
|
|
||||||
+ "WHERE operation='UPSERT' AND (chunk_id, sync_version) IN "
|
|
||||||
+ "(SELECT id, index_sync_version FROM tb_document_chunk WHERE id IN "
|
|
||||||
+ "<foreach collection='ids' item='id' open='(' separator=',' close=')'>#{id}</foreach>)"
|
|
||||||
+ "</script>")
|
|
||||||
List<DocumentChunkSyncTask> selectCurrentForChunks(@Param("ids") List<BigInteger> ids);
|
|
||||||
|
|
||||||
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_chunk_sync_task "
|
@Select("SELECT " + SELECT_COLUMNS + " FROM tb_document_chunk_sync_task "
|
||||||
+ "WHERE status='PENDING' AND next_retry_at <= #{now} "
|
+ "WHERE status='PENDING' AND next_retry_at <= #{now} "
|
||||||
+ "AND (last_dispatched_at IS NULL OR last_dispatched_at <= #{redispatchBefore}) "
|
+ "AND (last_dispatched_at IS NULL OR last_dispatched_at <= #{redispatchBefore}) "
|
||||||
@@ -74,7 +67,7 @@ public interface DocumentChunkSyncTaskMapper extends BaseMapper<DocumentChunkSyn
|
|||||||
|
|
||||||
@Update("UPDATE tb_document_chunk_sync_task SET status='RUNNING', "
|
@Update("UPDATE tb_document_chunk_sync_task SET status='RUNNING', "
|
||||||
+ "attempt_count=attempt_count + 1, execution_token=#{token}, "
|
+ "attempt_count=attempt_count + 1, execution_token=#{token}, "
|
||||||
+ "lease_until=#{leaseUntil}, modified=#{now} "
|
+ "lease_until=#{leaseUntil}, error_code=NULL, error_message=NULL, modified=#{now} "
|
||||||
+ "WHERE id=#{id} AND status='PENDING' AND next_retry_at <= #{now}")
|
+ "WHERE id=#{id} AND status='PENDING' AND next_retry_at <= #{now}")
|
||||||
int claim(@Param("id") BigInteger id,
|
int claim(@Param("id") BigInteger id,
|
||||||
@Param("token") String token,
|
@Param("token") String token,
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.easyagents.flow.core.chain.Chain;
|
|||||||
import com.easyagents.flow.core.chain.runtime.RetryableTriggerException;
|
import com.easyagents.flow.core.chain.runtime.RetryableTriggerException;
|
||||||
import com.easyagents.flow.core.node.BaseNode;
|
import com.easyagents.flow.core.node.BaseNode;
|
||||||
import com.easyagents.flow.core.util.IoBulkhead;
|
import com.easyagents.flow.core.util.IoBulkhead;
|
||||||
|
import com.mybatisflex.core.tenant.TenantManager;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import tech.easyflow.ai.utils.WorkFlowUtil;
|
import tech.easyflow.ai.utils.WorkFlowUtil;
|
||||||
@@ -54,6 +55,7 @@ public class SaveDatasetNode extends BaseNode {
|
|||||||
rows.add(item instanceof JSONObject json ? json : JSONObject.from(item));
|
rows.add(item instanceof JSONObject json ? json : JSONObject.from(item));
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
TenantManager.ignoreTenantCondition();
|
||||||
try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) {
|
try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) {
|
||||||
writeService.saveRowsIdempotently(
|
writeService.saveRowsIdempotently(
|
||||||
datasetRef,
|
datasetRef,
|
||||||
@@ -75,6 +77,8 @@ public class SaveDatasetNode extends BaseNode {
|
|||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.error("工作流保存数据到统一数据集失败,datasetRef={}", datasetRef, ex);
|
log.error("工作流保存数据到统一数据集失败,datasetRef={}", datasetRef, ex);
|
||||||
throw ex;
|
throw ex;
|
||||||
|
} finally {
|
||||||
|
TenantManager.restoreTenantCondition();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,13 +7,12 @@ import com.easyagents.flow.core.chain.repository.LoopInputReference;
|
|||||||
import com.easyagents.flow.core.node.BaseNode;
|
import com.easyagents.flow.core.node.BaseNode;
|
||||||
import com.easyagents.flow.core.util.IoBulkhead;
|
import com.easyagents.flow.core.util.IoBulkhead;
|
||||||
import com.mybatisflex.core.row.Row;
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import com.mybatisflex.core.tenant.TenantManager;
|
||||||
import tech.easyflow.common.util.SpringContextUtil;
|
import tech.easyflow.common.util.SpringContextUtil;
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
import tech.easyflow.ai.easyagentsflow.cancellation.WorkflowDatasetQueryCancellationRegistry;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -56,38 +55,32 @@ public class SearchDatasetNode extends BaseNode {
|
|||||||
Map<String, Object> params =
|
Map<String, Object> params =
|
||||||
chain.getExecutionState().resolveParameters(this);
|
chain.getExecutionState().resolveParameters(this);
|
||||||
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
|
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
|
||||||
WorkflowDatasetQueryCancellationRegistry cancellationRegistry =
|
|
||||||
SpringContextUtil.getBean(
|
|
||||||
WorkflowDatasetQueryCancellationRegistry.class);
|
|
||||||
LoginAccount account = tech.easyflow.ai.utils.WorkFlowUtil.getOperator(chain);
|
|
||||||
DatacenterSqlQueryRequest request = buildRuntimeRequest(params);
|
DatacenterSqlQueryRequest request = buildRuntimeRequest(params);
|
||||||
Map<String, Object> result = new HashMap<>();
|
Map<String, Object> result = new HashMap<>();
|
||||||
String queryId = UUID.randomUUID().toString();
|
try {
|
||||||
try (WorkflowDatasetQueryCancellationRegistry.Registration registration =
|
TenantManager.ignoreTenantCondition();
|
||||||
cancellationRegistry.register(
|
try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) {
|
||||||
chain.getStateInstanceId(), account, queryId);
|
String resultId = chain.getStateInstanceId()
|
||||||
IoBulkhead.Permit ignored = IoBulkhead.dataset()
|
+ ":dataset:"
|
||||||
.acquire(resolveIoTarget())) {
|
+ UUID.randomUUID();
|
||||||
String resultId = chain.getStateInstanceId()
|
int rowCount =
|
||||||
+ ":dataset:"
|
chain.storeProducedLoopInputOutsideLock(
|
||||||
+ queryId;
|
resultId,
|
||||||
int rowCount =
|
sink -> queryService.consumeBySql(
|
||||||
chain.storeProducedLoopInputOutsideLock(
|
request,
|
||||||
resultId,
|
QUERY_PAGE_SIZE,
|
||||||
sink -> queryService.consumeBySql(
|
sink::accept),
|
||||||
request,
|
0L,
|
||||||
QUERY_PAGE_SIZE,
|
chain.currentFencingClaimId(),
|
||||||
account,
|
chain.currentClaimGeneration());
|
||||||
queryId,
|
result.put(
|
||||||
sink::accept),
|
resolveOutputKey("data"),
|
||||||
0L,
|
new LoopInputReference(
|
||||||
chain.currentFencingClaimId(),
|
resultId, rowCount));
|
||||||
chain.currentClaimGeneration());
|
return result;
|
||||||
result.put(
|
}
|
||||||
resolveOutputKey("data"),
|
} finally {
|
||||||
new LoopInputReference(
|
TenantManager.restoreTenantCondition();
|
||||||
resultId, rowCount));
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -104,9 +97,6 @@ public class SearchDatasetNode extends BaseNode {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private DatacenterSqlQueryRequest buildRuntimeRequest(Map<String, Object> params) {
|
private DatacenterSqlQueryRequest buildRuntimeRequest(Map<String, Object> params) {
|
||||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
|
||||||
throw new BusinessException("数据集绑定缺少连接信息,请重新选择数据集");
|
|
||||||
}
|
|
||||||
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
|
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
|
||||||
request.setDatasetRef(copyDatasetRef());
|
request.setDatasetRef(copyDatasetRef());
|
||||||
request.setSql(resolveQuerySql(params));
|
request.setSql(resolveQuerySql(params));
|
||||||
@@ -138,13 +128,12 @@ public class SearchDatasetNode extends BaseNode {
|
|||||||
|
|
||||||
private DatasetRef copyDatasetRef() {
|
private DatasetRef copyDatasetRef() {
|
||||||
DatasetRef copy = new DatasetRef();
|
DatasetRef copy = new DatasetRef();
|
||||||
copy.setTenantId(datasetRef == null ? null : datasetRef.getTenantId());
|
|
||||||
copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId());
|
copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId());
|
||||||
copy.setCatalogId(datasetRef == null ? null : datasetRef.getCatalogId());
|
copy.setCatalogId(datasetRef == null ? null : datasetRef.getCatalogId());
|
||||||
copy.setCatalogName(datasetRef == null ? null : datasetRef.getCatalogName());
|
copy.setCatalogName(datasetRef == null ? null : datasetRef.getCatalogName());
|
||||||
copy.setTableId(datasetRef == null ? null : datasetRef.getTableId());
|
copy.setTableId(null);
|
||||||
copy.setTableName(datasetRef == null ? null : datasetRef.getTableName());
|
copy.setTableName(null);
|
||||||
copy.setVersionId(datasetRef == null ? null : datasetRef.getVersionId());
|
copy.setVersionId(null);
|
||||||
return copy;
|
return copy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package tech.easyflow.ai.service;
|
|||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.enums.PublishStatus;
|
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
import tech.easyflow.system.enums.CategoryResourceType;
|
import tech.easyflow.system.enums.CategoryResourceType;
|
||||||
@@ -15,7 +15,7 @@ import java.util.Objects;
|
|||||||
/**
|
/**
|
||||||
* 工作流使用权限校验服务。
|
* 工作流使用权限校验服务。
|
||||||
*
|
*
|
||||||
* <p>统一封装工作流存在性、租户、发布快照和资源使用权限校验,供页面能力和后台任务复用。</p>
|
* <p>统一封装工作流存在性、租户、启用状态和资源使用权限校验,供页面能力和后台任务复用。</p>
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
public class WorkflowUsageAuthorizationService {
|
public class WorkflowUsageAuthorizationService {
|
||||||
@@ -40,20 +40,20 @@ public class WorkflowUsageAuthorizationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取当前账号可使用的已发布工作流视图。
|
* 获取当前账号可使用的启用工作流。
|
||||||
*
|
*
|
||||||
* @param workflowId 工作流 ID
|
* @param workflowId 工作流 ID
|
||||||
* @param account 使用工作流的账号
|
* @param account 使用工作流的账号
|
||||||
* @param denyMessage 校验失败提示
|
* @param denyMessage 校验失败提示
|
||||||
* @return 可使用的工作流发布视图
|
* @return 可使用的工作流
|
||||||
* @throws BusinessException 工作流不存在、未发布、缺少发布快照、跨租户或无使用权限时抛出
|
* @throws BusinessException 工作流不存在、未启用、跨租户或无使用权限时抛出
|
||||||
*/
|
*/
|
||||||
public Workflow requireUsableWorkflow(
|
public Workflow requireUsableWorkflow(
|
||||||
BigInteger workflowId,
|
BigInteger workflowId,
|
||||||
LoginAccount account,
|
LoginAccount account,
|
||||||
String denyMessage) {
|
String denyMessage) {
|
||||||
String message = denyMessage == null || denyMessage.isBlank()
|
String message = denyMessage == null || denyMessage.isBlank()
|
||||||
? "工作流不存在、未发布或无权使用"
|
? "工作流不存在、已禁用或无权使用"
|
||||||
: denyMessage;
|
: denyMessage;
|
||||||
if (workflowId == null || account == null || account.getId() == null
|
if (workflowId == null || account == null || account.getId() == null
|
||||||
|| account.getTenantId() == null) {
|
|| account.getTenantId() == null) {
|
||||||
@@ -62,9 +62,7 @@ public class WorkflowUsageAuthorizationService {
|
|||||||
Workflow workflow = workflowService.getById(workflowId);
|
Workflow workflow = workflowService.getById(workflowId);
|
||||||
boolean usable = workflow != null
|
boolean usable = workflow != null
|
||||||
&& Objects.equals(workflow.getTenantId(), account.getTenantId())
|
&& Objects.equals(workflow.getTenantId(), account.getTenantId())
|
||||||
&& PublishStatus.PUBLISHED.getCode().equals(workflow.getPublishStatus())
|
&& EnumDataStatus.AVAILABLE.getCode().equals(workflow.getStatus())
|
||||||
&& workflow.getPublishedSnapshotJson() != null
|
|
||||||
&& !workflow.getPublishedSnapshotJson().isEmpty()
|
|
||||||
&& resourceAccessService.canAccess(
|
&& resourceAccessService.canAccess(
|
||||||
account,
|
account,
|
||||||
CategoryResourceType.WORKFLOW,
|
CategoryResourceType.WORKFLOW,
|
||||||
@@ -73,10 +71,6 @@ public class WorkflowUsageAuthorizationService {
|
|||||||
if (!usable) {
|
if (!usable) {
|
||||||
throw new BusinessException(403, 403, message);
|
throw new BusinessException(403, 403, message);
|
||||||
}
|
}
|
||||||
Workflow published = workflowService.toPublishedView(workflow);
|
return workflow;
|
||||||
if (published == null) {
|
|
||||||
throw new BusinessException(403, 403, message);
|
|
||||||
}
|
|
||||||
return published;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -263,7 +263,13 @@ public class DocumentChunkServiceImpl
|
|||||||
throw new BusinessException("分块不存在");
|
throw new BusinessException("分块不存在");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return syncTaskAppService.listSyncStatuses(chunks);
|
return chunks.stream().map(chunk -> new DocumentChunkSyncStatus(
|
||||||
|
chunk.getId(),
|
||||||
|
chunk.getIndexSyncStatus(),
|
||||||
|
chunk.getIndexSyncVersion(),
|
||||||
|
chunk.getIndexSyncErrorCode(),
|
||||||
|
chunk.getIndexSyncErrorMessage()
|
||||||
|
)).toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
private DocumentChunk requireChunk(BigInteger knowledgeId, BigInteger chunkId) {
|
private DocumentChunk requireChunk(BigInteger knowledgeId, BigInteger chunkId) {
|
||||||
|
|||||||
@@ -1,14 +1,6 @@
|
|||||||
package tech.easyflow.ai.documentchunk;
|
package tech.easyflow.ai.documentchunk;
|
||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.Assert;
|
|
||||||
import com.easyagents.core.document.Document;
|
|
||||||
import com.easyagents.core.model.embedding.EmbeddingModel;
|
|
||||||
import com.easyagents.core.model.exception.ModelException;
|
|
||||||
import com.easyagents.core.store.DocumentStore;
|
|
||||||
import com.easyagents.core.store.StoreResult;
|
|
||||||
import com.easyagents.core.store.VectorData;
|
|
||||||
import com.easyagents.search.engine.service.DocumentSearcher;
|
|
||||||
import org.mockito.InOrder;
|
import org.mockito.InOrder;
|
||||||
import org.mockito.Mockito;
|
import org.mockito.Mockito;
|
||||||
import org.springframework.transaction.PlatformTransactionManager;
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
@@ -18,8 +10,6 @@ import tech.easyflow.ai.config.AiMilvusConfig;
|
|||||||
import tech.easyflow.ai.config.SearcherFactory;
|
import tech.easyflow.ai.config.SearcherFactory;
|
||||||
import tech.easyflow.ai.entity.DocumentChunk;
|
import tech.easyflow.ai.entity.DocumentChunk;
|
||||||
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
import tech.easyflow.ai.entity.DocumentChunkSyncTask;
|
||||||
import tech.easyflow.ai.entity.DocumentCollection;
|
|
||||||
import tech.easyflow.ai.entity.Model;
|
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkMapper;
|
||||||
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
import tech.easyflow.ai.mapper.DocumentChunkSyncTaskMapper;
|
||||||
import tech.easyflow.ai.service.DocumentCollectionService;
|
import tech.easyflow.ai.service.DocumentCollectionService;
|
||||||
@@ -35,144 +25,6 @@ import java.util.function.Supplier;
|
|||||||
*/
|
*/
|
||||||
public class DocumentChunkSyncTaskAppServiceTest {
|
public class DocumentChunkSyncTaskAppServiceTest {
|
||||||
|
|
||||||
@Test
|
|
||||||
public void pollingShouldExposeCurrentRetryReasonWhileWaitingAndRunning() {
|
|
||||||
Fixture fixture = fixture(1);
|
|
||||||
fixture.task.setErrorCode("EMBEDDING_REQUEST_FAILED");
|
|
||||||
fixture.task.setErrorMessage("向量模型服务调用失败");
|
|
||||||
Mockito.when(fixture.taskMapper.selectCurrentForChunks(List.of(fixture.chunkId)))
|
|
||||||
.thenReturn(List.of(fixture.task));
|
|
||||||
|
|
||||||
var pending = fixture.service.listSyncStatuses(List.of(fixture.chunk)).get(0);
|
|
||||||
Assert.assertEquals("PENDING", pending.indexSyncStatus());
|
|
||||||
Assert.assertEquals("EMBEDDING_REQUEST_FAILED", pending.indexSyncErrorCode());
|
|
||||||
Assert.assertEquals(Integer.valueOf(1), pending.indexSyncAttemptCount());
|
|
||||||
Assert.assertEquals(5, pending.indexSyncMaxAttempts());
|
|
||||||
|
|
||||||
fixture.task.setStatus(DocumentChunkSyncState.TASK_RUNNING);
|
|
||||||
fixture.task.setAttemptCount(2);
|
|
||||||
var running = fixture.service.listSyncStatuses(List.of(fixture.chunk)).get(0);
|
|
||||||
Assert.assertEquals("EMBEDDING_REQUEST_FAILED", running.indexSyncErrorCode());
|
|
||||||
Assert.assertEquals(Integer.valueOf(2), running.indexSyncAttemptCount());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void pollingShouldNotExposeAnotherVersionsOrFinishedTasksFailure() {
|
|
||||||
Fixture fixture = fixture(2);
|
|
||||||
fixture.task.setErrorCode("EMBEDDING_REQUEST_FAILED");
|
|
||||||
Mockito.when(fixture.taskMapper.selectCurrentForChunks(List.of(fixture.chunkId)))
|
|
||||||
.thenReturn(List.of(fixture.task));
|
|
||||||
fixture.task.setSyncVersion(2L);
|
|
||||||
var changed = fixture.service.listSyncStatuses(List.of(fixture.chunk)).get(0);
|
|
||||||
Assert.assertNull(changed.indexSyncErrorCode());
|
|
||||||
Assert.assertNull(changed.indexSyncAttemptCount());
|
|
||||||
|
|
||||||
fixture.task.setSyncVersion(1L);
|
|
||||||
fixture.task.setStatus(DocumentChunkSyncState.TASK_SUCCEEDED);
|
|
||||||
var finished = fixture.service.listSyncStatuses(List.of(fixture.chunk)).get(0);
|
|
||||||
Assert.assertNull(finished.indexSyncErrorCode());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void pollingShouldHandleChunksWithoutTasksAndSkipEmptyBatch() {
|
|
||||||
Fixture fixture = fixture(1);
|
|
||||||
Assert.assertTrue(fixture.service.listSyncStatuses(List.of()).isEmpty());
|
|
||||||
Mockito.verify(fixture.taskMapper, Mockito.never()).selectCurrentForChunks(Mockito.anyList());
|
|
||||||
Mockito.when(fixture.taskMapper.selectCurrentForChunks(List.of(fixture.chunkId)))
|
|
||||||
.thenReturn(List.of());
|
|
||||||
var state = fixture.service.listSyncStatuses(List.of(fixture.chunk)).get(0);
|
|
||||||
Assert.assertEquals("PENDING", state.indexSyncStatus());
|
|
||||||
Assert.assertNull(state.indexSyncAttemptCount());
|
|
||||||
Assert.assertNull(state.indexSyncErrorCode());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void embeddingFailureShouldBeReportedWithoutCallingIndexes() {
|
|
||||||
Fixture fixture = fixture(1);
|
|
||||||
IndexFixture indexes = prepareIndexes(fixture);
|
|
||||||
Mockito.when(indexes.embeddingModel.embed(Mockito.any(Document.class), Mockito.any()))
|
|
||||||
.thenThrow(new ModelException("response is null or empty."));
|
|
||||||
|
|
||||||
fixture.service.handleTask(fixture.taskId);
|
|
||||||
|
|
||||||
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
|
||||||
Mockito.eq(fixture.taskId), Mockito.anyString(), Mockito.eq("PENDING"), Mockito.any(),
|
|
||||||
Mockito.eq("EMBEDDING_REQUEST_FAILED"), Mockito.eq("向量模型服务调用失败"), Mockito.any()
|
|
||||||
);
|
|
||||||
Mockito.verify(indexes.store, Mockito.never()).doUpdate(Mockito.anyList(), Mockito.any());
|
|
||||||
Mockito.verifyNoInteractions(indexes.searcher);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void vectorFailureShouldHaveItsOwnReasonAndEmbedOnlyOnce() {
|
|
||||||
Fixture fixture = fixture(1);
|
|
||||||
IndexFixture indexes = prepareIndexes(fixture);
|
|
||||||
Mockito.when(indexes.store.doUpdate(Mockito.anyList(), Mockito.any()))
|
|
||||||
.thenReturn(StoreResult.fail("vector write failed"));
|
|
||||||
|
|
||||||
fixture.service.handleTask(fixture.taskId);
|
|
||||||
|
|
||||||
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
|
||||||
Mockito.eq(fixture.taskId), Mockito.anyString(), Mockito.eq("PENDING"), Mockito.any(),
|
|
||||||
Mockito.eq("VECTOR_UPSERT_FAILED"), Mockito.anyString(), Mockito.any()
|
|
||||||
);
|
|
||||||
Mockito.verify(indexes.embeddingModel).embed(Mockito.any(Document.class), Mockito.any());
|
|
||||||
Mockito.verifyNoInteractions(indexes.searcher);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void keywordFailureShouldHaveItsOwnReason() {
|
|
||||||
Fixture fixture = fixture(1);
|
|
||||||
IndexFixture indexes = prepareIndexes(fixture);
|
|
||||||
Mockito.when(indexes.searcher.addDocuments(Mockito.anyList())).thenReturn(false);
|
|
||||||
|
|
||||||
fixture.service.handleTask(fixture.taskId);
|
|
||||||
|
|
||||||
Mockito.verify(fixture.taskMapper).failOrRetryOwned(
|
|
||||||
Mockito.eq(fixture.taskId), Mockito.anyString(), Mockito.eq("PENDING"), Mockito.any(),
|
|
||||||
Mockito.eq("KEYWORD_UPSERT_FAILED"), Mockito.anyString(), Mockito.any()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void successfulIndexUpdateShouldClearFailureAndMarkChunkSynced() {
|
|
||||||
Fixture fixture = fixture(2);
|
|
||||||
fixture.task.setErrorCode("EMBEDDING_REQUEST_FAILED");
|
|
||||||
IndexFixture indexes = prepareIndexes(fixture);
|
|
||||||
Mockito.when(indexes.searcher.addDocuments(Mockito.anyList())).thenReturn(true);
|
|
||||||
|
|
||||||
fixture.service.handleTask(fixture.taskId);
|
|
||||||
|
|
||||||
Mockito.verify(fixture.taskMapper).finishOwned(
|
|
||||||
Mockito.eq(fixture.taskId), Mockito.anyString(), Mockito.eq("SUCCEEDED"),
|
|
||||||
Mockito.isNull(), Mockito.isNull(), Mockito.any()
|
|
||||||
);
|
|
||||||
Mockito.verify(fixture.chunkMapper).updateSyncState(fixture.chunkId, 1L, "SYNCED", null, null);
|
|
||||||
Mockito.verify(indexes.embeddingModel).embed(Mockito.any(Document.class), Mockito.any());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static IndexFixture prepareIndexes(Fixture fixture) {
|
|
||||||
DocumentCollection collection = Mockito.mock(DocumentCollection.class);
|
|
||||||
DocumentStore store = Mockito.mock(DocumentStore.class, Mockito.CALLS_REAL_METHODS);
|
|
||||||
Model model = Mockito.mock(Model.class);
|
|
||||||
EmbeddingModel embeddingModel = Mockito.mock(EmbeddingModel.class);
|
|
||||||
DocumentSearcher searcher = Mockito.mock(DocumentSearcher.class);
|
|
||||||
Mockito.when(fixture.collectionService.getById(fixture.task.getDocumentCollectionId()))
|
|
||||||
.thenReturn(collection);
|
|
||||||
Mockito.when(collection.toDocumentStore()).thenReturn(store);
|
|
||||||
Mockito.when(fixture.modelService.getModelInstance(Mockito.any())).thenReturn(model);
|
|
||||||
Mockito.when(model.toEmbeddingModel()).thenReturn(embeddingModel);
|
|
||||||
Mockito.when(fixture.searcherFactory.getSearcher()).thenReturn(searcher);
|
|
||||||
VectorData vector = new VectorData();
|
|
||||||
vector.setVector(new float[] { 0.1f, 0.2f });
|
|
||||||
Mockito.when(embeddingModel.embed(Mockito.any(Document.class), Mockito.any())).thenReturn(vector);
|
|
||||||
Mockito.when(store.doUpdate(Mockito.anyList(), Mockito.any())).thenReturn(StoreResult.success());
|
|
||||||
return new IndexFixture(store, embeddingModel, searcher);
|
|
||||||
}
|
|
||||||
|
|
||||||
private record IndexFixture(DocumentStore store, EmbeddingModel embeddingModel, DocumentSearcher searcher) {
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void dispatchPendingShouldRecoverExpiredTasksAndSendClaimedRows() {
|
public void dispatchPendingShouldRecoverExpiredTasksAndSendClaimedRows() {
|
||||||
Fixture fixture = fixture(1);
|
Fixture fixture = fixture(1);
|
||||||
@@ -363,7 +215,7 @@ public class DocumentChunkSyncTaskAppServiceTest {
|
|||||||
Mockito.mock(ObjectProvider.class)
|
Mockito.mock(ObjectProvider.class)
|
||||||
);
|
);
|
||||||
return new Fixture(service, taskMapper, chunkMapper, collectionService,
|
return new Fixture(service, taskMapper, chunkMapper, collectionService,
|
||||||
producer, task, chunk, taskId, chunkId, modelService, searcherFactory);
|
producer, task, chunk, taskId, chunkId);
|
||||||
}
|
}
|
||||||
|
|
||||||
private record Fixture(
|
private record Fixture(
|
||||||
@@ -375,9 +227,7 @@ public class DocumentChunkSyncTaskAppServiceTest {
|
|||||||
DocumentChunkSyncTask task,
|
DocumentChunkSyncTask task,
|
||||||
DocumentChunk chunk,
|
DocumentChunk chunk,
|
||||||
BigInteger taskId,
|
BigInteger taskId,
|
||||||
BigInteger chunkId,
|
BigInteger chunkId
|
||||||
ModelService modelService,
|
|
||||||
SearcherFactory searcherFactory
|
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,131 +0,0 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
|
||||||
|
|
||||||
import com.easyagents.flow.core.chain.Chain;
|
|
||||||
import com.easyagents.flow.core.chain.ChainStatus;
|
|
||||||
import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.UUID;
|
|
||||||
import org.junit.Assert;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.mockito.ArgumentCaptor;
|
|
||||||
import org.mockito.Mockito;
|
|
||||||
import org.springframework.beans.factory.ObjectProvider;
|
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* {@link WorkflowDatasetQueryCancellationRegistry} 取消竞态回归测试。
|
|
||||||
*/
|
|
||||||
public class WorkflowDatasetQueryCancellationRegistryTest {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 验证工作流取消会使用登记租户取消活动 QueryId。
|
|
||||||
*/
|
|
||||||
@Test
|
|
||||||
public void shouldCancelRegisteredQuery() {
|
|
||||||
Fixture fixture = fixture();
|
|
||||||
String queryId = UUID.randomUUID().toString();
|
|
||||||
LoginAccount account = account(BigInteger.valueOf(7L));
|
|
||||||
|
|
||||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignored =
|
|
||||||
fixture.registry.register("instance-a", account, queryId)) {
|
|
||||||
Assert.assertTrue(fixture.registry.cancelExecution("instance-a"));
|
|
||||||
}
|
|
||||||
|
|
||||||
ArgumentCaptor<LoginAccount> accountCaptor =
|
|
||||||
ArgumentCaptor.forClass(LoginAccount.class);
|
|
||||||
Mockito.verify(fixture.cancellationService).cancel(
|
|
||||||
Mockito.eq(queryId), accountCaptor.capture());
|
|
||||||
Assert.assertEquals(BigInteger.valueOf(7L),
|
|
||||||
accountCaptor.getValue().getTenantId());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 验证单个查询取消失败不会阻断同一工作流的其他活动查询。
|
|
||||||
*/
|
|
||||||
@Test
|
|
||||||
public void shouldIsolateCancellationFailuresBetweenQueries() {
|
|
||||||
Fixture fixture = fixture();
|
|
||||||
String failedQueryId = UUID.randomUUID().toString();
|
|
||||||
String succeedingQueryId = UUID.randomUUID().toString();
|
|
||||||
LoginAccount account = account(BigInteger.valueOf(7L));
|
|
||||||
Mockito.doThrow(new IllegalStateException("simulated cancellation failure"))
|
|
||||||
.when(fixture.cancellationService)
|
|
||||||
.cancel(Mockito.eq(failedQueryId), Mockito.any(LoginAccount.class));
|
|
||||||
|
|
||||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignoredA =
|
|
||||||
fixture.registry.register("instance-failure", account, failedQueryId);
|
|
||||||
WorkflowDatasetQueryCancellationRegistry.Registration ignoredB =
|
|
||||||
fixture.registry.register("instance-failure", account, succeedingQueryId)) {
|
|
||||||
Assert.assertTrue(fixture.registry.cancelExecution("instance-failure"));
|
|
||||||
}
|
|
||||||
|
|
||||||
Mockito.verify(fixture.cancellationService).cancel(
|
|
||||||
Mockito.eq(failedQueryId), Mockito.any(LoginAccount.class));
|
|
||||||
Mockito.verify(fixture.cancellationService).cancel(
|
|
||||||
Mockito.eq(succeedingQueryId), Mockito.any(LoginAccount.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 验证取消先于节点登记发生时,后续查询仍会立即收到取消。
|
|
||||||
*/
|
|
||||||
@Test
|
|
||||||
public void shouldCancelQueryRegisteredAfterWorkflowCancellation() {
|
|
||||||
Fixture fixture = fixture();
|
|
||||||
String queryId = UUID.randomUUID().toString();
|
|
||||||
|
|
||||||
Assert.assertFalse(fixture.registry.cancelExecution("instance-b"));
|
|
||||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignored =
|
|
||||||
fixture.registry.register(
|
|
||||||
"instance-b", account(BigInteger.ONE), queryId)) {
|
|
||||||
Mockito.verify(fixture.cancellationService).cancel(
|
|
||||||
Mockito.eq(queryId), Mockito.any(LoginAccount.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 验证监听器只桥接取消终态。
|
|
||||||
*/
|
|
||||||
@Test
|
|
||||||
public void listenerShouldBridgeOnlyCancelledStatus() {
|
|
||||||
WorkflowDatasetQueryCancellationRegistry registry = Mockito.mock(
|
|
||||||
WorkflowDatasetQueryCancellationRegistry.class);
|
|
||||||
WorkflowDatasetQueryCancellationListener listener =
|
|
||||||
new WorkflowDatasetQueryCancellationListener(registry);
|
|
||||||
Chain chain = Mockito.mock(Chain.class);
|
|
||||||
Mockito.when(chain.getStateInstanceId()).thenReturn("instance-c");
|
|
||||||
|
|
||||||
listener.onEvent(new ChainStatusChangeEvent(
|
|
||||||
chain, ChainStatus.SUCCEEDED, ChainStatus.RUNNING), chain);
|
|
||||||
listener.onEvent(new ChainStatusChangeEvent(
|
|
||||||
chain, ChainStatus.CANCELLED, ChainStatus.RUNNING), chain);
|
|
||||||
|
|
||||||
Mockito.verify(registry).cancelExecution("instance-c");
|
|
||||||
}
|
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
private Fixture fixture() {
|
|
||||||
DatacenterFederationQueryCancellationService cancellationService =
|
|
||||||
Mockito.mock(
|
|
||||||
DatacenterFederationQueryCancellationService.class);
|
|
||||||
ObjectProvider<StringRedisTemplate> redisProvider =
|
|
||||||
Mockito.mock(ObjectProvider.class);
|
|
||||||
Mockito.when(redisProvider.getIfAvailable()).thenReturn(null);
|
|
||||||
return new Fixture(
|
|
||||||
new WorkflowDatasetQueryCancellationRegistry(
|
|
||||||
cancellationService, redisProvider),
|
|
||||||
cancellationService);
|
|
||||||
}
|
|
||||||
|
|
||||||
private LoginAccount account(BigInteger tenantId) {
|
|
||||||
LoginAccount account = new LoginAccount();
|
|
||||||
account.setTenantId(tenantId);
|
|
||||||
return account;
|
|
||||||
}
|
|
||||||
|
|
||||||
private record Fixture(
|
|
||||||
WorkflowDatasetQueryCancellationRegistry registry,
|
|
||||||
DatacenterFederationQueryCancellationService cancellationService) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,8 +2,6 @@ package tech.easyflow.ai.easyagentsflow.service;
|
|||||||
|
|
||||||
import com.easyagents.document.core.exception.DocumentParseException;
|
import com.easyagents.document.core.exception.DocumentParseException;
|
||||||
import com.easyagents.flow.core.chain.ChainState;
|
import com.easyagents.flow.core.chain.ChainState;
|
||||||
import com.easyagents.flow.core.chain.WorkflowErrorReason;
|
|
||||||
import com.easyagents.flow.core.chain.WorkflowExecutionException;
|
|
||||||
import com.easyagents.flow.core.chain.ChainStatus;
|
import com.easyagents.flow.core.chain.ChainStatus;
|
||||||
import com.easyagents.flow.core.chain.ExceptionSummary;
|
import com.easyagents.flow.core.chain.ExceptionSummary;
|
||||||
import com.easyagents.flow.core.chain.NodeState;
|
import com.easyagents.flow.core.chain.NodeState;
|
||||||
@@ -207,7 +205,7 @@ public class TinyFlowServiceTest {
|
|||||||
* @throws Exception 测试依赖注入失败时抛出
|
* @throws Exception 测试依赖注入失败时抛出
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void shouldHideRawJavascriptExceptionDetails()
|
public void shouldExposeJavascriptExecutionMessage()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||||
ChainStateRepository chainStateRepository =
|
ChainStateRepository chainStateRepository =
|
||||||
@@ -239,9 +237,9 @@ public class TinyFlowServiceTest {
|
|||||||
ChainInfo result = service.getChainStatus(
|
ChainInfo result = service.getChainStatus(
|
||||||
EXECUTE_ID, List.of(node(NodeStatus.READY)));
|
EXECUTE_ID, List.of(node(NodeStatus.READY)));
|
||||||
|
|
||||||
Assert.assertEquals(WorkflowErrorReason.WORKFLOW_INTERNAL_ERROR.getDefaultMessage(), result.getMessage());
|
Assert.assertEquals(message, result.getMessage());
|
||||||
Assert.assertEquals(
|
Assert.assertEquals(
|
||||||
WorkflowErrorReason.WORKFLOW_INTERNAL_ERROR.getDefaultMessage(),
|
message,
|
||||||
result.getNodes().get(NODE_ID).getMessage());
|
result.getNodes().get(NODE_ID).getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,7 +249,7 @@ public class TinyFlowServiceTest {
|
|||||||
* @throws Exception 测试依赖注入失败时抛出
|
* @throws Exception 测试依赖注入失败时抛出
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void shouldHideUnclassifiedDocumentCauseDetails()
|
public void shouldExposeDocumentParseMessageWithoutExceptionClass()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||||
ChainStateRepository chainStateRepository =
|
ChainStateRepository chainStateRepository =
|
||||||
@@ -276,47 +274,7 @@ public class TinyFlowServiceTest {
|
|||||||
|
|
||||||
ChainInfo result = service.getChainStatus(EXECUTE_ID, null);
|
ChainInfo result = service.getChainStatus(EXECUTE_ID, null);
|
||||||
|
|
||||||
Assert.assertEquals(WorkflowErrorReason.WORKFLOW_INTERNAL_ERROR.getDefaultMessage(), result.getMessage());
|
Assert.assertEquals(message, result.getMessage());
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void shouldKeepFailureNodeWithEmptyNodeSelection() throws Exception {
|
|
||||||
ChainExecutor executor = mock(ChainExecutor.class);
|
|
||||||
ChainStateRepository states = mock(ChainStateRepository.class);
|
|
||||||
when(executor.getChainStateRepository()).thenReturn(states);
|
|
||||||
ChainState state = new ChainState();
|
|
||||||
state.setStatus(ChainStatus.FAILED);
|
|
||||||
state.setError(new ExceptionSummary(new WorkflowExecutionException(WorkflowErrorReason.MODEL_RATE_LIMITED,
|
|
||||||
"raw credentials"), EXECUTE_ID, NODE_ID, "模型分析"));
|
|
||||||
when(states.load(EXECUTE_ID)).thenReturn(state);
|
|
||||||
ChainInfo result = service(executor).getChainStatus(EXECUTE_ID, List.of());
|
|
||||||
Assert.assertTrue(result.getNodes().isEmpty());
|
|
||||||
Assert.assertEquals(NODE_ID, result.getError().getNodeId());
|
|
||||||
Assert.assertEquals("MODEL_RATE_LIMITED", result.getError().getReasonCode());
|
|
||||||
Assert.assertFalse(result.getMessage().contains("credentials"));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void terminalWorkflowMustStopAdvertisingPendingRetry() throws Exception {
|
|
||||||
ChainExecutor executor = mock(ChainExecutor.class);
|
|
||||||
ChainStateRepository states = mock(ChainStateRepository.class);
|
|
||||||
NodeStateRepository nodes = mock(NodeStateRepository.class);
|
|
||||||
when(executor.getChainStateRepository()).thenReturn(states);
|
|
||||||
when(executor.getNodeStateRepository()).thenReturn(nodes);
|
|
||||||
NodeState failedAttempt = new NodeState();
|
|
||||||
failedAttempt.setStatus(NodeStatus.ERROR);
|
|
||||||
failedAttempt.setError(new ExceptionSummary(new WorkflowExecutionException(
|
|
||||||
WorkflowErrorReason.MODEL_TIMEOUT, "private cause"), EXECUTE_ID, NODE_ID, "模型分析"));
|
|
||||||
when(nodes.load(EXECUTE_ID, NODE_ID)).thenReturn(failedAttempt);
|
|
||||||
TinyFlowService service = service(executor);
|
|
||||||
for (ChainStatus status : new ChainStatus[]{ChainStatus.RUNNING, ChainStatus.FAILED, ChainStatus.CANCELLED}) {
|
|
||||||
ChainState state = new ChainState();
|
|
||||||
state.setStatus(status);
|
|
||||||
when(states.load(EXECUTE_ID)).thenReturn(state);
|
|
||||||
ChainInfo result = service.getChainStatus(EXECUTE_ID, List.of(node(NodeStatus.READY)));
|
|
||||||
Assert.assertEquals(status == ChainStatus.RUNNING, result.getNodes().get(NODE_ID).getError().isRetryable());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -9,8 +9,6 @@ import org.mockito.Mockito;
|
|||||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
|
||||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||||
|
|
||||||
import java.lang.reflect.Field;
|
import java.lang.reflect.Field;
|
||||||
@@ -43,22 +41,13 @@ public class WorkflowDatacenterContentServiceTest {
|
|||||||
DatacenterSource source = Mockito.mock(DatacenterSource.class);
|
DatacenterSource source = Mockito.mock(DatacenterSource.class);
|
||||||
Mockito.when(source.getSourceName()).thenReturn("ama 实验基线模型预算");
|
Mockito.when(source.getSourceName()).thenReturn("ama 实验基线模型预算");
|
||||||
Mockito.when(source.getSourceType()).thenReturn("EXCEL");
|
Mockito.when(source.getSourceType()).thenReturn("EXCEL");
|
||||||
Mockito.when(source.getTenantId()).thenReturn(BigInteger.ONE);
|
|
||||||
|
|
||||||
DatacenterTableField modelId = mockField("col_id", "模型ID", "VARCHAR");
|
DatacenterTableField modelId = mockField("col_id", "模型ID", "VARCHAR");
|
||||||
DatacenterTableField inputPrice = mockField("token", "输入价格", "DECIMAL");
|
DatacenterTableField inputPrice = mockField("token", "输入价格", "DECIMAL");
|
||||||
DatacenterTableField hidden = mockField("hidden_col", "受限字段", "VARCHAR");
|
|
||||||
Mockito.when(hidden.getQueryable()).thenReturn(0);
|
|
||||||
DatacenterTable table = Mockito.mock(DatacenterTable.class);
|
DatacenterTable table = Mockito.mock(DatacenterTable.class);
|
||||||
Mockito.when(table.getId()).thenReturn(TABLE_ID);
|
Mockito.when(table.getId()).thenReturn(TABLE_ID);
|
||||||
Mockito.when(table.getSourceId()).thenReturn(SOURCE_ID);
|
|
||||||
Mockito.when(table.getTenantId()).thenReturn(BigInteger.ONE);
|
|
||||||
Mockito.when(table.getTableName()).thenReturn("Sheet1");
|
Mockito.when(table.getTableName()).thenReturn("Sheet1");
|
||||||
Mockito.when(table.getQueryable()).thenReturn(1);
|
Mockito.when(table.getFields()).thenReturn(List.of(modelId, inputPrice));
|
||||||
Mockito.when(table.getMetadataStatus()).thenReturn(
|
|
||||||
DatacenterMetadataStatus.ACTIVE.name());
|
|
||||||
Mockito.when(table.getFields()).thenReturn(List.of(
|
|
||||||
modelId, inputPrice, hidden));
|
|
||||||
|
|
||||||
Mockito.when(registryService.getSourceRequired(SOURCE_ID)).thenReturn(source);
|
Mockito.when(registryService.getSourceRequired(SOURCE_ID)).thenReturn(source);
|
||||||
Mockito.when(registryService.listManagedTables(SOURCE_ID, null))
|
Mockito.when(registryService.listManagedTables(SOURCE_ID, null))
|
||||||
@@ -88,7 +77,6 @@ public class WorkflowDatacenterContentServiceTest {
|
|||||||
Assert.assertTrue(contextValue.contains("col_id"));
|
Assert.assertTrue(contextValue.contains("col_id"));
|
||||||
Assert.assertTrue(contextValue.contains("模型ID"));
|
Assert.assertTrue(contextValue.contains("模型ID"));
|
||||||
Assert.assertTrue(contextValue.contains("token AS input_price"));
|
Assert.assertTrue(contextValue.contains("token AS input_price"));
|
||||||
Assert.assertFalse(contextValue.contains("hidden_col"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -115,7 +103,6 @@ public class WorkflowDatacenterContentServiceTest {
|
|||||||
private JSONObject buildWorkflowRoot() {
|
private JSONObject buildWorkflowRoot() {
|
||||||
JSONObject datasetRef = new JSONObject();
|
JSONObject datasetRef = new JSONObject();
|
||||||
datasetRef.put("sourceId", SOURCE_ID);
|
datasetRef.put("sourceId", SOURCE_ID);
|
||||||
datasetRef.put("tableId", TABLE_ID);
|
|
||||||
|
|
||||||
JSONObject queryData = new JSONObject();
|
JSONObject queryData = new JSONObject();
|
||||||
queryData.put("datasetRef", datasetRef);
|
queryData.put("datasetRef", datasetRef);
|
||||||
@@ -173,11 +160,6 @@ public class WorkflowDatacenterContentServiceTest {
|
|||||||
Mockito.when(field.getFieldName()).thenReturn(fieldName);
|
Mockito.when(field.getFieldName()).thenReturn(fieldName);
|
||||||
Mockito.when(field.getFieldDesc()).thenReturn(fieldDesc);
|
Mockito.when(field.getFieldDesc()).thenReturn(fieldDesc);
|
||||||
Mockito.when(field.getJdbcType()).thenReturn(jdbcType);
|
Mockito.when(field.getJdbcType()).thenReturn(jdbcType);
|
||||||
Mockito.when(field.getQueryable()).thenReturn(1);
|
|
||||||
Mockito.when(field.getMetadataStatus()).thenReturn(
|
|
||||||
DatacenterMetadataStatus.ACTIVE.name());
|
|
||||||
Mockito.when(field.getSensitivityLevel()).thenReturn(
|
|
||||||
DatacenterSensitivityLevel.PUBLIC.name());
|
|
||||||
return field;
|
return field;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,165 +0,0 @@
|
|||||||
package tech.easyflow.ai.easyagentsflow.service;
|
|
||||||
|
|
||||||
import ch.qos.logback.classic.Logger;
|
|
||||||
import ch.qos.logback.classic.spi.ILoggingEvent;
|
|
||||||
import ch.qos.logback.core.read.ListAppender;
|
|
||||||
import com.easyagents.flow.core.chain.*;
|
|
||||||
import com.easyagents.flow.core.chain.event.ChainEndEvent;
|
|
||||||
import com.easyagents.flow.core.chain.repository.InMemoryChainStateRepository;
|
|
||||||
import com.easyagents.flow.core.chain.repository.InMemoryNodeStateRepository;
|
|
||||||
import com.easyagents.flow.core.chain.runtime.*;
|
|
||||||
import com.easyagents.flow.core.node.StartNode;
|
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
|
||||||
import org.junit.Assert;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.web.context.request.RequestContextHolder;
|
|
||||||
import org.springframework.web.context.request.ServletRequestAttributes;
|
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowExecutionError;
|
|
||||||
import tech.easyflow.common.web.error.RequestErrorProfile;
|
|
||||||
import tech.easyflow.common.web.error.WebErrorMapping;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.CountDownLatch;
|
|
||||||
import java.util.concurrent.Executors;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.Mockito.*;
|
|
||||||
|
|
||||||
public class WorkflowFailureDiagnosticsTest {
|
|
||||||
@Test
|
|
||||||
public void singleRunResponseMustMatchLoggedExecutionId() {
|
|
||||||
Logger logger = (Logger) LoggerFactory.getLogger(ChainExecutor.class);
|
|
||||||
ListAppender<ILoggingEvent> logs = new ListAppender<>();
|
|
||||||
logs.start();
|
|
||||||
logger.addAppender(logs);
|
|
||||||
HttpServletRequest request = request();
|
|
||||||
RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(request));
|
|
||||||
try (Fixture fixture = new Fixture()) {
|
|
||||||
WorkflowExecutionErrorMapper.installRequestProfile();
|
|
||||||
RequestErrorProfile profile = (RequestErrorProfile) request.getAttribute(RequestErrorProfile.ATTRIBUTE_NAME);
|
|
||||||
String previousId = null;
|
|
||||||
for (int run = 0; run < 2; run++) {
|
|
||||||
WorkflowExecutionException failure = Assert.assertThrows(WorkflowExecutionException.class,
|
|
||||||
() -> fixture.executor.executeNode("diagnostics", "worker", Map.of("privateInput", "do-not-log")));
|
|
||||||
WebErrorMapping response = profile.map(request, failure);
|
|
||||||
Map<?, ?> data = (Map<?, ?>) response.data();
|
|
||||||
String executeId = (String) data.get("executeId");
|
|
||||||
Assert.assertNotNull(executeId);
|
|
||||||
Assert.assertNotEquals(previousId, executeId);
|
|
||||||
Assert.assertEquals(failure.getChainId(), executeId);
|
|
||||||
Assert.assertEquals(500, response.httpStatus());
|
|
||||||
WorkflowExecutionError error = (WorkflowExecutionError) data.get("error");
|
|
||||||
Assert.assertEquals("MODEL_TIMEOUT", error.getReasonCode());
|
|
||||||
Assert.assertEquals("worker", error.getNodeId());
|
|
||||||
ILoggingEvent event = logs.list.get(run);
|
|
||||||
assertLogContext(event, executeId);
|
|
||||||
Assert.assertTrue(event.getFormattedMessage().contains("attemptKey=" + executeId + ":worker:single"));
|
|
||||||
previousId = executeId;
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
RequestContextHolder.resetRequestAttributes();
|
|
||||||
logger.detachAppender(logs);
|
|
||||||
logs.stop();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void optionalExecutionIdMustPreserveExistingRequestMapping() {
|
|
||||||
HttpServletRequest request = request();
|
|
||||||
WebErrorMapping fallback = new WebErrorMapping(400, 400, "原请求错误", null);
|
|
||||||
request.setAttribute(RequestErrorProfile.ATTRIBUTE_NAME, (RequestErrorProfile) (req, failure) -> fallback);
|
|
||||||
RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(request));
|
|
||||||
try {
|
|
||||||
WorkflowExecutionErrorMapper.installRequestProfile();
|
|
||||||
RequestErrorProfile profile = (RequestErrorProfile) request.getAttribute(RequestErrorProfile.ATTRIBUTE_NAME);
|
|
||||||
Assert.assertSame(fallback, profile.map(request, new IllegalArgumentException()));
|
|
||||||
WebErrorMapping response = profile.map(request,
|
|
||||||
new WorkflowExecutionException(WorkflowErrorReason.INPUT_INVALID, "internal"));
|
|
||||||
Map<?, ?> data = (Map<?, ?>) response.data();
|
|
||||||
Assert.assertFalse(data.containsKey("executeId"));
|
|
||||||
Assert.assertEquals("INPUT_INVALID", ((WorkflowExecutionError) data.get("error")).getReasonCode());
|
|
||||||
} finally {
|
|
||||||
RequestContextHolder.resetRequestAttributes();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void concurrentExecutionsAndRetriesMustHaveDistinctLogContexts() throws Exception {
|
|
||||||
Logger logger = (Logger) LoggerFactory.getLogger(Chain.class);
|
|
||||||
ListAppender<ILoggingEvent> logs = new ListAppender<>();
|
|
||||||
logs.start();
|
|
||||||
logger.addAppender(logs);
|
|
||||||
try (Fixture fixture = new Fixture()) {
|
|
||||||
CountDownLatch ended = new CountDownLatch(2);
|
|
||||||
fixture.executor.addEventListener((event, chain) -> {
|
|
||||||
if (event instanceof ChainEndEvent) ended.countDown();
|
|
||||||
});
|
|
||||||
String first = fixture.executor.executeAsync("diagnostics", Map.of());
|
|
||||||
String second = fixture.executor.executeAsync("diagnostics", Map.of());
|
|
||||||
Assert.assertTrue(ended.await(5, TimeUnit.SECONDS));
|
|
||||||
for (String executeId : List.of(first, second)) {
|
|
||||||
List<ILoggingEvent> attempts = logs.list.stream()
|
|
||||||
.filter(event -> event.getFormattedMessage().contains("executeId=" + executeId + ","))
|
|
||||||
.toList();
|
|
||||||
Assert.assertEquals(2, attempts.size());
|
|
||||||
attempts.forEach(event -> assertLogContext(event, executeId));
|
|
||||||
Assert.assertNotEquals(attempts.get(0).getArgumentArray()[4], attempts.get(1).getArgumentArray()[4]);
|
|
||||||
Assert.assertNotNull(attempts.get(0).getArgumentArray()[4]);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
logger.detachAppender(logs);
|
|
||||||
logs.stop();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private HttpServletRequest request() {
|
|
||||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
|
||||||
Map<String, Object> attributes = new HashMap<>();
|
|
||||||
when(request.getAttribute(anyString())).thenAnswer(call -> attributes.get(call.getArgument(0)));
|
|
||||||
doAnswer(call -> {
|
|
||||||
attributes.put(call.getArgument(0), call.getArgument(1));
|
|
||||||
return null;
|
|
||||||
}).when(request).setAttribute(anyString(), any());
|
|
||||||
return request;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void assertLogContext(ILoggingEvent event, String executeId) {
|
|
||||||
String message = event.getFormattedMessage();
|
|
||||||
Assert.assertTrue(message.contains("executeId=" + executeId + ","));
|
|
||||||
Assert.assertTrue(message.contains("chainInstanceId=" + executeId + ","));
|
|
||||||
Assert.assertTrue(message.contains("nodeId=worker, nodeName=模型分析,"));
|
|
||||||
Assert.assertFalse(message.contains("do-not-log"));
|
|
||||||
Assert.assertNotNull(event.getThrowableProxy());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static class Fixture implements AutoCloseable {
|
|
||||||
final TriggerScheduler scheduler = new TriggerScheduler(new InMemoryTriggerStore(),
|
|
||||||
Executors.newSingleThreadScheduledExecutor(), Executors.newFixedThreadPool(3), 1000);
|
|
||||||
final ChainExecutor executor;
|
|
||||||
|
|
||||||
Fixture() {
|
|
||||||
ChainDefinition definition = new ChainDefinition();
|
|
||||||
definition.setId("diagnostics");
|
|
||||||
StartNode start = new StartNode(); start.setId("start"); definition.addNode(start);
|
|
||||||
Node worker = new Node() {
|
|
||||||
public Map<String, Object> execute(Chain chain) {
|
|
||||||
throw new WorkflowExecutionException(WorkflowErrorReason.MODEL_TIMEOUT, "synthetic cause");
|
|
||||||
}
|
|
||||||
};
|
|
||||||
worker.setId("worker"); worker.setName("模型分析");
|
|
||||||
worker.setRetryEnable(true); worker.setMaxRetryCount(1); worker.setRetryIntervalMs(5);
|
|
||||||
definition.addNode(worker);
|
|
||||||
Edge edge = new Edge(); edge.setId("start-worker"); edge.setSource("start"); edge.setTarget("worker");
|
|
||||||
definition.addEdge(edge);
|
|
||||||
executor = new ChainExecutor(id -> definition, new InMemoryChainStateRepository(),
|
|
||||||
new InMemoryNodeStateRepository(), scheduler);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void close() { scheduler.shutdown(); }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,7 @@ package tech.easyflow.ai.service;
|
|||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.enums.PublishStatus;
|
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
import tech.easyflow.system.enums.CategoryResourceType;
|
import tech.easyflow.system.enums.CategoryResourceType;
|
||||||
@@ -11,7 +11,6 @@ import tech.easyflow.system.enums.ResourceAction;
|
|||||||
import tech.easyflow.system.service.ResourceAccessService;
|
import tech.easyflow.system.service.ResourceAccessService;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
@@ -22,14 +21,14 @@ import static org.mockito.Mockito.when;
|
|||||||
public class WorkflowUsageAuthorizationServiceTest {
|
public class WorkflowUsageAuthorizationServiceTest {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证未发布工作流即使资源权限允许也不能被使用。
|
* 验证禁用工作流即使资源权限允许也不能被使用。
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void shouldRejectUnpublishedWorkflow() {
|
public void shouldRejectDisabledWorkflow() {
|
||||||
BigInteger workflowId = BigInteger.valueOf(101);
|
BigInteger workflowId = BigInteger.valueOf(101);
|
||||||
WorkflowService workflowService = mock(WorkflowService.class);
|
WorkflowService workflowService = mock(WorkflowService.class);
|
||||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||||
Workflow workflow = workflow(workflowId, BigInteger.TEN, PublishStatus.DRAFT, Map.of());
|
Workflow workflow = workflow(workflowId, BigInteger.TEN, EnumDataStatus.UNAVAILABLE.getCode());
|
||||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||||
when(resourceAccessService.canAccess(
|
when(resourceAccessService.canAccess(
|
||||||
@@ -59,8 +58,7 @@ public class WorkflowUsageAuthorizationServiceTest {
|
|||||||
Workflow workflow = workflow(
|
Workflow workflow = workflow(
|
||||||
workflowId,
|
workflowId,
|
||||||
BigInteger.valueOf(20),
|
BigInteger.valueOf(20),
|
||||||
PublishStatus.PUBLISHED,
|
EnumDataStatus.AVAILABLE.getCode());
|
||||||
Map.of("title", "published"));
|
|
||||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||||
WorkflowUsageAuthorizationService service =
|
WorkflowUsageAuthorizationService service =
|
||||||
@@ -73,21 +71,17 @@ public class WorkflowUsageAuthorizationServiceTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 验证已发布、同租户且具有使用权限的工作流返回发布视图。
|
* 验证启用、同租户且具有使用权限的工作流可以返回。
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void shouldReturnPublishedWorkflowView() {
|
public void shouldReturnUsableWorkflow() {
|
||||||
BigInteger workflowId = BigInteger.valueOf(103);
|
BigInteger workflowId = BigInteger.valueOf(103);
|
||||||
WorkflowService workflowService = mock(WorkflowService.class);
|
WorkflowService workflowService = mock(WorkflowService.class);
|
||||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||||
Workflow workflow = workflow(
|
Workflow workflow = workflow(
|
||||||
workflowId,
|
workflowId,
|
||||||
BigInteger.TEN,
|
BigInteger.TEN,
|
||||||
PublishStatus.PUBLISHED,
|
EnumDataStatus.AVAILABLE.getCode());
|
||||||
Map.of("title", "published"));
|
|
||||||
Workflow published = new Workflow();
|
|
||||||
published.setId(workflowId);
|
|
||||||
published.setTitle("发布版");
|
|
||||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||||
when(resourceAccessService.canAccess(
|
when(resourceAccessService.canAccess(
|
||||||
@@ -95,42 +89,12 @@ public class WorkflowUsageAuthorizationServiceTest {
|
|||||||
CategoryResourceType.WORKFLOW,
|
CategoryResourceType.WORKFLOW,
|
||||||
workflow,
|
workflow,
|
||||||
ResourceAction.USE)).thenReturn(true);
|
ResourceAction.USE)).thenReturn(true);
|
||||||
when(workflowService.toPublishedView(workflow)).thenReturn(published);
|
|
||||||
WorkflowUsageAuthorizationService service =
|
WorkflowUsageAuthorizationService service =
|
||||||
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
|
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
|
||||||
|
|
||||||
Workflow result = service.requireUsableWorkflow(workflowId, account, "工作流不可用");
|
Workflow result = service.requireUsableWorkflow(workflowId, account, "工作流不可用");
|
||||||
|
|
||||||
Assert.assertSame(result, published);
|
Assert.assertSame(result, workflow);
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 验证发布状态异常但缺少快照的工作流不可被后台任务使用。
|
|
||||||
*/
|
|
||||||
@Test
|
|
||||||
public void shouldRejectPublishedWorkflowWithoutSnapshot() {
|
|
||||||
BigInteger workflowId = BigInteger.valueOf(104);
|
|
||||||
WorkflowService workflowService = mock(WorkflowService.class);
|
|
||||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
|
||||||
Workflow workflow = workflow(
|
|
||||||
workflowId,
|
|
||||||
BigInteger.TEN,
|
|
||||||
PublishStatus.PUBLISHED,
|
|
||||||
Map.of());
|
|
||||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
|
||||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
|
||||||
when(resourceAccessService.canAccess(
|
|
||||||
account,
|
|
||||||
CategoryResourceType.WORKFLOW,
|
|
||||||
workflow,
|
|
||||||
ResourceAction.USE)).thenReturn(true);
|
|
||||||
WorkflowUsageAuthorizationService service =
|
|
||||||
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
|
|
||||||
|
|
||||||
Assert.assertThrows(
|
|
||||||
BusinessException.class,
|
|
||||||
() -> service.requireUsableWorkflow(workflowId, account, "工作流不可用")
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -138,18 +102,14 @@ public class WorkflowUsageAuthorizationServiceTest {
|
|||||||
*
|
*
|
||||||
* @param id 工作流 ID
|
* @param id 工作流 ID
|
||||||
* @param tenantId 租户 ID
|
* @param tenantId 租户 ID
|
||||||
* @param publishStatus 工作流发布状态
|
* @param status 工作流状态
|
||||||
* @param snapshot 工作流发布快照
|
|
||||||
* @return 工作流
|
* @return 工作流
|
||||||
*/
|
*/
|
||||||
private Workflow workflow(BigInteger id, BigInteger tenantId,
|
private Workflow workflow(BigInteger id, BigInteger tenantId, Integer status) {
|
||||||
PublishStatus publishStatus,
|
|
||||||
Map<String, Object> snapshot) {
|
|
||||||
Workflow workflow = new Workflow();
|
Workflow workflow = new Workflow();
|
||||||
workflow.setId(id);
|
workflow.setId(id);
|
||||||
workflow.setTenantId(tenantId);
|
workflow.setTenantId(tenantId);
|
||||||
workflow.setPublishStatus(publishStatus.getCode());
|
workflow.setStatus(status);
|
||||||
workflow.setPublishedSnapshotJson(snapshot);
|
|
||||||
return workflow;
|
return workflow;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -154,11 +154,6 @@ public class DocumentChunkServiceImplTest {
|
|||||||
Mockito.when(fixture.chunkMapper.selectSyncStates(
|
Mockito.when(fixture.chunkMapper.selectSyncStates(
|
||||||
fixture.documentId, List.of(fixture.chunkId)
|
fixture.documentId, List.of(fixture.chunkId)
|
||||||
)).thenReturn(List.of(state));
|
)).thenReturn(List.of(state));
|
||||||
Mockito.when(fixture.syncTaskAppService.listSyncStatuses(List.of(state)))
|
|
||||||
.thenReturn(List.of(new tech.easyflow.ai.dto.DocumentChunkSyncStatus(
|
|
||||||
state.getId(), state.getIndexSyncStatus(), state.getIndexSyncVersion(),
|
|
||||||
null, null, 0, 5
|
|
||||||
)));
|
|
||||||
|
|
||||||
List<tech.easyflow.ai.dto.DocumentChunkSyncStatus> result =
|
List<tech.easyflow.ai.dto.DocumentChunkSyncStatus> result =
|
||||||
fixture.service.listIndexSyncStatus(
|
fixture.service.listIndexSyncStatus(
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ tech.easyflow.approval.config.ApprovalModuleConfig
|
|||||||
tech.easyflow.auth.config.AuthModuleConfig
|
tech.easyflow.auth.config.AuthModuleConfig
|
||||||
tech.easyflow.chatlog.config.ChatlogModuleConfig
|
tech.easyflow.chatlog.config.ChatlogModuleConfig
|
||||||
tech.easyflow.datacenter.config.DatacenterModuleConfig
|
tech.easyflow.datacenter.config.DatacenterModuleConfig
|
||||||
tech.easyflow.dataspace.config.DataspaceModuleConfig
|
|
||||||
tech.easyflow.job.config.JobModuleConfig
|
tech.easyflow.job.config.JobModuleConfig
|
||||||
tech.easyflow.log.config.LogModuleConfig
|
tech.easyflow.log.config.LogModuleConfig
|
||||||
tech.easyflow.skill.config.SkillModuleConfig
|
tech.easyflow.skill.config.SkillModuleConfig
|
||||||
|
|||||||
@@ -20,10 +20,6 @@
|
|||||||
<groupId>com.zaxxer</groupId>
|
<groupId>com.zaxxer</groupId>
|
||||||
<artifactId>HikariCP</artifactId>
|
<artifactId>HikariCP</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<groupId>com.easyagents</groupId>
|
|
||||||
<artifactId>easy-agents-federation-sql-adapter-jdbc</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.postgresql</groupId>
|
<groupId>org.postgresql</groupId>
|
||||||
<artifactId>postgresql</artifactId>
|
<artifactId>postgresql</artifactId>
|
||||||
|
|||||||
@@ -1,168 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.audit;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
|
||||||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
|
|
||||||
import com.mybatisflex.annotation.Column;
|
|
||||||
import com.mybatisflex.annotation.Id;
|
|
||||||
import com.mybatisflex.annotation.KeyType;
|
|
||||||
import com.mybatisflex.annotation.Table;
|
|
||||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.Date;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据中枢统一只读查询审计记录。
|
|
||||||
*/
|
|
||||||
@Table(value = "tb_datacenter_query_audit", comment = "数据中枢统一查询审计")
|
|
||||||
public class DatacenterQueryAudit implements Serializable {
|
|
||||||
|
|
||||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
|
||||||
private BigInteger id;
|
|
||||||
@Column(comment = "查询标识")
|
|
||||||
private String queryId;
|
|
||||||
@Column(tenantId = true, comment = "租户ID")
|
|
||||||
private BigInteger tenantId;
|
|
||||||
@Column(comment = "部门ID")
|
|
||||||
private BigInteger deptId;
|
|
||||||
@Column(comment = "执行账号ID")
|
|
||||||
private BigInteger executorAccountId;
|
|
||||||
@Column(comment = "调用方类型")
|
|
||||||
private String callerType;
|
|
||||||
@Column(comment = "调用方标识")
|
|
||||||
private String callerId;
|
|
||||||
@Column(comment = "数据源ID")
|
|
||||||
private BigInteger sourceId;
|
|
||||||
@Column(comment = "数据源版本")
|
|
||||||
private Long sourceRevision;
|
|
||||||
@Column(comment = "纳管范围版本")
|
|
||||||
private Long scopeRevision;
|
|
||||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "引用表摘要")
|
|
||||||
private List<String> referencedTablesJson;
|
|
||||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "引用字段摘要")
|
|
||||||
private List<String> referencedFieldsJson;
|
|
||||||
@Column(comment = "参数化或脱敏SQL")
|
|
||||||
private String parameterizedSql;
|
|
||||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "脱敏参数摘要")
|
|
||||||
private Map<String, Object> maskedParametersJson;
|
|
||||||
@Column(comment = "执行状态")
|
|
||||||
private String status;
|
|
||||||
@Column(comment = "稳定错误码")
|
|
||||||
private String errorCode;
|
|
||||||
@Column(comment = "有界错误说明")
|
|
||||||
private String errorMessage;
|
|
||||||
@Column(comment = "开始时间")
|
|
||||||
private Date startedAt;
|
|
||||||
@Column(comment = "结束时间")
|
|
||||||
private Date finishedAt;
|
|
||||||
@Column(comment = "总耗时毫秒")
|
|
||||||
private Long durationMs;
|
|
||||||
@Column(comment = "数据库耗时毫秒")
|
|
||||||
private Long databaseDurationMs;
|
|
||||||
@Column(comment = "返回行数")
|
|
||||||
private Long returnedRows;
|
|
||||||
@Column(comment = "是否截断")
|
|
||||||
private Integer truncated;
|
|
||||||
@Column(comment = "创建时间")
|
|
||||||
private Date created;
|
|
||||||
|
|
||||||
/** @return 主键 */
|
|
||||||
@JsonSerialize(using = ToStringSerializer.class)
|
|
||||||
public BigInteger getId() { return id; }
|
|
||||||
/** @param id 主键 */
|
|
||||||
public void setId(BigInteger id) { this.id = id; }
|
|
||||||
/** @return 查询标识 */
|
|
||||||
public String getQueryId() { return queryId; }
|
|
||||||
/** @param queryId 查询标识 */
|
|
||||||
public void setQueryId(String queryId) { this.queryId = queryId; }
|
|
||||||
/** @return 租户 ID */
|
|
||||||
public BigInteger getTenantId() { return tenantId; }
|
|
||||||
/** @param tenantId 租户 ID */
|
|
||||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
|
||||||
/** @return 部门 ID */
|
|
||||||
public BigInteger getDeptId() { return deptId; }
|
|
||||||
/** @param deptId 部门 ID */
|
|
||||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
|
||||||
/** @return 执行账号 ID;系统任务为空 */
|
|
||||||
public BigInteger getExecutorAccountId() { return executorAccountId; }
|
|
||||||
/** @param executorAccountId 执行账号 ID */
|
|
||||||
public void setExecutorAccountId(BigInteger executorAccountId) { this.executorAccountId = executorAccountId; }
|
|
||||||
/** @return 调用方类型 */
|
|
||||||
public String getCallerType() { return callerType; }
|
|
||||||
/** @param callerType 调用方类型 */
|
|
||||||
public void setCallerType(String callerType) { this.callerType = callerType; }
|
|
||||||
/** @return 调用方标识 */
|
|
||||||
public String getCallerId() { return callerId; }
|
|
||||||
/** @param callerId 调用方标识 */
|
|
||||||
public void setCallerId(String callerId) { this.callerId = callerId; }
|
|
||||||
/** @return 数据源 ID */
|
|
||||||
public BigInteger getSourceId() { return sourceId; }
|
|
||||||
/** @param sourceId 数据源 ID */
|
|
||||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
|
||||||
/** @return 数据源版本 */
|
|
||||||
public Long getSourceRevision() { return sourceRevision; }
|
|
||||||
/** @param sourceRevision 数据源版本 */
|
|
||||||
public void setSourceRevision(Long sourceRevision) { this.sourceRevision = sourceRevision; }
|
|
||||||
/** @return 纳管范围版本 */
|
|
||||||
public Long getScopeRevision() { return scopeRevision; }
|
|
||||||
/** @param scopeRevision 纳管范围版本 */
|
|
||||||
public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; }
|
|
||||||
/** @return 引用表 */
|
|
||||||
public List<String> getReferencedTablesJson() { return referencedTablesJson; }
|
|
||||||
/** @param referencedTablesJson 引用表 */
|
|
||||||
public void setReferencedTablesJson(List<String> referencedTablesJson) { this.referencedTablesJson = referencedTablesJson; }
|
|
||||||
/** @return 引用字段 */
|
|
||||||
public List<String> getReferencedFieldsJson() { return referencedFieldsJson; }
|
|
||||||
/** @param referencedFieldsJson 引用字段 */
|
|
||||||
public void setReferencedFieldsJson(List<String> referencedFieldsJson) { this.referencedFieldsJson = referencedFieldsJson; }
|
|
||||||
/** @return 参数化 SQL */
|
|
||||||
public String getParameterizedSql() { return parameterizedSql; }
|
|
||||||
/** @param parameterizedSql 参数化 SQL */
|
|
||||||
public void setParameterizedSql(String parameterizedSql) { this.parameterizedSql = parameterizedSql; }
|
|
||||||
/** @return 脱敏参数 */
|
|
||||||
public Map<String, Object> getMaskedParametersJson() { return maskedParametersJson; }
|
|
||||||
/** @param maskedParametersJson 脱敏参数 */
|
|
||||||
public void setMaskedParametersJson(Map<String, Object> maskedParametersJson) { this.maskedParametersJson = maskedParametersJson; }
|
|
||||||
/** @return 状态 */
|
|
||||||
public String getStatus() { return status; }
|
|
||||||
/** @param status 状态 */
|
|
||||||
public void setStatus(String status) { this.status = status; }
|
|
||||||
/** @return 错误码 */
|
|
||||||
public String getErrorCode() { return errorCode; }
|
|
||||||
/** @param errorCode 错误码 */
|
|
||||||
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
|
|
||||||
/** @return 错误说明 */
|
|
||||||
public String getErrorMessage() { return errorMessage; }
|
|
||||||
/** @param errorMessage 错误说明 */
|
|
||||||
public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
|
|
||||||
/** @return 开始时间 */
|
|
||||||
public Date getStartedAt() { return startedAt; }
|
|
||||||
/** @param startedAt 开始时间 */
|
|
||||||
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
|
|
||||||
/** @return 结束时间 */
|
|
||||||
public Date getFinishedAt() { return finishedAt; }
|
|
||||||
/** @param finishedAt 结束时间 */
|
|
||||||
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
|
|
||||||
/** @return 总耗时 */
|
|
||||||
public Long getDurationMs() { return durationMs; }
|
|
||||||
/** @param durationMs 总耗时 */
|
|
||||||
public void setDurationMs(Long durationMs) { this.durationMs = durationMs; }
|
|
||||||
/** @return 数据库耗时 */
|
|
||||||
public Long getDatabaseDurationMs() { return databaseDurationMs; }
|
|
||||||
/** @param databaseDurationMs 数据库耗时 */
|
|
||||||
public void setDatabaseDurationMs(Long databaseDurationMs) { this.databaseDurationMs = databaseDurationMs; }
|
|
||||||
/** @return 返回行数 */
|
|
||||||
public Long getReturnedRows() { return returnedRows; }
|
|
||||||
/** @param returnedRows 返回行数 */
|
|
||||||
public void setReturnedRows(Long returnedRows) { this.returnedRows = returnedRows; }
|
|
||||||
/** @return 是否截断 */
|
|
||||||
public Integer getTruncated() { return truncated; }
|
|
||||||
/** @param truncated 是否截断 */
|
|
||||||
public void setTruncated(Integer truncated) { this.truncated = truncated; }
|
|
||||||
/** @return 创建时间 */
|
|
||||||
public Date getCreated() { return created; }
|
|
||||||
/** @param created 创建时间 */
|
|
||||||
public void setCreated(Date created) { this.created = created; }
|
|
||||||
}
|
|
||||||
@@ -1,301 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.audit;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.Date;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterQueryAuditMapper;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在查询前落账并在资源释放后收口状态的审计服务。
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class DatacenterQueryAuditService {
|
|
||||||
|
|
||||||
private final DatacenterQueryAuditMapper auditMapper;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建审计服务。
|
|
||||||
*
|
|
||||||
* @param auditMapper 审计 Mapper
|
|
||||||
*/
|
|
||||||
public DatacenterQueryAuditService(DatacenterQueryAuditMapper auditMapper) {
|
|
||||||
this.auditMapper = auditMapper;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在数据库查询开始前写入 RUNNING 记录。
|
|
||||||
*
|
|
||||||
* @param queryId 查询标识
|
|
||||||
* @param source 数据源快照
|
|
||||||
* @param sql 参数化或无参数只读 SQL
|
|
||||||
* @param parameters 脱敏参数摘要
|
|
||||||
* @param account 执行账号
|
|
||||||
* @param callerType 调用方类型
|
|
||||||
* @param callerId 调用方标识
|
|
||||||
* @return 已持久化审计记录
|
|
||||||
*/
|
|
||||||
public DatacenterQueryAudit start(
|
|
||||||
String queryId,
|
|
||||||
DatacenterSource source,
|
|
||||||
String sql,
|
|
||||||
Map<String, Object> parameters,
|
|
||||||
LoginAccount account,
|
|
||||||
String callerType,
|
|
||||||
String callerId) {
|
|
||||||
Date now = new Date();
|
|
||||||
DatacenterQueryAudit audit = new DatacenterQueryAudit();
|
|
||||||
audit.setQueryId(queryId);
|
|
||||||
audit.setTenantId(account == null || account.getTenantId() == null
|
|
||||||
? value(source.getTenantId()) : account.getTenantId());
|
|
||||||
audit.setDeptId(account == null || account.getDeptId() == null
|
|
||||||
? value(source.getDeptId()) : account.getDeptId());
|
|
||||||
audit.setExecutorAccountId(account == null ? null : account.getId());
|
|
||||||
audit.setCallerType(callerType == null ? "SYSTEM" : callerType);
|
|
||||||
audit.setCallerId(callerId);
|
|
||||||
audit.setSourceId(source.getId());
|
|
||||||
audit.setSourceRevision(value(source.getDefinitionRevision()));
|
|
||||||
audit.setScopeRevision(value(source.getScopeRevision()));
|
|
||||||
audit.setReferencedTablesJson(List.of());
|
|
||||||
audit.setReferencedFieldsJson(List.of());
|
|
||||||
audit.setParameterizedSql(redactSql(sql));
|
|
||||||
audit.setMaskedParametersJson(parameters == null ? Map.of() : Map.copyOf(parameters));
|
|
||||||
audit.setStatus("RUNNING");
|
|
||||||
audit.setStartedAt(now);
|
|
||||||
audit.setReturnedRows(0L);
|
|
||||||
audit.setTruncated(0);
|
|
||||||
audit.setCreated(now);
|
|
||||||
auditMapper.insert(audit);
|
|
||||||
return audit;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 标记查询成功。
|
|
||||||
*
|
|
||||||
* @param audit 审计记录
|
|
||||||
* @param referencedTables Calcite 解析出的引用表
|
|
||||||
* @param referencedFields Calcite 解析出的引用字段
|
|
||||||
* @param returnedRows 返回行数
|
|
||||||
* @param truncated 是否截断
|
|
||||||
* @param durationMs 总耗时
|
|
||||||
* @param databaseDurationMs 数据库耗时
|
|
||||||
*/
|
|
||||||
public void succeed(
|
|
||||||
DatacenterQueryAudit audit,
|
|
||||||
List<String> referencedTables,
|
|
||||||
List<String> referencedFields,
|
|
||||||
long returnedRows,
|
|
||||||
boolean truncated,
|
|
||||||
long durationMs,
|
|
||||||
long databaseDurationMs) {
|
|
||||||
finish(audit, "SUCCESS", null, null, referencedTables, referencedFields,
|
|
||||||
returnedRows, truncated, durationMs, databaseDurationMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 标记查询失败。
|
|
||||||
*
|
|
||||||
* @param audit 审计记录
|
|
||||||
* @param errorCode 稳定错误码
|
|
||||||
* @param message 有界错误说明
|
|
||||||
* @param referencedTables Calcite 已解析出的引用表
|
|
||||||
* @param referencedFields Calcite 已解析出的引用字段
|
|
||||||
* @param durationMs 总耗时
|
|
||||||
*/
|
|
||||||
public void fail(
|
|
||||||
DatacenterQueryAudit audit,
|
|
||||||
String errorCode,
|
|
||||||
String message,
|
|
||||||
List<String> referencedTables,
|
|
||||||
List<String> referencedFields,
|
|
||||||
long durationMs) {
|
|
||||||
finish(audit, "FAILED", errorCode, bounded(message), referencedTables, referencedFields,
|
|
||||||
0L, false, durationMs, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void finish(
|
|
||||||
DatacenterQueryAudit audit,
|
|
||||||
String status,
|
|
||||||
String errorCode,
|
|
||||||
String errorMessage,
|
|
||||||
List<String> referencedTables,
|
|
||||||
List<String> referencedFields,
|
|
||||||
long returnedRows,
|
|
||||||
boolean truncated,
|
|
||||||
long durationMs,
|
|
||||||
Long databaseDurationMs) {
|
|
||||||
audit.setStatus(status);
|
|
||||||
audit.setErrorCode(errorCode);
|
|
||||||
audit.setErrorMessage(errorMessage);
|
|
||||||
audit.setReferencedTablesJson(List.copyOf(
|
|
||||||
referencedTables == null ? List.of() : referencedTables));
|
|
||||||
audit.setReferencedFieldsJson(List.copyOf(
|
|
||||||
referencedFields == null ? List.of() : referencedFields));
|
|
||||||
audit.setReturnedRows(returnedRows);
|
|
||||||
audit.setTruncated(truncated ? 1 : 0);
|
|
||||||
audit.setDurationMs(durationMs);
|
|
||||||
audit.setDatabaseDurationMs(databaseDurationMs);
|
|
||||||
audit.setFinishedAt(new Date());
|
|
||||||
auditMapper.update(audit);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 将超过十分钟仍为 RUNNING 的遗留审计收口为失败,覆盖进程异常退出场景。
|
|
||||||
*/
|
|
||||||
@Scheduled(fixedDelayString = "${easyflow.datacenter.audit-reconcile-ms:60000}")
|
|
||||||
public void reconcileAbandonedQueries() {
|
|
||||||
Date now = new Date();
|
|
||||||
DatacenterQueryAudit patch = new DatacenterQueryAudit();
|
|
||||||
patch.setStatus("FAILED");
|
|
||||||
patch.setErrorCode("QUERY_ABANDONED");
|
|
||||||
patch.setErrorMessage("查询进程异常结束或超出审计存活时间");
|
|
||||||
patch.setFinishedAt(now);
|
|
||||||
QueryWrapper guard = QueryWrapper.create()
|
|
||||||
.eq(DatacenterQueryAudit::getStatus, "RUNNING")
|
|
||||||
.lt(DatacenterQueryAudit::getStartedAt, new Date(now.getTime() - 600_000L));
|
|
||||||
auditMapper.updateByQuery(patch, guard);
|
|
||||||
}
|
|
||||||
|
|
||||||
private long value(Long value) {
|
|
||||||
return value == null ? 0L : value;
|
|
||||||
}
|
|
||||||
|
|
||||||
private BigInteger value(BigInteger value) {
|
|
||||||
return value == null ? BigInteger.ZERO : value;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String redactSql(String sql) {
|
|
||||||
if (sql == null || sql.isBlank()) {
|
|
||||||
return sql;
|
|
||||||
}
|
|
||||||
StringBuilder result = new StringBuilder(Math.min(sql.length(), 100_000));
|
|
||||||
boolean quoted = false;
|
|
||||||
for (int index = 0; index < sql.length(); index++) {
|
|
||||||
char current = sql.charAt(index);
|
|
||||||
if (quoted) {
|
|
||||||
if (current == '\'' && index + 1 < sql.length() && sql.charAt(index + 1) == '\'') {
|
|
||||||
index++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (current == '\'') {
|
|
||||||
quoted = false;
|
|
||||||
result.append('?');
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (current == '\'') {
|
|
||||||
quoted = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (current == '-' && index + 1 < sql.length() && sql.charAt(index + 1) == '-') {
|
|
||||||
result.append("--?");
|
|
||||||
index += 2;
|
|
||||||
while (index < sql.length() && sql.charAt(index) != '\n'
|
|
||||||
&& sql.charAt(index) != '\r') {
|
|
||||||
index++;
|
|
||||||
}
|
|
||||||
if (index < sql.length()) {
|
|
||||||
result.append(sql.charAt(index));
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (current == '/' && index + 1 < sql.length() && sql.charAt(index + 1) == '*') {
|
|
||||||
result.append("/*?*/");
|
|
||||||
index = skipBlockComment(sql, index + 2);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String dollarDelimiter = current == '$' ? dollarDelimiter(sql, index) : null;
|
|
||||||
if (dollarDelimiter != null) {
|
|
||||||
int end = sql.indexOf(dollarDelimiter, index + dollarDelimiter.length());
|
|
||||||
result.append('?');
|
|
||||||
index = end < 0 ? sql.length() : end + dollarDelimiter.length() - 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (Character.isDigit(current)
|
|
||||||
&& (index == 0 || !Character.isJavaIdentifierPart(sql.charAt(index - 1)))) {
|
|
||||||
result.append('?');
|
|
||||||
while (index + 1 < sql.length()) {
|
|
||||||
char next = sql.charAt(index + 1);
|
|
||||||
if (!Character.isDigit(next) && next != '.' && next != 'e' && next != 'E'
|
|
||||||
&& next != '+' && next != '-') {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
index++;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
result.append(current);
|
|
||||||
if (result.length() >= 100_000) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (quoted) {
|
|
||||||
result.append('?');
|
|
||||||
}
|
|
||||||
return result.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 跳过支持嵌套的块注释内容。
|
|
||||||
*
|
|
||||||
* @param sql 原始 SQL
|
|
||||||
* @param contentStart 首个注释内容字符位置
|
|
||||||
* @return 外层注释结束标记的末尾位置;未闭合时返回末尾
|
|
||||||
*/
|
|
||||||
private int skipBlockComment(String sql, int contentStart) {
|
|
||||||
int depth = 1;
|
|
||||||
for (int index = contentStart; index < sql.length() - 1; index++) {
|
|
||||||
if (sql.charAt(index) == '/' && sql.charAt(index + 1) == '*') {
|
|
||||||
depth++;
|
|
||||||
index++;
|
|
||||||
} else if (sql.charAt(index) == '*' && sql.charAt(index + 1) == '/') {
|
|
||||||
depth--;
|
|
||||||
if (depth == 0) {
|
|
||||||
return index + 1;
|
|
||||||
}
|
|
||||||
index++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return sql.length();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 识别 PostgreSQL dollar-quoted 字面量起始分隔符。
|
|
||||||
*
|
|
||||||
* @param sql 原始 SQL
|
|
||||||
* @param start 美元符号位置
|
|
||||||
* @return 完整分隔符;当前位置不是合法起始符时为空
|
|
||||||
*/
|
|
||||||
private String dollarDelimiter(String sql, int start) {
|
|
||||||
int end = sql.indexOf('$', start + 1);
|
|
||||||
if (end < 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
String tag = sql.substring(start + 1, end);
|
|
||||||
if (tag.isEmpty()) {
|
|
||||||
return "$$";
|
|
||||||
}
|
|
||||||
if (!Character.isLetter(tag.charAt(0)) && tag.charAt(0) != '_') {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
for (int index = 1; index < tag.length(); index++) {
|
|
||||||
char character = tag.charAt(index);
|
|
||||||
if (!Character.isLetterOrDigit(character) && character != '_') {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return '$' + tag + '$';
|
|
||||||
}
|
|
||||||
|
|
||||||
private String bounded(String message) {
|
|
||||||
if (message == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return message.length() <= 500 ? message : message.substring(0, 500);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.config;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.source.SourceId;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
|
||||||
import org.springframework.context.annotation.Bean;
|
|
||||||
import org.springframework.context.annotation.Configuration;
|
|
||||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
|
||||||
import org.springframework.data.redis.listener.ChannelTopic;
|
|
||||||
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
|
|
||||||
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
|
|
||||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
|
||||||
import tech.easyflow.datacenter.federation.DatacenterFederationSourceStateProvider;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据中枢无凭据 Definition 变更提示订阅配置。
|
|
||||||
*/
|
|
||||||
@Configuration(proxyBeanMethods = false)
|
|
||||||
public class DatacenterFederationRedisConfig {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationRedisConfig.class);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建独立 Redis 监听容器,收到提示后始终回源数据库。
|
|
||||||
*
|
|
||||||
* @param connectionFactory Redis 连接工厂
|
|
||||||
* @param stateProvider 数据库权威状态 Provider
|
|
||||||
* @param queryCancellationService 查询取消服务
|
|
||||||
* @return 监听容器
|
|
||||||
*/
|
|
||||||
@Bean(name = "datacenterFederationRedisListenerContainer")
|
|
||||||
@ConditionalOnBean(RedisConnectionFactory.class)
|
|
||||||
public RedisMessageListenerContainer datacenterFederationRedisListenerContainer(
|
|
||||||
RedisConnectionFactory connectionFactory,
|
|
||||||
DatacenterFederationSourceStateProvider stateProvider,
|
|
||||||
DatacenterFederationQueryCancellationService queryCancellationService) {
|
|
||||||
RedisMessageListenerContainer container = new RedisMessageListenerContainer();
|
|
||||||
container.setConnectionFactory(connectionFactory);
|
|
||||||
container.addMessageListener((message, pattern) -> {
|
|
||||||
String value = new String(message.getBody(), StandardCharsets.UTF_8);
|
|
||||||
try {
|
|
||||||
stateProvider.refresh(new SourceId(value));
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to refresh datacenter Federation source {}", value, exception);
|
|
||||||
}
|
|
||||||
}, new ChannelTopic(DatacenterFederationChangeNotifier.CHANNEL));
|
|
||||||
container.addMessageListener((message, pattern) ->
|
|
||||||
queryCancellationService.acceptCancellationHint(
|
|
||||||
new String(message.getBody(), StandardCharsets.UTF_8)),
|
|
||||||
new ChannelTopic(DatacenterFederationQueryCancellationService.CHANNEL));
|
|
||||||
return container;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,8 +3,6 @@ package tech.easyflow.datacenter.connector;
|
|||||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -12,97 +10,7 @@ import java.util.List;
|
|||||||
public interface MetadataExplorer {
|
public interface MetadataExplorer {
|
||||||
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
|
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
|
||||||
|
|
||||||
/**
|
|
||||||
* 分页浏览 Catalog 或 Schema。
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @param keyword 名称搜索词
|
|
||||||
* @param pageNumber 页码
|
|
||||||
* @param pageSize 每页大小
|
|
||||||
* @return 有界命名空间列表
|
|
||||||
*/
|
|
||||||
default DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
|
|
||||||
DatacenterSource source,
|
|
||||||
String keyword,
|
|
||||||
long pageNumber,
|
|
||||||
long pageSize) {
|
|
||||||
String normalizedKeyword = keyword == null
|
|
||||||
? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT);
|
|
||||||
List<DatacenterCatalogMeta> filtered = listCatalogs(source).stream()
|
|
||||||
.filter(catalog -> normalizedKeyword.isEmpty()
|
|
||||||
|| String.valueOf(catalog.getCatalogName())
|
|
||||||
.toLowerCase(java.util.Locale.ROOT)
|
|
||||||
.contains(normalizedKeyword))
|
|
||||||
.toList();
|
|
||||||
return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
|
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
|
||||||
|
|
||||||
/**
|
|
||||||
* 分页浏览表或视图。
|
|
||||||
*
|
|
||||||
* <p>非 JDBC Connector 默认复用已有列表能力;外部 JDBC Connector 应覆盖此方法,
|
|
||||||
* 直接在元数据 ResultSet 上做有界读取。</p>
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @param catalogName 物理命名空间
|
|
||||||
* @param keyword 表名搜索词
|
|
||||||
* @param pageNumber 页码
|
|
||||||
* @param pageSize 每页大小
|
|
||||||
* @return 有界表列表
|
|
||||||
*/
|
|
||||||
default DatacenterMetadataPage<DatacenterTable> listTablesPage(
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName,
|
|
||||||
String keyword,
|
|
||||||
long pageNumber,
|
|
||||||
long pageSize) {
|
|
||||||
String normalizedKeyword = keyword == null ? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT);
|
|
||||||
List<DatacenterTable> filtered = listTables(source, catalogName).stream()
|
|
||||||
.filter(table -> normalizedKeyword.isEmpty()
|
|
||||||
|| String.valueOf(table.getTableName())
|
|
||||||
.toLowerCase(java.util.Locale.ROOT)
|
|
||||||
.contains(normalizedKeyword))
|
|
||||||
.toList();
|
|
||||||
return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
|
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
|
||||||
|
|
||||||
/**
|
|
||||||
* 批量读取表详情。
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @param catalogName 物理命名空间
|
|
||||||
* @param tableNames 表名集合
|
|
||||||
* @return 表详情集合
|
|
||||||
*/
|
|
||||||
default List<DatacenterTableDetailMeta> getTableDetails(
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName,
|
|
||||||
List<String> tableNames) {
|
|
||||||
return tableNames.stream()
|
|
||||||
.map(tableName -> getTableDetail(source, catalogName, tableName))
|
|
||||||
.toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 读取已纳管表的刷新快照。
|
|
||||||
*
|
|
||||||
* <p>默认实现保留 Connector 的既有逐表语义;JDBC Connector 应覆盖此方法以复用
|
|
||||||
* 单个连接,并明确区分物理表缺失与连接故障。</p>
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @param catalogName 物理命名空间
|
|
||||||
* @param tableNames 已纳管表名集合
|
|
||||||
* @return 存在与缺失对象的快照
|
|
||||||
*/
|
|
||||||
default DatacenterManagedMetadataSnapshot inspectManagedTables(
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName,
|
|
||||||
List<String> tableNames) {
|
|
||||||
return new DatacenterManagedMetadataSnapshot(
|
|
||||||
getTableDetails(source, catalogName, tableNames), java.util.Set.of());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,12 +28,13 @@ public class MysqlConnector extends AbstractJdbcConnector {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
// MySQL 查询 Runtime 已由 Federation SQL 独占管理;此 Connector 只保留短生命周期元数据访问。
|
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||||
HikariDataSource dataSource = datasourceManager.createExternalDatasource(source);
|
|
||||||
try (Connection connection = dataSource.getConnection()) {
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
return callback.apply(connection);
|
return callback.apply(connection);
|
||||||
} finally {
|
} finally {
|
||||||
dataSource.close();
|
if (!cacheable || source.getId() == null) {
|
||||||
|
dataSource.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,12 +28,13 @@ public class PostgresqlConnector extends AbstractJdbcConnector {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
// PostgreSQL 查询 Runtime 已由 Federation SQL 独占管理;此 Connector 只保留短生命周期元数据访问。
|
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||||
HikariDataSource dataSource = datasourceManager.createExternalDatasource(source);
|
|
||||||
try (Connection connection = dataSource.getConnection()) {
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
return callback.apply(connection);
|
return callback.apply(connection);
|
||||||
} finally {
|
} finally {
|
||||||
dataSource.close();
|
if (!cacheable || source.getId() == null) {
|
||||||
|
dataSource.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package tech.easyflow.datacenter.connector.support;
|
|||||||
import cn.hutool.core.util.StrUtil;
|
import cn.hutool.core.util.StrUtil;
|
||||||
import com.alibaba.fastjson2.JSONObject;
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.query.QueryColumn;
|
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import com.mybatisflex.core.row.Db;
|
import com.mybatisflex.core.row.Db;
|
||||||
import com.mybatisflex.core.row.Row;
|
import com.mybatisflex.core.row.Row;
|
||||||
@@ -16,7 +15,6 @@ import tech.easyflow.datacenter.connector.DatacenterConnector;
|
|||||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
@@ -87,180 +85,35 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||||
if (request == null || request.getPageSize() == null
|
|
||||||
|| request.getPageSize() < 1L
|
|
||||||
|| request.getPageSize() > 500L) {
|
|
||||||
throw new BusinessException("pageSize 必须在 1 到 500 之间");
|
|
||||||
}
|
|
||||||
String actualTable = resolveTableName(table);
|
String actualTable = resolveTableName(table);
|
||||||
long count = Db.selectCountByQuery(
|
long count = Db.selectCountByQuery(
|
||||||
actualTable, createQueryWrapper(table, request, false));
|
actualTable, createQueryWrapper(request.getWhere()));
|
||||||
if (count == 0) {
|
if (count == 0) {
|
||||||
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
|
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
|
||||||
}
|
}
|
||||||
// selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*),分页查询必须使用独立实例。
|
// selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*),分页查询必须使用独立实例。
|
||||||
QueryWrapper pageQuery = createQueryWrapper(table, request, true);
|
|
||||||
pageQuery.select(resolveSelectedColumns(table, request).stream()
|
|
||||||
.map(this::quoteInternalIdentifier)
|
|
||||||
.toArray(String[]::new));
|
|
||||||
Page<Row> page = Db.paginate(
|
Page<Row> page = Db.paginate(
|
||||||
actualTable,
|
actualTable,
|
||||||
new Page<>(request.getPageNumber(), request.getPageSize(), count),
|
new Page<>(request.getPageNumber(), request.getPageSize(), count),
|
||||||
pageQuery);
|
createQueryWrapper(request.getWhere()));
|
||||||
normalizeRows(page.getRecords());
|
normalizeRows(page.getRecords());
|
||||||
return page;
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 将已在服务层校验的逻辑字段映射为内部物理列。
|
|
||||||
*
|
|
||||||
* @param table 绑定表及字段
|
|
||||||
* @param request 查询请求
|
|
||||||
* @return 按请求顺序排列的物理列名
|
|
||||||
* @throws BusinessException 字段元数据缺失时抛出
|
|
||||||
*/
|
|
||||||
private List<String> resolveSelectedColumns(
|
|
||||||
DatacenterTable table,
|
|
||||||
DatacenterQueryRequest request) {
|
|
||||||
Map<String, DatacenterTableField> fields = new LinkedHashMap<>();
|
|
||||||
for (DatacenterTableField field : table.getFields()) {
|
|
||||||
fields.put(field.getFieldName(), field);
|
|
||||||
}
|
|
||||||
List<String> columns = new ArrayList<>();
|
|
||||||
for (String selected : request.getSelectedColumns()) {
|
|
||||||
DatacenterTableField field = fields.get(selected);
|
|
||||||
if (field == null) {
|
|
||||||
throw new BusinessException("字段不存在: " + selected);
|
|
||||||
}
|
|
||||||
String sourceColumn = StrUtil.blankToDefault(
|
|
||||||
field.getSourceColumnName(), field.getFieldName());
|
|
||||||
columns.add(sourceColumn);
|
|
||||||
}
|
|
||||||
if (columns.isEmpty()) {
|
|
||||||
throw new BusinessException("当前数据集没有可查询字段");
|
|
||||||
}
|
|
||||||
return columns;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 引用可信元数据列名,避免保留字或特殊字符改变 SQL 结构。
|
|
||||||
*
|
|
||||||
* @param identifier 物理列名
|
|
||||||
* @return MySQL 反引号标识符
|
|
||||||
*/
|
|
||||||
private String quoteInternalIdentifier(String identifier) {
|
|
||||||
return "`" + identifier.replace("`", "``") + "`";
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建用于动态表查询的独立条件包装器。
|
* 创建用于动态表查询的独立条件包装器。
|
||||||
*
|
*
|
||||||
* @param where 调用方传入的旧原始筛选表达式;非空时拒绝
|
* @param where 已校验的筛选表达式
|
||||||
* @return 新建的查询条件包装器
|
* @return 新建的查询条件包装器
|
||||||
*/
|
*/
|
||||||
static QueryWrapper createQueryWrapper(String where) {
|
static QueryWrapper createQueryWrapper(String where) {
|
||||||
if (StrUtil.isNotBlank(where)) {
|
|
||||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
|
||||||
}
|
|
||||||
QueryWrapper wrapper = QueryWrapper.create();
|
QueryWrapper wrapper = QueryWrapper.create();
|
||||||
return wrapper;
|
if (StrUtil.isNotBlank(where)) {
|
||||||
}
|
wrapper.where(where);
|
||||||
|
|
||||||
private QueryWrapper createQueryWrapper(
|
|
||||||
DatacenterTable table,
|
|
||||||
DatacenterQueryRequest request,
|
|
||||||
boolean includeSorts) {
|
|
||||||
QueryWrapper wrapper = createQueryWrapper(request.getWhere());
|
|
||||||
Map<String, DatacenterTableField> fields = table.getFields().stream()
|
|
||||||
.collect(java.util.stream.Collectors.toMap(
|
|
||||||
DatacenterTableField::getFieldName,
|
|
||||||
field -> field,
|
|
||||||
(first, ignored) -> first,
|
|
||||||
LinkedHashMap::new));
|
|
||||||
List<Object> parameters = new ArrayList<>();
|
|
||||||
StringBuilder condition = new StringBuilder();
|
|
||||||
if (request.getFilters() != null) {
|
|
||||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
|
||||||
DatacenterTableField field = fields.get(filter.getColumn());
|
|
||||||
if (field == null) {
|
|
||||||
throw new BusinessException("字段不存在: " + filter.getColumn());
|
|
||||||
}
|
|
||||||
appendFilter(condition, parameters, field, filter);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!condition.isEmpty()) {
|
|
||||||
wrapper.where(condition.toString(), parameters.toArray());
|
|
||||||
}
|
|
||||||
if (includeSorts && request.getSorts() != null && !request.getSorts().isEmpty()) {
|
|
||||||
request.getSorts().forEach(sort -> {
|
|
||||||
DatacenterTableField field = fields.get(sort.getColumn());
|
|
||||||
if (field == null) {
|
|
||||||
throw new BusinessException("字段不存在: " + sort.getColumn());
|
|
||||||
}
|
|
||||||
wrapper.orderBy(
|
|
||||||
new QueryColumn(sourceColumnName(field)),
|
|
||||||
!"DESC".equalsIgnoreCase(sort.getDirection()));
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
return wrapper;
|
return wrapper;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void appendFilter(
|
|
||||||
StringBuilder condition,
|
|
||||||
List<Object> parameters,
|
|
||||||
DatacenterTableField field,
|
|
||||||
DatacenterQueryFilter filter) {
|
|
||||||
if (!condition.isEmpty()) {
|
|
||||||
condition.append(" AND ");
|
|
||||||
}
|
|
||||||
String column = quoteInternalIdentifier(sourceColumnName(field));
|
|
||||||
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ")
|
|
||||||
.toUpperCase(Locale.ROOT);
|
|
||||||
switch (operator) {
|
|
||||||
case "EQ" -> appendComparison(condition, parameters, column, "=", filter.getValue());
|
|
||||||
case "LIKE" -> appendComparison(
|
|
||||||
condition, parameters, column, "LIKE",
|
|
||||||
"%" + String.valueOf(filter.getValue()) + "%");
|
|
||||||
case "GT" -> appendComparison(condition, parameters, column, ">", filter.getValue());
|
|
||||||
case "GTE" -> appendComparison(condition, parameters, column, ">=", filter.getValue());
|
|
||||||
case "LT" -> appendComparison(condition, parameters, column, "<", filter.getValue());
|
|
||||||
case "LTE" -> appendComparison(condition, parameters, column, "<=", filter.getValue());
|
|
||||||
case "IS_NULL" -> condition.append(column).append(" IS NULL");
|
|
||||||
case "IN" -> appendInFilter(condition, parameters, column, filter.getValues());
|
|
||||||
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void appendComparison(
|
|
||||||
StringBuilder condition,
|
|
||||||
List<Object> parameters,
|
|
||||||
String column,
|
|
||||||
String operator,
|
|
||||||
Object value) {
|
|
||||||
condition.append(column).append(' ').append(operator).append(" ?");
|
|
||||||
parameters.add(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void appendInFilter(
|
|
||||||
StringBuilder condition,
|
|
||||||
List<Object> parameters,
|
|
||||||
String column,
|
|
||||||
List<Object> values) {
|
|
||||||
if (values == null || values.isEmpty()) {
|
|
||||||
condition.append("1 = 0");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
condition.append(column).append(" IN (")
|
|
||||||
.append(Collections.nCopies(values.size(), "?").stream()
|
|
||||||
.collect(java.util.stream.Collectors.joining(", ")))
|
|
||||||
.append(')');
|
|
||||||
parameters.addAll(values);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String sourceColumnName(DatacenterTableField field) {
|
|
||||||
return StrUtil.blankToDefault(field.getSourceColumnName(), field.getFieldName());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||||
List<Row> rows = Db.selectListBySql(sql);
|
List<Row> rows = Db.selectListBySql(sql);
|
||||||
|
|||||||
@@ -16,17 +16,12 @@ import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
|||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||||
import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
|
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
|
||||||
import java.math.BigDecimal;
|
import java.math.BigDecimal;
|
||||||
@@ -110,39 +105,30 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
return withConnection(source, true, connection -> {
|
return withConnection(source, true, connection -> {
|
||||||
List<DatacenterCatalogMeta> result = new ArrayList<>();
|
List<DatacenterCatalogMeta> result = new ArrayList<>();
|
||||||
DatabaseMetaData metaData = connection.getMetaData();
|
DatabaseMetaData metaData = connection.getMetaData();
|
||||||
if (usesCatalogNamespace()) {
|
try (ResultSet catalogs = metaData.getCatalogs()) {
|
||||||
try (ResultSet catalogs = metaData.getCatalogs()) {
|
while (catalogs.next()) {
|
||||||
while (catalogs.next()) {
|
String name = catalogs.getString("TABLE_CAT");
|
||||||
String name = catalogs.getString("TABLE_CAT");
|
if (StrUtil.isBlank(name)) {
|
||||||
if (StrUtil.isBlank(name)) {
|
continue;
|
||||||
continue;
|
|
||||||
}
|
|
||||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
|
||||||
meta.setSourceId(source.getId());
|
|
||||||
meta.setCatalogName(name);
|
|
||||||
meta.setCatalogType("CATALOG");
|
|
||||||
meta.setLogicalSchemaName(name);
|
|
||||||
meta.setPhysicalCatalogName(name);
|
|
||||||
result.add(meta);
|
|
||||||
}
|
}
|
||||||
|
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||||
|
meta.setSourceId(source.getId());
|
||||||
|
meta.setCatalogName(name);
|
||||||
|
meta.setCatalogType("CATALOG");
|
||||||
|
result.add(meta);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!usesCatalogNamespace()) {
|
try (ResultSet schemas = metaData.getSchemas()) {
|
||||||
try (ResultSet schemas = metaData.getSchemas()) {
|
while (schemas.next()) {
|
||||||
while (schemas.next()) {
|
String name = schemas.getString("TABLE_SCHEM");
|
||||||
String name = schemas.getString("TABLE_SCHEM");
|
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
|
||||||
if (StrUtil.isBlank(name)) {
|
continue;
|
||||||
continue;
|
|
||||||
}
|
|
||||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
|
||||||
meta.setSourceId(source.getId());
|
|
||||||
meta.setCatalogName(name);
|
|
||||||
meta.setCatalogType("SCHEMA");
|
|
||||||
meta.setLogicalSchemaName(name);
|
|
||||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
|
||||||
meta.setPhysicalSchemaName(name);
|
|
||||||
result.add(meta);
|
|
||||||
}
|
}
|
||||||
|
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||||
|
meta.setSourceId(source.getId());
|
||||||
|
meta.setCatalogName(name);
|
||||||
|
meta.setCatalogType("SCHEMA");
|
||||||
|
result.add(meta);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result = filterConfiguredCatalogs(source, result);
|
result = filterConfiguredCatalogs(source, result);
|
||||||
@@ -153,13 +139,6 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
meta.setSourceId(source.getId());
|
meta.setSourceId(source.getId());
|
||||||
meta.setCatalogName(fallback);
|
meta.setCatalogName(fallback);
|
||||||
meta.setCatalogType("DEFAULT");
|
meta.setCatalogType("DEFAULT");
|
||||||
meta.setLogicalSchemaName(fallback);
|
|
||||||
if (usesCatalogNamespace()) {
|
|
||||||
meta.setPhysicalCatalogName(fallback);
|
|
||||||
} else {
|
|
||||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
|
||||||
meta.setPhysicalSchemaName(fallback);
|
|
||||||
}
|
|
||||||
result.add(meta);
|
result.add(meta);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -170,98 +149,24 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 直接在 JDBC Catalog/Schema 元数据游标上执行有界分页。
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @param keyword 名称搜索词
|
|
||||||
* @param pageNumber 页码
|
|
||||||
* @param pageSize 每页大小
|
|
||||||
* @return 有界命名空间列表
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
|
|
||||||
DatacenterSource source,
|
|
||||||
String keyword,
|
|
||||||
long pageNumber,
|
|
||||||
long pageSize) {
|
|
||||||
try {
|
|
||||||
return withConnection(source, true, connection -> {
|
|
||||||
DatabaseMetaData metadata = connection.getMetaData();
|
|
||||||
String normalizedKeyword = keyword == null
|
|
||||||
? "" : keyword.trim().toLowerCase(Locale.ROOT);
|
|
||||||
String configuredName = usesCatalogNamespace()
|
|
||||||
? StrUtil.trimToNull(source.getDatabaseName())
|
|
||||||
: StrUtil.trimToNull(source.getSchemaName());
|
|
||||||
long offset = (pageNumber - 1L) * pageSize;
|
|
||||||
long matched = 0L;
|
|
||||||
boolean hasMore = false;
|
|
||||||
List<DatacenterCatalogMeta> records = new ArrayList<>((int) pageSize);
|
|
||||||
try (ResultSet names = usesCatalogNamespace()
|
|
||||||
? metadata.getCatalogs() : metadata.getSchemas()) {
|
|
||||||
String column = usesCatalogNamespace() ? "TABLE_CAT" : "TABLE_SCHEM";
|
|
||||||
while (names.next()) {
|
|
||||||
String name = names.getString(column);
|
|
||||||
if (StrUtil.isBlank(name)
|
|
||||||
|| (configuredName != null && !configuredName.equals(name))
|
|
||||||
|| (!normalizedKeyword.isEmpty()
|
|
||||||
&& !name.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (matched++ < offset) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (records.size() >= pageSize) {
|
|
||||||
hasMore = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
records.add(catalogMeta(source, name));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (records.isEmpty() && offset == 0L && normalizedKeyword.isEmpty()) {
|
|
||||||
String fallback = resolveCatalogName(source, null);
|
|
||||||
if (StrUtil.isNotBlank(fallback)) {
|
|
||||||
records.add(catalogMeta(source, fallback));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new DatacenterMetadataPage<>(
|
|
||||||
records, pageNumber, pageSize, hasMore);
|
|
||||||
});
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private DatacenterCatalogMeta catalogMeta(
|
|
||||||
DatacenterSource source,
|
|
||||||
String name) {
|
|
||||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
|
||||||
meta.setSourceId(source.getId());
|
|
||||||
meta.setCatalogName(name);
|
|
||||||
meta.setCatalogType(usesCatalogNamespace() ? "CATALOG" : "SCHEMA");
|
|
||||||
meta.setLogicalSchemaName(name);
|
|
||||||
if (usesCatalogNamespace()) {
|
|
||||||
meta.setPhysicalCatalogName(name);
|
|
||||||
} else {
|
|
||||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
|
||||||
meta.setPhysicalSchemaName(name);
|
|
||||||
}
|
|
||||||
return meta;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||||
try {
|
try {
|
||||||
return withConnection(source, true, connection -> {
|
return withConnection(source, true, connection -> {
|
||||||
DatabaseMetaData metaData = connection.getMetaData();
|
DatabaseMetaData metaData = connection.getMetaData();
|
||||||
List<DatacenterTable> tables = new ArrayList<>();
|
List<DatacenterTable> tables = new ArrayList<>();
|
||||||
try (ResultSet resultSet = metaData.getTables(
|
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
|
||||||
resolveCatalogArgument(source, catalogName),
|
|
||||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
|
||||||
"%",
|
|
||||||
new String[]{"TABLE", "VIEW"})) {
|
|
||||||
while (resultSet.next()) {
|
while (resultSet.next()) {
|
||||||
tables.add(tableFromMetadata(source, resultSet));
|
DatacenterTable table = new DatacenterTable();
|
||||||
|
table.setSourceId(source.getId());
|
||||||
|
table.setTableName(resultSet.getString("TABLE_NAME"));
|
||||||
|
table.setTableDesc(resultSet.getString("REMARKS"));
|
||||||
|
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
||||||
|
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
||||||
|
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||||
|
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
||||||
|
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||||
|
tables.add(table);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return tables;
|
return tables;
|
||||||
@@ -271,166 +176,10 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 直接在 JDBC 元数据游标上完成表名筛选和有界分页。
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @param catalogName 物理命名空间
|
|
||||||
* @param keyword 表名搜索词
|
|
||||||
* @param pageNumber 页码
|
|
||||||
* @param pageSize 每页大小
|
|
||||||
* @return 有界表列表
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public DatacenterMetadataPage<DatacenterTable> listTablesPage(
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName,
|
|
||||||
String keyword,
|
|
||||||
long pageNumber,
|
|
||||||
long pageSize) {
|
|
||||||
try {
|
|
||||||
return withConnection(source, true, connection -> {
|
|
||||||
DatabaseMetaData metaData = connection.getMetaData();
|
|
||||||
String normalizedKeyword = keyword == null
|
|
||||||
? "" : keyword.trim().toLowerCase(Locale.ROOT);
|
|
||||||
long offset = (pageNumber - 1L) * pageSize;
|
|
||||||
long matched = 0L;
|
|
||||||
List<DatacenterTable> tables = new ArrayList<>((int) pageSize);
|
|
||||||
boolean hasMore = false;
|
|
||||||
try (ResultSet resultSet = metaData.getTables(
|
|
||||||
resolveCatalogArgument(source, catalogName),
|
|
||||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
|
||||||
"%",
|
|
||||||
new String[]{"TABLE", "VIEW"})) {
|
|
||||||
while (resultSet.next()) {
|
|
||||||
String tableName = resultSet.getString("TABLE_NAME");
|
|
||||||
if (!normalizedKeyword.isEmpty()
|
|
||||||
&& (tableName == null
|
|
||||||
|| !tableName.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (matched++ < offset) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (tables.size() >= pageSize) {
|
|
||||||
hasMore = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
tables.add(tableFromMetadata(source, resultSet));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new DatacenterMetadataPage<>(
|
|
||||||
tables, pageNumber, pageSize, hasMore);
|
|
||||||
});
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private DatacenterTable tableFromMetadata(
|
|
||||||
DatacenterSource source,
|
|
||||||
ResultSet resultSet) throws SQLException {
|
|
||||||
DatacenterTable table = new DatacenterTable();
|
|
||||||
table.setSourceId(source.getId());
|
|
||||||
table.setTableName(resultSet.getString("TABLE_NAME"));
|
|
||||||
table.setTableDesc(resultSet.getString("REMARKS"));
|
|
||||||
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
|
||||||
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
|
||||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION)
|
|
||||||
? "READ_WRITE" : "READ_ONLY");
|
|
||||||
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
|
||||||
table.setQueryable(1);
|
|
||||||
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
|
||||||
table.setLastSeenAt(new java.util.Date());
|
|
||||||
table.setTimeSemantics("NONE");
|
|
||||||
table.setMetadataRevision(1L);
|
|
||||||
table.setCapabilitiesJson(Map.of(
|
|
||||||
"capabilities", capabilities.stream().map(Enum::name).toList()));
|
|
||||||
return table;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||||
try {
|
|
||||||
return withConnection(source, true,
|
|
||||||
connection -> readTableDetail(connection, source, catalogName, tableName));
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用单个元数据连接批量读取表详情,避免每张表重复创建连接池。
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @param catalogName 物理命名空间
|
|
||||||
* @param tableNames 表名集合
|
|
||||||
* @return 表详情集合
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public List<DatacenterTableDetailMeta> getTableDetails(
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName,
|
|
||||||
List<String> tableNames) {
|
|
||||||
if (tableNames == null || tableNames.isEmpty()) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
return withConnection(source, true, connection -> {
|
return withConnection(source, true, connection -> {
|
||||||
List<DatacenterTableDetailMeta> details = new ArrayList<>(tableNames.size());
|
|
||||||
for (String tableName : tableNames) {
|
|
||||||
details.add(readTableDetail(connection, source, catalogName, tableName));
|
|
||||||
}
|
|
||||||
return List.copyOf(details);
|
|
||||||
});
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("批量读取表详情失败", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用单个连接刷新已纳管表,并把物理缺失作为快照状态返回。
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @param catalogName 物理命名空间
|
|
||||||
* @param tableNames 已纳管表名集合
|
|
||||||
* @return 已发现详情与缺失表名
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public DatacenterManagedMetadataSnapshot inspectManagedTables(
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName,
|
|
||||||
List<String> tableNames) {
|
|
||||||
if (tableNames == null || tableNames.isEmpty()) {
|
|
||||||
return new DatacenterManagedMetadataSnapshot(List.of(), Set.of());
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return withConnection(source, true, connection -> {
|
|
||||||
List<DatacenterTableDetailMeta> details = new ArrayList<>(tableNames.size());
|
|
||||||
Set<String> missing = new java.util.LinkedHashSet<>();
|
|
||||||
for (String tableName : tableNames) {
|
|
||||||
try {
|
|
||||||
details.add(readTableDetail(connection, source, catalogName, tableName));
|
|
||||||
} catch (BusinessException exception) {
|
|
||||||
if (exception.getMessage() == null
|
|
||||||
|| !exception.getMessage().startsWith("所选数据表已变化")) {
|
|
||||||
throw exception;
|
|
||||||
}
|
|
||||||
missing.add(tableName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new DatacenterManagedMetadataSnapshot(details, missing);
|
|
||||||
});
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("刷新已纳管表失败", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private DatacenterTableDetailMeta readTableDetail(
|
|
||||||
Connection connection,
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName,
|
|
||||||
String tableName) throws SQLException {
|
|
||||||
DatabaseMetaData metaData = connection.getMetaData();
|
DatabaseMetaData metaData = connection.getMetaData();
|
||||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||||
DatacenterTable table = new DatacenterTable();
|
DatacenterTable table = new DatacenterTable();
|
||||||
@@ -440,35 +189,24 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
table.setMaterializedTable(tableName);
|
table.setMaterializedTable(tableName);
|
||||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||||
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
|
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
|
||||||
table.setQueryable(1);
|
|
||||||
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
|
||||||
table.setLastSeenAt(new java.util.Date());
|
|
||||||
table.setTimeSemantics("NONE");
|
|
||||||
table.setMetadataRevision(1L);
|
|
||||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||||
detail.setTable(table);
|
detail.setTable(table);
|
||||||
|
|
||||||
boolean tableFound = false;
|
|
||||||
try (ResultSet tableSet = metaData.getTables(
|
try (ResultSet tableSet = metaData.getTables(
|
||||||
resolveCatalogArgument(source, catalogName),
|
resolveCatalogArgument(source, catalogName),
|
||||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
resolveSchemaArgument(source, catalogName),
|
||||||
metadataPattern(metaData, tableName),
|
tableName,
|
||||||
new String[]{"TABLE", "VIEW"})) {
|
new String[]{"TABLE", "VIEW"})) {
|
||||||
while (tableSet.next()) {
|
while (tableSet.next()) {
|
||||||
if (!matchesMetadataTable(
|
String currentTableName = tableSet.getString("TABLE_NAME");
|
||||||
tableSet, source, catalogName, tableName)) {
|
if (!matchesTableName(currentTableName, tableName)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
table.setTableDesc(tableSet.getString("REMARKS"));
|
table.setTableDesc(tableSet.getString("REMARKS"));
|
||||||
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
|
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
|
||||||
tableFound = true;
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!tableFound) {
|
|
||||||
throw new BusinessException(
|
|
||||||
"所选数据表已变化,请刷新后重试: " + tableName);
|
|
||||||
}
|
|
||||||
|
|
||||||
Set<String> primaryKeys = new HashSet<>();
|
Set<String> primaryKeys = new HashSet<>();
|
||||||
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
|
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
|
||||||
@@ -478,24 +216,13 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
}
|
}
|
||||||
|
|
||||||
List<DatacenterTableField> fields = new ArrayList<>();
|
List<DatacenterTableField> fields = new ArrayList<>();
|
||||||
try (ResultSet columns = metaData.getColumns(
|
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
|
||||||
resolveCatalogArgument(source, catalogName),
|
|
||||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
|
||||||
metadataPattern(metaData, tableName),
|
|
||||||
"%")) {
|
|
||||||
while (columns.next()) {
|
while (columns.next()) {
|
||||||
if (!matchesMetadataTable(
|
|
||||||
columns, source, catalogName, tableName)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
DatacenterTableField field = new DatacenterTableField();
|
DatacenterTableField field = new DatacenterTableField();
|
||||||
field.setFieldName(columns.getString("COLUMN_NAME"));
|
field.setFieldName(columns.getString("COLUMN_NAME"));
|
||||||
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
|
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
|
||||||
field.setFieldDesc(columns.getString("REMARKS"));
|
field.setFieldDesc(columns.getString("REMARKS"));
|
||||||
field.setJdbcType(columns.getString("TYPE_NAME"));
|
field.setJdbcType(columns.getString("TYPE_NAME"));
|
||||||
field.setJdbcTypeCode(columns.getInt("DATA_TYPE"));
|
|
||||||
field.setNativeTypeName(columns.getString("TYPE_NAME"));
|
|
||||||
field.setOrdinalPosition(columns.getInt("ORDINAL_POSITION"));
|
|
||||||
field.setPrecision(columns.getInt("COLUMN_SIZE"));
|
field.setPrecision(columns.getInt("COLUMN_SIZE"));
|
||||||
field.setScale(columns.getInt("DECIMAL_DIGITS"));
|
field.setScale(columns.getInt("DECIMAL_DIGITS"));
|
||||||
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
|
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
|
||||||
@@ -504,21 +231,15 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
|
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
|
||||||
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
|
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
|
||||||
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
|
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
|
||||||
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
|
||||||
field.setLastSeenAt(new java.util.Date());
|
|
||||||
field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name());
|
|
||||||
field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field));
|
|
||||||
fields.add(field);
|
fields.add(field);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (fields.isEmpty()) {
|
|
||||||
throw new BusinessException(
|
|
||||||
"无法读取数据表字段,请检查权限: " + tableName);
|
|
||||||
}
|
|
||||||
table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
|
|
||||||
table.getTableName(), table.getTableKind(), fields));
|
|
||||||
detail.setFields(fields);
|
detail.setFields(fields);
|
||||||
return detail;
|
return detail;
|
||||||
|
});
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -526,11 +247,6 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||||
throw new BusinessException("当前数据源暂不支持查询");
|
throw new BusinessException("当前数据源暂不支持查询");
|
||||||
}
|
}
|
||||||
if (request == null || request.getPageSize() == null
|
|
||||||
|| request.getPageSize() < 1L
|
|
||||||
|| request.getPageSize() > 500L) {
|
|
||||||
throw new BusinessException("pageSize 必须在 1 到 500 之间");
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
|
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
@@ -609,7 +325,7 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
|
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
|
||||||
StringBuilder whereClause = new StringBuilder();
|
StringBuilder whereClause = new StringBuilder();
|
||||||
if (StrUtil.isNotBlank(request.getWhere())) {
|
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
whereClause.append(" WHERE ").append(request.getWhere());
|
||||||
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
|
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||||
whereClause.append(" WHERE 1=1 ");
|
whereClause.append(" WHERE 1=1 ");
|
||||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||||
@@ -906,9 +622,13 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
return items;
|
return items;
|
||||||
}
|
}
|
||||||
List<DatacenterCatalogMeta> matched = items.stream()
|
List<DatacenterCatalogMeta> matched = items.stream()
|
||||||
.filter(item -> configuredName.equals(item.getCatalogName()))
|
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
return matched;
|
return matched.isEmpty() ? items : matched;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
|
||||||
|
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private DatacenterTableKind resolveTableKind(String tableType) {
|
private DatacenterTableKind resolveTableKind(String tableType) {
|
||||||
@@ -919,58 +639,8 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
|||||||
if (currentTableName == null || targetTableName == null) {
|
if (currentTableName == null || targetTableName == null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return currentTableName.equals(targetTableName);
|
return currentTableName.equals(targetTableName)
|
||||||
}
|
|| currentTableName.equalsIgnoreCase(targetTableName);
|
||||||
|
|
||||||
/**
|
|
||||||
* 将精确标识符转换为 JDBC 元数据 Pattern,转义驱动声明的通配符字符。
|
|
||||||
*
|
|
||||||
* @param metaData JDBC 元数据
|
|
||||||
* @param value 精确 Schema 或表名
|
|
||||||
* @return 可安全用于元数据 Pattern 参数的文本
|
|
||||||
* @throws SQLException 驱动无法返回转义字符时抛出
|
|
||||||
*/
|
|
||||||
private String metadataPattern(DatabaseMetaData metaData, String value)
|
|
||||||
throws SQLException {
|
|
||||||
if (value == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
String escape = metaData.getSearchStringEscape();
|
|
||||||
if (escape == null || escape.isEmpty()) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
return value.replace(escape, escape + escape)
|
|
||||||
.replace("%", escape + "%")
|
|
||||||
.replace("_", escape + "_");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 对元数据结果再次执行精确 Catalog、Schema 与表名校验,防御驱动忽略 Pattern 转义。
|
|
||||||
*
|
|
||||||
* @param resultSet 元数据结果集
|
|
||||||
* @param source 数据源
|
|
||||||
* @param catalogName 用户选择的物理命名空间
|
|
||||||
* @param tableName 目标表名
|
|
||||||
* @return 当前结果行是否属于精确目标
|
|
||||||
* @throws SQLException 读取元数据列失败时抛出
|
|
||||||
*/
|
|
||||||
private boolean matchesMetadataTable(
|
|
||||||
ResultSet resultSet,
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName,
|
|
||||||
String tableName) throws SQLException {
|
|
||||||
if (!matchesTableName(resultSet.getString("TABLE_NAME"), tableName)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
String expectedCatalog = resolveCatalogArgument(source, catalogName);
|
|
||||||
String expectedSchema = resolveSchemaArgument(source, catalogName);
|
|
||||||
String actualCatalog = resultSet.getString("TABLE_CAT");
|
|
||||||
String actualSchema = resultSet.getString("TABLE_SCHEM");
|
|
||||||
// Schema 型数据库已由当前连接锁定 Database;部分 PostgreSQL 驱动不会回填 TABLE_CAT。
|
|
||||||
return (!usesCatalogNamespace()
|
|
||||||
|| expectedCatalog == null
|
|
||||||
|| expectedCatalog.equals(actualCatalog))
|
|
||||||
&& (expectedSchema == null || expectedSchema.equals(actualSchema));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Integer mapFieldType(int jdbcType) {
|
private Integer mapFieldType(int jdbcType) {
|
||||||
|
|||||||
@@ -65,38 +65,6 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
|||||||
@Column(comment = "物理表名")
|
@Column(comment = "物理表名")
|
||||||
private String actualTable;
|
private String actualTable;
|
||||||
|
|
||||||
/** 物理对象稳定摘要。 */
|
|
||||||
@Column(comment = "物理对象稳定摘要")
|
|
||||||
private String physicalIdentityKey;
|
|
||||||
|
|
||||||
/** 表结构摘要。 */
|
|
||||||
@Column(comment = "表结构摘要")
|
|
||||||
private String metadataFingerprint;
|
|
||||||
|
|
||||||
/** 元数据状态。 */
|
|
||||||
@Column(comment = "元数据状态")
|
|
||||||
private String metadataStatus;
|
|
||||||
|
|
||||||
/** 最近发现时间。 */
|
|
||||||
@Column(comment = "最近发现时间")
|
|
||||||
private Date lastSeenAt;
|
|
||||||
|
|
||||||
/** 是否允许统一只读查询。 */
|
|
||||||
@Column(comment = "是否允许统一只读查询")
|
|
||||||
private Integer queryable;
|
|
||||||
|
|
||||||
/** 时间语义。 */
|
|
||||||
@Column(comment = "时间语义")
|
|
||||||
private String timeSemantics;
|
|
||||||
|
|
||||||
/** 默认时间字段 ID。 */
|
|
||||||
@Column(comment = "默认时间字段ID")
|
|
||||||
private BigInteger defaultTimeFieldId;
|
|
||||||
|
|
||||||
/** 表元数据版本。 */
|
|
||||||
@Column(comment = "表元数据版本")
|
|
||||||
private Long metadataRevision;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 表类型
|
* 表类型
|
||||||
*/
|
*/
|
||||||
@@ -230,39 +198,6 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
|||||||
this.actualTable = actualTable;
|
this.actualTable = actualTable;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getPhysicalIdentityKey() { return physicalIdentityKey; }
|
|
||||||
|
|
||||||
public void setPhysicalIdentityKey(String physicalIdentityKey) { this.physicalIdentityKey = physicalIdentityKey; }
|
|
||||||
|
|
||||||
public String getMetadataFingerprint() { return metadataFingerprint; }
|
|
||||||
|
|
||||||
public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; }
|
|
||||||
|
|
||||||
public String getMetadataStatus() { return metadataStatus; }
|
|
||||||
|
|
||||||
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
|
|
||||||
|
|
||||||
public Date getLastSeenAt() { return lastSeenAt; }
|
|
||||||
|
|
||||||
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
|
|
||||||
|
|
||||||
public Integer getQueryable() { return queryable; }
|
|
||||||
|
|
||||||
public void setQueryable(Integer queryable) { this.queryable = queryable; }
|
|
||||||
|
|
||||||
public String getTimeSemantics() { return timeSemantics; }
|
|
||||||
|
|
||||||
public void setTimeSemantics(String timeSemantics) { this.timeSemantics = timeSemantics; }
|
|
||||||
|
|
||||||
@JsonSerialize(using = ToStringSerializer.class)
|
|
||||||
public BigInteger getDefaultTimeFieldId() { return defaultTimeFieldId; }
|
|
||||||
|
|
||||||
public void setDefaultTimeFieldId(BigInteger defaultTimeFieldId) { this.defaultTimeFieldId = defaultTimeFieldId; }
|
|
||||||
|
|
||||||
public Long getMetadataRevision() { return metadataRevision; }
|
|
||||||
|
|
||||||
public void setMetadataRevision(Long metadataRevision) { this.metadataRevision = metadataRevision; }
|
|
||||||
|
|
||||||
public String getTableKind() {
|
public String getTableKind() {
|
||||||
return tableKind;
|
return tableKind;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,10 +41,6 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
@Column(comment = "源字段名")
|
@Column(comment = "源字段名")
|
||||||
private String sourceColumnName;
|
private String sourceColumnName;
|
||||||
|
|
||||||
/** JDBC 字段顺序。 */
|
|
||||||
@Column(comment = "JDBC字段顺序")
|
|
||||||
private Integer ordinalPosition;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 字段描述
|
* 字段描述
|
||||||
*/
|
*/
|
||||||
@@ -63,14 +59,6 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
@Column(comment = "JDBC类型")
|
@Column(comment = "JDBC类型")
|
||||||
private String jdbcType;
|
private String jdbcType;
|
||||||
|
|
||||||
/** java.sql.Types 数值。 */
|
|
||||||
@Column(comment = "JDBC类型编码")
|
|
||||||
private Integer jdbcTypeCode;
|
|
||||||
|
|
||||||
/** 数据库原生类型名。 */
|
|
||||||
@Column(comment = "数据库原生类型名")
|
|
||||||
private String nativeTypeName;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 精度
|
* 精度
|
||||||
*/
|
*/
|
||||||
@@ -83,26 +71,6 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
@Column(comment = "小数位")
|
@Column(comment = "小数位")
|
||||||
private Integer scale;
|
private Integer scale;
|
||||||
|
|
||||||
/** 字段元数据摘要。 */
|
|
||||||
@Column(comment = "字段元数据摘要")
|
|
||||||
private String metadataFingerprint;
|
|
||||||
|
|
||||||
/** 元数据状态。 */
|
|
||||||
@Column(comment = "元数据状态")
|
|
||||||
private String metadataStatus;
|
|
||||||
|
|
||||||
/** 最近发现时间。 */
|
|
||||||
@Column(comment = "最近发现时间")
|
|
||||||
private Date lastSeenAt;
|
|
||||||
|
|
||||||
/** 敏感级别。 */
|
|
||||||
@Column(comment = "敏感级别")
|
|
||||||
private String sensitivityLevel;
|
|
||||||
|
|
||||||
/** 脱敏策略。 */
|
|
||||||
@Column(comment = "脱敏策略")
|
|
||||||
private String maskingStrategy;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 是否必填
|
* 是否必填
|
||||||
*/
|
*/
|
||||||
@@ -197,10 +165,6 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
this.sourceColumnName = sourceColumnName;
|
this.sourceColumnName = sourceColumnName;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Integer getOrdinalPosition() { return ordinalPosition; }
|
|
||||||
|
|
||||||
public void setOrdinalPosition(Integer ordinalPosition) { this.ordinalPosition = ordinalPosition; }
|
|
||||||
|
|
||||||
public String getFieldDesc() {
|
public String getFieldDesc() {
|
||||||
return fieldDesc;
|
return fieldDesc;
|
||||||
}
|
}
|
||||||
@@ -225,14 +189,6 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
this.jdbcType = jdbcType;
|
this.jdbcType = jdbcType;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Integer getJdbcTypeCode() { return jdbcTypeCode; }
|
|
||||||
|
|
||||||
public void setJdbcTypeCode(Integer jdbcTypeCode) { this.jdbcTypeCode = jdbcTypeCode; }
|
|
||||||
|
|
||||||
public String getNativeTypeName() { return nativeTypeName; }
|
|
||||||
|
|
||||||
public void setNativeTypeName(String nativeTypeName) { this.nativeTypeName = nativeTypeName; }
|
|
||||||
|
|
||||||
public Integer getPrecision() {
|
public Integer getPrecision() {
|
||||||
return precision;
|
return precision;
|
||||||
}
|
}
|
||||||
@@ -249,26 +205,6 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
this.scale = scale;
|
this.scale = scale;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getMetadataFingerprint() { return metadataFingerprint; }
|
|
||||||
|
|
||||||
public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; }
|
|
||||||
|
|
||||||
public String getMetadataStatus() { return metadataStatus; }
|
|
||||||
|
|
||||||
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
|
|
||||||
|
|
||||||
public Date getLastSeenAt() { return lastSeenAt; }
|
|
||||||
|
|
||||||
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
|
|
||||||
|
|
||||||
public String getSensitivityLevel() { return sensitivityLevel; }
|
|
||||||
|
|
||||||
public void setSensitivityLevel(String sensitivityLevel) { this.sensitivityLevel = sensitivityLevel; }
|
|
||||||
|
|
||||||
public String getMaskingStrategy() { return maskingStrategy; }
|
|
||||||
|
|
||||||
public void setMaskingStrategy(String maskingStrategy) { this.maskingStrategy = maskingStrategy; }
|
|
||||||
|
|
||||||
public Integer getRequired() {
|
public Integer getRequired() {
|
||||||
return required;
|
return required;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -346,7 +346,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
|||||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||||
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||||
final int[] rowIndex = {1};
|
final int[] rowIndex = {1};
|
||||||
totalRows += iterateRows(queryRequest, account, row -> {
|
totalRows += iterateRows(queryRequest, row -> {
|
||||||
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
|
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
|
||||||
for (int i = 0; i < table.getFields().size(); i++) {
|
for (int i = 0; i < table.getFields().size(); i++) {
|
||||||
Cell cell = excelRow.createCell(i);
|
Cell cell = excelRow.createCell(i);
|
||||||
@@ -434,7 +434,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
|||||||
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
|
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||||
List<BigInteger> derivedIds = new ArrayList<>();
|
List<BigInteger> derivedIds = new ArrayList<>();
|
||||||
final Holder holder = new Holder();
|
final Holder holder = new Holder();
|
||||||
long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> {
|
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||||
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
|
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
|
||||||
holder.batchNo++;
|
holder.batchNo++;
|
||||||
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||||
@@ -466,7 +466,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
|||||||
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
|
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||||
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
|
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
|
||||||
List<BigInteger> derivedIds = new ArrayList<>();
|
List<BigInteger> derivedIds = new ArrayList<>();
|
||||||
long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> {
|
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||||
String fieldValue = stringify(row.get(splitField.getFieldName()));
|
String fieldValue = stringify(row.get(splitField.getFieldName()));
|
||||||
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
|
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
|
||||||
DatacenterTable targetTable = targets.get(bucket);
|
DatacenterTable targetTable = targets.get(bucket);
|
||||||
@@ -543,7 +543,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
|||||||
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
|
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
|
||||||
for (DatacenterTable table : tables) {
|
for (DatacenterTable table : tables) {
|
||||||
Map<String, String> mapping = fieldMappings.get(table.getId());
|
Map<String, String> mapping = fieldMappings.get(table.getId());
|
||||||
iterateRows(buildFullQuery(table), account, row -> {
|
iterateRows(buildFullQuery(table), row -> {
|
||||||
String joinValue = stringify(row.get(request.getJoinKey()));
|
String joinValue = stringify(row.get(request.getJoinKey()));
|
||||||
if (joinValue == null || joinValue.isBlank()) {
|
if (joinValue == null || joinValue.isBlank()) {
|
||||||
return;
|
return;
|
||||||
@@ -729,22 +729,16 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
|||||||
}
|
}
|
||||||
|
|
||||||
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
|
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
|
||||||
return iterateRows(
|
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
|
||||||
queryRequest,
|
|
||||||
account,
|
|
||||||
row -> saveToTable(targetTable, mapper.map(row), account));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private long iterateRows(
|
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
|
||||||
DatacenterQueryRequest queryRequest,
|
|
||||||
LoginAccount account,
|
|
||||||
RowConsumer consumer) {
|
|
||||||
long total = 0L;
|
long total = 0L;
|
||||||
long pageNumber = 1L;
|
long pageNumber = 1L;
|
||||||
while (true) {
|
while (true) {
|
||||||
queryRequest.setPageNumber(pageNumber);
|
queryRequest.setPageNumber(pageNumber);
|
||||||
queryRequest.setPageSize(QUERY_BATCH_SIZE);
|
queryRequest.setPageSize(QUERY_BATCH_SIZE);
|
||||||
Page<Row> page = queryService.queryPage(queryRequest, account);
|
Page<Row> page = queryService.queryPage(queryRequest);
|
||||||
if (page.getRecords() == null || page.getRecords().isEmpty()) {
|
if (page.getRecords() == null || page.getRecords().isEmpty()) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,46 +5,31 @@ import tech.easyflow.datacenter.entity.DatacenterTableField;
|
|||||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public class DatacenterSchemaResponse {
|
public class DatacenterSchemaResponse {
|
||||||
private DatasetRef datasetRef;
|
private DatasetRef datasetRef;
|
||||||
private DatacenterSourceView source;
|
private DatacenterSource source;
|
||||||
private DatacenterCatalog catalog;
|
private DatacenterCatalog catalog;
|
||||||
private DatacenterTable table;
|
private DatacenterTable table;
|
||||||
private List<DatacenterTableField> fields = new ArrayList<>();
|
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||||
private long fieldPageNumber = 1L;
|
|
||||||
private long fieldPageSize;
|
|
||||||
private boolean hasMoreFields;
|
|
||||||
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
|
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
|
||||||
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
|
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
|
||||||
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
|
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
|
||||||
|
|
||||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||||
public DatacenterSourceView getSource() { return source; }
|
public DatacenterSource getSource() { return source; }
|
||||||
public void setSource(DatacenterSourceView source) { this.source = source; }
|
public void setSource(DatacenterSource source) { this.source = source; }
|
||||||
public DatacenterCatalog getCatalog() { return catalog; }
|
public DatacenterCatalog getCatalog() { return catalog; }
|
||||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||||
public DatacenterTable getTable() { return table; }
|
public DatacenterTable getTable() { return table; }
|
||||||
public void setTable(DatacenterTable table) { this.table = table; }
|
public void setTable(DatacenterTable table) { this.table = table; }
|
||||||
public List<DatacenterTableField> getFields() { return fields; }
|
public List<DatacenterTableField> getFields() { return fields; }
|
||||||
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||||
/** @return 当前字段页码 */
|
|
||||||
public long getFieldPageNumber() { return fieldPageNumber; }
|
|
||||||
/** @param fieldPageNumber 当前字段页码 */
|
|
||||||
public void setFieldPageNumber(long fieldPageNumber) { this.fieldPageNumber = fieldPageNumber; }
|
|
||||||
/** @return 当前字段页大小 */
|
|
||||||
public long getFieldPageSize() { return fieldPageSize; }
|
|
||||||
/** @param fieldPageSize 当前字段页大小 */
|
|
||||||
public void setFieldPageSize(long fieldPageSize) { this.fieldPageSize = fieldPageSize; }
|
|
||||||
/** @return 是否还有下一页字段 */
|
|
||||||
public boolean isHasMoreFields() { return hasMoreFields; }
|
|
||||||
/** @param hasMoreFields 是否还有下一页字段 */
|
|
||||||
public void setHasMoreFields(boolean hasMoreFields) { this.hasMoreFields = hasMoreFields; }
|
|
||||||
public List<DatacenterDatasetVersion> getVersions() { return versions; }
|
public List<DatacenterDatasetVersion> getVersions() { return versions; }
|
||||||
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
|
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
|
||||||
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
|
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.execution.model;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 管理端取消只读 SQL 查询请求。
|
|
||||||
*
|
|
||||||
* @param queryId 客户端在执行前生成的查询 UUID
|
|
||||||
*/
|
|
||||||
public record DatacenterSqlCancelRequest(String queryId) {
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.execution.model;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* SQL 控制台结果列。
|
|
||||||
*
|
|
||||||
* @param key 前端稳定列键
|
|
||||||
* @param label JDBC 列标签
|
|
||||||
* @param jdbcType JDBC 类型编码
|
|
||||||
* @param typeName 数据库类型名
|
|
||||||
* @param nullable 是否允许空值
|
|
||||||
*/
|
|
||||||
public record DatacenterSqlColumnView(
|
|
||||||
String key,
|
|
||||||
String label,
|
|
||||||
int jdbcType,
|
|
||||||
String typeName,
|
|
||||||
boolean nullable) {
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.execution.model;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 管理端只读 SQL 控制台请求。
|
|
||||||
*
|
|
||||||
* @param sourceId 已激活数据源 ID
|
|
||||||
* @param queryId 客户端在执行前生成的查询 UUID
|
|
||||||
* @param sql 单条只读 SQL
|
|
||||||
* @param maxRows 最大返回行数
|
|
||||||
*/
|
|
||||||
public record DatacenterSqlConsoleRequest(
|
|
||||||
BigInteger sourceId,
|
|
||||||
String queryId,
|
|
||||||
String sql,
|
|
||||||
Integer maxRows) {
|
|
||||||
}
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.execution.model;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 有界 SQL 控制台结果。
|
|
||||||
*
|
|
||||||
* @param queryId 查询标识
|
|
||||||
* @param columns 结果列
|
|
||||||
* @param rows 结果行
|
|
||||||
* @param returnedRows 返回行数
|
|
||||||
* @param truncated 是否因上限截断
|
|
||||||
* @param durationMs 总耗时毫秒
|
|
||||||
*/
|
|
||||||
public record DatacenterSqlConsoleResult(
|
|
||||||
String queryId,
|
|
||||||
List<DatacenterSqlColumnView> columns,
|
|
||||||
List<Map<String, Object>> rows,
|
|
||||||
long returnedRows,
|
|
||||||
boolean truncated,
|
|
||||||
long durationMs) {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 防御性复制结果集合。
|
|
||||||
*/
|
|
||||||
public DatacenterSqlConsoleResult {
|
|
||||||
columns = List.copyOf(columns == null ? List.of() : columns);
|
|
||||||
rows = List.copyOf(rows == null ? List.of() : rows);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,7 +5,6 @@ import java.math.BigInteger;
|
|||||||
public class DatasetRef implements java.io.Serializable {
|
public class DatasetRef implements java.io.Serializable {
|
||||||
private static final long serialVersionUID = 1L;
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
private BigInteger tenantId;
|
|
||||||
private BigInteger sourceId;
|
private BigInteger sourceId;
|
||||||
private BigInteger catalogId;
|
private BigInteger catalogId;
|
||||||
private String catalogName;
|
private String catalogName;
|
||||||
@@ -13,8 +12,6 @@ public class DatasetRef implements java.io.Serializable {
|
|||||||
private String tableName;
|
private String tableName;
|
||||||
private BigInteger versionId;
|
private BigInteger versionId;
|
||||||
|
|
||||||
public BigInteger getTenantId() { return tenantId; }
|
|
||||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
|
||||||
public BigInteger getSourceId() { return sourceId; }
|
public BigInteger getSourceId() { return sourceId; }
|
||||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||||
public BigInteger getCatalogId() { return catalogId; }
|
public BigInteger getCatalogId() { return catalogId; }
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package tech.easyflow.datacenter.execution.service;
|
|||||||
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.row.Row;
|
import com.mybatisflex.core.row.Row;
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||||
@@ -21,19 +20,6 @@ public interface DatacenterDatasetQueryService {
|
|||||||
*/
|
*/
|
||||||
Page<Row> queryPage(DatacenterQueryRequest request);
|
Page<Row> queryPage(DatacenterQueryRequest request);
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用明确执行账号分页查询结构化数据集。
|
|
||||||
*
|
|
||||||
* @param request 查询请求
|
|
||||||
* @param account 执行账号
|
|
||||||
* @return 分页结果
|
|
||||||
*/
|
|
||||||
default Page<Row> queryPage(
|
|
||||||
DatacenterQueryRequest request,
|
|
||||||
LoginAccount account) {
|
|
||||||
return queryPage(request);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 执行原生 SQL 并返回完整结果。
|
* 执行原生 SQL 并返回完整结果。
|
||||||
*
|
*
|
||||||
@@ -49,43 +35,9 @@ public interface DatacenterDatasetQueryService {
|
|||||||
* @param fetchSize JDBC 建议拉取行数
|
* @param fetchSize JDBC 建议拉取行数
|
||||||
* @param consumer 单行消费者
|
* @param consumer 单行消费者
|
||||||
*/
|
*/
|
||||||
default void consumeBySql(
|
|
||||||
DatacenterSqlQueryRequest request,
|
|
||||||
int fetchSize,
|
|
||||||
Consumer<Row> consumer) {
|
|
||||||
consumeBySql(request, fetchSize, null, consumer);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用明确执行账号流式消费 SQL,后台工作流据此绑定权威租户。
|
|
||||||
*
|
|
||||||
* @param request SQL 查询请求
|
|
||||||
* @param fetchSize JDBC 建议拉取行数
|
|
||||||
* @param account 执行账号
|
|
||||||
* @param consumer 单行消费者
|
|
||||||
*/
|
|
||||||
default void consumeBySql(
|
|
||||||
DatacenterSqlQueryRequest request,
|
|
||||||
int fetchSize,
|
|
||||||
LoginAccount account,
|
|
||||||
Consumer<Row> consumer) {
|
|
||||||
consumeBySql(request, fetchSize, account, null, consumer);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用明确执行账号和调用方 QueryId 流式消费 SQL。
|
|
||||||
*
|
|
||||||
* @param request SQL 查询请求
|
|
||||||
* @param fetchSize JDBC 建议拉取行数
|
|
||||||
* @param account 执行账号
|
|
||||||
* @param requestedQueryId 调用方预生成的查询 UUID;为空时服务端生成
|
|
||||||
* @param consumer 单行消费者
|
|
||||||
*/
|
|
||||||
void consumeBySql(
|
void consumeBySql(
|
||||||
DatacenterSqlQueryRequest request,
|
DatacenterSqlQueryRequest request,
|
||||||
int fetchSize,
|
int fetchSize,
|
||||||
LoginAccount account,
|
|
||||||
String requestedQueryId,
|
|
||||||
Consumer<Row> consumer);
|
Consumer<Row> consumer);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -96,19 +48,6 @@ public interface DatacenterDatasetQueryService {
|
|||||||
*/
|
*/
|
||||||
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
|
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取有界字段页的数据集结构。
|
|
||||||
*
|
|
||||||
* @param datasetRef 数据集引用
|
|
||||||
* @param fieldPageNumber 字段页码
|
|
||||||
* @param fieldPageSize 字段页大小
|
|
||||||
* @return 数据集结构和当前字段页
|
|
||||||
*/
|
|
||||||
DatacenterSchemaResponse getSchema(
|
|
||||||
DatasetRef datasetRef,
|
|
||||||
Long fieldPageNumber,
|
|
||||||
Long fieldPageSize);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 仅解析数据集定位信息,不加载版本和血缘。
|
* 仅解析数据集定位信息,不加载版本和血缘。
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -1,43 +1,22 @@
|
|||||||
package tech.easyflow.datacenter.execution.service.impl;
|
package tech.easyflow.datacenter.execution.service.impl;
|
||||||
|
|
||||||
import com.easyagents.federation.sql.execute.SqlParameter;
|
|
||||||
import com.easyagents.federation.sql.execute.QueryId;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
import com.mybatisflex.core.paginate.Page;
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
import com.mybatisflex.core.row.Row;
|
import com.mybatisflex.core.row.Row;
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.sql.Types;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.LinkedHashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.function.Consumer;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||||
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||||
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
|
|
||||||
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||||
@@ -46,14 +25,19 @@ import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
|||||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceViews;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
|
||||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||||
import tech.easyflow.datacenter.utils.SqlSupportUtils;
|
import tech.easyflow.datacenter.utils.SqlSupportUtils;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
|
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
|
||||||
@@ -70,34 +54,18 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||||
@Resource
|
@Resource
|
||||||
private DatacenterDerivedTableMapper derivedTableMapper;
|
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||||
@Resource
|
|
||||||
private DatacenterFederationQueryService federationQueryService;
|
|
||||||
@Resource
|
|
||||||
private DatacenterQueryAuditService queryAuditService;
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Page<Row> queryPage(DatacenterQueryRequest request) {
|
public Page<Row> queryPage(DatacenterQueryRequest request) {
|
||||||
throw new BusinessException("结构化查询必须提供执行账号");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* {@inheritDoc}
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public Page<Row> queryPage(
|
|
||||||
DatacenterQueryRequest request,
|
|
||||||
LoginAccount account) {
|
|
||||||
if (request == null || request.getDatasetRef() == null) {
|
if (request == null || request.getDatasetRef() == null) {
|
||||||
throw new BusinessException("datasetRef 不能为空");
|
throw new BusinessException("datasetRef 不能为空");
|
||||||
}
|
}
|
||||||
normalizePage(request);
|
normalizePage(request);
|
||||||
DatacenterTable table = resolveTable(request.getDatasetRef());
|
DatacenterTable table = resolveTable(request.getDatasetRef());
|
||||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
validateStructuredQueryTenant(
|
|
||||||
request.getDatasetRef(), table, source, account);
|
|
||||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||||
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
|
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
|
||||||
validateRequest(queryTable, request);
|
validateRequest(queryTable, request, source);
|
||||||
request.getDatasetRef().setSourceId(table.getSourceId());
|
request.getDatasetRef().setSourceId(table.getSourceId());
|
||||||
request.getDatasetRef().setCatalogId(table.getCatalogId());
|
request.getDatasetRef().setCatalogId(table.getCatalogId());
|
||||||
request.getDatasetRef().setTableId(table.getId());
|
request.getDatasetRef().setTableId(table.getId());
|
||||||
@@ -105,199 +73,15 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
if (catalog != null) {
|
if (catalog != null) {
|
||||||
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
|
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
|
||||||
}
|
}
|
||||||
if (isFederated(source)) {
|
|
||||||
return queryFederatedPage(
|
|
||||||
source, catalog, queryTable, request, account);
|
|
||||||
}
|
|
||||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
QueryId queryId = QueryId.create();
|
return connector.queryPage(source, queryTable, request);
|
||||||
long startedNanos = System.nanoTime();
|
|
||||||
DatacenterQueryAudit audit = queryAuditService.start(
|
|
||||||
queryId.value(), source, structuredQueryDescription(queryTable, request),
|
|
||||||
structuredParameterSummary(request), account,
|
|
||||||
"DATASET_PAGE", table.getId().toString());
|
|
||||||
try {
|
|
||||||
Page<Row> result = connector.queryPage(source, queryTable, request);
|
|
||||||
long durationMs = elapsedMillis(startedNanos);
|
|
||||||
queryAuditService.succeed(
|
|
||||||
audit,
|
|
||||||
List.of(table.getTableName()),
|
|
||||||
structuredReferencedFields(request),
|
|
||||||
result.getRecords().size(),
|
|
||||||
false,
|
|
||||||
durationMs,
|
|
||||||
durationMs);
|
|
||||||
return result;
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
queryAuditService.fail(
|
|
||||||
audit,
|
|
||||||
"DATASET_PAGE_QUERY_FAILED",
|
|
||||||
safeMessage(exception),
|
|
||||||
List.of(table.getTableName()),
|
|
||||||
structuredReferencedFields(request),
|
|
||||||
elapsedMillis(startedNanos));
|
|
||||||
throw exception;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用执行账号绑定结构化查询的权威租户。
|
|
||||||
*
|
|
||||||
* @param datasetRef 数据集引用
|
|
||||||
* @param table 已解析表
|
|
||||||
* @param source 已解析数据源
|
|
||||||
* @param account 执行账号
|
|
||||||
* @throws BusinessException 账号或租户边界不合法时抛出
|
|
||||||
*/
|
|
||||||
private void validateStructuredQueryTenant(
|
|
||||||
DatasetRef datasetRef,
|
|
||||||
DatacenterTable table,
|
|
||||||
DatacenterSource source,
|
|
||||||
LoginAccount account) {
|
|
||||||
if (account == null || account.getTenantId() == null) {
|
|
||||||
throw new BusinessException("结构化查询缺少执行账号租户");
|
|
||||||
}
|
|
||||||
BigInteger tenantId = account.getTenantId();
|
|
||||||
if ((datasetRef.getTenantId() != null
|
|
||||||
&& !tenantId.equals(datasetRef.getTenantId()))
|
|
||||||
|| !tenantId.equals(table.getTenantId())
|
|
||||||
|| !tenantId.equals(source.getTenantId())) {
|
|
||||||
throw new BusinessException("数据集不属于当前租户");
|
|
||||||
}
|
|
||||||
datasetRef.setTenantId(tenantId);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String structuredQueryDescription(
|
|
||||||
DatacenterTable table,
|
|
||||||
DatacenterQueryRequest request) {
|
|
||||||
StringBuilder sql = new StringBuilder("SELECT ")
|
|
||||||
.append(String.join(", ", request.getSelectedColumns()))
|
|
||||||
.append(" FROM ").append(table.getTableName());
|
|
||||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
|
||||||
sql.append(" WHERE ");
|
|
||||||
for (int index = 0; index < request.getFilters().size(); index++) {
|
|
||||||
DatacenterQueryFilter filter = request.getFilters().get(index);
|
|
||||||
if (index > 0) {
|
|
||||||
sql.append(" AND ");
|
|
||||||
}
|
|
||||||
String operator = normalizeFilterOperator(filter.getOperator());
|
|
||||||
sql.append(filter.getColumn()).append(' ');
|
|
||||||
switch (operator) {
|
|
||||||
case "IS_NULL" -> sql.append("IS NULL");
|
|
||||||
case "IN" -> sql.append("IN (?)");
|
|
||||||
default -> sql.append(operator).append(" ?");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
|
||||||
sql.append(" ORDER BY ");
|
|
||||||
for (int index = 0; index < request.getSorts().size(); index++) {
|
|
||||||
DatacenterQuerySort sort = request.getSorts().get(index);
|
|
||||||
if (index > 0) {
|
|
||||||
sql.append(", ");
|
|
||||||
}
|
|
||||||
sql.append(sort.getColumn()).append(' ')
|
|
||||||
.append("DESC".equalsIgnoreCase(sort.getDirection())
|
|
||||||
? "DESC" : "ASC");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return sql.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 构建不包含实际值的结构化查询参数摘要。
|
|
||||||
*
|
|
||||||
* @param request 查询请求
|
|
||||||
* @return 脱敏参数摘要
|
|
||||||
*/
|
|
||||||
private Map<String, Object> structuredParameterSummary(
|
|
||||||
DatacenterQueryRequest request) {
|
|
||||||
List<Map<String, Object>> filters = new ArrayList<>();
|
|
||||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
|
||||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
|
||||||
String operator = normalizeFilterOperator(filter.getOperator());
|
|
||||||
int valueCount = "IN".equals(operator)
|
|
||||||
? (filter.getValues() == null ? 0 : filter.getValues().size())
|
|
||||||
: "IS_NULL".equals(operator) ? 0 : 1;
|
|
||||||
filters.add(Map.of(
|
|
||||||
"column", filter.getColumn(),
|
|
||||||
"operator", operator,
|
|
||||||
"valueCount", valueCount));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Map.of(
|
|
||||||
"filters", List.copyOf(filters),
|
|
||||||
"pageNumber", request.getPageNumber(),
|
|
||||||
"pageSize", request.getPageSize());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 收集结构化查询实际引用的投影、筛选和排序字段。
|
|
||||||
*
|
|
||||||
* @param request 查询请求
|
|
||||||
* @return 去重后的字段列表
|
|
||||||
*/
|
|
||||||
private List<String> structuredReferencedFields(
|
|
||||||
DatacenterQueryRequest request) {
|
|
||||||
Set<String> fields = new LinkedHashSet<>(request.getSelectedColumns());
|
|
||||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
|
||||||
request.getFilters().stream()
|
|
||||||
.map(DatacenterQueryFilter::getColumn)
|
|
||||||
.forEach(fields::add);
|
|
||||||
}
|
|
||||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
|
||||||
request.getSorts().stream()
|
|
||||||
.map(DatacenterQuerySort::getColumn)
|
|
||||||
.forEach(fields::add);
|
|
||||||
}
|
|
||||||
return List.copyOf(fields);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 将结构化筛选操作符归一化为审计中的稳定名称。
|
|
||||||
*
|
|
||||||
* @param operator 原始操作符
|
|
||||||
* @return 大写稳定操作符
|
|
||||||
*/
|
|
||||||
private String normalizeFilterOperator(String operator) {
|
|
||||||
return trimToNull(operator) == null
|
|
||||||
? "EQ" : operator.trim().toUpperCase(java.util.Locale.ROOT);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
|
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
|
||||||
ResolvedSqlQuery query = resolveSqlQuery(request);
|
ResolvedSqlQuery query = resolveSqlQuery(request);
|
||||||
if (isFederated(query.source)) {
|
return query.connector.queryBySql(
|
||||||
DatacenterSqlConsoleResult result = federationQueryService.execute(
|
query.source, query.sql);
|
||||||
query.source,
|
|
||||||
query.sql,
|
|
||||||
List.of(),
|
|
||||||
1_000,
|
|
||||||
null,
|
|
||||||
"DATASET",
|
|
||||||
request.getDatasetRef().getTableId() == null
|
|
||||||
? null : request.getDatasetRef().getTableId().toString());
|
|
||||||
if (result.truncated()) {
|
|
||||||
throw new BusinessException("数据集查询结果超过返回上限,请使用流式消费");
|
|
||||||
}
|
|
||||||
return toRows(result);
|
|
||||||
}
|
|
||||||
QueryId queryId = QueryId.create();
|
|
||||||
long startedNanos = System.nanoTime();
|
|
||||||
DatacenterQueryAudit audit = queryAuditService.start(
|
|
||||||
queryId.value(), query.source, query.sql, Map.of(), null,
|
|
||||||
"DATASET", request.getDatasetRef().getTableId().toString());
|
|
||||||
try {
|
|
||||||
List<Row> rows = query.connector.queryBySql(query.source, query.sql);
|
|
||||||
long durationMs = elapsedMillis(startedNanos);
|
|
||||||
queryAuditService.succeed(audit, query.referencedTables, List.of(),
|
|
||||||
rows.size(), false, durationMs, durationMs);
|
|
||||||
return rows;
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception),
|
|
||||||
query.referencedTables, List.of(), elapsedMillis(startedNanos));
|
|
||||||
throw exception;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -307,38 +91,12 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
public void consumeBySql(
|
public void consumeBySql(
|
||||||
DatacenterSqlQueryRequest request,
|
DatacenterSqlQueryRequest request,
|
||||||
int fetchSize,
|
int fetchSize,
|
||||||
LoginAccount account,
|
|
||||||
String requestedQueryId,
|
|
||||||
Consumer<Row> consumer) {
|
Consumer<Row> consumer) {
|
||||||
if (fetchSize <= 0 || consumer == null) {
|
if (fetchSize <= 0 || consumer == null) {
|
||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException(
|
||||||
"fetchSize and consumer must be valid");
|
"fetchSize and consumer must be valid");
|
||||||
}
|
}
|
||||||
ResolvedSqlQuery query = resolveSqlQuery(request, account);
|
ResolvedSqlQuery query = resolveSqlQuery(request);
|
||||||
if (isFederated(query.source)) {
|
|
||||||
int maxRows = Integer.getInteger(
|
|
||||||
"easyflow.datacenter.query.max-rows", 1_000_000);
|
|
||||||
long maxBytes = Long.getLong(
|
|
||||||
"easyflow.datacenter.query.max-bytes", 512L * 1024L * 1024L);
|
|
||||||
federationQueryService.consume(
|
|
||||||
query.source,
|
|
||||||
query.sql,
|
|
||||||
List.of(),
|
|
||||||
fetchSize,
|
|
||||||
maxRows,
|
|
||||||
maxBytes,
|
|
||||||
account,
|
|
||||||
"DATASET",
|
|
||||||
request.getDatasetRef().getTableId() == null
|
|
||||||
? null : request.getDatasetRef().getTableId().toString(),
|
|
||||||
requestedQueryId,
|
|
||||||
sourceRow -> {
|
|
||||||
Row row = new Row();
|
|
||||||
sourceRow.forEach(row::put);
|
|
||||||
consumer.accept(row);
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
int maxRows = Integer.getInteger(
|
int maxRows = Integer.getInteger(
|
||||||
"easyflow.datacenter.query.max-rows",
|
"easyflow.datacenter.query.max-rows",
|
||||||
1_000_000);
|
1_000_000);
|
||||||
@@ -347,48 +105,34 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
512L * 1024L * 1024L);
|
512L * 1024L * 1024L);
|
||||||
long[] accumulatedRows = {0L};
|
long[] accumulatedRows = {0L};
|
||||||
long[] accumulatedBytes = {0L};
|
long[] accumulatedBytes = {0L};
|
||||||
QueryId queryId = resolveQueryId(requestedQueryId);
|
query.connector.consumeBySql(
|
||||||
long startedNanos = System.nanoTime();
|
query.source,
|
||||||
DatacenterQueryAudit audit = queryAuditService.start(
|
query.sql,
|
||||||
queryId.value(), query.source, query.sql, Map.of(), account,
|
fetchSize,
|
||||||
"DATASET", request.getDatasetRef().getTableId().toString());
|
row -> {
|
||||||
try {
|
accumulatedRows[0]++;
|
||||||
query.connector.consumeBySql(
|
if (maxRows > 0
|
||||||
query.source,
|
&& accumulatedRows[0] > maxRows) {
|
||||||
query.sql,
|
throw new BusinessException(
|
||||||
fetchSize,
|
"数据集查询结果超过行数上限: "
|
||||||
row -> {
|
+ maxRows);
|
||||||
accumulatedRows[0]++;
|
|
||||||
if (maxRows > 0
|
|
||||||
&& accumulatedRows[0] > maxRows) {
|
|
||||||
throw new BusinessException(
|
|
||||||
"数据集查询结果超过行数上限: "
|
|
||||||
+ maxRows);
|
|
||||||
}
|
|
||||||
for (Map.Entry<String, Object> entry
|
|
||||||
: row.entrySet()) {
|
|
||||||
accumulatedBytes[0] +=
|
|
||||||
estimateQueryValueBytes(
|
|
||||||
entry.getKey(),
|
|
||||||
entry.getValue());
|
|
||||||
if (maxBytes > 0L
|
|
||||||
&& accumulatedBytes[0] > maxBytes) {
|
|
||||||
throw new BusinessException(
|
|
||||||
"数据集查询结果超过字节上限: "
|
|
||||||
+ maxBytes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
consumer.accept(row);
|
|
||||||
}
|
}
|
||||||
);
|
for (Map.Entry<String, Object> entry
|
||||||
long durationMs = elapsedMillis(startedNanos);
|
: row.entrySet()) {
|
||||||
queryAuditService.succeed(audit, query.referencedTables, List.of(),
|
accumulatedBytes[0] +=
|
||||||
accumulatedRows[0], false, durationMs, durationMs);
|
estimateQueryValueBytes(
|
||||||
} catch (RuntimeException exception) {
|
entry.getKey(),
|
||||||
queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception),
|
entry.getValue());
|
||||||
query.referencedTables, List.of(), elapsedMillis(startedNanos));
|
if (maxBytes > 0L
|
||||||
throw exception;
|
&& accumulatedBytes[0] > maxBytes) {
|
||||||
}
|
throw new BusinessException(
|
||||||
|
"数据集查询结果超过字节上限: "
|
||||||
|
+ maxBytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
consumer.accept(row);
|
||||||
|
}
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -427,19 +171,6 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
*/
|
*/
|
||||||
private ResolvedSqlQuery resolveSqlQuery(
|
private ResolvedSqlQuery resolveSqlQuery(
|
||||||
DatacenterSqlQueryRequest request) {
|
DatacenterSqlQueryRequest request) {
|
||||||
return resolveSqlQuery(request, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 校验请求、执行账号租户并解析实际连接器与可执行 SQL。
|
|
||||||
*
|
|
||||||
* @param request SQL 查询请求
|
|
||||||
* @param account 后台执行账号;普通同步调用可为空
|
|
||||||
* @return 已解析查询
|
|
||||||
*/
|
|
||||||
private ResolvedSqlQuery resolveSqlQuery(
|
|
||||||
DatacenterSqlQueryRequest request,
|
|
||||||
LoginAccount account) {
|
|
||||||
if (request == null || request.getDatasetRef() == null) {
|
if (request == null || request.getDatasetRef() == null) {
|
||||||
throw new BusinessException("datasetRef 不能为空");
|
throw new BusinessException("datasetRef 不能为空");
|
||||||
}
|
}
|
||||||
@@ -452,28 +183,8 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
throw new BusinessException("缺少连接服务配置");
|
throw new BusinessException("缺少连接服务配置");
|
||||||
}
|
}
|
||||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||||
BigInteger authoritativeTenantId = account != null && account.getTenantId() != null
|
|
||||||
? account.getTenantId() : datasetRef.getTenantId();
|
|
||||||
if (authoritativeTenantId == null) {
|
|
||||||
throw new BusinessException("数据集绑定缺少租户信息");
|
|
||||||
}
|
|
||||||
if ((datasetRef.getTenantId() != null
|
|
||||||
&& !datasetRef.getTenantId().equals(authoritativeTenantId))
|
|
||||||
|| !authoritativeTenantId.equals(source.getTenantId())) {
|
|
||||||
throw new BusinessException("数据集不属于当前租户");
|
|
||||||
}
|
|
||||||
datasetRef.setTenantId(authoritativeTenantId);
|
|
||||||
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
|
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
|
||||||
if (datasetRef.getTableId() == null) {
|
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
|
||||||
throw new BusinessException("数据集 SQL 查询必须绑定具体表");
|
|
||||||
}
|
|
||||||
DatacenterTable boundTable = registryService.getTableWithFields(datasetRef.getTableId());
|
|
||||||
if (!datasetRef.getSourceId().equals(boundTable.getSourceId())
|
|
||||||
|| (catalogId != null && !catalogId.equals(boundTable.getCatalogId()))
|
|
||||||
|| !authoritativeTenantId.equals(boundTable.getTenantId())) {
|
|
||||||
throw new BusinessException("数据集引用与绑定表不一致");
|
|
||||||
}
|
|
||||||
List<DatacenterTable> managedTables = List.of(boundTable);
|
|
||||||
if (CollectionUtils.isEmpty(managedTables)) {
|
if (CollectionUtils.isEmpty(managedTables)) {
|
||||||
throw new BusinessException("当前连接下没有已接入表");
|
throw new BusinessException("当前连接下没有已接入表");
|
||||||
}
|
}
|
||||||
@@ -487,15 +198,12 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
// 内部连接的 catalog 是逻辑命名空间,底层项目 MySQL 只执行物理表名。
|
// 内部连接的 catalog 是逻辑命名空间,底层项目 MySQL 只执行物理表名。
|
||||||
SqlSupportUtils.ResolvedSql resolvedSql =
|
SqlSupportUtils.ResolvedSql resolvedSql =
|
||||||
connector instanceof AbstractInternalTableConnector
|
connector instanceof AbstractInternalTableConnector
|
||||||
|| DatacenterSourceType.PROJECT_MYSQL.name()
|
|
||||||
.equals(source.getSourceType())
|
|
||||||
? SqlSupportUtils.resolveInternalMysql(sql, sqlTables)
|
? SqlSupportUtils.resolveInternalMysql(sql, sqlTables)
|
||||||
: SqlSupportUtils.resolve(sql, sqlTables);
|
: SqlSupportUtils.resolve(sql, sqlTables);
|
||||||
return new ResolvedSqlQuery(
|
return new ResolvedSqlQuery(
|
||||||
source,
|
source,
|
||||||
connector,
|
connector,
|
||||||
isFederated(source) ? sql : resolvedSql.getExecutableSql(),
|
resolvedSql.getExecutableSql());
|
||||||
resolvedSql.getLogicalTables());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -504,123 +212,30 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
* @param source 数据源
|
* @param source 数据源
|
||||||
* @param connector 数据连接器
|
* @param connector 数据连接器
|
||||||
* @param sql 可执行 SQL
|
* @param sql 可执行 SQL
|
||||||
* @param referencedTables 已校验引用表
|
|
||||||
*/
|
*/
|
||||||
private record ResolvedSqlQuery(
|
private record ResolvedSqlQuery(
|
||||||
DatacenterSource source,
|
DatacenterSource source,
|
||||||
DatacenterConnector connector,
|
DatacenterConnector connector,
|
||||||
String sql,
|
String sql) {
|
||||||
List<String> referencedTables) {
|
|
||||||
}
|
|
||||||
|
|
||||||
private long elapsedMillis(long startedNanos) {
|
|
||||||
return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 校验调用方查询 UUID,内部连接也使用同一标识完成审计关联。
|
|
||||||
*
|
|
||||||
* @param requestedQueryId 调用方标识;为空时生成
|
|
||||||
* @return 查询标识
|
|
||||||
*/
|
|
||||||
private QueryId resolveQueryId(String requestedQueryId) {
|
|
||||||
if (!StringUtils.hasText(requestedQueryId)) {
|
|
||||||
return QueryId.create();
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return new QueryId(UUID.fromString(
|
|
||||||
requestedQueryId.trim()).toString());
|
|
||||||
} catch (IllegalArgumentException exception) {
|
|
||||||
throw new BusinessException("queryId 必须是 UUID");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String safeMessage(RuntimeException exception) {
|
|
||||||
String message = exception.getMessage();
|
|
||||||
if (message == null || message.isBlank()) {
|
|
||||||
return "数据集查询失败";
|
|
||||||
}
|
|
||||||
return message.length() <= 500 ? message : message.substring(0, 500);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
|
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
|
||||||
return getSchema(datasetRef, null, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* {@inheritDoc}
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public DatacenterSchemaResponse getSchema(
|
|
||||||
DatasetRef datasetRef,
|
|
||||||
Long fieldPageNumber,
|
|
||||||
Long fieldPageSize) {
|
|
||||||
DatacenterTable table = resolveTable(datasetRef);
|
DatacenterTable table = resolveTable(datasetRef);
|
||||||
List<DatacenterTableField> allFields = List.copyOf(table.getFields());
|
|
||||||
long actualPage = fieldPageNumber == null ? 1L : Math.max(1L, fieldPageNumber);
|
|
||||||
long actualSize = fieldPageSize == null
|
|
||||||
? Math.max(1L, allFields.size())
|
|
||||||
: Math.min(200L, Math.max(1L, fieldPageSize));
|
|
||||||
List<DatacenterTableField> pageFields = sliceFields(
|
|
||||||
allFields, actualPage, actualSize);
|
|
||||||
long offset = safePageOffset(actualPage, actualSize);
|
|
||||||
boolean hasMoreFields = offset < allFields.size()
|
|
||||||
&& offset + pageFields.size() < allFields.size();
|
|
||||||
// 字段只通过顶层分页字段返回,避免在 table 内重复序列化完整列表。
|
|
||||||
table.setFields(List.of());
|
|
||||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||||
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
|
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
|
||||||
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||||
response.setSource(DatacenterSourceViews.from(source));
|
response.setSource(source);
|
||||||
response.setCatalog(catalog);
|
response.setCatalog(catalog);
|
||||||
response.setTable(table);
|
response.setTable(table);
|
||||||
response.setFields(pageFields);
|
response.setFields(table.getFields());
|
||||||
response.setFieldPageNumber(actualPage);
|
|
||||||
response.setFieldPageSize(actualSize);
|
|
||||||
response.setHasMoreFields(hasMoreFields);
|
|
||||||
response.setVersions(listVersions(table.getId()));
|
response.setVersions(listVersions(table.getId()));
|
||||||
response.setUpstreamLineage(listUpstream(table.getId()));
|
response.setUpstreamLineage(listUpstream(table.getId()));
|
||||||
response.setDownstreamLineage(listDownstream(table.getId()));
|
response.setDownstreamLineage(listDownstream(table.getId()));
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 截取字段页并避免向响应暴露可变子列表。
|
|
||||||
*
|
|
||||||
* @param fields 完整字段列表
|
|
||||||
* @param pageNumber 页码
|
|
||||||
* @param pageSize 页大小
|
|
||||||
* @return 当前页字段
|
|
||||||
*/
|
|
||||||
private List<DatacenterTableField> sliceFields(
|
|
||||||
List<DatacenterTableField> fields,
|
|
||||||
long pageNumber,
|
|
||||||
long pageSize) {
|
|
||||||
long offset = safePageOffset(pageNumber, pageSize);
|
|
||||||
if (offset >= fields.size()) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
int fromIndex = Math.toIntExact(offset);
|
|
||||||
int toIndex = (int) Math.min((long) fields.size(), offset + pageSize);
|
|
||||||
return List.copyOf(fields.subList(fromIndex, toIndex));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 计算字段页偏移并防止长整型乘法溢出。
|
|
||||||
*
|
|
||||||
* @param pageNumber 页码
|
|
||||||
* @param pageSize 页大小
|
|
||||||
* @return 安全偏移;溢出时返回最大长整型
|
|
||||||
*/
|
|
||||||
private long safePageOffset(long pageNumber, long pageSize) {
|
|
||||||
if (pageNumber - 1L > Long.MAX_VALUE / pageSize) {
|
|
||||||
return Long.MAX_VALUE;
|
|
||||||
}
|
|
||||||
return (pageNumber - 1L) * pageSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*/
|
*/
|
||||||
@@ -630,8 +245,8 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
DatacenterSchemaResponse response =
|
DatacenterSchemaResponse response =
|
||||||
new DatacenterSchemaResponse();
|
new DatacenterSchemaResponse();
|
||||||
response.setDatasetRef(datasetRef);
|
response.setDatasetRef(datasetRef);
|
||||||
response.setSource(DatacenterSourceViews.from(
|
response.setSource(
|
||||||
registryService.getSourceRequired(table.getSourceId())));
|
registryService.getSourceRequired(table.getSourceId()));
|
||||||
response.setCatalog(
|
response.setCatalog(
|
||||||
registryService.getCatalogById(table.getCatalogId()));
|
registryService.getCatalogById(table.getCatalogId()));
|
||||||
response.setTable(table);
|
response.setTable(table);
|
||||||
@@ -704,29 +319,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
DatacenterCatalog catalog = catalogId == null
|
DatacenterCatalog catalog = catalogId == null
|
||||||
? null
|
? null
|
||||||
: catalogsById.get(catalogId);
|
: catalogsById.get(catalogId);
|
||||||
List<String> knownColumns = table.getFields().stream()
|
|
||||||
.filter(field -> DatacenterMetadataStatus.ACTIVE.name()
|
|
||||||
.equals(field.getMetadataStatus()))
|
|
||||||
.map(this::sourceColumnName)
|
|
||||||
.toList();
|
|
||||||
List<String> queryableColumns = table.getFields().stream()
|
|
||||||
.filter(this::isQueryable)
|
|
||||||
.map(this::sourceColumnName)
|
|
||||||
.toList();
|
|
||||||
return new SqlSupportUtils.ManagedTable(
|
return new SqlSupportUtils.ManagedTable(
|
||||||
catalog == null ? null : catalog.getCatalogName(),
|
catalog == null ? null : catalog.getCatalogName(),
|
||||||
table.getTableName(),
|
table.getTableName(),
|
||||||
resolvePhysicalTableName(table),
|
resolvePhysicalTableName(table)
|
||||||
knownColumns,
|
|
||||||
queryableColumns
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String sourceColumnName(DatacenterTableField field) {
|
|
||||||
String sourceColumnName = trimToNull(field.getSourceColumnName());
|
|
||||||
return sourceColumnName == null ? field.getFieldName() : sourceColumnName;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 一次批量加载 SQL 白名单表关联的目录,避免逐表查询。
|
* 一次批量加载 SQL 白名单表关联的目录,避免逐表查询。
|
||||||
*
|
*
|
||||||
@@ -768,153 +367,6 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
return actualTable != null ? actualTable : table.getTableName();
|
return actualTable != null ? actualTable : table.getTableName();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用逻辑 Schema 构建参数化 ANSI SQL,再由 Calcite 转换为目标方言。
|
|
||||||
*/
|
|
||||||
private Page<Row> queryFederatedPage(
|
|
||||||
DatacenterSource source,
|
|
||||||
DatacenterCatalog catalog,
|
|
||||||
DatacenterTable table,
|
|
||||||
DatacenterQueryRequest request,
|
|
||||||
LoginAccount account) {
|
|
||||||
if (catalog == null || catalog.getLogicalSchemaName() == null) {
|
|
||||||
throw new BusinessException("数据集缺少 Federation Schema 绑定");
|
|
||||||
}
|
|
||||||
if (request.getPageSize() > 500L) {
|
|
||||||
throw new BusinessException("外部数据源单页最多返回 500 行");
|
|
||||||
}
|
|
||||||
Map<String, DatacenterTableField> fields = table.getFields().stream()
|
|
||||||
.collect(Collectors.toMap(
|
|
||||||
DatacenterTableField::getFieldName,
|
|
||||||
Function.identity(),
|
|
||||||
(first, ignored) -> first,
|
|
||||||
LinkedHashMap::new));
|
|
||||||
FederatedWhere where = buildFederatedWhere(request.getFilters(), fields);
|
|
||||||
String from = quote(catalog.getLogicalSchemaName()) + "." + quote(table.getTableName());
|
|
||||||
String countSql = "SELECT COUNT(*) AS total FROM " + from + where.sql();
|
|
||||||
DatacenterSqlConsoleResult countResult = federationQueryService.execute(
|
|
||||||
source, countSql, where.parameters(), 1, account,
|
|
||||||
"DATASET", table.getId().toString());
|
|
||||||
long total = countResult.rows().isEmpty()
|
|
||||||
? 0L : Long.parseLong(String.valueOf(countResult.rows().get(0).get("c1")));
|
|
||||||
|
|
||||||
if (total == 0L) {
|
|
||||||
return new Page<>(List.of(), request.getPageNumber(), request.getPageSize(), 0L);
|
|
||||||
}
|
|
||||||
String columns = request.getSelectedColumns().stream()
|
|
||||||
.map(name -> quote(requireField(fields, name).getSourceColumnName()))
|
|
||||||
.collect(Collectors.joining(", "));
|
|
||||||
String orderBy = buildFederatedOrderBy(request.getSorts(), fields);
|
|
||||||
long offset = (request.getPageNumber() - 1L) * request.getPageSize();
|
|
||||||
String pageSql = "SELECT " + columns + " FROM " + from + where.sql() + orderBy
|
|
||||||
+ " OFFSET " + offset + " ROWS FETCH NEXT " + request.getPageSize() + " ROWS ONLY";
|
|
||||||
DatacenterSqlConsoleResult pageResult = federationQueryService.execute(
|
|
||||||
source, pageSql, where.parameters(), request.getPageSize().intValue(),
|
|
||||||
account, "DATASET", table.getId().toString());
|
|
||||||
return new Page<>(toRows(pageResult), request.getPageNumber(), request.getPageSize(), total);
|
|
||||||
}
|
|
||||||
|
|
||||||
private FederatedWhere buildFederatedWhere(
|
|
||||||
List<DatacenterQueryFilter> filters,
|
|
||||||
Map<String, DatacenterTableField> fields) {
|
|
||||||
if (CollectionUtils.isEmpty(filters)) {
|
|
||||||
return new FederatedWhere("", List.of());
|
|
||||||
}
|
|
||||||
StringBuilder sql = new StringBuilder(" WHERE 1 = 1");
|
|
||||||
List<SqlParameter> parameters = new ArrayList<>();
|
|
||||||
for (DatacenterQueryFilter filter : filters) {
|
|
||||||
DatacenterTableField field = requireField(fields, filter.getColumn());
|
|
||||||
String column = quote(field.getSourceColumnName());
|
|
||||||
String operator = trimToNull(filter.getOperator()) == null
|
|
||||||
? "EQ" : filter.getOperator().trim().toUpperCase(java.util.Locale.ROOT);
|
|
||||||
switch (operator) {
|
|
||||||
case "EQ", "GT", "GTE", "LT", "LTE", "LIKE" -> {
|
|
||||||
String sqlOperator = switch (operator) {
|
|
||||||
case "GT" -> ">";
|
|
||||||
case "GTE" -> ">=";
|
|
||||||
case "LT" -> "<";
|
|
||||||
case "LTE" -> "<=";
|
|
||||||
case "LIKE" -> "LIKE";
|
|
||||||
default -> "=";
|
|
||||||
};
|
|
||||||
sql.append(" AND ").append(column).append(' ').append(sqlOperator).append(" ?");
|
|
||||||
Object value = "LIKE".equals(operator)
|
|
||||||
? "%" + filter.getValue() + "%" : filter.getValue();
|
|
||||||
parameters.add(parameter(field, value));
|
|
||||||
}
|
|
||||||
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
|
|
||||||
case "IN" -> {
|
|
||||||
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
|
|
||||||
if (values.isEmpty()) {
|
|
||||||
sql.append(" AND 1 = 0");
|
|
||||||
} else {
|
|
||||||
sql.append(" AND ").append(column).append(" IN (")
|
|
||||||
.append(values.stream().map(ignored -> "?").collect(Collectors.joining(",")))
|
|
||||||
.append(')');
|
|
||||||
values.forEach(value -> parameters.add(parameter(field, value)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new FederatedWhere(sql.toString(), List.copyOf(parameters));
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildFederatedOrderBy(
|
|
||||||
List<DatacenterQuerySort> sorts,
|
|
||||||
Map<String, DatacenterTableField> fields) {
|
|
||||||
if (CollectionUtils.isEmpty(sorts)) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
return " ORDER BY " + sorts.stream()
|
|
||||||
.map(sort -> quote(requireField(fields, sort.getColumn()).getSourceColumnName())
|
|
||||||
+ ("DESC".equalsIgnoreCase(sort.getDirection()) ? " DESC" : " ASC"))
|
|
||||||
.collect(Collectors.joining(", "));
|
|
||||||
}
|
|
||||||
|
|
||||||
private DatacenterTableField requireField(
|
|
||||||
Map<String, DatacenterTableField> fields,
|
|
||||||
String name) {
|
|
||||||
DatacenterTableField field = fields.get(name);
|
|
||||||
if (field == null || !isEnabled(field.getQueryable())) {
|
|
||||||
throw new BusinessException("字段不可查询: " + name);
|
|
||||||
}
|
|
||||||
return field;
|
|
||||||
}
|
|
||||||
|
|
||||||
private SqlParameter parameter(DatacenterTableField field, Object value) {
|
|
||||||
int jdbcType = field.getJdbcTypeCode() == null ? Types.VARCHAR : field.getJdbcTypeCode();
|
|
||||||
return value == null ? new SqlParameter(jdbcType, null) : new SqlParameter(jdbcType, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String quote(String identifier) {
|
|
||||||
if (!StringUtils.hasText(identifier)) {
|
|
||||||
throw new BusinessException("数据集物理名称为空");
|
|
||||||
}
|
|
||||||
return '"' + identifier.replace("\"", "\"\"") + '"';
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<Row> toRows(DatacenterSqlConsoleResult result) {
|
|
||||||
List<Row> rows = new ArrayList<>(result.rows().size());
|
|
||||||
for (Map<String, Object> sourceRow : result.rows()) {
|
|
||||||
Row row = new Row();
|
|
||||||
for (int index = 0; index < result.columns().size(); index++) {
|
|
||||||
var column = result.columns().get(index);
|
|
||||||
row.put(column.label(), sourceRow.get(column.key()));
|
|
||||||
}
|
|
||||||
rows.add(row);
|
|
||||||
}
|
|
||||||
return rows;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isFederated(DatacenterSource source) {
|
|
||||||
return source != null && (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|
|
||||||
|| DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private record FederatedWhere(String sql, List<SqlParameter> parameters) {
|
|
||||||
}
|
|
||||||
|
|
||||||
private void normalizePage(DatacenterQueryRequest request) {
|
private void normalizePage(DatacenterQueryRequest request) {
|
||||||
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
|
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
|
||||||
request.setPageNumber(1L);
|
request.setPageNumber(1L);
|
||||||
@@ -922,9 +374,6 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
if (request.getPageSize() == null || request.getPageSize() < 1L) {
|
if (request.getPageSize() == null || request.getPageSize() < 1L) {
|
||||||
throw new BusinessException("pageSize 必须大于 0");
|
throw new BusinessException("pageSize 必须大于 0");
|
||||||
}
|
}
|
||||||
if (request.getPageSize() > 500L) {
|
|
||||||
throw new BusinessException("单页最多返回 500 行");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
|
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
|
||||||
@@ -951,7 +400,7 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
return queryTable;
|
return queryTable;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request) {
|
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
|
||||||
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||||
for (DatacenterTableField field : table.getFields()) {
|
for (DatacenterTableField field : table.getFields()) {
|
||||||
fieldMap.put(field.getFieldName(), field);
|
fieldMap.put(field.getFieldName(), field);
|
||||||
@@ -959,25 +408,22 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
||||||
for (String column : request.getSelectedColumns()) {
|
for (String column : request.getSelectedColumns()) {
|
||||||
DatacenterTableField field = fieldMap.get(column);
|
DatacenterTableField field = fieldMap.get(column);
|
||||||
if (field == null || !isQueryable(field)) {
|
if (field == null || !isEnabled(field.getQueryable())) {
|
||||||
throw new BusinessException("字段不可查询: " + column);
|
throw new BusinessException("字段不可查询: " + column);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
request.setSelectedColumns(
|
request.setSelectedColumns(
|
||||||
table.getFields().stream()
|
table.getFields().stream()
|
||||||
.filter(this::isQueryable)
|
.filter(field -> isEnabled(field.getQueryable()))
|
||||||
.map(DatacenterTableField::getFieldName)
|
.map(DatacenterTableField::getFieldName)
|
||||||
.toList()
|
.toList()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
|
||||||
throw new BusinessException("当前数据集没有可查询字段");
|
|
||||||
}
|
|
||||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||||
request.getFilters().forEach(filter -> {
|
request.getFilters().forEach(filter -> {
|
||||||
DatacenterTableField field = fieldMap.get(filter.getColumn());
|
DatacenterTableField field = fieldMap.get(filter.getColumn());
|
||||||
if (field == null || !isQueryable(field)) {
|
if (field == null || !isEnabled(field.getQueryable())) {
|
||||||
throw new BusinessException("字段不可过滤: " + filter.getColumn());
|
throw new BusinessException("字段不可过滤: " + filter.getColumn());
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -985,13 +431,18 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||||
request.getSorts().forEach(sort -> {
|
request.getSorts().forEach(sort -> {
|
||||||
DatacenterTableField field = fieldMap.get(sort.getColumn());
|
DatacenterTableField field = fieldMap.get(sort.getColumn());
|
||||||
if (field == null || !isQueryable(field) || !isEnabled(field.getSortable())) {
|
if (field == null || !isEnabled(field.getSortable())) {
|
||||||
throw new BusinessException("字段不可排序: " + sort.getColumn());
|
throw new BusinessException("字段不可排序: " + sort.getColumn());
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (request.getWhere() != null && !request.getWhere().isBlank()) {
|
if (request.getWhere() != null && !request.getWhere().isBlank()) {
|
||||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|
||||||
|
|| "MYSQL".equals(source.getSourceType())
|
||||||
|
|| "POSTGRESQL".equals(source.getSourceType());
|
||||||
|
if (!allowLegacyWhere) {
|
||||||
|
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -999,13 +450,6 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
|||||||
return value == null || value == 1;
|
return value == null || value == 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isQueryable(DatacenterTableField field) {
|
|
||||||
return field != null
|
|
||||||
&& isEnabled(field.getQueryable())
|
|
||||||
&& DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus())
|
|
||||||
&& DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel());
|
|
||||||
}
|
|
||||||
|
|
||||||
private String trimToNull(String value) {
|
private String trimToNull(String value) {
|
||||||
if (!StringUtils.hasText(value)) {
|
if (!StringUtils.hasText(value)) {
|
||||||
return null;
|
return null;
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
|||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
|
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
|
||||||
DatacenterTable table = resolveTable(datasetRef, account);
|
DatacenterTable table = resolveTable(datasetRef);
|
||||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
connector.saveRow(source, table, data, account);
|
connector.saveRow(source, table, data, account);
|
||||||
@@ -52,7 +52,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
|||||||
if (rows == null || rows.isEmpty()) {
|
if (rows == null || rows.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
DatacenterTable table = resolveTable(datasetRef, account);
|
DatacenterTable table = resolveTable(datasetRef);
|
||||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
connector.saveRows(source, table, rows, account, Math.max(1, batchSize));
|
connector.saveRows(source, table, rows, account, Math.max(1, batchSize));
|
||||||
@@ -70,7 +70,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
|||||||
if (rows == null || rows.isEmpty()) {
|
if (rows == null || rows.isEmpty()) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
DatacenterTable table = resolveTable(datasetRef, account);
|
DatacenterTable table = resolveTable(datasetRef);
|
||||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
String payloadHash = sha256Rows(rows);
|
String payloadHash = sha256Rows(rows);
|
||||||
@@ -95,27 +95,17 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
|||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
|
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
|
||||||
DatacenterTable table = resolveTable(datasetRef, account);
|
DatacenterTable table = resolveTable(datasetRef);
|
||||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
connector.deleteRow(source, table, id, account);
|
connector.deleteRow(source, table, id, account);
|
||||||
}
|
}
|
||||||
|
|
||||||
private DatacenterTable resolveTable(DatasetRef datasetRef, LoginAccount account) {
|
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||||
throw new BusinessException("缺少 tableId");
|
throw new BusinessException("缺少 tableId");
|
||||||
}
|
}
|
||||||
if (account == null || account.getTenantId() == null) {
|
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||||
throw new BusinessException("数据集写入缺少执行租户");
|
|
||||||
}
|
|
||||||
DatacenterTable table = registryService.getTableWithFields(datasetRef.getTableId());
|
|
||||||
if (!account.getTenantId().equals(table.getTenantId())
|
|
||||||
|| (datasetRef.getTenantId() != null
|
|
||||||
&& !account.getTenantId().equals(datasetRef.getTenantId()))) {
|
|
||||||
throw new BusinessException("数据集不属于当前租户");
|
|
||||||
}
|
|
||||||
datasetRef.setTenantId(account.getTenantId());
|
|
||||||
return table;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,308 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
|
|
||||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
|
||||||
import com.easyagents.federation.sql.execute.FederationQueryAdmissionController;
|
|
||||||
import com.easyagents.federation.sql.execute.FederationQueryPermit;
|
|
||||||
import com.easyagents.federation.sql.execute.LocalFederationQueryAdmissionController;
|
|
||||||
import com.easyagents.federation.sql.execute.QueryId;
|
|
||||||
import com.easyagents.federation.sql.source.SourceId;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
|
||||||
import java.util.function.BooleanSupplier;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 对查询执行租户、数据源与节点三级本地并发准入。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationAdmissionController
|
|
||||||
implements FederationQueryAdmissionController {
|
|
||||||
|
|
||||||
private final LocalFederationQueryAdmissionController globalAdmission;
|
|
||||||
private final int maximumConcurrentQueriesPerTenant;
|
|
||||||
private final int maximumConcurrentQueriesPerSource;
|
|
||||||
private final Map<String, AdmissionEntry>
|
|
||||||
tenantAdmissions = new ConcurrentHashMap<>();
|
|
||||||
private final Map<SourceId, AdmissionEntry>
|
|
||||||
sourceAdmissions = new ConcurrentHashMap<>();
|
|
||||||
private final Object lifecycleLock = new Object();
|
|
||||||
private final AtomicBoolean closed = new AtomicBoolean();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建三级准入控制器。
|
|
||||||
*
|
|
||||||
* @param maximumConcurrentQueries 节点查询上限
|
|
||||||
* @param maximumConcurrentQueriesPerTenant 单租户查询上限
|
|
||||||
* @param maximumConcurrentQueriesPerSource 单数据源查询上限
|
|
||||||
*/
|
|
||||||
public DatacenterFederationAdmissionController(
|
|
||||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries:64}")
|
|
||||||
int maximumConcurrentQueries,
|
|
||||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-tenant:20}")
|
|
||||||
int maximumConcurrentQueriesPerTenant,
|
|
||||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-source:10}")
|
|
||||||
int maximumConcurrentQueriesPerSource) {
|
|
||||||
if (maximumConcurrentQueries <= 0
|
|
||||||
|| maximumConcurrentQueriesPerTenant <= 0
|
|
||||||
|| maximumConcurrentQueriesPerSource <= 0) {
|
|
||||||
throw new IllegalArgumentException(
|
|
||||||
"query admission limits must be positive");
|
|
||||||
}
|
|
||||||
this.globalAdmission = new LocalFederationQueryAdmissionController(
|
|
||||||
maximumConcurrentQueries);
|
|
||||||
this.maximumConcurrentQueriesPerTenant =
|
|
||||||
maximumConcurrentQueriesPerTenant;
|
|
||||||
this.maximumConcurrentQueriesPerSource =
|
|
||||||
maximumConcurrentQueriesPerSource;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取三级查询许可。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源标识
|
|
||||||
* @param queryId 查询标识
|
|
||||||
* @param timeout 最大等待时间
|
|
||||||
* @return 查询许可
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public FederationQueryPermit acquire(
|
|
||||||
SourceId sourceId,
|
|
||||||
QueryId queryId,
|
|
||||||
Duration timeout) {
|
|
||||||
return acquire(sourceId, queryId, timeout, () -> false);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 按数据源、租户、节点顺序获取许可,避免热点连接池等待占满全局配额。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源标识
|
|
||||||
* @param queryId 查询标识
|
|
||||||
* @param timeout 最大等待时间
|
|
||||||
* @param cancellationRequested 查询取消状态
|
|
||||||
* @return 查询许可
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public FederationQueryPermit acquire(
|
|
||||||
SourceId sourceId,
|
|
||||||
QueryId queryId,
|
|
||||||
Duration timeout,
|
|
||||||
BooleanSupplier cancellationRequested) {
|
|
||||||
long started = System.nanoTime();
|
|
||||||
String tenantKey = tenantKey(sourceId);
|
|
||||||
AdmissionEntry sourceEntry;
|
|
||||||
AdmissionEntry tenantEntry;
|
|
||||||
synchronized (lifecycleLock) {
|
|
||||||
ensureOpen();
|
|
||||||
sourceEntry = retain(sourceAdmissions, sourceId,
|
|
||||||
maximumConcurrentQueriesPerSource);
|
|
||||||
tenantEntry = retain(tenantAdmissions, tenantKey,
|
|
||||||
maximumConcurrentQueriesPerTenant);
|
|
||||||
}
|
|
||||||
FederationQueryPermit sourcePermit = null;
|
|
||||||
FederationQueryPermit tenantPermit = null;
|
|
||||||
FederationQueryPermit globalPermit = null;
|
|
||||||
try {
|
|
||||||
sourcePermit = sourceEntry.controller.acquire(
|
|
||||||
sourceId, queryId, remaining(timeout, started),
|
|
||||||
cancellationRequested);
|
|
||||||
tenantPermit = tenantEntry.controller.acquire(
|
|
||||||
sourceId, queryId, remaining(timeout, started),
|
|
||||||
cancellationRequested);
|
|
||||||
globalPermit = globalAdmission.acquire(
|
|
||||||
sourceId, queryId, remaining(timeout, started),
|
|
||||||
cancellationRequested);
|
|
||||||
FederationQueryPermit acquiredSource = sourcePermit;
|
|
||||||
FederationQueryPermit acquiredTenant = tenantPermit;
|
|
||||||
FederationQueryPermit acquiredGlobal = globalPermit;
|
|
||||||
AtomicBoolean released = new AtomicBoolean();
|
|
||||||
return () -> {
|
|
||||||
if (released.compareAndSet(false, true)) {
|
|
||||||
try {
|
|
||||||
acquiredGlobal.close();
|
|
||||||
} finally {
|
|
||||||
try {
|
|
||||||
acquiredTenant.close();
|
|
||||||
} finally {
|
|
||||||
try {
|
|
||||||
acquiredSource.close();
|
|
||||||
} finally {
|
|
||||||
releaseEntries(
|
|
||||||
sourceId, sourceEntry,
|
|
||||||
tenantKey, tenantEntry);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
closeQuietly(globalPermit);
|
|
||||||
closeQuietly(tenantPermit);
|
|
||||||
closeQuietly(sourcePermit);
|
|
||||||
releaseEntries(sourceId, sourceEntry, tenantKey, tenantEntry);
|
|
||||||
throw exception;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 关闭全部准入控制器并唤醒等待线程。
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public void close() {
|
|
||||||
synchronized (lifecycleLock) {
|
|
||||||
if (!closed.compareAndSet(false, true)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
globalAdmission.close();
|
|
||||||
tenantAdmissions.values().forEach(
|
|
||||||
entry -> entry.controller.close());
|
|
||||||
sourceAdmissions.values().forEach(
|
|
||||||
entry -> entry.controller.close());
|
|
||||||
tenantAdmissions.clear();
|
|
||||||
sourceAdmissions.clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Duration remaining(Duration timeout, long started) {
|
|
||||||
if (timeout == null || timeout.isZero() || timeout.isNegative()) {
|
|
||||||
return Duration.ZERO;
|
|
||||||
}
|
|
||||||
long remaining = timeout.toNanos() - (System.nanoTime() - started);
|
|
||||||
return remaining <= 0L ? Duration.ZERO : Duration.ofNanos(remaining);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String tenantKey(SourceId sourceId) {
|
|
||||||
String value = sourceId == null ? "" : sourceId.value();
|
|
||||||
int marker = value.lastIndexOf("-source-");
|
|
||||||
return marker <= 0 ? value : value.substring(0, marker);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取或创建分层准入项并登记一个活动查询或等待者。
|
|
||||||
*
|
|
||||||
* @param admissions 准入项映射
|
|
||||||
* @param key 租户或数据源键
|
|
||||||
* @param maximumConcurrentQueries 并发上限
|
|
||||||
* @param <K> 键类型
|
|
||||||
* @return 已登记引用的准入项
|
|
||||||
*/
|
|
||||||
private <K> AdmissionEntry retain(
|
|
||||||
Map<K, AdmissionEntry> admissions,
|
|
||||||
K key,
|
|
||||||
int maximumConcurrentQueries) {
|
|
||||||
AdmissionEntry entry = admissions.computeIfAbsent(
|
|
||||||
key,
|
|
||||||
ignored -> new AdmissionEntry(
|
|
||||||
new LocalFederationQueryAdmissionController(
|
|
||||||
maximumConcurrentQueries)));
|
|
||||||
entry.references++;
|
|
||||||
return entry;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 同步释放数据源与租户准入项引用。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源标识
|
|
||||||
* @param sourceEntry 数据源准入项
|
|
||||||
* @param tenantKey 租户键
|
|
||||||
* @param tenantEntry 租户准入项
|
|
||||||
*/
|
|
||||||
private void releaseEntries(
|
|
||||||
SourceId sourceId,
|
|
||||||
AdmissionEntry sourceEntry,
|
|
||||||
String tenantKey,
|
|
||||||
AdmissionEntry tenantEntry) {
|
|
||||||
synchronized (lifecycleLock) {
|
|
||||||
release(sourceAdmissions, sourceId, sourceEntry);
|
|
||||||
release(tenantAdmissions, tenantKey, tenantEntry);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 释放一个准入项引用,并在无查询或等待者时移除并关闭该项。
|
|
||||||
*
|
|
||||||
* @param admissions 准入项映射
|
|
||||||
* @param key 租户或数据源键
|
|
||||||
* @param entry 待释放准入项
|
|
||||||
* @param <K> 键类型
|
|
||||||
*/
|
|
||||||
private <K> void release(
|
|
||||||
Map<K, AdmissionEntry> admissions,
|
|
||||||
K key,
|
|
||||||
AdmissionEntry entry) {
|
|
||||||
entry.references--;
|
|
||||||
if (entry.references < 0) {
|
|
||||||
throw new IllegalStateException(
|
|
||||||
"query admission reference count became negative");
|
|
||||||
}
|
|
||||||
if (entry.references == 0 && admissions.remove(key, entry)) {
|
|
||||||
entry.controller.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回当前仍有查询或等待者的数据源准入项数量。
|
|
||||||
*
|
|
||||||
* @return 活动数据源准入项数量
|
|
||||||
*/
|
|
||||||
int trackedSourceAdmissionCount() {
|
|
||||||
return sourceAdmissions.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回当前仍有查询或等待者的租户准入项数量。
|
|
||||||
*
|
|
||||||
* @return 活动租户准入项数量
|
|
||||||
*/
|
|
||||||
int trackedTenantAdmissionCount() {
|
|
||||||
return tenantAdmissions.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回指定数据源当前活动查询与等待者的引用总数。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源标识
|
|
||||||
* @return 当前引用数
|
|
||||||
*/
|
|
||||||
int trackedSourceReferenceCount(SourceId sourceId) {
|
|
||||||
synchronized (lifecycleLock) {
|
|
||||||
AdmissionEntry entry = sourceAdmissions.get(sourceId);
|
|
||||||
return entry == null ? 0 : entry.references;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ensureOpen() {
|
|
||||||
if (closed.get()) {
|
|
||||||
throw new FederationSqlException(
|
|
||||||
FederationSqlErrorCode.ENGINE_CLOSED,
|
|
||||||
"datacenter query admission controller is closed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void closeQuietly(FederationQueryPermit permit) {
|
|
||||||
if (permit != null) {
|
|
||||||
permit.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 共享公平信号量及其活动查询、等待者引用计数。
|
|
||||||
*/
|
|
||||||
private static final class AdmissionEntry {
|
|
||||||
|
|
||||||
private final LocalFederationQueryAdmissionController controller;
|
|
||||||
private int references;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建准入项。
|
|
||||||
*
|
|
||||||
* @param controller 本地公平准入控制器
|
|
||||||
*/
|
|
||||||
private AdmissionEntry(LocalFederationQueryAdmissionController controller) {
|
|
||||||
this.controller = controller;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.ObjectProvider;
|
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import org.springframework.transaction.support.TransactionSynchronization;
|
|
||||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 发布不含凭据的 Federation Definition 变更提示。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationChangeNotifier {
|
|
||||||
|
|
||||||
/** Redis Pub/Sub 频道。 */
|
|
||||||
public static final String CHANNEL = "easyflow:datacenter:federation-source-changed";
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationChangeNotifier.class);
|
|
||||||
|
|
||||||
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
|
|
||||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建通知器。
|
|
||||||
*
|
|
||||||
* @param redisTemplateProvider 可选 Redis 模板
|
|
||||||
* @param definitionFactory Definition 工厂
|
|
||||||
*/
|
|
||||||
public DatacenterFederationChangeNotifier(
|
|
||||||
ObjectProvider<StringRedisTemplate> redisTemplateProvider,
|
|
||||||
DatacenterFederationDefinitionFactory definitionFactory) {
|
|
||||||
this.redisTemplateProvider = redisTemplateProvider;
|
|
||||||
this.definitionFactory = definitionFactory;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在当前事务提交后发布 SourceId;无 Redis 时不影响数据库权威状态。
|
|
||||||
*
|
|
||||||
* @param source 已提交或即将提交的数据源
|
|
||||||
*/
|
|
||||||
public void publishAfterCommit(DatacenterSource source) {
|
|
||||||
String sourceId = definitionFactory.sourceId(source).value();
|
|
||||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
|
||||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
|
||||||
@Override
|
|
||||||
public void afterCommit() {
|
|
||||||
publish(sourceId);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
publish(sourceId);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void publish(String sourceId) {
|
|
||||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
|
||||||
if (redisTemplate == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
redisTemplate.convertAndSend(CHANNEL, sourceId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
// Redis 仅缩短传播时间;数据库按需读取继续保证 minimumRevision 正确性。
|
|
||||||
log.warn("Failed to publish datacenter source change hint for {}", sourceId, exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,64 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.zaxxer.hikari.HikariConfig;
|
|
||||||
import com.zaxxer.hikari.HikariDataSource;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建由 Federation Runtime 独占的只读 HikariCP 连接池。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationDataSourceFactory {
|
|
||||||
|
|
||||||
private final DatacenterCredentialCipher credentialCipher;
|
|
||||||
private final int maximumPoolSize;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建数据源工厂。
|
|
||||||
*
|
|
||||||
* @param credentialCipher 凭据解密器
|
|
||||||
* @param maximumPoolSize 单数据源连接池上限,与单源查询准入共用配置
|
|
||||||
*/
|
|
||||||
public DatacenterFederationDataSourceFactory(
|
|
||||||
DatacenterCredentialCipher credentialCipher,
|
|
||||||
@Value("${easyflow.datacenter.connection-pool.maximum-size:10}")
|
|
||||||
int maximumPoolSize) {
|
|
||||||
if (maximumPoolSize <= 0) {
|
|
||||||
throw new IllegalArgumentException(
|
|
||||||
"datacenter connection pool maximum size must be positive");
|
|
||||||
}
|
|
||||||
this.credentialCipher = credentialCipher;
|
|
||||||
this.maximumPoolSize = maximumPoolSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 根据持久化数据源快照创建只读池。
|
|
||||||
*
|
|
||||||
* @param source 数据源快照
|
|
||||||
* @return 未共享的 HikariCP 池
|
|
||||||
*/
|
|
||||||
public HikariDataSource create(DatacenterSource source) {
|
|
||||||
HikariConfig config = new HikariConfig();
|
|
||||||
config.setPoolName("dc-fed-" + source.getId() + "-r" + source.getDefinitionRevision());
|
|
||||||
config.setJdbcUrl(source.getJdbcUrl());
|
|
||||||
config.setUsername(source.getUsername());
|
|
||||||
config.setPassword(credentialCipher.decrypt(source.getCredentialCipher()));
|
|
||||||
if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) {
|
|
||||||
config.setDriverClassName(source.getDriverClassName());
|
|
||||||
}
|
|
||||||
config.setReadOnly(true);
|
|
||||||
config.setAutoCommit(true);
|
|
||||||
config.setMaximumPoolSize(maximumPoolSize);
|
|
||||||
config.setMinimumIdle(Integer.getInteger("easyflow.datacenter.pool.minimum-idle", 0));
|
|
||||||
config.setConnectionTimeout(Long.getLong("easyflow.datacenter.pool.connection-timeout-ms", 5_000L));
|
|
||||||
config.setValidationTimeout(Long.getLong("easyflow.datacenter.pool.validation-timeout-ms", 3_000L));
|
|
||||||
config.setIdleTimeout(Long.getLong("easyflow.datacenter.pool.idle-timeout-ms", 600_000L));
|
|
||||||
config.setMaxLifetime(Long.getLong("easyflow.datacenter.pool.max-lifetime-ms", 1_800_000L));
|
|
||||||
config.setInitializationFailTimeout(
|
|
||||||
Long.getLong("easyflow.datacenter.pool.initialization-timeout-ms", 5_000L));
|
|
||||||
return new HikariDataSource(config);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,140 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.source.FederationDataSourceHandle;
|
|
||||||
import com.easyagents.federation.sql.source.FederationDataSourceHandles;
|
|
||||||
import com.easyagents.federation.sql.source.FederationDataSourceResolver;
|
|
||||||
import com.easyagents.federation.sql.source.FederationSourceDefinition;
|
|
||||||
import com.easyagents.federation.sql.source.RuntimeFingerprint;
|
|
||||||
import com.zaxxer.hikari.HikariDataSource;
|
|
||||||
import java.sql.Connection;
|
|
||||||
import java.sql.DatabaseMetaData;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.function.Supplier;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 从 EasyFlow 权威配置解析节点本地 Federation DataSource Handle。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationDataSourceResolver implements FederationDataSourceResolver {
|
|
||||||
|
|
||||||
private final DatacenterSourceMapper sourceMapper;
|
|
||||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
|
||||||
private final DatacenterFederationDataSourceFactory dataSourceFactory;
|
|
||||||
private final ThreadLocal<CandidateResolution> candidateSource = new ThreadLocal<>();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建 Resolver。
|
|
||||||
*
|
|
||||||
* @param sourceMapper 数据源 Mapper
|
|
||||||
* @param definitionFactory Definition 工厂
|
|
||||||
* @param dataSourceFactory 连接池工厂
|
|
||||||
*/
|
|
||||||
public DatacenterFederationDataSourceResolver(
|
|
||||||
DatacenterSourceMapper sourceMapper,
|
|
||||||
DatacenterFederationDefinitionFactory definitionFactory,
|
|
||||||
DatacenterFederationDataSourceFactory dataSourceFactory) {
|
|
||||||
this.sourceMapper = sourceMapper;
|
|
||||||
this.definitionFactory = definitionFactory;
|
|
||||||
this.dataSourceFactory = dataSourceFactory;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建由 Runtime 独占并在 revision 淘汰后关闭的 Handle。
|
|
||||||
*
|
|
||||||
* @param definition 无凭据 Definition
|
|
||||||
* @return 独占 Handle
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public FederationDataSourceHandle resolve(FederationSourceDefinition definition) {
|
|
||||||
CandidateResolution candidate = candidateSource.get();
|
|
||||||
DatacenterSource source = candidate != null
|
|
||||||
&& Objects.equals(candidate.source().getId(), definitionFactory.sourceRecordId(definition))
|
|
||||||
? candidate.source() : sourceMapper.selectOneById(definitionFactory.sourceRecordId(definition));
|
|
||||||
if (source == null) {
|
|
||||||
throw new IllegalStateException("data source no longer exists");
|
|
||||||
}
|
|
||||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
|
|
||||||
boolean draftProbe = status == DatacenterSourceStatus.DRAFT;
|
|
||||||
if (!draftProbe
|
|
||||||
&& status != DatacenterSourceStatus.READY
|
|
||||||
&& status != DatacenterSourceStatus.DEGRADED) {
|
|
||||||
throw new IllegalStateException("data source cannot be resolved in its current state");
|
|
||||||
}
|
|
||||||
if (!definition.sourceId().equals(definitionFactory.sourceId(source))) {
|
|
||||||
throw new IllegalStateException("data source tenant identity does not match definition");
|
|
||||||
}
|
|
||||||
long revision = source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision();
|
|
||||||
if (revision != definition.revision()) {
|
|
||||||
throw new IllegalStateException("data source revision changed while runtime was resolving");
|
|
||||||
}
|
|
||||||
FederationSourceDefinition currentDefinition = candidate != null
|
|
||||||
? candidate.definition()
|
|
||||||
: definitionFactory.create(source);
|
|
||||||
if (!currentDefinition.checksum().equals(definition.checksum())) {
|
|
||||||
throw new IllegalStateException("data source definition checksum mismatch");
|
|
||||||
}
|
|
||||||
// 草稿只通过 probe 创建短生命周期连接池,尚未发布持久化 checksum;
|
|
||||||
// READY/DEGRADED Runtime 必须同时匹配已发布 checksum。
|
|
||||||
if (!draftProbe && (source.getDefinitionChecksum() == null
|
|
||||||
|| !source.getDefinitionChecksum().equals(definition.checksum()))) {
|
|
||||||
throw new IllegalStateException("published data source definition checksum mismatch");
|
|
||||||
}
|
|
||||||
HikariDataSource pool = dataSourceFactory.create(source);
|
|
||||||
try (Connection connection = pool.getConnection()) {
|
|
||||||
DatabaseMetaData metadata = connection.getMetaData();
|
|
||||||
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
|
|
||||||
metadata.getDatabaseProductName(),
|
|
||||||
metadata.getDatabaseProductVersion(),
|
|
||||||
metadata.getDriverName(),
|
|
||||||
metadata.getDriverVersion(),
|
|
||||||
"easyflow-xl01-wp1");
|
|
||||||
return FederationDataSourceHandles.owned(pool, fingerprint, pool::close);
|
|
||||||
} catch (SQLException | RuntimeException exception) {
|
|
||||||
try {
|
|
||||||
pool.close();
|
|
||||||
} catch (RuntimeException closeException) {
|
|
||||||
exception.addSuppressed(closeException);
|
|
||||||
}
|
|
||||||
throw new IllegalStateException("failed to initialize data source runtime", exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在当前线程内使用未发布的候选配置完成一次 XL15 probe。
|
|
||||||
*
|
|
||||||
* <p>候选配置不会进入共享状态、Redis 或持久化记录,并在操作结束后立即移除。</p>
|
|
||||||
*
|
|
||||||
* @param source 已完成凭据加密和连接归一化的候选配置
|
|
||||||
* @param action 需要解析候选配置的同步操作
|
|
||||||
* @param <T> 操作结果类型
|
|
||||||
* @return 操作结果
|
|
||||||
* @throws IllegalStateException 当前线程已经存在候选配置时抛出
|
|
||||||
*/
|
|
||||||
public <T> T withCandidate(
|
|
||||||
DatacenterSource source,
|
|
||||||
FederationSourceDefinition definition,
|
|
||||||
Supplier<T> action) {
|
|
||||||
Objects.requireNonNull(source, "source");
|
|
||||||
Objects.requireNonNull(definition, "definition");
|
|
||||||
Objects.requireNonNull(action, "action");
|
|
||||||
if (candidateSource.get() != null) {
|
|
||||||
throw new IllegalStateException("nested data source candidate resolution is not supported");
|
|
||||||
}
|
|
||||||
candidateSource.set(new CandidateResolution(source, definition));
|
|
||||||
try {
|
|
||||||
return action.get();
|
|
||||||
} finally {
|
|
||||||
candidateSource.remove();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private record CandidateResolution(
|
|
||||||
DatacenterSource source,
|
|
||||||
FederationSourceDefinition definition) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,208 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider;
|
|
||||||
import com.easyagents.federation.sql.adapter.jdbc.JdbcSchemaDefinition;
|
|
||||||
import com.easyagents.federation.sql.source.FederationSourceDefinition;
|
|
||||||
import com.easyagents.federation.sql.source.SourceId;
|
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 将 EasyFlow 持久化数据源快照转换为无凭据 Federation Definition。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationDefinitionFactory {
|
|
||||||
|
|
||||||
/** Definition 中用于解析业务数据源主键的选项。 */
|
|
||||||
public static final String SOURCE_RECORD_ID_OPTION = "sourceRecordId";
|
|
||||||
|
|
||||||
private final DatacenterCatalogMapper catalogMapper;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建 Definition 工厂。
|
|
||||||
*
|
|
||||||
* @param catalogMapper 命名空间 Mapper
|
|
||||||
*/
|
|
||||||
public DatacenterFederationDefinitionFactory(DatacenterCatalogMapper catalogMapper) {
|
|
||||||
this.catalogMapper = catalogMapper;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建用于探测或执行的 Definition。
|
|
||||||
*
|
|
||||||
* @param source 数据源当前快照
|
|
||||||
* @return 无凭据 Definition
|
|
||||||
* @throws IllegalArgumentException 数据源不是当前 JDBC 支持类型
|
|
||||||
*/
|
|
||||||
public FederationSourceDefinition create(DatacenterSource source) {
|
|
||||||
return create(source, loadCatalogs(source));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 为尚未发布的连接候选创建只包含默认命名空间的 Definition。
|
|
||||||
*
|
|
||||||
* @param source 候选数据源
|
|
||||||
* @return 不读取旧持久化 Catalog 的候选 Definition
|
|
||||||
*/
|
|
||||||
public FederationSourceDefinition createCandidate(DatacenterSource source) {
|
|
||||||
return create(source, List.of());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 为即将发布的单命名空间配置创建与持久化结果一致的 Definition。
|
|
||||||
*
|
|
||||||
* @param source 候选数据源
|
|
||||||
* @param catalogName 即将纳管的物理 Catalog 或 Schema
|
|
||||||
* @return 不读取旧 Catalog 的候选 Definition
|
|
||||||
*/
|
|
||||||
public FederationSourceDefinition createCandidate(
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName) {
|
|
||||||
if (catalogName == null || catalogName.isBlank()) {
|
|
||||||
throw new IllegalArgumentException("candidate catalog name is required");
|
|
||||||
}
|
|
||||||
String normalized = catalogName.trim();
|
|
||||||
DatacenterCatalog catalog = new DatacenterCatalog();
|
|
||||||
catalog.setLogicalSchemaName(normalized);
|
|
||||||
if (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())) {
|
|
||||||
catalog.setPhysicalCatalogName(normalized);
|
|
||||||
} else {
|
|
||||||
catalog.setPhysicalCatalogName(source.getDatabaseName());
|
|
||||||
catalog.setPhysicalSchemaName(normalized);
|
|
||||||
}
|
|
||||||
return create(source, List.of(catalog));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 批量创建 Definition,一次加载全部活动 Catalog,供周期 reconcile 避免 N+1。
|
|
||||||
*
|
|
||||||
* @param sources 数据源快照
|
|
||||||
* @return 按数据源记录 ID 索引的 Definition
|
|
||||||
*/
|
|
||||||
public Map<BigInteger, FederationSourceDefinition> createBatch(
|
|
||||||
Collection<DatacenterSource> sources) {
|
|
||||||
List<DatacenterSource> sourceList = sources == null
|
|
||||||
? List.of() : sources.stream().filter(java.util.Objects::nonNull).toList();
|
|
||||||
if (sourceList.isEmpty()) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
List<BigInteger> sourceIds = sourceList.stream()
|
|
||||||
.map(DatacenterSource::getId)
|
|
||||||
.filter(java.util.Objects::nonNull)
|
|
||||||
.toList();
|
|
||||||
Map<BigInteger, List<DatacenterCatalog>> catalogsBySource = sourceIds.isEmpty()
|
|
||||||
? Map.of()
|
|
||||||
: catalogMapper.selectListByQuery(QueryWrapper.create()
|
|
||||||
.in(DatacenterCatalog::getSourceId, sourceIds)
|
|
||||||
.eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name())
|
|
||||||
.orderBy("created asc"))
|
|
||||||
.stream()
|
|
||||||
.collect(Collectors.groupingBy(DatacenterCatalog::getSourceId));
|
|
||||||
Map<BigInteger, FederationSourceDefinition> definitions = new LinkedHashMap<>();
|
|
||||||
for (DatacenterSource source : sourceList) {
|
|
||||||
definitions.put(source.getId(), create(
|
|
||||||
source, catalogsBySource.getOrDefault(source.getId(), List.of())));
|
|
||||||
}
|
|
||||||
return Map.copyOf(definitions);
|
|
||||||
}
|
|
||||||
|
|
||||||
private FederationSourceDefinition create(
|
|
||||||
DatacenterSource source,
|
|
||||||
List<DatacenterCatalog> catalogs) {
|
|
||||||
requireExternalJdbc(source);
|
|
||||||
long revision = Math.max(1L, source.getDefinitionRevision() == null
|
|
||||||
? 0L : source.getDefinitionRevision());
|
|
||||||
List<JdbcSchemaDefinition> schemas = schemas(source, catalogs);
|
|
||||||
Map<String, String> options = new LinkedHashMap<>();
|
|
||||||
if (source.getAdapterOptionsJson() != null) {
|
|
||||||
options.putAll(source.getAdapterOptionsJson());
|
|
||||||
}
|
|
||||||
options.put(SOURCE_RECORD_ID_OPTION, source.getId().toString());
|
|
||||||
return new FederationSourceDefinition(
|
|
||||||
sourceId(source),
|
|
||||||
revision,
|
|
||||||
source.getAdapterId() == null || source.getAdapterId().isBlank()
|
|
||||||
? JdbcFederationSqlAdapterProvider.ADAPTER_ID
|
|
||||||
: source.getAdapterId(),
|
|
||||||
new ArrayList<>(schemas),
|
|
||||||
options);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 派生租户隔离且不随显示名称变化的 SourceId。
|
|
||||||
*
|
|
||||||
* @param source 数据源
|
|
||||||
* @return Federation SourceId
|
|
||||||
*/
|
|
||||||
public SourceId sourceId(DatacenterSource source) {
|
|
||||||
BigInteger tenantId = source.getTenantId() == null ? BigInteger.ZERO : source.getTenantId();
|
|
||||||
return new SourceId("tenant-" + tenantId + "-source-" + source.getId());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 从 Definition 读取 EasyFlow 数据源主键。
|
|
||||||
*
|
|
||||||
* @param definition Definition
|
|
||||||
* @return 数据源主键
|
|
||||||
*/
|
|
||||||
public BigInteger sourceRecordId(FederationSourceDefinition definition) {
|
|
||||||
String value = definition.adapterOptions().get(SOURCE_RECORD_ID_OPTION);
|
|
||||||
if (value == null || value.isBlank()) {
|
|
||||||
throw new IllegalArgumentException("sourceRecordId is missing from Federation Definition");
|
|
||||||
}
|
|
||||||
return new BigInteger(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<DatacenterCatalog> loadCatalogs(DatacenterSource source) {
|
|
||||||
QueryWrapper query = QueryWrapper.create()
|
|
||||||
.eq(DatacenterCatalog::getSourceId, source.getId())
|
|
||||||
.eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name())
|
|
||||||
.orderBy("created asc");
|
|
||||||
return catalogMapper.selectListByQuery(query);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<JdbcSchemaDefinition> schemas(
|
|
||||||
DatacenterSource source,
|
|
||||||
List<DatacenterCatalog> catalogs) {
|
|
||||||
if (catalogs.isEmpty()) {
|
|
||||||
return List.of(defaultSchema(source));
|
|
||||||
}
|
|
||||||
return catalogs.stream()
|
|
||||||
.map(catalog -> new JdbcSchemaDefinition(
|
|
||||||
catalog.getLogicalSchemaName(),
|
|
||||||
catalog.getPhysicalCatalogName(),
|
|
||||||
catalog.getPhysicalSchemaName()))
|
|
||||||
.toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
private JdbcSchemaDefinition defaultSchema(DatacenterSource source) {
|
|
||||||
if (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())) {
|
|
||||||
return new JdbcSchemaDefinition("MAIN", source.getDatabaseName(), null);
|
|
||||||
}
|
|
||||||
String schema = source.getSchemaName() == null || source.getSchemaName().isBlank()
|
|
||||||
? "public" : source.getSchemaName();
|
|
||||||
return new JdbcSchemaDefinition("MAIN", source.getDatabaseName(), schema);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void requireExternalJdbc(DatacenterSource source) {
|
|
||||||
if (source == null || source.getId() == null) {
|
|
||||||
throw new IllegalArgumentException("persisted source is required");
|
|
||||||
}
|
|
||||||
if (!DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|
|
||||||
&& !DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType())) {
|
|
||||||
throw new IllegalArgumentException("Federation JDBC currently supports MySQL and PostgreSQL sources");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,292 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.execute.QueryId;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.ObjectProvider;
|
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 管理查询从编译到游标关闭的取消状态,并通过 Redis 将取消提示广播到执行节点。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationQueryCancellationService {
|
|
||||||
|
|
||||||
/** 查询取消 Redis Pub/Sub 频道。 */
|
|
||||||
public static final String CHANNEL = "easyflow:datacenter:federation-query-cancel";
|
|
||||||
private static final String CANCEL_KEY_PREFIX =
|
|
||||||
"easyflow:datacenter:federation-query-cancelled:";
|
|
||||||
private static final Duration CANCEL_TTL = Duration.ofMinutes(10);
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(
|
|
||||||
DatacenterFederationQueryCancellationService.class);
|
|
||||||
|
|
||||||
private final DatacenterFederationRuntime runtime;
|
|
||||||
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
|
|
||||||
private final ConcurrentHashMap<String, ActiveQuery> activeQueries = new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建查询取消服务。
|
|
||||||
*
|
|
||||||
* @param runtime 节点本地 Federation Runtime
|
|
||||||
* @param redisTemplateProvider 可选 Redis 模板
|
|
||||||
*/
|
|
||||||
public DatacenterFederationQueryCancellationService(
|
|
||||||
DatacenterFederationRuntime runtime,
|
|
||||||
ObjectProvider<StringRedisTemplate> redisTemplateProvider) {
|
|
||||||
this.runtime = runtime;
|
|
||||||
this.redisTemplateProvider = redisTemplateProvider;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 校验或创建查询标识。
|
|
||||||
*
|
|
||||||
* @param requestedQueryId 调用方预生成的 UUID;为空时由服务端生成
|
|
||||||
* @return 已校验的查询标识
|
|
||||||
* @throws BusinessException 标识格式不合法
|
|
||||||
*/
|
|
||||||
public QueryId resolveQueryId(String requestedQueryId) {
|
|
||||||
if (requestedQueryId == null || requestedQueryId.isBlank()) {
|
|
||||||
return QueryId.create();
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return new QueryId(UUID.fromString(requestedQueryId.trim()).toString());
|
|
||||||
} catch (IllegalArgumentException exception) {
|
|
||||||
throw new BusinessException("queryId 必须是 UUID");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在编译前登记查询,防止重复 QueryId 混淆取消目标。
|
|
||||||
*
|
|
||||||
* @param queryId 查询标识
|
|
||||||
* @param account 执行账号
|
|
||||||
* @return 必须关闭的登记句柄
|
|
||||||
* @throws BusinessException QueryId 正在使用
|
|
||||||
*/
|
|
||||||
public QueryRegistration register(QueryId queryId, LoginAccount account) {
|
|
||||||
ActiveQuery activeQuery = new ActiveQuery(tenantId(account));
|
|
||||||
if (activeQueries.putIfAbsent(queryId.value(), activeQuery) != null) {
|
|
||||||
throw new BusinessException(409, 14123, "queryId 正在使用", null);
|
|
||||||
}
|
|
||||||
if (isCancellationPersisted(activeQuery.tenantId, queryId)) {
|
|
||||||
try {
|
|
||||||
cancelLocal(activeQuery.tenantId, queryId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
activeQueries.remove(queryId.value(), activeQuery);
|
|
||||||
throw exception;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new QueryRegistration(queryId, activeQuery);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 向所有节点广播取消提示,并立即取消当前节点已登记的 Statement。
|
|
||||||
*
|
|
||||||
* @param requestedQueryId 查询标识
|
|
||||||
* @param account 当前账号
|
|
||||||
* @return 当前节点已找到查询或 Redis 已接收广播
|
|
||||||
*/
|
|
||||||
public boolean cancel(String requestedQueryId, LoginAccount account) {
|
|
||||||
if (requestedQueryId == null || requestedQueryId.isBlank()) {
|
|
||||||
throw new BusinessException("queryId 不能为空");
|
|
||||||
}
|
|
||||||
QueryId queryId = resolveQueryId(requestedQueryId);
|
|
||||||
BigInteger tenantId = tenantId(account);
|
|
||||||
boolean distributedCancellationPersisted = persistAndPublishCancellation(
|
|
||||||
tenantId, queryId);
|
|
||||||
boolean localCancelled;
|
|
||||||
try {
|
|
||||||
localCancelled = cancelLocal(tenantId, queryId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.error("Failed to cancel local datacenter query for tenant {} query {}",
|
|
||||||
tenantId, queryId.value(), exception);
|
|
||||||
if (!distributedCancellationPersisted) {
|
|
||||||
throw exception;
|
|
||||||
}
|
|
||||||
localCancelled = false;
|
|
||||||
}
|
|
||||||
return localCancelled || distributedCancellationPersisted;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 先持久化取消终态,再尽力发布低延迟提示。
|
|
||||||
*
|
|
||||||
* @param tenantId 租户 ID
|
|
||||||
* @param queryId 查询标识
|
|
||||||
* @return Redis 是否已可靠保存取消终态
|
|
||||||
*/
|
|
||||||
private boolean persistAndPublishCancellation(
|
|
||||||
BigInteger tenantId,
|
|
||||||
QueryId queryId) {
|
|
||||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
|
||||||
if (redisTemplate == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
redisTemplate.opsForValue().set(
|
|
||||||
cancellationKey(tenantId, queryId), "1", CANCEL_TTL);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to persist datacenter query cancellation for tenant {} query {}",
|
|
||||||
tenantId, queryId.value(), exception);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
redisTemplate.convertAndSend(
|
|
||||||
CHANNEL, tenantId + "|" + queryId.value());
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
// Pub/Sub 仅缩短生效延迟;持久取消终态已由周期核对保证交付。
|
|
||||||
log.warn("Failed to publish datacenter query cancellation hint for tenant {} query {}",
|
|
||||||
tenantId, queryId.value(), exception);
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 处理 Redis 取消提示;格式无效时安全忽略。
|
|
||||||
*
|
|
||||||
* @param payload 租户与 QueryId 组合文本
|
|
||||||
*/
|
|
||||||
public void acceptCancellationHint(String payload) {
|
|
||||||
if (payload == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
int separator = payload.indexOf('|');
|
|
||||||
if (separator <= 0 || separator == payload.length() - 1) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
BigInteger tenantId = new BigInteger(payload.substring(0, separator));
|
|
||||||
QueryId queryId = resolveQueryId(payload.substring(separator + 1));
|
|
||||||
cancelLocal(tenantId, queryId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Ignored invalid datacenter query cancellation hint", exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 在编译或取数边界检查查询是否已取消。
|
|
||||||
*
|
|
||||||
* @param queryId 查询标识
|
|
||||||
* @throws BusinessException 查询已取消
|
|
||||||
*/
|
|
||||||
public void throwIfCancelled(QueryId queryId) {
|
|
||||||
ActiveQuery activeQuery = activeQueries.get(queryId.value());
|
|
||||||
if (activeQuery != null && activeQuery.cancelled.get()) {
|
|
||||||
throw new BusinessException(409, 14118, "查询已取消", null);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回查询当前是否已收到取消请求。
|
|
||||||
*
|
|
||||||
* @param queryId 查询标识
|
|
||||||
* @return 是否取消
|
|
||||||
*/
|
|
||||||
public boolean isCancelled(QueryId queryId) {
|
|
||||||
ActiveQuery activeQuery = activeQueries.get(queryId.value());
|
|
||||||
return activeQuery != null && activeQuery.cancelled.get();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 周期核对 Redis 持久取消标记,覆盖 Pub/Sub 提示丢失或节点短暂断连。
|
|
||||||
*/
|
|
||||||
@Scheduled(fixedDelayString =
|
|
||||||
"${easyflow.datacenter.query-cancel-reconcile-ms:1000}")
|
|
||||||
public void reconcilePersistedCancellations() {
|
|
||||||
activeQueries.forEach((queryIdValue, activeQuery) -> {
|
|
||||||
if (activeQuery.localCancellationApplied.get()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
QueryId queryId = new QueryId(queryIdValue);
|
|
||||||
if (activeQuery.cancelled.get()
|
|
||||||
|| isCancellationPersisted(activeQuery.tenantId, queryId)) {
|
|
||||||
try {
|
|
||||||
cancelLocal(activeQuery.tenantId, queryId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.error("Failed to reconcile datacenter query cancellation for tenant {} query {}",
|
|
||||||
activeQuery.tenantId, queryId.value(), exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean cancelLocal(BigInteger tenantId, QueryId queryId) {
|
|
||||||
ActiveQuery activeQuery = activeQueries.get(queryId.value());
|
|
||||||
if (activeQuery == null || !activeQuery.tenantId.equals(tenantId)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
activeQuery.cancelled.set(true);
|
|
||||||
runtime.engine().cancel(queryId);
|
|
||||||
activeQuery.localCancellationApplied.set(true);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isCancellationPersisted(BigInteger tenantId, QueryId queryId) {
|
|
||||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
|
||||||
if (redisTemplate == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return Boolean.TRUE.equals(redisTemplate.hasKey(
|
|
||||||
cancellationKey(tenantId, queryId)));
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to read datacenter query cancellation for tenant {} query {}",
|
|
||||||
tenantId, queryId.value(), exception);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String cancellationKey(BigInteger tenantId, QueryId queryId) {
|
|
||||||
return CANCEL_KEY_PREFIX + tenantId + ":" + queryId.value();
|
|
||||||
}
|
|
||||||
|
|
||||||
private BigInteger tenantId(LoginAccount account) {
|
|
||||||
return account == null || account.getTenantId() == null
|
|
||||||
? BigInteger.ZERO : account.getTenantId();
|
|
||||||
}
|
|
||||||
|
|
||||||
private record ActiveQuery(
|
|
||||||
BigInteger tenantId,
|
|
||||||
AtomicBoolean cancelled,
|
|
||||||
AtomicBoolean localCancellationApplied) {
|
|
||||||
|
|
||||||
private ActiveQuery(BigInteger tenantId) {
|
|
||||||
this(tenantId, new AtomicBoolean(), new AtomicBoolean());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 查询活动登记句柄。
|
|
||||||
*/
|
|
||||||
public final class QueryRegistration implements AutoCloseable {
|
|
||||||
|
|
||||||
private final QueryId queryId;
|
|
||||||
private final ActiveQuery activeQuery;
|
|
||||||
private final AtomicBoolean closed = new AtomicBoolean();
|
|
||||||
|
|
||||||
private QueryRegistration(QueryId queryId, ActiveQuery activeQuery) {
|
|
||||||
this.queryId = queryId;
|
|
||||||
this.activeQuery = activeQuery;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 仅移除当前登记实例,避免误删随后复用的同名查询。
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public void close() {
|
|
||||||
if (closed.compareAndSet(false, true)) {
|
|
||||||
activeQueries.remove(queryId.value(), activeQuery);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,731 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
|
||||||
import com.easyagents.federation.sql.api.SqlExecutionContext;
|
|
||||||
import com.easyagents.federation.sql.compile.FederationSqlPlan;
|
|
||||||
import com.easyagents.federation.sql.compile.SqlCompileRequest;
|
|
||||||
import com.easyagents.federation.sql.execute.FederationColumn;
|
|
||||||
import com.easyagents.federation.sql.execute.FederationResultCursor;
|
|
||||||
import com.easyagents.federation.sql.execute.QueryId;
|
|
||||||
import com.easyagents.federation.sql.execute.SqlExecutionOptions;
|
|
||||||
import com.easyagents.federation.sql.execute.SqlParameter;
|
|
||||||
import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition;
|
|
||||||
import com.easyagents.federation.sql.source.SourceId;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.Reader;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.sql.Blob;
|
|
||||||
import java.sql.Clob;
|
|
||||||
import java.sql.SQLXML;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.sql.Struct;
|
|
||||||
import java.sql.Types;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Base64;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.LinkedHashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.function.Consumer;
|
|
||||||
import org.apache.calcite.rel.RelNode;
|
|
||||||
import org.apache.calcite.rel.RelVisitor;
|
|
||||||
import org.apache.calcite.rel.core.TableScan;
|
|
||||||
import org.apache.calcite.sql.SqlIdentifier;
|
|
||||||
import org.apache.calcite.sql.util.SqlBasicVisitor;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
|
|
||||||
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlColumnView;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 通过 Federation SQL 完成有界只读查询、资源关闭和统一审计。
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class DatacenterFederationQueryService {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationQueryService.class);
|
|
||||||
private static final int DEFAULT_MAX_ROWS = 200;
|
|
||||||
private static final int MAX_ALLOWED_ROWS = 1_000;
|
|
||||||
private static final long MAX_RESULT_BYTES = 4L * 1024L * 1024L;
|
|
||||||
private static final int MAX_TEXT_CELL_CHARS = 65_536;
|
|
||||||
|
|
||||||
private final DatacenterFederationRuntime runtime;
|
|
||||||
private final DatacenterQueryAuditService auditService;
|
|
||||||
private final DatacenterFederationQueryCancellationService cancellationService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建查询服务。
|
|
||||||
*
|
|
||||||
* @param runtime Federation Runtime
|
|
||||||
* @param auditService 查询审计服务
|
|
||||||
* @param cancellationService 跨节点查询取消服务
|
|
||||||
*/
|
|
||||||
public DatacenterFederationQueryService(
|
|
||||||
DatacenterFederationRuntime runtime,
|
|
||||||
DatacenterQueryAuditService auditService,
|
|
||||||
DatacenterFederationQueryCancellationService cancellationService) {
|
|
||||||
this.runtime = runtime;
|
|
||||||
this.auditService = auditService;
|
|
||||||
this.cancellationService = cancellationService;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 执行一条单数据源只读 SQL。
|
|
||||||
*
|
|
||||||
* @param source 已激活数据源
|
|
||||||
* @param sql 只读 SQL
|
|
||||||
* @param parameters 显式 JDBC 参数
|
|
||||||
* @param requestedMaxRows 请求行数上限
|
|
||||||
* @param account 执行账号
|
|
||||||
* @param callerType 调用方类型
|
|
||||||
* @param callerId 调用方标识
|
|
||||||
* @return 有界查询结果
|
|
||||||
*/
|
|
||||||
public DatacenterSqlConsoleResult execute(
|
|
||||||
DatacenterSource source,
|
|
||||||
String sql,
|
|
||||||
List<SqlParameter> parameters,
|
|
||||||
Integer requestedMaxRows,
|
|
||||||
LoginAccount account,
|
|
||||||
String callerType,
|
|
||||||
String callerId) {
|
|
||||||
return execute(source, sql, parameters, requestedMaxRows, account,
|
|
||||||
callerType, callerId, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用调用方预生成的 QueryId 执行只读 SQL,以支持请求完成前取消。
|
|
||||||
*
|
|
||||||
* @param source 已激活数据源
|
|
||||||
* @param sql 只读 SQL
|
|
||||||
* @param parameters 显式 JDBC 参数
|
|
||||||
* @param requestedMaxRows 请求行数上限
|
|
||||||
* @param account 执行账号
|
|
||||||
* @param callerType 调用方类型
|
|
||||||
* @param callerId 调用方标识
|
|
||||||
* @param requestedQueryId 调用方预生成的查询 UUID;为空时服务端生成
|
|
||||||
* @return 有界查询结果
|
|
||||||
*/
|
|
||||||
public DatacenterSqlConsoleResult execute(
|
|
||||||
DatacenterSource source,
|
|
||||||
String sql,
|
|
||||||
List<SqlParameter> parameters,
|
|
||||||
Integer requestedMaxRows,
|
|
||||||
LoginAccount account,
|
|
||||||
String callerType,
|
|
||||||
String callerId,
|
|
||||||
String requestedQueryId) {
|
|
||||||
requireExecutable(source, sql);
|
|
||||||
int maxRows = normalizeMaxRows(requestedMaxRows);
|
|
||||||
List<SqlParameter> safeParameters = List.copyOf(parameters == null ? List.of() : parameters);
|
|
||||||
QueryId queryId = cancellationService.resolveQueryId(requestedQueryId);
|
|
||||||
DatacenterFederationQueryCancellationService.QueryRegistration registration =
|
|
||||||
cancellationService.register(queryId, account);
|
|
||||||
long startedNanos = System.nanoTime();
|
|
||||||
DatacenterQueryAudit audit;
|
|
||||||
try {
|
|
||||||
audit = auditService.start(
|
|
||||||
queryId.value(), source, sql, maskedParameterSummary(safeParameters),
|
|
||||||
account, callerType, callerId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
registration.close();
|
|
||||||
throw exception;
|
|
||||||
}
|
|
||||||
List<String> referencedTables = List.of();
|
|
||||||
List<String> referencedFields = List.of();
|
|
||||||
try {
|
|
||||||
cancellationService.throwIfCancelled(queryId);
|
|
||||||
SqlCompileRequest compileRequest = compileRequest(source, sql, safeParameters);
|
|
||||||
FederationSqlPlan plan = runtime.engine().compile(compileRequest);
|
|
||||||
cancellationService.throwIfCancelled(queryId);
|
|
||||||
referencedTables = referencedTables(plan.relRoot().rel);
|
|
||||||
referencedFields = referencedFields(plan.sqlNode());
|
|
||||||
SqlExecutionOptions options = new SqlExecutionOptions(
|
|
||||||
200,
|
|
||||||
maxRows + 1,
|
|
||||||
30,
|
|
||||||
true);
|
|
||||||
long databaseStartedNanos = System.nanoTime();
|
|
||||||
List<DatacenterSqlColumnView> columns;
|
|
||||||
List<Map<String, Object>> rows = new ArrayList<>(maxRows);
|
|
||||||
boolean truncated = false;
|
|
||||||
long resultBytes = 0L;
|
|
||||||
try (FederationResultCursor cursor = runtime.engine().execute(
|
|
||||||
plan,
|
|
||||||
new SqlExecutionContext(
|
|
||||||
queryId,
|
|
||||||
safeParameters,
|
|
||||||
options,
|
|
||||||
Duration.ofSeconds(5)))) {
|
|
||||||
columns = columns(cursor.columns());
|
|
||||||
while (cursor.next()) {
|
|
||||||
cancellationService.throwIfCancelled(queryId);
|
|
||||||
if (rows.size() >= maxRows) {
|
|
||||||
truncated = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Map<String, Object> row = new LinkedHashMap<>();
|
|
||||||
for (int index = 0; index < columns.size(); index++) {
|
|
||||||
Object value = readColumn(cursor, cursor.columns().get(index), index + 1);
|
|
||||||
resultBytes += estimateBytes(columns.get(index).label(), value);
|
|
||||||
if (resultBytes > MAX_RESULT_BYTES) {
|
|
||||||
truncated = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
row.put(columns.get(index).key(), value);
|
|
||||||
}
|
|
||||||
if (truncated && resultBytes > MAX_RESULT_BYTES) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
rows.add(Collections.unmodifiableMap(new LinkedHashMap<>(row)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
long databaseDurationMs = elapsedMillis(databaseStartedNanos);
|
|
||||||
long durationMs = elapsedMillis(startedNanos);
|
|
||||||
auditService.succeed(audit, referencedTables, referencedFields, rows.size(), truncated,
|
|
||||||
durationMs, databaseDurationMs);
|
|
||||||
return new DatacenterSqlConsoleResult(
|
|
||||||
queryId.value(), columns, rows, rows.size(), truncated, durationMs);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
long durationMs = elapsedMillis(startedNanos);
|
|
||||||
String errorCode = cancellationService.isCancelled(queryId)
|
|
||||||
? "QUERY_CANCELLED"
|
|
||||||
: exception instanceof FederationSqlException federationException
|
|
||||||
? federationException.errorCode().name()
|
|
||||||
: "DATACENTER_QUERY_FAILED";
|
|
||||||
try {
|
|
||||||
auditService.fail(audit, errorCode, safeMessage(exception),
|
|
||||||
referencedTables, referencedFields, durationMs);
|
|
||||||
} catch (RuntimeException auditException) {
|
|
||||||
exception.addSuppressed(auditException);
|
|
||||||
log.error("Failed to finalize datacenter query audit {}", queryId.value(), auditException);
|
|
||||||
}
|
|
||||||
if (exception instanceof BusinessException businessException) {
|
|
||||||
throw businessException;
|
|
||||||
}
|
|
||||||
logUnexpectedFailure(queryId, source, exception);
|
|
||||||
throw toBusinessException(exception);
|
|
||||||
} finally {
|
|
||||||
registration.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用 Federation 游标流式消费单数据源只读 SQL,并在触及资源上限时显式失败。
|
|
||||||
*
|
|
||||||
* @param source 已激活数据源
|
|
||||||
* @param sql 只读 SQL
|
|
||||||
* @param parameters 显式 JDBC 参数
|
|
||||||
* @param fetchSize JDBC 拉取批次
|
|
||||||
* @param maxRows 最大消费行数
|
|
||||||
* @param maxBytes 最大估算字节数
|
|
||||||
* @param account 执行账号
|
|
||||||
* @param callerType 调用方类型
|
|
||||||
* @param callerId 调用方标识
|
|
||||||
* @param consumer 单行消费者
|
|
||||||
* @return 已消费行数
|
|
||||||
*/
|
|
||||||
public long consume(
|
|
||||||
DatacenterSource source,
|
|
||||||
String sql,
|
|
||||||
List<SqlParameter> parameters,
|
|
||||||
int fetchSize,
|
|
||||||
long maxRows,
|
|
||||||
long maxBytes,
|
|
||||||
LoginAccount account,
|
|
||||||
String callerType,
|
|
||||||
String callerId,
|
|
||||||
Consumer<Map<String, Object>> consumer) {
|
|
||||||
return consume(source, sql, parameters, fetchSize, maxRows, maxBytes,
|
|
||||||
account, callerType, callerId, null, consumer);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用调用方预生成的 QueryId 流式消费只读 SQL,以支持流消费期间取消。
|
|
||||||
*
|
|
||||||
* @param source 已激活数据源
|
|
||||||
* @param sql 只读 SQL
|
|
||||||
* @param parameters 显式 JDBC 参数
|
|
||||||
* @param fetchSize JDBC 拉取批次
|
|
||||||
* @param maxRows 最大消费行数
|
|
||||||
* @param maxBytes 最大估算字节数
|
|
||||||
* @param account 执行账号
|
|
||||||
* @param callerType 调用方类型
|
|
||||||
* @param callerId 调用方标识
|
|
||||||
* @param requestedQueryId 调用方预生成的查询 UUID;为空时服务端生成
|
|
||||||
* @param consumer 单行消费者
|
|
||||||
* @return 已消费行数
|
|
||||||
*/
|
|
||||||
public long consume(
|
|
||||||
DatacenterSource source,
|
|
||||||
String sql,
|
|
||||||
List<SqlParameter> parameters,
|
|
||||||
int fetchSize,
|
|
||||||
long maxRows,
|
|
||||||
long maxBytes,
|
|
||||||
LoginAccount account,
|
|
||||||
String callerType,
|
|
||||||
String callerId,
|
|
||||||
String requestedQueryId,
|
|
||||||
Consumer<Map<String, Object>> consumer) {
|
|
||||||
requireExecutable(source, sql);
|
|
||||||
if (fetchSize <= 0 || maxRows <= 0 || maxRows >= Integer.MAX_VALUE
|
|
||||||
|| maxBytes <= 0 || consumer == null) {
|
|
||||||
throw new IllegalArgumentException("stream query limits and consumer must be valid");
|
|
||||||
}
|
|
||||||
List<SqlParameter> safeParameters = List.copyOf(parameters == null ? List.of() : parameters);
|
|
||||||
QueryId queryId = cancellationService.resolveQueryId(requestedQueryId);
|
|
||||||
DatacenterFederationQueryCancellationService.QueryRegistration registration =
|
|
||||||
cancellationService.register(queryId, account);
|
|
||||||
long startedNanos = System.nanoTime();
|
|
||||||
DatacenterQueryAudit audit;
|
|
||||||
try {
|
|
||||||
audit = auditService.start(
|
|
||||||
queryId.value(), source, sql,
|
|
||||||
maskedParameterSummary(safeParameters),
|
|
||||||
account, callerType, callerId);
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
registration.close();
|
|
||||||
throw exception;
|
|
||||||
}
|
|
||||||
List<String> referencedTables = List.of();
|
|
||||||
List<String> referencedFields = List.of();
|
|
||||||
long consumedRows = 0L;
|
|
||||||
try {
|
|
||||||
cancellationService.throwIfCancelled(queryId);
|
|
||||||
SqlCompileRequest compileRequest = compileRequest(source, sql, safeParameters);
|
|
||||||
FederationSqlPlan plan = runtime.engine().compile(compileRequest);
|
|
||||||
cancellationService.throwIfCancelled(queryId);
|
|
||||||
referencedTables = referencedTables(plan.relRoot().rel);
|
|
||||||
referencedFields = referencedFields(plan.sqlNode());
|
|
||||||
SqlExecutionOptions options = new SqlExecutionOptions(
|
|
||||||
fetchSize, Math.toIntExact(maxRows + 1L), 30, true);
|
|
||||||
long databaseStartedNanos = System.nanoTime();
|
|
||||||
long resultBytes = 0L;
|
|
||||||
try (FederationResultCursor cursor = runtime.engine().execute(
|
|
||||||
plan,
|
|
||||||
new SqlExecutionContext(queryId, safeParameters, options, Duration.ofSeconds(5)))) {
|
|
||||||
List<DatacenterSqlColumnView> columns = columns(cursor.columns());
|
|
||||||
while (cursor.next()) {
|
|
||||||
cancellationService.throwIfCancelled(queryId);
|
|
||||||
if (consumedRows >= maxRows) {
|
|
||||||
throw new BusinessException("数据集查询结果超过行数上限: " + maxRows);
|
|
||||||
}
|
|
||||||
Map<String, Object> row = new LinkedHashMap<>();
|
|
||||||
for (int index = 0; index < columns.size(); index++) {
|
|
||||||
Object value = readColumn(cursor, cursor.columns().get(index), index + 1);
|
|
||||||
resultBytes += estimateBytes(columns.get(index).label(), value);
|
|
||||||
if (resultBytes > maxBytes) {
|
|
||||||
throw new BusinessException("数据集查询结果超过字节上限: " + maxBytes);
|
|
||||||
}
|
|
||||||
row.put(columns.get(index).label(), value);
|
|
||||||
}
|
|
||||||
consumer.accept(Collections.unmodifiableMap(row));
|
|
||||||
consumedRows++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
long durationMs = elapsedMillis(startedNanos);
|
|
||||||
auditService.succeed(
|
|
||||||
audit, referencedTables, referencedFields, consumedRows, false,
|
|
||||||
durationMs, elapsedMillis(databaseStartedNanos));
|
|
||||||
return consumedRows;
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
long durationMs = elapsedMillis(startedNanos);
|
|
||||||
String errorCode = cancellationService.isCancelled(queryId)
|
|
||||||
? "QUERY_CANCELLED"
|
|
||||||
: exception instanceof FederationSqlException federationException
|
|
||||||
? federationException.errorCode().name()
|
|
||||||
: "DATACENTER_QUERY_FAILED";
|
|
||||||
try {
|
|
||||||
auditService.fail(audit, errorCode, safeMessage(exception),
|
|
||||||
referencedTables, referencedFields, durationMs);
|
|
||||||
} catch (RuntimeException auditException) {
|
|
||||||
exception.addSuppressed(auditException);
|
|
||||||
log.error("Failed to finalize datacenter stream query audit {}", queryId.value(), auditException);
|
|
||||||
}
|
|
||||||
if (exception instanceof BusinessException businessException) {
|
|
||||||
throw businessException;
|
|
||||||
}
|
|
||||||
logUnexpectedFailure(queryId, source, exception);
|
|
||||||
throw toBusinessException(exception);
|
|
||||||
} finally {
|
|
||||||
registration.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private SqlCompileRequest compileRequest(
|
|
||||||
DatacenterSource source,
|
|
||||||
String sql,
|
|
||||||
List<SqlParameter> parameters) {
|
|
||||||
SourceId sourceId = runtime.definitions().sourceId(source);
|
|
||||||
return new SqlCompileRequest(
|
|
||||||
sql,
|
|
||||||
FederationQueryScopeDefinition.single(
|
|
||||||
"datacenter-source:" + source.getId(),
|
|
||||||
source.getScopeRevision(),
|
|
||||||
sourceId.value(),
|
|
||||||
sourceId,
|
|
||||||
source.getDefinitionRevision()),
|
|
||||||
parameters.stream().map(SqlParameter::jdbcType).toList(),
|
|
||||||
"scope-" + source.getScopeRevision());
|
|
||||||
}
|
|
||||||
|
|
||||||
private void requireExecutable(DatacenterSource source, String sql) {
|
|
||||||
if (source == null || DatacenterSourceStatus.fromCode(source.getStatus())
|
|
||||||
!= DatacenterSourceStatus.READY) {
|
|
||||||
throw new BusinessException("数据连接尚未就绪");
|
|
||||||
}
|
|
||||||
if (sql == null || sql.isBlank()) {
|
|
||||||
throw new BusinessException("SQL 不能为空");
|
|
||||||
}
|
|
||||||
if (sql.length() > 100_000) {
|
|
||||||
throw new BusinessException("SQL 长度超过限制");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private int normalizeMaxRows(Integer requested) {
|
|
||||||
if (requested == null) {
|
|
||||||
return DEFAULT_MAX_ROWS;
|
|
||||||
}
|
|
||||||
if (requested <= 0) {
|
|
||||||
throw new BusinessException("maxRows 必须大于 0");
|
|
||||||
}
|
|
||||||
return Math.min(requested, MAX_ALLOWED_ROWS);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<DatacenterSqlColumnView> columns(List<FederationColumn> columns) {
|
|
||||||
List<DatacenterSqlColumnView> result = new ArrayList<>(columns.size());
|
|
||||||
for (FederationColumn column : columns) {
|
|
||||||
result.add(new DatacenterSqlColumnView(
|
|
||||||
"c" + column.index(),
|
|
||||||
column.label(),
|
|
||||||
column.jdbcType(),
|
|
||||||
column.typeName(),
|
|
||||||
column.nullable()));
|
|
||||||
}
|
|
||||||
return List.copyOf(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<String> referencedTables(RelNode root) {
|
|
||||||
Set<String> names = new LinkedHashSet<>();
|
|
||||||
new RelVisitor() {
|
|
||||||
@Override
|
|
||||||
public void visit(RelNode node, int ordinal, RelNode parent) {
|
|
||||||
if (node instanceof TableScan scan) {
|
|
||||||
names.add(String.join(".", scan.getTable().getQualifiedName()));
|
|
||||||
}
|
|
||||||
super.visit(node, ordinal, parent);
|
|
||||||
}
|
|
||||||
}.go(root);
|
|
||||||
return List.copyOf(names);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<String> referencedFields(org.apache.calcite.sql.SqlNode sqlNode) {
|
|
||||||
Set<String> names = new LinkedHashSet<>();
|
|
||||||
sqlNode.accept(new SqlBasicVisitor<Void>() {
|
|
||||||
@Override
|
|
||||||
public Void visit(SqlIdentifier identifier) {
|
|
||||||
if (!identifier.isStar() && !identifier.names.isEmpty()) {
|
|
||||||
names.add(String.join(".", identifier.names));
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return List.copyOf(names);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, Object> maskedParameterSummary(List<SqlParameter> parameters) {
|
|
||||||
Map<String, Object> summary = new LinkedHashMap<>();
|
|
||||||
for (int index = 0; index < parameters.size(); index++) {
|
|
||||||
SqlParameter parameter = parameters.get(index);
|
|
||||||
summary.put("p" + (index + 1), Map.of(
|
|
||||||
"jdbcType", parameter.jdbcType(),
|
|
||||||
"present", parameter.value() != null));
|
|
||||||
}
|
|
||||||
return Map.copyOf(summary);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Object normalizeValue(Object value) {
|
|
||||||
try {
|
|
||||||
if (value instanceof byte[] bytes) {
|
|
||||||
if (bytes.length > MAX_TEXT_CELL_CHARS) {
|
|
||||||
return "[binary " + bytes.length + " bytes]";
|
|
||||||
}
|
|
||||||
return Base64.getEncoder().encodeToString(bytes);
|
|
||||||
}
|
|
||||||
if (value instanceof Blob blob) {
|
|
||||||
return "[binary " + blob.length() + " bytes]";
|
|
||||||
}
|
|
||||||
if (value instanceof Clob clob) {
|
|
||||||
long length = Math.min(clob.length(), 4_096L);
|
|
||||||
return clob.getSubString(1L, (int) length);
|
|
||||||
}
|
|
||||||
if (value instanceof SQLXML sqlxml) {
|
|
||||||
String text = sqlxml.getString();
|
|
||||||
return boundedText(text);
|
|
||||||
}
|
|
||||||
if (value instanceof java.sql.Array array) {
|
|
||||||
return "[array " + array.getBaseTypeName() + "]";
|
|
||||||
}
|
|
||||||
if (value instanceof Struct struct) {
|
|
||||||
return "[struct " + struct.getSQLTypeName() + "]";
|
|
||||||
}
|
|
||||||
if (value instanceof CharSequence text) {
|
|
||||||
return boundedText(text.toString());
|
|
||||||
}
|
|
||||||
if (value instanceof BigInteger || value instanceof Long) {
|
|
||||||
return value.toString();
|
|
||||||
}
|
|
||||||
Package valuePackage = value.getClass().getPackage();
|
|
||||||
if (valuePackage != null && valuePackage.getName().startsWith("org.postgresql")) {
|
|
||||||
return boundedText(String.valueOf(value));
|
|
||||||
}
|
|
||||||
return value instanceof Number || value instanceof Boolean
|
|
||||||
|| value instanceof java.time.temporal.TemporalAccessor
|
|
||||||
|| value instanceof java.util.Date
|
|
||||||
? value : boundedText(String.valueOf(value));
|
|
||||||
} catch (SQLException exception) {
|
|
||||||
throw new BusinessException("查询结果包含无法读取的大对象");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 按 JDBC 类型选择有界流式读取或普通对象读取。
|
|
||||||
*
|
|
||||||
* @param cursor 当前结果游标
|
|
||||||
* @param column 列元数据
|
|
||||||
* @param columnIndex 从 1 开始的列序号
|
|
||||||
* @return 可安全序列化的有界值
|
|
||||||
*/
|
|
||||||
private Object readColumn(
|
|
||||||
FederationResultCursor cursor,
|
|
||||||
FederationColumn column,
|
|
||||||
int columnIndex) {
|
|
||||||
try {
|
|
||||||
if (isBinaryType(column.jdbcType())) {
|
|
||||||
try (InputStream stream = cursor.getBinaryStream(columnIndex)) {
|
|
||||||
return readBoundedBinary(stream);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (isCharacterType(column.jdbcType(), column.typeName())) {
|
|
||||||
try (Reader reader = cursor.getCharacterStream(columnIndex)) {
|
|
||||||
return readBoundedText(reader);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (UnsupportedOperationException ignored) {
|
|
||||||
// 第三方 Adapter 可暂不实现流式列读取;保留兼容的对象读取路径。
|
|
||||||
} catch (IOException exception) {
|
|
||||||
throw new BusinessException("查询结果流读取失败");
|
|
||||||
}
|
|
||||||
return normalizeValue(cursor.getObject(columnIndex));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 有界读取二进制流,小值编码为 Base64,大值返回摘要。
|
|
||||||
*
|
|
||||||
* @param stream 二进制流;可为空
|
|
||||||
* @return Base64 文本、大小摘要或 null
|
|
||||||
* @throws IOException 流读取失败
|
|
||||||
*/
|
|
||||||
private Object readBoundedBinary(InputStream stream) throws IOException {
|
|
||||||
if (stream == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
byte[] bytes = stream.readNBytes(MAX_TEXT_CELL_CHARS + 1);
|
|
||||||
if (bytes.length > MAX_TEXT_CELL_CHARS) {
|
|
||||||
return "[binary exceeds " + MAX_TEXT_CELL_CHARS + " bytes]";
|
|
||||||
}
|
|
||||||
return Base64.getEncoder().encodeToString(bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 有界读取字符流,超过单元格限制时附加截断标识。
|
|
||||||
*
|
|
||||||
* @param reader 字符流;可为空
|
|
||||||
* @return 有界文本或 null
|
|
||||||
* @throws IOException 流读取失败
|
|
||||||
*/
|
|
||||||
private String readBoundedText(Reader reader) throws IOException {
|
|
||||||
if (reader == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
StringBuilder text = new StringBuilder(Math.min(MAX_TEXT_CELL_CHARS, 4_096));
|
|
||||||
char[] buffer = new char[4_096];
|
|
||||||
while (text.length() <= MAX_TEXT_CELL_CHARS) {
|
|
||||||
int remaining = MAX_TEXT_CELL_CHARS + 1 - text.length();
|
|
||||||
int read = reader.read(buffer, 0, Math.min(buffer.length, remaining));
|
|
||||||
if (read < 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
text.append(buffer, 0, read);
|
|
||||||
}
|
|
||||||
if (text.length() > MAX_TEXT_CELL_CHARS) {
|
|
||||||
return text.substring(0, MAX_TEXT_CELL_CHARS) + "…[truncated]";
|
|
||||||
}
|
|
||||||
return text.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断 JDBC 类型是否应使用二进制流读取。
|
|
||||||
*
|
|
||||||
* @param jdbcType JDBC 类型
|
|
||||||
* @return 是否为二进制大字段类型
|
|
||||||
*/
|
|
||||||
private boolean isBinaryType(int jdbcType) {
|
|
||||||
return jdbcType == Types.BINARY
|
|
||||||
|| jdbcType == Types.VARBINARY
|
|
||||||
|| jdbcType == Types.LONGVARBINARY
|
|
||||||
|| jdbcType == Types.BLOB;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断 JDBC 类型或数据库类型名是否应使用字符流读取。
|
|
||||||
*
|
|
||||||
* @param jdbcType JDBC 类型
|
|
||||||
* @param typeName 数据库类型名
|
|
||||||
* @return 是否为字符大字段类型
|
|
||||||
*/
|
|
||||||
private boolean isCharacterType(int jdbcType, String typeName) {
|
|
||||||
if (jdbcType == Types.CHAR
|
|
||||||
|| jdbcType == Types.VARCHAR
|
|
||||||
|| jdbcType == Types.LONGVARCHAR
|
|
||||||
|| jdbcType == Types.NCHAR
|
|
||||||
|| jdbcType == Types.NVARCHAR
|
|
||||||
|| jdbcType == Types.LONGNVARCHAR
|
|
||||||
|| jdbcType == Types.CLOB
|
|
||||||
|| jdbcType == Types.NCLOB
|
|
||||||
|| jdbcType == Types.SQLXML) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
String normalized = typeName == null ? "" : typeName.toLowerCase(java.util.Locale.ROOT);
|
|
||||||
return normalized.equals("json") || normalized.equals("jsonb") || normalized.equals("xml");
|
|
||||||
}
|
|
||||||
|
|
||||||
private String boundedText(String value) {
|
|
||||||
if (value == null || value.length() <= MAX_TEXT_CELL_CHARS) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
return value.substring(0, MAX_TEXT_CELL_CHARS)
|
|
||||||
+ "…[truncated " + value.length() + " chars]";
|
|
||||||
}
|
|
||||||
|
|
||||||
private long estimateBytes(String label, Object value) {
|
|
||||||
long bytes = label == null ? 0L : (long) label.length() * Character.BYTES;
|
|
||||||
if (value == null) {
|
|
||||||
return bytes + 8L;
|
|
||||||
}
|
|
||||||
if (value instanceof CharSequence text) {
|
|
||||||
return bytes + (long) text.length() * Character.BYTES;
|
|
||||||
}
|
|
||||||
return bytes + 64L;
|
|
||||||
}
|
|
||||||
|
|
||||||
private long elapsedMillis(long startedNanos) {
|
|
||||||
return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String safeMessage(Throwable exception) {
|
|
||||||
String message = exception.getMessage();
|
|
||||||
if (message == null || message.isBlank()) {
|
|
||||||
return "数据查询失败";
|
|
||||||
}
|
|
||||||
return message.length() <= 500 ? message : message.substring(0, 500);
|
|
||||||
}
|
|
||||||
|
|
||||||
private BusinessException toBusinessException(RuntimeException exception) {
|
|
||||||
if (exception instanceof FederationSqlException federationException) {
|
|
||||||
int status = switch (federationException.errorCode()) {
|
|
||||||
case QUERY_ADMISSION_TIMEOUT, QUERY_TIMEOUT,
|
|
||||||
SQL_COMPILE_TIMEOUT -> 408;
|
|
||||||
case NODE_MEMORY_ADMISSION_TIMEOUT -> 429;
|
|
||||||
case SOURCE_NOT_FOUND, SOURCE_REMOVED -> 404;
|
|
||||||
case SOURCE_REVISION_NOT_READY, SOURCE_DEFINITION_CONFLICT -> 409;
|
|
||||||
case PLAN_STALE -> 409;
|
|
||||||
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> 422;
|
|
||||||
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
|
|
||||||
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED,
|
|
||||||
EXPLAIN_FAILED, SQL_COMPLETION_FAILED,
|
|
||||||
CONNECTION_ACQUISITION_TIMEOUT,
|
|
||||||
CONNECTION_ACQUISITION_FAILED -> 503;
|
|
||||||
default -> 400;
|
|
||||||
};
|
|
||||||
int code = switch (federationException.errorCode()) {
|
|
||||||
case INVALID_ARGUMENT -> 14101;
|
|
||||||
case SQL_PARSE_FAILED -> 14102;
|
|
||||||
case SQL_VALIDATION_FAILED -> 14103;
|
|
||||||
case SQL_NOT_READ_ONLY -> 14104;
|
|
||||||
case SQL_COMPILE_FAILED -> 14105;
|
|
||||||
case SQL_NOT_FULLY_PUSHDOWN -> 14106;
|
|
||||||
case CROSS_SOURCE_DISABLED -> 14107;
|
|
||||||
case CROSS_SOURCE_EXECUTION_UNSUPPORTED -> 14108;
|
|
||||||
case SOURCE_NOT_FOUND -> 14109;
|
|
||||||
case SOURCE_REMOVED -> 14110;
|
|
||||||
case SOURCE_REVISION_NOT_READY -> 14111;
|
|
||||||
case SOURCE_DEFINITION_CONFLICT -> 14112;
|
|
||||||
case SOURCE_INITIALIZATION_FAILED -> 14113;
|
|
||||||
case ADAPTER_NOT_FOUND -> 14114;
|
|
||||||
case ADAPTER_UNSUPPORTED -> 14115;
|
|
||||||
case PARAMETER_COUNT_MISMATCH -> 14116;
|
|
||||||
case QUERY_ADMISSION_TIMEOUT -> 14117;
|
|
||||||
case QUERY_CANCELLED -> 14118;
|
|
||||||
case QUERY_TIMEOUT -> 14119;
|
|
||||||
case EXECUTION_FAILED -> 14120;
|
|
||||||
case RESOURCE_CLOSE_FAILED -> 14121;
|
|
||||||
case ENGINE_CLOSED -> 14122;
|
|
||||||
case INVALID_QUERY_SCOPE -> 14124;
|
|
||||||
case FEDERATION_OPERATOR_UNSUPPORTED -> 14125;
|
|
||||||
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> 14126;
|
|
||||||
case PLAN_STALE -> 14127;
|
|
||||||
case EXPLAIN_FAILED -> 14128;
|
|
||||||
case SQL_COMPLETION_FAILED -> 14129;
|
|
||||||
case SQL_COMPILE_TIMEOUT -> 14130;
|
|
||||||
case NODE_MEMORY_ADMISSION_TIMEOUT -> 14131;
|
|
||||||
case CONNECTION_ACQUISITION_TIMEOUT -> 14132;
|
|
||||||
case CONNECTION_ACQUISITION_FAILED -> 14133;
|
|
||||||
};
|
|
||||||
String message = switch (federationException.errorCode()) {
|
|
||||||
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
|
|
||||||
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED -> "数据库查询暂时不可用";
|
|
||||||
case QUERY_TIMEOUT, QUERY_ADMISSION_TIMEOUT -> "数据库查询超时";
|
|
||||||
case QUERY_CANCELLED -> "查询已取消";
|
|
||||||
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> "联邦查询超过资源限制";
|
|
||||||
case EXPLAIN_FAILED -> "数据库查询计划分析失败";
|
|
||||||
case SQL_COMPLETION_FAILED -> "SQL 补全暂时不可用";
|
|
||||||
case SQL_COMPILE_TIMEOUT -> "SQL 编译超时,请简化语句后重试";
|
|
||||||
case NODE_MEMORY_ADMISSION_TIMEOUT -> "节点查询资源繁忙,请稍后重试";
|
|
||||||
case CONNECTION_ACQUISITION_TIMEOUT -> "数据库连接池繁忙,请稍后重试";
|
|
||||||
case CONNECTION_ACQUISITION_FAILED -> "无法获取数据库连接,请检查连接状态";
|
|
||||||
default -> safeMessage(exception);
|
|
||||||
};
|
|
||||||
return new BusinessException(status, code, message, exception);
|
|
||||||
}
|
|
||||||
return new BusinessException(500, 14999, "数据查询失败", exception);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void logUnexpectedFailure(
|
|
||||||
QueryId queryId,
|
|
||||||
DatacenterSource source,
|
|
||||||
RuntimeException exception) {
|
|
||||||
if (!(exception instanceof FederationSqlException federationException)
|
|
||||||
|| switch (federationException.errorCode()) {
|
|
||||||
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
|
|
||||||
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED,
|
|
||||||
EXPLAIN_FAILED, CONNECTION_ACQUISITION_FAILED -> true;
|
|
||||||
default -> false;
|
|
||||||
}) {
|
|
||||||
log.error("Datacenter query failed, queryId={}, sourceId={}",
|
|
||||||
queryId.value(), source == null ? null : source.getId(), exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,175 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider;
|
|
||||||
import com.easyagents.federation.sql.api.FederationSqlEngine;
|
|
||||||
import com.easyagents.federation.sql.api.FederationSqlEngines;
|
|
||||||
import com.easyagents.federation.sql.source.FederationSourceDefinition;
|
|
||||||
import com.easyagents.federation.sql.source.PreparedSourceRuntime;
|
|
||||||
import com.easyagents.federation.sql.source.SourceApplyOptions;
|
|
||||||
import com.easyagents.federation.sql.source.SourceApplyResult;
|
|
||||||
import com.easyagents.federation.sql.source.SourceApplyStatus;
|
|
||||||
import com.easyagents.federation.sql.source.SourceProbeResult;
|
|
||||||
import com.easyagents.federation.sql.source.SourceRemoveResult;
|
|
||||||
import com.easyagents.federation.sql.source.SourceTombstone;
|
|
||||||
import jakarta.annotation.PreDestroy;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* EasyFlow 数据中枢持有的单例 Federation SQL Engine。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationRuntime {
|
|
||||||
|
|
||||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
|
||||||
private final DatacenterFederationDataSourceResolver resolver;
|
|
||||||
private final FederationSqlEngine engine;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 组装 DataSource Resolver、数据库状态 Provider 与业务 Policy。
|
|
||||||
*
|
|
||||||
* @param definitionFactory Definition 工厂
|
|
||||||
* @param resolver HikariCP Resolver
|
|
||||||
* @param stateProvider 数据库权威状态 Provider
|
|
||||||
* @param policy 纳管范围 Policy
|
|
||||||
* @param admissionController 租户与数据源分层准入控制器
|
|
||||||
*/
|
|
||||||
public DatacenterFederationRuntime(
|
|
||||||
DatacenterFederationDefinitionFactory definitionFactory,
|
|
||||||
DatacenterFederationDataSourceResolver resolver,
|
|
||||||
DatacenterFederationSourceStateProvider stateProvider,
|
|
||||||
DatacenterFederationSqlPolicy policy,
|
|
||||||
DatacenterFederationAdmissionController admissionController) {
|
|
||||||
this.definitionFactory = definitionFactory;
|
|
||||||
this.resolver = resolver;
|
|
||||||
this.engine = FederationSqlEngines.builder()
|
|
||||||
.dataSourceResolver(resolver)
|
|
||||||
.stateProvider(stateProvider)
|
|
||||||
.adapter(new JdbcFederationSqlAdapterProvider())
|
|
||||||
.policy(policy)
|
|
||||||
.admissionController(admissionController)
|
|
||||||
.maximumPlanCacheEntries(512)
|
|
||||||
.maximumConcurrentCompilations(Math.max(1, Math.min(4,
|
|
||||||
Runtime.getRuntime().availableProcessors())))
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 探测草稿数据源,不保留临时连接池。
|
|
||||||
*
|
|
||||||
* @param source 数据源草稿
|
|
||||||
* @return 探测结果
|
|
||||||
*/
|
|
||||||
public SourceProbeResult probe(DatacenterSource source) {
|
|
||||||
return engine.sources().probe(definitionFactory.create(source));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用未发布候选配置执行短生命周期探测。
|
|
||||||
*
|
|
||||||
* @param source 候选数据源
|
|
||||||
* @return 探测结果
|
|
||||||
*/
|
|
||||||
public SourceProbeResult probeCandidate(DatacenterSource source) {
|
|
||||||
FederationSourceDefinition definition = definitionFactory.createCandidate(source);
|
|
||||||
return resolver.withCandidate(source, definition,
|
|
||||||
() -> engine.sources().probe(definition));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 预构建单命名空间候选 Runtime,且不修改共享 Source Slot。
|
|
||||||
*
|
|
||||||
* @param source 候选数据源
|
|
||||||
* @param catalogName 即将发布的 Catalog 或 Schema
|
|
||||||
* @return 由调用方暂时持有的候选 Runtime
|
|
||||||
*/
|
|
||||||
public PreparedSourceRuntime prepareCandidate(
|
|
||||||
DatacenterSource source,
|
|
||||||
String catalogName) {
|
|
||||||
FederationSourceDefinition definition =
|
|
||||||
definitionFactory.createCandidate(source, catalogName);
|
|
||||||
return resolver.withCandidate(source, definition,
|
|
||||||
() -> engine.sources().prepare(definition));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用当前持久化命名空间预构建候选 Runtime,且不修改共享 Source Slot。
|
|
||||||
*
|
|
||||||
* @param source 候选数据源
|
|
||||||
* @return 由调用方暂时持有的候选 Runtime
|
|
||||||
*/
|
|
||||||
public PreparedSourceRuntime prepareCandidate(DatacenterSource source) {
|
|
||||||
FederationSourceDefinition definition = definitionFactory.create(source);
|
|
||||||
return resolver.withCandidate(source, definition,
|
|
||||||
() -> engine.sources().prepare(definition));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 原子接管已经通过业务 revision CAS 的候选 Runtime。
|
|
||||||
*
|
|
||||||
* @param prepared 预构建 Runtime
|
|
||||||
* @return Definition 应用结果
|
|
||||||
* @throws IllegalStateException Slot 已被更高版本或冲突版本占用时抛出
|
|
||||||
*/
|
|
||||||
public SourceApplyResult commitPrepared(PreparedSourceRuntime prepared) {
|
|
||||||
SourceApplyResult result = engine.sources().commit(prepared);
|
|
||||||
if (result.status() == SourceApplyStatus.CONFLICT
|
|
||||||
|| result.status() == SourceApplyStatus.IGNORED_STALE) {
|
|
||||||
throw new IllegalStateException(
|
|
||||||
"prepared runtime was superseded before commit: " + result.status());
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 应用已持久化 Definition。
|
|
||||||
*
|
|
||||||
* @param source 数据源当前快照
|
|
||||||
* @param prewarm 是否立即预热
|
|
||||||
* @return 应用结果
|
|
||||||
*/
|
|
||||||
public SourceApplyResult apply(DatacenterSource source, boolean prewarm) {
|
|
||||||
FederationSourceDefinition definition = definitionFactory.create(source);
|
|
||||||
return engine.sources().apply(
|
|
||||||
definition,
|
|
||||||
prewarm ? SourceApplyOptions.prewarmNow() : SourceApplyOptions.lazy());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 应用禁用或删除墓碑。
|
|
||||||
*
|
|
||||||
* @param source 数据源当前快照
|
|
||||||
* @return 删除结果
|
|
||||||
*/
|
|
||||||
public SourceRemoveResult remove(DatacenterSource source) {
|
|
||||||
return engine.sources().remove(SourceTombstone.of(
|
|
||||||
definitionFactory.sourceId(source),
|
|
||||||
source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回统一查询 Engine。
|
|
||||||
*
|
|
||||||
* @return Engine
|
|
||||||
*/
|
|
||||||
public FederationSqlEngine engine() {
|
|
||||||
return engine;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回 Definition 工厂。
|
|
||||||
*
|
|
||||||
* @return Definition 工厂
|
|
||||||
*/
|
|
||||||
public DatacenterFederationDefinitionFactory definitions() {
|
|
||||||
return definitionFactory;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 应用停止时关闭查询、计划缓存和所有 Runtime 句柄。
|
|
||||||
*/
|
|
||||||
@PreDestroy
|
|
||||||
public void close() {
|
|
||||||
engine.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,177 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.source.ActiveSourceState;
|
|
||||||
import com.easyagents.federation.sql.source.FederationSourceState;
|
|
||||||
import com.easyagents.federation.sql.source.FederationSourceStateProvider;
|
|
||||||
import com.easyagents.federation.sql.source.SourceId;
|
|
||||||
import com.easyagents.federation.sql.source.SourceStateSubscription;
|
|
||||||
import com.easyagents.federation.sql.source.SourceTombstone;
|
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.concurrent.CopyOnWriteArrayList;
|
|
||||||
import java.util.function.Consumer;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据库权威、Redis 仅提示的 Federation Source State Provider。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationSourceStateProvider implements FederationSourceStateProvider {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationSourceStateProvider.class);
|
|
||||||
|
|
||||||
private final DatacenterSourceMapper sourceMapper;
|
|
||||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
|
||||||
private final CopyOnWriteArrayList<Consumer<FederationSourceState>> subscribers =
|
|
||||||
new CopyOnWriteArrayList<>();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建状态 Provider。
|
|
||||||
*
|
|
||||||
* @param sourceMapper 数据源 Mapper
|
|
||||||
* @param definitionFactory Definition 工厂
|
|
||||||
*/
|
|
||||||
public DatacenterFederationSourceStateProvider(
|
|
||||||
DatacenterSourceMapper sourceMapper,
|
|
||||||
DatacenterFederationDefinitionFactory definitionFactory) {
|
|
||||||
this.sourceMapper = sourceMapper;
|
|
||||||
this.definitionFactory = definitionFactory;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 从数据库读取指定数据源的最新 Definition 或墓碑。
|
|
||||||
*
|
|
||||||
* @param sourceId Federation SourceId
|
|
||||||
* @return 当前共享状态
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public Optional<FederationSourceState> find(SourceId sourceId) {
|
|
||||||
BigInteger recordId = parseRecordId(sourceId);
|
|
||||||
if (recordId == null) {
|
|
||||||
return Optional.empty();
|
|
||||||
}
|
|
||||||
return toState(sourceMapper.selectOneById(recordId));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 加载当前数据库上下文中可见的外部数据源快照。
|
|
||||||
*
|
|
||||||
* @return Definition 与墓碑集合
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public Collection<FederationSourceState> loadSnapshot() {
|
|
||||||
QueryWrapper query = QueryWrapper.create()
|
|
||||||
.in(DatacenterSource::getSourceType, List.of(
|
|
||||||
DatacenterSourceType.MYSQL.name(),
|
|
||||||
DatacenterSourceType.POSTGRESQL.name()))
|
|
||||||
.in(DatacenterSource::getStatus, List.of(
|
|
||||||
DatacenterSourceStatus.DRAFT.code(),
|
|
||||||
DatacenterSourceStatus.READY.code(),
|
|
||||||
DatacenterSourceStatus.DEGRADED.code(),
|
|
||||||
DatacenterSourceStatus.DISABLED.code(),
|
|
||||||
DatacenterSourceStatus.DELETED.code()));
|
|
||||||
List<DatacenterSource> sources = sourceMapper.selectListByQuery(query);
|
|
||||||
Map<BigInteger, com.easyagents.federation.sql.source.FederationSourceDefinition> definitions =
|
|
||||||
definitionFactory.createBatch(sources.stream()
|
|
||||||
.filter(this::isActive)
|
|
||||||
.toList());
|
|
||||||
return sources.stream()
|
|
||||||
.map(source -> toState(source, definitions.get(source.getId())))
|
|
||||||
.flatMap(Optional::stream)
|
|
||||||
.toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 注册节点本地变更消费者。
|
|
||||||
*
|
|
||||||
* @param consumer 状态消费者
|
|
||||||
* @return 可关闭订阅
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public SourceStateSubscription subscribe(Consumer<FederationSourceState> consumer) {
|
|
||||||
subscribers.add(consumer);
|
|
||||||
return () -> subscribers.remove(consumer);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 响应无凭据 Redis 提示,从数据库重读并发布最新状态。
|
|
||||||
*
|
|
||||||
* @param sourceId Federation SourceId
|
|
||||||
*/
|
|
||||||
public void refresh(SourceId sourceId) {
|
|
||||||
find(sourceId).ifPresent(state -> subscribers.forEach(consumer -> consumer.accept(state)));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 周期性回源数据库,弥补 Redis Pub/Sub 在节点离线或网络抖动时可能丢失的提示。
|
|
||||||
*/
|
|
||||||
@Scheduled(fixedDelayString = "${easyflow.datacenter.federation-reconcile-ms:30000}")
|
|
||||||
public void reconcile() {
|
|
||||||
try {
|
|
||||||
loadSnapshot().forEach(state -> subscribers.forEach(consumer -> consumer.accept(state)));
|
|
||||||
} catch (RuntimeException exception) {
|
|
||||||
log.warn("Failed to reconcile datacenter Federation source states", exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Optional<FederationSourceState> toState(DatacenterSource source) {
|
|
||||||
return toState(source, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Optional<FederationSourceState> toState(
|
|
||||||
DatacenterSource source,
|
|
||||||
com.easyagents.federation.sql.source.FederationSourceDefinition preloadedDefinition) {
|
|
||||||
if (source == null || source.getId() == null) {
|
|
||||||
return Optional.empty();
|
|
||||||
}
|
|
||||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
|
|
||||||
long revision = source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision();
|
|
||||||
if (status == DatacenterSourceStatus.READY || status == DatacenterSourceStatus.DEGRADED) {
|
|
||||||
if (revision <= 0L) {
|
|
||||||
return Optional.empty();
|
|
||||||
}
|
|
||||||
return Optional.of(ActiveSourceState.of(preloadedDefinition == null
|
|
||||||
? definitionFactory.create(source) : preloadedDefinition));
|
|
||||||
}
|
|
||||||
if ((status == DatacenterSourceStatus.DRAFT && revision > 0L)
|
|
||||||
|| status == DatacenterSourceStatus.DISABLED
|
|
||||||
|| status == DatacenterSourceStatus.DELETED) {
|
|
||||||
return Optional.of(SourceTombstone.of(definitionFactory.sourceId(source), revision));
|
|
||||||
}
|
|
||||||
return Optional.empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isActive(DatacenterSource source) {
|
|
||||||
DatacenterSourceStatus status = source == null
|
|
||||||
? DatacenterSourceStatus.DRAFT
|
|
||||||
: DatacenterSourceStatus.fromCode(source.getStatus());
|
|
||||||
return status == DatacenterSourceStatus.READY || status == DatacenterSourceStatus.DEGRADED;
|
|
||||||
}
|
|
||||||
|
|
||||||
private BigInteger parseRecordId(SourceId sourceId) {
|
|
||||||
if (sourceId == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
String value = sourceId.value();
|
|
||||||
int marker = value.lastIndexOf("-source-");
|
|
||||||
if (marker < 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return new BigInteger(value.substring(marker + "-source-".length()));
|
|
||||||
} catch (NumberFormatException ignored) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,859 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
|
|
||||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
|
||||||
import com.easyagents.federation.sql.compile.FederationSqlPolicy;
|
|
||||||
import com.easyagents.federation.sql.compile.SqlPolicyContext;
|
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.ArrayDeque;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.Deque;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.IdentityHashMap;
|
|
||||||
import java.util.LinkedHashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Locale;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
|
||||||
import org.apache.calcite.rel.RelNode;
|
|
||||||
import org.apache.calcite.rel.RelVisitor;
|
|
||||||
import org.apache.calcite.rel.core.TableScan;
|
|
||||||
import org.apache.calcite.sql.SqlIdentifier;
|
|
||||||
import org.apache.calcite.sql.SqlCall;
|
|
||||||
import org.apache.calcite.sql.SqlKind;
|
|
||||||
import org.apache.calcite.sql.SqlJoin;
|
|
||||||
import org.apache.calcite.sql.SqlNode;
|
|
||||||
import org.apache.calcite.sql.SqlNodeList;
|
|
||||||
import org.apache.calcite.sql.SqlOrderBy;
|
|
||||||
import org.apache.calcite.sql.SqlSelect;
|
|
||||||
import org.apache.calcite.sql.SqlWith;
|
|
||||||
import org.apache.calcite.sql.SqlWithItem;
|
|
||||||
import org.apache.calcite.sql.util.SqlBasicVisitor;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
|
|
||||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
|
||||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 将 Calcite 解析出的表与字段引用限制在当前数据源的可查询纳管范围内。
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class DatacenterFederationSqlPolicy implements FederationSqlPolicy {
|
|
||||||
|
|
||||||
private static final int MAX_POLICY_CACHE_ENTRIES = 512;
|
|
||||||
|
|
||||||
private final DatacenterTableMapper tableMapper;
|
|
||||||
private final DatacenterTableFieldMapper tableFieldMapper;
|
|
||||||
private final DatacenterCatalogMapper catalogMapper;
|
|
||||||
private final DatacenterSourceMapper sourceMapper;
|
|
||||||
private final Map<PolicyCacheKey, PolicySnapshot> policyCache = new ConcurrentHashMap<>();
|
|
||||||
private final ConcurrentLinkedQueue<PolicyCacheKey> policyCacheOrder = new ConcurrentLinkedQueue<>();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建业务 SQL Policy。
|
|
||||||
*
|
|
||||||
* @param tableMapper 表 Mapper
|
|
||||||
* @param tableFieldMapper 字段 Mapper
|
|
||||||
* @param catalogMapper 命名空间 Mapper
|
|
||||||
* @param sourceMapper 数据源 Mapper
|
|
||||||
*/
|
|
||||||
public DatacenterFederationSqlPolicy(
|
|
||||||
DatacenterTableMapper tableMapper,
|
|
||||||
DatacenterTableFieldMapper tableFieldMapper,
|
|
||||||
DatacenterCatalogMapper catalogMapper,
|
|
||||||
DatacenterSourceMapper sourceMapper) {
|
|
||||||
this.tableMapper = tableMapper;
|
|
||||||
this.tableFieldMapper = tableFieldMapper;
|
|
||||||
this.catalogMapper = catalogMapper;
|
|
||||||
this.sourceMapper = sourceMapper;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回 Policy 代码版本。
|
|
||||||
*
|
|
||||||
* @return 稳定版本
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public String version() {
|
|
||||||
return "xl01-wp1-v6";
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 拒绝未纳管表、不可查询字段和任何尚未支持脱敏的敏感字段。
|
|
||||||
*
|
|
||||||
* @param context Calcite 策略上下文
|
|
||||||
*/
|
|
||||||
@Override
|
|
||||||
public void validate(SqlPolicyContext context) {
|
|
||||||
if (context.request().queryScope().bindings().size() != 1
|
|
||||||
|| context.referencedSources().size() != 1
|
|
||||||
|| !context.referencedSources().contains(
|
|
||||||
context.request().sourceId())) {
|
|
||||||
throw validationFailure("当前数据中枢尚未开放跨数据源授权范围");
|
|
||||||
}
|
|
||||||
BigInteger sourceId = parseRecordId(context.request().sourceId().value());
|
|
||||||
requireCurrentScopeRevision(
|
|
||||||
sourceId, context.request().policyVersion());
|
|
||||||
PolicyCacheKey cacheKey = new PolicyCacheKey(sourceId, context.request().policyVersion());
|
|
||||||
PolicySnapshot snapshot = cachedSnapshot(cacheKey);
|
|
||||||
Set<String> referencedNames = referencedTables(context.relRoot().rel);
|
|
||||||
Set<BigInteger> referencedTableIds = new LinkedHashSet<>();
|
|
||||||
for (String referencedName : referencedNames) {
|
|
||||||
Set<BigInteger> tableIds = snapshot.tableIdsByName().get(referencedName);
|
|
||||||
if (tableIds == null || tableIds.isEmpty()) {
|
|
||||||
throw validationFailure("SQL 引用了未纳管表: " + referencedName);
|
|
||||||
}
|
|
||||||
referencedTableIds.addAll(tableIds);
|
|
||||||
}
|
|
||||||
validatePhysicalColumns(context.relRoot().rel, snapshot);
|
|
||||||
validateFields(context, snapshot, referencedTableIds);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 直接核对数据库权威范围版本,避免跨节点通知延迟复用旧策略。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源记录 ID
|
|
||||||
* @param policyVersion 编译请求携带的范围版本
|
|
||||||
*/
|
|
||||||
private void requireCurrentScopeRevision(
|
|
||||||
BigInteger sourceId,
|
|
||||||
String policyVersion) {
|
|
||||||
long requestedRevision = parseScopeRevision(policyVersion);
|
|
||||||
DatacenterSource current = sourceMapper.selectOneById(sourceId);
|
|
||||||
if (current == null
|
|
||||||
|| DatacenterSourceStatus.fromCode(current.getStatus())
|
|
||||||
!= DatacenterSourceStatus.READY
|
|
||||||
|| current.getScopeRevision() == null
|
|
||||||
|| current.getScopeRevision() != requestedRevision) {
|
|
||||||
throw validationFailure("数据范围已变化,请重新执行查询");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 解析范围策略版本。
|
|
||||||
*
|
|
||||||
* @param policyVersion 策略版本
|
|
||||||
* @return 范围 revision
|
|
||||||
*/
|
|
||||||
private long parseScopeRevision(String policyVersion) {
|
|
||||||
if (policyVersion == null || !policyVersion.startsWith("scope-")) {
|
|
||||||
throw validationFailure("数据范围版本无效");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return Long.parseLong(policyVersion.substring("scope-".length()));
|
|
||||||
} catch (NumberFormatException exception) {
|
|
||||||
throw validationFailure("数据范围版本无效");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private PolicySnapshot cachedSnapshot(PolicyCacheKey cacheKey) {
|
|
||||||
PolicySnapshot existing = policyCache.get(cacheKey);
|
|
||||||
if (existing != null) {
|
|
||||||
return existing;
|
|
||||||
}
|
|
||||||
PolicySnapshot loaded = loadSnapshot(cacheKey.sourceId());
|
|
||||||
PolicySnapshot raced = policyCache.putIfAbsent(cacheKey, loaded);
|
|
||||||
if (raced != null) {
|
|
||||||
return raced;
|
|
||||||
}
|
|
||||||
policyCacheOrder.add(cacheKey);
|
|
||||||
trimCache();
|
|
||||||
return loaded;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void trimCache() {
|
|
||||||
while (policyCache.size() > MAX_POLICY_CACHE_ENTRIES) {
|
|
||||||
PolicyCacheKey eldest = policyCacheOrder.poll();
|
|
||||||
if (eldest == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
policyCache.remove(eldest);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private PolicySnapshot loadSnapshot(BigInteger sourceId) {
|
|
||||||
QueryWrapper tableQuery = QueryWrapper.create()
|
|
||||||
.eq(DatacenterTable::getSourceId, sourceId)
|
|
||||||
.eq(DatacenterTable::getQueryable, 1)
|
|
||||||
.eq(DatacenterTable::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name());
|
|
||||||
List<DatacenterTable> tables = tableMapper.selectListByQuery(tableQuery);
|
|
||||||
if (tables.isEmpty()) {
|
|
||||||
return PolicySnapshot.empty();
|
|
||||||
}
|
|
||||||
Set<BigInteger> catalogIds = new HashSet<>();
|
|
||||||
Set<BigInteger> tableIds = new HashSet<>();
|
|
||||||
for (DatacenterTable table : tables) {
|
|
||||||
tableIds.add(table.getId());
|
|
||||||
if (table.getCatalogId() != null) {
|
|
||||||
catalogIds.add(table.getCatalogId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Map<BigInteger, String> logicalSchemas = new HashMap<>();
|
|
||||||
if (!catalogIds.isEmpty()) {
|
|
||||||
QueryWrapper catalogQuery = QueryWrapper.create().in(DatacenterCatalog::getId, catalogIds);
|
|
||||||
for (DatacenterCatalog catalog : catalogMapper.selectListByQuery(catalogQuery)) {
|
|
||||||
logicalSchemas.put(catalog.getId(), catalog.getLogicalSchemaName());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Map<String, Set<BigInteger>> idsByName = new HashMap<>();
|
|
||||||
Set<String> tableReferenceNames = new HashSet<>();
|
|
||||||
for (DatacenterTable table : tables) {
|
|
||||||
addTableName(idsByName, tableReferenceNames, table.getTableName(), table.getId());
|
|
||||||
addTableName(idsByName, tableReferenceNames, table.getActualTable(), table.getId());
|
|
||||||
String schema = logicalSchemas.get(table.getCatalogId());
|
|
||||||
if (schema != null && !schema.isBlank()) {
|
|
||||||
addTableName(idsByName, tableReferenceNames, schema + "." + table.getTableName(), table.getId());
|
|
||||||
addTableName(idsByName, tableReferenceNames, schema + "." + table.getActualTable(), table.getId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<BigInteger, ColumnPolicy> columnsByTable = new HashMap<>();
|
|
||||||
QueryWrapper fieldQuery = QueryWrapper.create().in(DatacenterTableField::getTableId, tableIds);
|
|
||||||
for (DatacenterTableField field : tableFieldMapper.selectListByQuery(fieldQuery)) {
|
|
||||||
ColumnPolicy columns = columnsByTable.computeIfAbsent(
|
|
||||||
field.getTableId(), ignored -> ColumnPolicy.mutable());
|
|
||||||
String columnName = sourceColumnName(field);
|
|
||||||
if (columnName != null && !columnName.isBlank()) {
|
|
||||||
columns.knownExact().add(columnName);
|
|
||||||
columns.knownFolded().add(columnName.toUpperCase(Locale.ROOT));
|
|
||||||
if (fieldAllowed(field)) {
|
|
||||||
columns.allowedExact().add(columnName);
|
|
||||||
columns.allowedFolded().add(columnName.toUpperCase(Locale.ROOT));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Map<String, Set<BigInteger>> immutableNames = new HashMap<>();
|
|
||||||
idsByName.forEach((name, ids) -> immutableNames.put(name, Set.copyOf(ids)));
|
|
||||||
return new PolicySnapshot(
|
|
||||||
Map.copyOf(immutableNames),
|
|
||||||
immutableColumnPolicies(columnsByTable),
|
|
||||||
Set.copyOf(tableReferenceNames));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<BigInteger, ColumnPolicy> immutableColumnPolicies(
|
|
||||||
Map<BigInteger, ColumnPolicy> mutablePolicies) {
|
|
||||||
Map<BigInteger, ColumnPolicy> immutable = new HashMap<>();
|
|
||||||
mutablePolicies.forEach((tableId, policy) -> immutable.put(tableId, new ColumnPolicy(
|
|
||||||
Set.copyOf(policy.knownExact()),
|
|
||||||
Set.copyOf(policy.knownFolded()),
|
|
||||||
Set.copyOf(policy.allowedExact()),
|
|
||||||
Set.copyOf(policy.allowedFolded()))));
|
|
||||||
return Map.copyOf(immutable);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void validatePhysicalColumns(RelNode root, PolicySnapshot snapshot) {
|
|
||||||
new RelVisitor() {
|
|
||||||
@Override
|
|
||||||
public void visit(RelNode node, int ordinal, RelNode parent) {
|
|
||||||
if (node instanceof TableScan scan) {
|
|
||||||
String referencedName = referencedTableName(scan);
|
|
||||||
Set<BigInteger> tableIds = snapshot.tableIdsByName().get(referencedName);
|
|
||||||
if (tableIds == null || tableIds.isEmpty()) {
|
|
||||||
throw validationFailure("SQL 引用了未纳管表: " + referencedName);
|
|
||||||
}
|
|
||||||
for (BigInteger tableId : tableIds) {
|
|
||||||
ColumnPolicy policy = snapshot.columnsByTable().get(tableId);
|
|
||||||
if (policy == null || policy.knownExact().isEmpty()) {
|
|
||||||
throw validationFailure("数据表尚未纳管任何字段: " + referencedName);
|
|
||||||
}
|
|
||||||
for (String fieldName : scan.getRowType().getFieldNames()) {
|
|
||||||
if (!policy.knownExact().contains(fieldName)
|
|
||||||
&& !policy.knownFolded().contains(fieldName.toUpperCase(Locale.ROOT))) {
|
|
||||||
throw validationFailure("物理表包含尚未纳管的字段: " + fieldName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
super.visit(node, ordinal, parent);
|
|
||||||
}
|
|
||||||
}.go(root);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void validateFields(
|
|
||||||
SqlPolicyContext context,
|
|
||||||
PolicySnapshot snapshot,
|
|
||||||
Set<BigInteger> referencedTableIds) {
|
|
||||||
Deque<AliasContext> scopes = new ArrayDeque<>();
|
|
||||||
Map<String, DerivedRelation> commonTables = collectCommonTables(
|
|
||||||
context.validatedSql(), snapshot, referencedTableIds);
|
|
||||||
context.validatedSql().accept(new SqlBasicVisitor<Void>() {
|
|
||||||
@Override
|
|
||||||
public Void visit(SqlCall call) {
|
|
||||||
AliasContext scope = scopeFor(
|
|
||||||
call, snapshot, referencedTableIds, commonTables);
|
|
||||||
if (scope == null) {
|
|
||||||
return super.visit(call);
|
|
||||||
}
|
|
||||||
scopes.push(scope);
|
|
||||||
try {
|
|
||||||
return super.visit(call);
|
|
||||||
} finally {
|
|
||||||
scopes.pop();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Void visit(SqlIdentifier identifier) {
|
|
||||||
if (scopes.isEmpty()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
AliasContext aliasContext = scopes.peek();
|
|
||||||
if (aliasContext.syntaxAliases().contains(identifier)
|
|
||||||
|| aliasContext.tableIdentifiers().contains(identifier)
|
|
||||||
|| aliasContext.outputAliasReferences().contains(identifier)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (identifier.isStar()) {
|
|
||||||
boolean allAllowed = referencedTableIds.stream()
|
|
||||||
.map(snapshot.columnsByTable()::get)
|
|
||||||
.allMatch(policy -> policy != null
|
|
||||||
&& policy.knownExact().equals(policy.allowedExact()));
|
|
||||||
if (!allAllowed) {
|
|
||||||
throw validationFailure("查询范围包含不可查询或敏感字段,请显式选择允许字段");
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
List<String> names = identifier.names;
|
|
||||||
if (names.isEmpty()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
int last = names.size() - 1;
|
|
||||||
String columnName = names.get(last);
|
|
||||||
boolean quoted = identifier.isComponentQuoted(last);
|
|
||||||
if (derivedColumnReference(identifier, aliasContext)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
Set<BigInteger> targetTableIds = resolveColumnTables(
|
|
||||||
identifier, snapshot, referencedTableIds, aliasContext);
|
|
||||||
boolean allowed = !targetTableIds.isEmpty() && targetTableIds.stream()
|
|
||||||
.allMatch(tableId -> columnAllowed(
|
|
||||||
snapshot.columnsByTable().get(tableId), columnName, quoted));
|
|
||||||
if (!allowed) {
|
|
||||||
throw validationFailure("SQL 引用了不可查询或敏感字段: " + columnName);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private AliasContext scopeFor(
|
|
||||||
SqlCall call,
|
|
||||||
PolicySnapshot snapshot,
|
|
||||||
Set<BigInteger> referencedTableIds,
|
|
||||||
Map<String, DerivedRelation> commonTables) {
|
|
||||||
if (call instanceof SqlOrderBy orderBy
|
|
||||||
&& orderBy.query instanceof SqlSelect select) {
|
|
||||||
return collectAliases(
|
|
||||||
select, orderBy.orderList, snapshot,
|
|
||||||
referencedTableIds, commonTables);
|
|
||||||
}
|
|
||||||
if (call instanceof SqlSelect select) {
|
|
||||||
return collectAliases(
|
|
||||||
select, null, snapshot, referencedTableIds, commonTables);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean derivedColumnReference(
|
|
||||||
SqlIdentifier identifier,
|
|
||||||
AliasContext aliasContext) {
|
|
||||||
List<String> names = identifier.names;
|
|
||||||
int columnIndex = names.size() - 1;
|
|
||||||
String columnKey = identifierKey(
|
|
||||||
names.get(columnIndex),
|
|
||||||
identifier.isComponentQuoted(columnIndex));
|
|
||||||
if (names.size() == 1) {
|
|
||||||
return aliasContext.unqualifiedDerivedColumns().contains(columnKey);
|
|
||||||
}
|
|
||||||
int qualifierIndex = names.size() - 2;
|
|
||||||
String qualifierKey = identifierKey(
|
|
||||||
names.get(qualifierIndex),
|
|
||||||
identifier.isComponentQuoted(qualifierIndex));
|
|
||||||
Set<String> derivedColumns = aliasContext
|
|
||||||
.derivedColumnsByAlias().get(qualifierKey);
|
|
||||||
return derivedColumns != null && derivedColumns.contains(columnKey);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<BigInteger> resolveColumnTables(
|
|
||||||
SqlIdentifier identifier,
|
|
||||||
PolicySnapshot snapshot,
|
|
||||||
Set<BigInteger> referencedTableIds,
|
|
||||||
AliasContext aliasContext) {
|
|
||||||
List<String> identifierNames = identifier.names;
|
|
||||||
if (identifierNames.size() >= 2) {
|
|
||||||
int qualifierIndex = identifierNames.size() - 2;
|
|
||||||
String qualifier = identifierNames.get(qualifierIndex);
|
|
||||||
Set<BigInteger> aliased = aliasContext.tableIdsByAlias().get(
|
|
||||||
identifierKey(qualifier, identifier.isComponentQuoted(qualifierIndex)));
|
|
||||||
if (aliased != null && !aliased.isEmpty()) {
|
|
||||||
return aliased;
|
|
||||||
}
|
|
||||||
Set<BigInteger> qualified = snapshot.tableIdsByName().get(qualifier);
|
|
||||||
if (qualified != null && !qualified.isEmpty()) {
|
|
||||||
Set<BigInteger> referencedQualified = new LinkedHashSet<>(qualified);
|
|
||||||
referencedQualified.retainAll(referencedTableIds);
|
|
||||||
if (!referencedQualified.isEmpty()) {
|
|
||||||
return referencedQualified;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (referencedTableIds.size() == 1) {
|
|
||||||
return referencedTableIds;
|
|
||||||
}
|
|
||||||
// 无法将别名还原到单表时采用交集语义,避免多表同名列扩大权限。
|
|
||||||
return referencedTableIds;
|
|
||||||
}
|
|
||||||
|
|
||||||
private AliasContext collectAliases(
|
|
||||||
SqlSelect select,
|
|
||||||
SqlNodeList externalOrderList,
|
|
||||||
PolicySnapshot snapshot,
|
|
||||||
Set<BigInteger> referencedTableIds,
|
|
||||||
Map<String, DerivedRelation> commonTables) {
|
|
||||||
Map<String, Set<BigInteger>> tableIdsByAlias = new HashMap<>();
|
|
||||||
Map<String, Set<String>> derivedColumnsByAlias = new HashMap<>();
|
|
||||||
Set<String> outputAliases = new HashSet<>();
|
|
||||||
Set<SqlIdentifier> syntaxAliases = Collections.newSetFromMap(
|
|
||||||
new IdentityHashMap<>());
|
|
||||||
Set<SqlIdentifier> tableIdentifiers = Collections.newSetFromMap(
|
|
||||||
new IdentityHashMap<>());
|
|
||||||
Set<SqlIdentifier> outputAliasReferences = Collections.newSetFromMap(
|
|
||||||
new IdentityHashMap<>());
|
|
||||||
collectOutputAliases(
|
|
||||||
select, outputAliases, syntaxAliases, outputAliasReferences);
|
|
||||||
FromSummary fromSummary = collectFrom(
|
|
||||||
select.getFrom(), snapshot, referencedTableIds,
|
|
||||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
|
||||||
syntaxAliases, tableIdentifiers);
|
|
||||||
markOutputAliasReferences(
|
|
||||||
externalOrderList, outputAliases, outputAliasReferences);
|
|
||||||
return new AliasContext(
|
|
||||||
Map.copyOf(tableIdsByAlias),
|
|
||||||
Map.copyOf(derivedColumnsByAlias),
|
|
||||||
fromSummary.singleDerived()
|
|
||||||
? fromSummary.outputColumns() : Set.of(),
|
|
||||||
Set.copyOf(outputAliases),
|
|
||||||
syntaxAliases,
|
|
||||||
tableIdentifiers,
|
|
||||||
outputAliasReferences);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void collectOutputAliases(
|
|
||||||
SqlSelect select,
|
|
||||||
Set<String> outputAliases,
|
|
||||||
Set<SqlIdentifier> syntaxAliases,
|
|
||||||
Set<SqlIdentifier> outputAliasReferences) {
|
|
||||||
if (select.getSelectList() == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
for (SqlNode item : select.getSelectList()) {
|
|
||||||
if (!(item instanceof SqlCall call)
|
|
||||||
|| call.getKind() != SqlKind.AS
|
|
||||||
|| call.operandCount() < 2
|
|
||||||
|| !(call.operand(1) instanceof SqlIdentifier alias)
|
|
||||||
|| alias.names.isEmpty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
int aliasIndex = alias.names.size() - 1;
|
|
||||||
outputAliases.add(identifierKey(
|
|
||||||
alias.names.get(aliasIndex),
|
|
||||||
alias.isComponentQuoted(aliasIndex)));
|
|
||||||
syntaxAliases.add(alias);
|
|
||||||
}
|
|
||||||
markOutputAliasReferences(
|
|
||||||
select.getOrderList(), outputAliases, outputAliasReferences);
|
|
||||||
markOutputAliasReferences(
|
|
||||||
select.getGroup(), outputAliases, outputAliasReferences);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void markOutputAliasReferences(
|
|
||||||
SqlNode node,
|
|
||||||
Set<String> outputAliases,
|
|
||||||
Set<SqlIdentifier> outputAliasReferences) {
|
|
||||||
if (node == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (node instanceof SqlNodeList nodes) {
|
|
||||||
for (SqlNode item : nodes) {
|
|
||||||
markOutputAliasReferences(item, outputAliases, outputAliasReferences);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (node instanceof SqlIdentifier identifier && identifier.names.size() == 1) {
|
|
||||||
if (outputAliases.contains(identifierKey(
|
|
||||||
identifier.names.get(0), identifier.isComponentQuoted(0)))) {
|
|
||||||
outputAliasReferences.add(identifier);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (node instanceof SqlCall call
|
|
||||||
&& (call.getKind() == SqlKind.DESCENDING
|
|
||||||
|| call.getKind() == SqlKind.NULLS_FIRST
|
|
||||||
|| call.getKind() == SqlKind.NULLS_LAST)
|
|
||||||
&& call.operandCount() > 0) {
|
|
||||||
markOutputAliasReferences(
|
|
||||||
call.operand(0), outputAliases, outputAliasReferences);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private FromSummary collectFrom(
|
|
||||||
SqlNode from,
|
|
||||||
PolicySnapshot snapshot,
|
|
||||||
Set<BigInteger> referencedTableIds,
|
|
||||||
Map<String, DerivedRelation> commonTables,
|
|
||||||
Map<String, Set<BigInteger>> tableIdsByAlias,
|
|
||||||
Map<String, Set<String>> derivedColumnsByAlias,
|
|
||||||
Set<SqlIdentifier> syntaxAliases,
|
|
||||||
Set<SqlIdentifier> tableIdentifiers) {
|
|
||||||
if (from == null) {
|
|
||||||
return FromSummary.empty();
|
|
||||||
}
|
|
||||||
if (from instanceof SqlIdentifier identifier) {
|
|
||||||
tableIdentifiers.add(identifier);
|
|
||||||
String relationKey = identifierKey(
|
|
||||||
identifier.names.get(identifier.names.size() - 1),
|
|
||||||
identifier.isComponentQuoted(identifier.names.size() - 1));
|
|
||||||
DerivedRelation commonTable = commonTables.get(relationKey);
|
|
||||||
if (commonTable != null) {
|
|
||||||
tableIdsByAlias.put(relationKey, commonTable.tableIds());
|
|
||||||
derivedColumnsByAlias.put(
|
|
||||||
relationKey, commonTable.outputColumns());
|
|
||||||
return new FromSummary(
|
|
||||||
commonTable.tableIds(), commonTable.outputColumns(), true);
|
|
||||||
}
|
|
||||||
return new FromSummary(
|
|
||||||
resolveReferencedTableIds(
|
|
||||||
identifier, snapshot, referencedTableIds),
|
|
||||||
Set.of(),
|
|
||||||
false);
|
|
||||||
}
|
|
||||||
if (!(from instanceof SqlCall call)) {
|
|
||||||
return FromSummary.empty();
|
|
||||||
}
|
|
||||||
if (call instanceof SqlSelect nestedSelect) {
|
|
||||||
FromSummary nested = collectFrom(
|
|
||||||
nestedSelect.getFrom(), snapshot, referencedTableIds,
|
|
||||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
|
||||||
syntaxAliases, tableIdentifiers);
|
|
||||||
return new FromSummary(
|
|
||||||
nested.tableIds(), outputColumnKeys(nestedSelect), true);
|
|
||||||
}
|
|
||||||
if (call instanceof SqlJoin join) {
|
|
||||||
FromSummary left = collectFrom(
|
|
||||||
join.getLeft(), snapshot, referencedTableIds,
|
|
||||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
|
||||||
syntaxAliases, tableIdentifiers);
|
|
||||||
FromSummary right = collectFrom(
|
|
||||||
join.getRight(), snapshot, referencedTableIds,
|
|
||||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
|
||||||
syntaxAliases, tableIdentifiers);
|
|
||||||
Set<BigInteger> tableIds = new LinkedHashSet<>(left.tableIds());
|
|
||||||
tableIds.addAll(right.tableIds());
|
|
||||||
return new FromSummary(Set.copyOf(tableIds), Set.of(), false);
|
|
||||||
}
|
|
||||||
if (call.getKind() == SqlKind.AS && call.operandCount() >= 2) {
|
|
||||||
SqlNode relationNode = call.operand(0);
|
|
||||||
FromSummary relation = collectFrom(
|
|
||||||
relationNode, snapshot, referencedTableIds,
|
|
||||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
|
||||||
syntaxAliases, tableIdentifiers);
|
|
||||||
SqlNode aliasNode = call.operand(1);
|
|
||||||
if (aliasNode instanceof SqlIdentifier alias && !alias.names.isEmpty()) {
|
|
||||||
int aliasIndex = alias.names.size() - 1;
|
|
||||||
String aliasKey = identifierKey(
|
|
||||||
alias.names.get(aliasIndex),
|
|
||||||
alias.isComponentQuoted(aliasIndex));
|
|
||||||
syntaxAliases.add(alias);
|
|
||||||
if (!relation.tableIds().isEmpty()) {
|
|
||||||
tableIdsByAlias.merge(aliasKey, relation.tableIds(), (left, right) -> {
|
|
||||||
Set<BigInteger> merged = new LinkedHashSet<>(left);
|
|
||||||
merged.addAll(right);
|
|
||||||
return Set.copyOf(merged);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (relation.singleDerived()
|
|
||||||
&& !relation.outputColumns().isEmpty()) {
|
|
||||||
derivedColumnsByAlias.put(
|
|
||||||
aliasKey, relation.outputColumns());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return relation;
|
|
||||||
}
|
|
||||||
if (call instanceof SqlOrderBy orderBy) {
|
|
||||||
return collectFrom(
|
|
||||||
orderBy.query, snapshot, referencedTableIds,
|
|
||||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
|
||||||
syntaxAliases, tableIdentifiers);
|
|
||||||
}
|
|
||||||
if (call instanceof SqlWith sqlWith) {
|
|
||||||
return collectFrom(
|
|
||||||
sqlWith.body, snapshot, referencedTableIds,
|
|
||||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
|
||||||
syntaxAliases, tableIdentifiers);
|
|
||||||
}
|
|
||||||
return FromSummary.empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, DerivedRelation> collectCommonTables(
|
|
||||||
SqlNode root,
|
|
||||||
PolicySnapshot snapshot,
|
|
||||||
Set<BigInteger> referencedTableIds) {
|
|
||||||
Map<String, DerivedRelation> commonTables = new HashMap<>();
|
|
||||||
collectCommonTables(
|
|
||||||
root, snapshot, referencedTableIds, commonTables);
|
|
||||||
return Map.copyOf(commonTables);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void collectCommonTables(
|
|
||||||
SqlNode node,
|
|
||||||
PolicySnapshot snapshot,
|
|
||||||
Set<BigInteger> referencedTableIds,
|
|
||||||
Map<String, DerivedRelation> commonTables) {
|
|
||||||
if (node instanceof SqlOrderBy orderBy) {
|
|
||||||
collectCommonTables(
|
|
||||||
orderBy.query, snapshot, referencedTableIds, commonTables);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!(node instanceof SqlWith sqlWith)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
for (SqlNode withNode : sqlWith.withList) {
|
|
||||||
if (!(withNode instanceof SqlWithItem withItem)
|
|
||||||
|| withItem.name == null
|
|
||||||
|| withItem.name.names.isEmpty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
collectCommonTables(
|
|
||||||
withItem.query, snapshot, referencedTableIds, commonTables);
|
|
||||||
SqlSelect select = selectBody(withItem.query);
|
|
||||||
if (select == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
FromSummary source = collectFrom(
|
|
||||||
select.getFrom(), snapshot, referencedTableIds,
|
|
||||||
commonTables, new HashMap<>(), new HashMap<>(),
|
|
||||||
Collections.newSetFromMap(new IdentityHashMap<>()),
|
|
||||||
Collections.newSetFromMap(new IdentityHashMap<>()));
|
|
||||||
Set<String> outputColumns = withItem.columnList == null
|
|
||||||
|| withItem.columnList.isEmpty()
|
|
||||||
? outputColumnKeys(select)
|
|
||||||
: identifierKeys(withItem.columnList);
|
|
||||||
int nameIndex = withItem.name.names.size() - 1;
|
|
||||||
commonTables.put(
|
|
||||||
identifierKey(
|
|
||||||
withItem.name.names.get(nameIndex),
|
|
||||||
withItem.name.isComponentQuoted(nameIndex)),
|
|
||||||
new DerivedRelation(
|
|
||||||
source.tableIds(), outputColumns));
|
|
||||||
}
|
|
||||||
collectCommonTables(
|
|
||||||
sqlWith.body, snapshot, referencedTableIds, commonTables);
|
|
||||||
}
|
|
||||||
|
|
||||||
private SqlSelect selectBody(SqlNode node) {
|
|
||||||
if (node instanceof SqlSelect select) {
|
|
||||||
return select;
|
|
||||||
}
|
|
||||||
if (node instanceof SqlOrderBy orderBy) {
|
|
||||||
return selectBody(orderBy.query);
|
|
||||||
}
|
|
||||||
if (node instanceof SqlWith sqlWith) {
|
|
||||||
return selectBody(sqlWith.body);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<String> outputColumnKeys(SqlNode query) {
|
|
||||||
SqlSelect select = selectBody(query);
|
|
||||||
if (select == null || select.getSelectList() == null) {
|
|
||||||
return Set.of();
|
|
||||||
}
|
|
||||||
Set<String> outputColumns = new LinkedHashSet<>();
|
|
||||||
for (SqlNode item : select.getSelectList()) {
|
|
||||||
if (item instanceof SqlCall call
|
|
||||||
&& call.getKind() == SqlKind.AS
|
|
||||||
&& call.operandCount() >= 2
|
|
||||||
&& call.operand(1) instanceof SqlIdentifier alias
|
|
||||||
&& !alias.names.isEmpty()) {
|
|
||||||
int index = alias.names.size() - 1;
|
|
||||||
outputColumns.add(identifierKey(
|
|
||||||
alias.names.get(index),
|
|
||||||
alias.isComponentQuoted(index)));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (item instanceof SqlIdentifier identifier
|
|
||||||
&& !identifier.isStar()
|
|
||||||
&& !identifier.names.isEmpty()) {
|
|
||||||
int index = identifier.names.size() - 1;
|
|
||||||
outputColumns.add(identifierKey(
|
|
||||||
identifier.names.get(index),
|
|
||||||
identifier.isComponentQuoted(index)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Set.copyOf(outputColumns);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<String> identifierKeys(SqlNodeList identifiers) {
|
|
||||||
Set<String> keys = new LinkedHashSet<>();
|
|
||||||
for (SqlNode node : identifiers) {
|
|
||||||
if (node instanceof SqlIdentifier identifier
|
|
||||||
&& !identifier.names.isEmpty()) {
|
|
||||||
int index = identifier.names.size() - 1;
|
|
||||||
keys.add(identifierKey(
|
|
||||||
identifier.names.get(index),
|
|
||||||
identifier.isComponentQuoted(index)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Set.copyOf(keys);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<BigInteger> resolveReferencedTableIds(
|
|
||||||
SqlIdentifier identifier,
|
|
||||||
PolicySnapshot snapshot,
|
|
||||||
Set<BigInteger> referencedTableIds) {
|
|
||||||
Set<BigInteger> candidates = snapshot.tableIdsByName().get(
|
|
||||||
String.join(".", identifier.names));
|
|
||||||
if (candidates == null || candidates.isEmpty()) {
|
|
||||||
return Set.of();
|
|
||||||
}
|
|
||||||
Set<BigInteger> referenced = new LinkedHashSet<>(candidates);
|
|
||||||
referenced.retainAll(referencedTableIds);
|
|
||||||
return Set.copyOf(referenced);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String identifierKey(String name, boolean quoted) {
|
|
||||||
return quoted ? "Q:" + name : "U:" + name.toUpperCase(Locale.ROOT);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean columnAllowed(ColumnPolicy policy, String columnName, boolean quoted) {
|
|
||||||
if (policy == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return quoted
|
|
||||||
? policy.allowedExact().contains(columnName)
|
|
||||||
: policy.allowedFolded().contains(columnName.toUpperCase(Locale.ROOT));
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean fieldAllowed(DatacenterTableField field) {
|
|
||||||
return Integer.valueOf(1).equals(field.getQueryable())
|
|
||||||
&& DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus())
|
|
||||||
&& DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel());
|
|
||||||
}
|
|
||||||
|
|
||||||
private void addTableName(
|
|
||||||
Map<String, Set<BigInteger>> idsByName,
|
|
||||||
Set<String> tableReferenceNames,
|
|
||||||
String name,
|
|
||||||
BigInteger tableId) {
|
|
||||||
if (name == null || name.isBlank()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
idsByName.computeIfAbsent(name, ignored -> new LinkedHashSet<>()).add(tableId);
|
|
||||||
tableReferenceNames.add(name);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<String> referencedTables(RelNode root) {
|
|
||||||
Set<String> names = new LinkedHashSet<>();
|
|
||||||
new RelVisitor() {
|
|
||||||
@Override
|
|
||||||
public void visit(RelNode node, int ordinal, RelNode parent) {
|
|
||||||
if (node instanceof TableScan scan) {
|
|
||||||
names.add(referencedTableName(scan));
|
|
||||||
}
|
|
||||||
super.visit(node, ordinal, parent);
|
|
||||||
}
|
|
||||||
}.go(root);
|
|
||||||
return names;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String referencedTableName(TableScan scan) {
|
|
||||||
List<String> qualified = scan.getTable().getQualifiedName();
|
|
||||||
if (qualified.isEmpty()) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
String table = qualified.get(qualified.size() - 1);
|
|
||||||
return qualified.size() >= 2
|
|
||||||
? qualified.get(qualified.size() - 2) + "." + table
|
|
||||||
: table;
|
|
||||||
}
|
|
||||||
|
|
||||||
private BigInteger parseRecordId(String sourceId) {
|
|
||||||
int marker = sourceId.lastIndexOf("-source-");
|
|
||||||
if (marker < 0) {
|
|
||||||
throw new FederationSqlException(
|
|
||||||
FederationSqlErrorCode.INVALID_ARGUMENT,
|
|
||||||
"invalid data source id");
|
|
||||||
}
|
|
||||||
return new BigInteger(sourceId.substring(marker + "-source-".length()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private String sourceColumnName(DatacenterTableField field) {
|
|
||||||
return field.getSourceColumnName() == null || field.getSourceColumnName().isBlank()
|
|
||||||
? field.getFieldName() : field.getSourceColumnName();
|
|
||||||
}
|
|
||||||
|
|
||||||
private FederationSqlException validationFailure(String message) {
|
|
||||||
return new FederationSqlException(FederationSqlErrorCode.SQL_VALIDATION_FAILED, message);
|
|
||||||
}
|
|
||||||
|
|
||||||
private record PolicyCacheKey(BigInteger sourceId, String policyVersion) {
|
|
||||||
}
|
|
||||||
|
|
||||||
private record AliasContext(
|
|
||||||
Map<String, Set<BigInteger>> tableIdsByAlias,
|
|
||||||
Map<String, Set<String>> derivedColumnsByAlias,
|
|
||||||
Set<String> unqualifiedDerivedColumns,
|
|
||||||
Set<String> outputAliases,
|
|
||||||
Set<SqlIdentifier> syntaxAliases,
|
|
||||||
Set<SqlIdentifier> tableIdentifiers,
|
|
||||||
Set<SqlIdentifier> outputAliasReferences) {
|
|
||||||
}
|
|
||||||
|
|
||||||
private record DerivedRelation(
|
|
||||||
Set<BigInteger> tableIds,
|
|
||||||
Set<String> outputColumns) {
|
|
||||||
}
|
|
||||||
|
|
||||||
private record FromSummary(
|
|
||||||
Set<BigInteger> tableIds,
|
|
||||||
Set<String> outputColumns,
|
|
||||||
boolean singleDerived) {
|
|
||||||
|
|
||||||
private static FromSummary empty() {
|
|
||||||
return new FromSummary(Set.of(), Set.of(), false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private record ColumnPolicy(
|
|
||||||
Set<String> knownExact,
|
|
||||||
Set<String> knownFolded,
|
|
||||||
Set<String> allowedExact,
|
|
||||||
Set<String> allowedFolded) {
|
|
||||||
|
|
||||||
private static ColumnPolicy mutable() {
|
|
||||||
return new ColumnPolicy(new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private record PolicySnapshot(
|
|
||||||
Map<String, Set<BigInteger>> tableIdsByName,
|
|
||||||
Map<BigInteger, ColumnPolicy> columnsByTable,
|
|
||||||
Set<String> tableReferenceNames) {
|
|
||||||
|
|
||||||
private static PolicySnapshot empty() {
|
|
||||||
return new PolicySnapshot(Map.of(), Map.of(), Set.of());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,174 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.federation;
|
|
||||||
|
|
||||||
import java.nio.ByteBuffer;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.security.MessageDigest;
|
|
||||||
import java.security.NoSuchAlgorithmException;
|
|
||||||
import java.util.HexFormat;
|
|
||||||
import java.util.List;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 使用 presence 与长度前缀生成稳定元数据摘要。
|
|
||||||
*/
|
|
||||||
public final class DatacenterMetadataIdentity {
|
|
||||||
|
|
||||||
private DatacenterMetadataIdentity() {
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 计算物理命名空间摘要。
|
|
||||||
*
|
|
||||||
* @param catalog 物理 Catalog
|
|
||||||
* @param schema 物理 Schema
|
|
||||||
* @return SHA-256 摘要
|
|
||||||
*/
|
|
||||||
public static String namespaceKey(String catalog, String schema) {
|
|
||||||
return digest(List.of(nullable(catalog), nullable(schema)));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 计算物理表身份摘要。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源 ID
|
|
||||||
* @param namespaceKey 命名空间摘要
|
|
||||||
* @param physicalTable 物理表名
|
|
||||||
* @param tableKind 对象类型
|
|
||||||
* @return SHA-256 摘要
|
|
||||||
*/
|
|
||||||
public static String tableIdentity(
|
|
||||||
String sourceId,
|
|
||||||
String namespaceKey,
|
|
||||||
String physicalTable,
|
|
||||||
String tableKind) {
|
|
||||||
return digest(List.of(
|
|
||||||
"table-v1",
|
|
||||||
nullable(sourceId),
|
|
||||||
nullable(namespaceKey),
|
|
||||||
nullable(physicalTable),
|
|
||||||
nullable(tableKind)));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 计算表结构观测摘要。
|
|
||||||
*
|
|
||||||
* @param tableName 表名
|
|
||||||
* @param tableKind 对象类型
|
|
||||||
* @param fields 按 JDBC 顺序排列的字段
|
|
||||||
* @return SHA-256 摘要
|
|
||||||
*/
|
|
||||||
public static String tableFingerprint(
|
|
||||||
String tableName,
|
|
||||||
String tableKind,
|
|
||||||
List<DatacenterTableField> fields) {
|
|
||||||
MessageDigest digest = newDigest();
|
|
||||||
update(digest, "table-metadata-v1");
|
|
||||||
update(digest, nullable(tableName));
|
|
||||||
update(digest, nullable(tableKind));
|
|
||||||
List<DatacenterTableField> safeFields = fields == null ? List.of() : fields;
|
|
||||||
digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(safeFields.size()).array());
|
|
||||||
for (DatacenterTableField field : safeFields) {
|
|
||||||
update(digest, nullable(field.getSourceColumnName()));
|
|
||||||
update(digest, nullable(field.getNativeTypeName()));
|
|
||||||
update(digest, nullable(field.getJdbcTypeCode()));
|
|
||||||
update(digest, nullable(field.getPrecision()));
|
|
||||||
update(digest, nullable(field.getScale()));
|
|
||||||
update(digest, nullable(field.getRequired()));
|
|
||||||
}
|
|
||||||
return HexFormat.of().formatHex(digest.digest());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 计算单字段元数据摘要。
|
|
||||||
*
|
|
||||||
* @param field 字段元数据
|
|
||||||
* @return SHA-256 摘要
|
|
||||||
*/
|
|
||||||
public static String fieldFingerprint(DatacenterTableField field) {
|
|
||||||
return digest(List.of(
|
|
||||||
"field-v1",
|
|
||||||
nullable(field.getSourceColumnName()),
|
|
||||||
nullable(field.getNativeTypeName()),
|
|
||||||
nullable(field.getJdbcTypeCode()),
|
|
||||||
nullable(field.getPrecision()),
|
|
||||||
nullable(field.getScale()),
|
|
||||||
nullable(field.getRequired())));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 复现 V58 的存量表身份回填算法,仅用于首次刷新时识别可安全升级的旧摘要。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源 ID
|
|
||||||
* @param catalogId 目录 ID
|
|
||||||
* @param physicalTable 物理表名
|
|
||||||
* @param tableKind 表类型
|
|
||||||
* @return V58 SHA-256 摘要
|
|
||||||
*/
|
|
||||||
public static String legacyV58TableIdentity(
|
|
||||||
Object sourceId,
|
|
||||||
Object catalogId,
|
|
||||||
String physicalTable,
|
|
||||||
String tableKind) {
|
|
||||||
String source = sourceId == null ? "0" : String.valueOf(sourceId);
|
|
||||||
String catalog = catalogId == null ? "0" : String.valueOf(catalogId);
|
|
||||||
String table = physicalTable == null ? "" : physicalTable;
|
|
||||||
String kind = tableKind == null ? "" : tableKind;
|
|
||||||
return rawDigest("S" + codePointLength(source) + ":" + source
|
|
||||||
+ "C" + codePointLength(catalog) + ":" + catalog
|
|
||||||
+ "T" + codePointLength(table) + ":" + table
|
|
||||||
+ "K" + codePointLength(kind) + ":" + kind);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 复现 V58 的存量表结构摘要回填算法。
|
|
||||||
*
|
|
||||||
* @param tableName 表名
|
|
||||||
* @param tableKind 表类型
|
|
||||||
* @return V58 SHA-256 摘要
|
|
||||||
*/
|
|
||||||
public static String legacyV58TableFingerprint(String tableName, String tableKind) {
|
|
||||||
String table = tableName == null ? "" : tableName;
|
|
||||||
String kind = tableKind == null ? "" : tableKind;
|
|
||||||
return rawDigest(codePointLength(table) + ":" + table
|
|
||||||
+ codePointLength(kind) + ":" + kind);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static int codePointLength(String value) {
|
|
||||||
return value.codePointCount(0, value.length());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String rawDigest(String value) {
|
|
||||||
MessageDigest digest = newDigest();
|
|
||||||
return HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8)));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String digest(List<String> values) {
|
|
||||||
MessageDigest digest = newDigest();
|
|
||||||
for (String value : values) {
|
|
||||||
update(digest, value);
|
|
||||||
}
|
|
||||||
return HexFormat.of().formatHex(digest.digest());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void update(MessageDigest digest, String value) {
|
|
||||||
byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
|
|
||||||
digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(bytes.length).array());
|
|
||||||
digest.update(bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String nullable(Object value) {
|
|
||||||
if (value == null) {
|
|
||||||
return "N";
|
|
||||||
}
|
|
||||||
String text = String.valueOf(value);
|
|
||||||
return "V" + text.length() + ":" + text;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static MessageDigest newDigest() {
|
|
||||||
try {
|
|
||||||
return MessageDigest.getInstance("SHA-256");
|
|
||||||
} catch (NoSuchAlgorithmException exception) {
|
|
||||||
throw new IllegalStateException("SHA-256 is not available", exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,18 +1,7 @@
|
|||||||
package tech.easyflow.datacenter.integration;
|
package tech.easyflow.datacenter.integration;
|
||||||
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
|
||||||
public interface AssistantDatacenterBridge {
|
public interface AssistantDatacenterBridge {
|
||||||
|
AssistantDatacenterResult queryPage(DatacenterQueryRequest request);
|
||||||
/**
|
|
||||||
* 使用明确执行账号查询数据中枢。
|
|
||||||
*
|
|
||||||
* @param request 查询请求
|
|
||||||
* @param account 执行账号
|
|
||||||
* @return 查询结果
|
|
||||||
*/
|
|
||||||
AssistantDatacenterResult queryPage(
|
|
||||||
DatacenterQueryRequest request,
|
|
||||||
LoginAccount account);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import com.mybatisflex.core.row.Row;
|
|||||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
@@ -13,7 +13,7 @@ import java.util.Map;
|
|||||||
|
|
||||||
public class AssistantDatacenterResult {
|
public class AssistantDatacenterResult {
|
||||||
private List<Row> rows = new ArrayList<>();
|
private List<Row> rows = new ArrayList<>();
|
||||||
private DatacenterSourceView source;
|
private DatacenterSource source;
|
||||||
private DatacenterCatalog catalog;
|
private DatacenterCatalog catalog;
|
||||||
private DatacenterTable table;
|
private DatacenterTable table;
|
||||||
private DatacenterDatasetVersion version;
|
private DatacenterDatasetVersion version;
|
||||||
@@ -21,8 +21,8 @@ public class AssistantDatacenterResult {
|
|||||||
|
|
||||||
public List<Row> getRows() { return rows; }
|
public List<Row> getRows() { return rows; }
|
||||||
public void setRows(List<Row> rows) { this.rows = rows; }
|
public void setRows(List<Row> rows) { this.rows = rows; }
|
||||||
public DatacenterSourceView getSource() { return source; }
|
public DatacenterSource getSource() { return source; }
|
||||||
public void setSource(DatacenterSourceView source) { this.source = source; }
|
public void setSource(DatacenterSource source) { this.source = source; }
|
||||||
public DatacenterCatalog getCatalog() { return catalog; }
|
public DatacenterCatalog getCatalog() { return catalog; }
|
||||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||||
public DatacenterTable getTable() { return table; }
|
public DatacenterTable getTable() { return table; }
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package tech.easyflow.datacenter.integration;
|
package tech.easyflow.datacenter.integration;
|
||||||
|
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
@@ -18,10 +17,8 @@ public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBrid
|
|||||||
private DatacenterDatasetQueryService queryService;
|
private DatacenterDatasetQueryService queryService;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public AssistantDatacenterResult queryPage(
|
public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) {
|
||||||
DatacenterQueryRequest request,
|
var page = queryService.queryPage(request);
|
||||||
LoginAccount account) {
|
|
||||||
var page = queryService.queryPage(request, account);
|
|
||||||
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
|
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
|
||||||
AssistantDatacenterResult result = new AssistantDatacenterResult();
|
AssistantDatacenterResult result = new AssistantDatacenterResult();
|
||||||
result.setRows(page.getRecords());
|
result.setRows(page.getRecords());
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.mapper;
|
|
||||||
|
|
||||||
import com.mybatisflex.core.BaseMapper;
|
|
||||||
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据中枢查询审计 Mapper。
|
|
||||||
*/
|
|
||||||
public interface DatacenterQueryAuditMapper extends BaseMapper<DatacenterQueryAudit> {
|
|
||||||
}
|
|
||||||
@@ -31,18 +31,6 @@ public class DatacenterCatalog extends DateEntity implements Serializable {
|
|||||||
private String catalogDesc;
|
private String catalogDesc;
|
||||||
@Column(comment = "目录类型")
|
@Column(comment = "目录类型")
|
||||||
private String catalogType;
|
private String catalogType;
|
||||||
@Column(comment = "Calcite 逻辑 Schema")
|
|
||||||
private String logicalSchemaName;
|
|
||||||
@Column(comment = "JDBC 物理 Catalog")
|
|
||||||
private String physicalCatalogName;
|
|
||||||
@Column(comment = "JDBC 物理 Schema")
|
|
||||||
private String physicalSchemaName;
|
|
||||||
@Column(comment = "物理命名空间稳定摘要")
|
|
||||||
private String namespaceKey;
|
|
||||||
@Column(comment = "元数据状态")
|
|
||||||
private String metadataStatus;
|
|
||||||
@Column(comment = "最近发现时间")
|
|
||||||
private Date lastSeenAt;
|
|
||||||
@Column(comment = "状态")
|
@Column(comment = "状态")
|
||||||
private Integer status;
|
private Integer status;
|
||||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||||
@@ -71,18 +59,6 @@ public class DatacenterCatalog extends DateEntity implements Serializable {
|
|||||||
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||||
public String getCatalogType() { return catalogType; }
|
public String getCatalogType() { return catalogType; }
|
||||||
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||||
public String getLogicalSchemaName() { return logicalSchemaName; }
|
|
||||||
public void setLogicalSchemaName(String logicalSchemaName) { this.logicalSchemaName = logicalSchemaName; }
|
|
||||||
public String getPhysicalCatalogName() { return physicalCatalogName; }
|
|
||||||
public void setPhysicalCatalogName(String physicalCatalogName) { this.physicalCatalogName = physicalCatalogName; }
|
|
||||||
public String getPhysicalSchemaName() { return physicalSchemaName; }
|
|
||||||
public void setPhysicalSchemaName(String physicalSchemaName) { this.physicalSchemaName = physicalSchemaName; }
|
|
||||||
public String getNamespaceKey() { return namespaceKey; }
|
|
||||||
public void setNamespaceKey(String namespaceKey) { this.namespaceKey = namespaceKey; }
|
|
||||||
public String getMetadataStatus() { return metadataStatus; }
|
|
||||||
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
|
|
||||||
public Date getLastSeenAt() { return lastSeenAt; }
|
|
||||||
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
|
|
||||||
public Integer getStatus() { return status; }
|
public Integer getStatus() { return status; }
|
||||||
public void setStatus(Integer status) { this.status = status; }
|
public void setStatus(Integer status) { this.status = status; }
|
||||||
public Map<String, Object> getOptions() { return options; }
|
public Map<String, Object> getOptions() { return options; }
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
package tech.easyflow.datacenter.meta.entity;
|
package tech.easyflow.datacenter.meta.entity;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
|
||||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
|
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
|
||||||
import com.mybatisflex.annotation.Column;
|
import com.mybatisflex.annotation.Column;
|
||||||
@@ -30,30 +29,6 @@ public class DatacenterSource extends DateEntity implements Serializable {
|
|||||||
private String sourceCode;
|
private String sourceCode;
|
||||||
@Column(comment = "数据源类型")
|
@Column(comment = "数据源类型")
|
||||||
private String sourceType;
|
private String sourceType;
|
||||||
@Column(comment = "Federation Adapter 标识")
|
|
||||||
private String adapterId;
|
|
||||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "不含凭据的 Adapter 选项")
|
|
||||||
private Map<String, String> adapterOptionsJson;
|
|
||||||
@Column(comment = "数据源 Definition 版本")
|
|
||||||
private Long definitionRevision;
|
|
||||||
@Column(comment = "Definition SHA-256 校验和")
|
|
||||||
private String definitionChecksum;
|
|
||||||
@Column(comment = "纳管范围版本")
|
|
||||||
private Long scopeRevision;
|
|
||||||
@Column(comment = "Adapter 兼容状态")
|
|
||||||
private String compatibilityStatus;
|
|
||||||
@Column(comment = "数据库产品名称")
|
|
||||||
private String databaseProductName;
|
|
||||||
@Column(comment = "数据库产品版本")
|
|
||||||
private String databaseProductVersion;
|
|
||||||
@Column(comment = "JDBC Driver 名称")
|
|
||||||
private String driverName;
|
|
||||||
@Column(comment = "JDBC Driver 版本")
|
|
||||||
private String driverVersion;
|
|
||||||
@Column(comment = "元数据刷新状态")
|
|
||||||
private String metadataRefreshStatus;
|
|
||||||
@Column(comment = "元数据最近刷新时间")
|
|
||||||
private Date metadataRefreshedAt;
|
|
||||||
@Column(comment = "访问模式")
|
@Column(comment = "访问模式")
|
||||||
private String accessMode;
|
private String accessMode;
|
||||||
@Column(comment = "是否内置")
|
@Column(comment = "是否内置")
|
||||||
@@ -110,30 +85,6 @@ public class DatacenterSource extends DateEntity implements Serializable {
|
|||||||
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
|
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
|
||||||
public String getSourceType() { return sourceType; }
|
public String getSourceType() { return sourceType; }
|
||||||
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
|
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
|
||||||
public String getAdapterId() { return adapterId; }
|
|
||||||
public void setAdapterId(String adapterId) { this.adapterId = adapterId; }
|
|
||||||
public Map<String, String> getAdapterOptionsJson() { return adapterOptionsJson; }
|
|
||||||
public void setAdapterOptionsJson(Map<String, String> adapterOptionsJson) { this.adapterOptionsJson = adapterOptionsJson; }
|
|
||||||
public Long getDefinitionRevision() { return definitionRevision; }
|
|
||||||
public void setDefinitionRevision(Long definitionRevision) { this.definitionRevision = definitionRevision; }
|
|
||||||
public String getDefinitionChecksum() { return definitionChecksum; }
|
|
||||||
public void setDefinitionChecksum(String definitionChecksum) { this.definitionChecksum = definitionChecksum; }
|
|
||||||
public Long getScopeRevision() { return scopeRevision; }
|
|
||||||
public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; }
|
|
||||||
public String getCompatibilityStatus() { return compatibilityStatus; }
|
|
||||||
public void setCompatibilityStatus(String compatibilityStatus) { this.compatibilityStatus = compatibilityStatus; }
|
|
||||||
public String getDatabaseProductName() { return databaseProductName; }
|
|
||||||
public void setDatabaseProductName(String databaseProductName) { this.databaseProductName = databaseProductName; }
|
|
||||||
public String getDatabaseProductVersion() { return databaseProductVersion; }
|
|
||||||
public void setDatabaseProductVersion(String databaseProductVersion) { this.databaseProductVersion = databaseProductVersion; }
|
|
||||||
public String getDriverName() { return driverName; }
|
|
||||||
public void setDriverName(String driverName) { this.driverName = driverName; }
|
|
||||||
public String getDriverVersion() { return driverVersion; }
|
|
||||||
public void setDriverVersion(String driverVersion) { this.driverVersion = driverVersion; }
|
|
||||||
public String getMetadataRefreshStatus() { return metadataRefreshStatus; }
|
|
||||||
public void setMetadataRefreshStatus(String metadataRefreshStatus) { this.metadataRefreshStatus = metadataRefreshStatus; }
|
|
||||||
public Date getMetadataRefreshedAt() { return metadataRefreshedAt; }
|
|
||||||
public void setMetadataRefreshedAt(Date metadataRefreshedAt) { this.metadataRefreshedAt = metadataRefreshedAt; }
|
|
||||||
public String getAccessMode() { return accessMode; }
|
public String getAccessMode() { return accessMode; }
|
||||||
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
|
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
|
||||||
public Integer getBuiltinFlag() { return builtinFlag; }
|
public Integer getBuiltinFlag() { return builtinFlag; }
|
||||||
@@ -152,10 +103,8 @@ public class DatacenterSource extends DateEntity implements Serializable {
|
|||||||
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
|
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
|
||||||
public String getUsername() { return username; }
|
public String getUsername() { return username; }
|
||||||
public void setUsername(String username) { this.username = username; }
|
public void setUsername(String username) { this.username = username; }
|
||||||
@JsonIgnore
|
|
||||||
public String getCredentialCipher() { return credentialCipher; }
|
public String getCredentialCipher() { return credentialCipher; }
|
||||||
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
|
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
|
||||||
@JsonIgnore
|
|
||||||
public Map<String, Object> getConfigJson() { return configJson; }
|
public Map<String, Object> getConfigJson() { return configJson; }
|
||||||
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
|
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
|
||||||
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }
|
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }
|
||||||
|
|||||||
@@ -1,15 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.meta.enums;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据库对象元数据状态。
|
|
||||||
*/
|
|
||||||
public enum DatacenterMetadataStatus {
|
|
||||||
/** 当前仍能从目标数据库发现。 */
|
|
||||||
ACTIVE,
|
|
||||||
/** 本次刷新未发现。 */
|
|
||||||
MISSING,
|
|
||||||
/** 结构已变化,等待用户确认。 */
|
|
||||||
CHANGED,
|
|
||||||
/** 已由用户停止纳管。 */
|
|
||||||
RETIRED
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.meta.enums;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据字段敏感级别。
|
|
||||||
*/
|
|
||||||
public enum DatacenterSensitivityLevel {
|
|
||||||
/** 可直接用于受控查询。 */
|
|
||||||
PUBLIC,
|
|
||||||
/** 仅限组织内部。 */
|
|
||||||
INTERNAL,
|
|
||||||
/** 需要脱敏后使用。 */
|
|
||||||
SENSITIVE,
|
|
||||||
/** 默认禁止进入查询结果。 */
|
|
||||||
RESTRICTED
|
|
||||||
}
|
|
||||||
@@ -1,56 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.meta.enums;
|
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据源持久化生命周期。
|
|
||||||
*/
|
|
||||||
public enum DatacenterSourceStatus {
|
|
||||||
|
|
||||||
/** 配置已保存,尚未激活。 */
|
|
||||||
DRAFT(0),
|
|
||||||
/** Definition 已发布且节点可查询。 */
|
|
||||||
READY(1),
|
|
||||||
/** 配置仍保留,但当前 Runtime 不可用。 */
|
|
||||||
DEGRADED(2),
|
|
||||||
/** 管理员主动禁用。 */
|
|
||||||
DISABLED(3),
|
|
||||||
/** 删除墓碑,禁止新查询。 */
|
|
||||||
DELETED(4);
|
|
||||||
|
|
||||||
private final int code;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建状态。
|
|
||||||
*
|
|
||||||
* @param code 持久化编码
|
|
||||||
*/
|
|
||||||
DatacenterSourceStatus(int code) {
|
|
||||||
this.code = code;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 返回持久化编码。
|
|
||||||
*
|
|
||||||
* @return 状态编码
|
|
||||||
*/
|
|
||||||
public int code() {
|
|
||||||
return code;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 按持久化编码解析状态。
|
|
||||||
*
|
|
||||||
* @param code 状态编码
|
|
||||||
* @return 对应状态;未知编码按草稿处理
|
|
||||||
*/
|
|
||||||
public static DatacenterSourceStatus fromCode(Integer code) {
|
|
||||||
if (code == null) {
|
|
||||||
return DRAFT;
|
|
||||||
}
|
|
||||||
return Arrays.stream(values())
|
|
||||||
.filter(status -> status.code == code)
|
|
||||||
.findFirst()
|
|
||||||
.orElse(DRAFT);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,9 +14,6 @@ public class DatacenterCatalogMeta {
|
|||||||
private String catalogName;
|
private String catalogName;
|
||||||
private String catalogType;
|
private String catalogType;
|
||||||
private String catalogDesc;
|
private String catalogDesc;
|
||||||
private String logicalSchemaName;
|
|
||||||
private String physicalCatalogName;
|
|
||||||
private String physicalSchemaName;
|
|
||||||
|
|
||||||
@JsonSerialize(using = ToStringSerializer.class)
|
@JsonSerialize(using = ToStringSerializer.class)
|
||||||
public BigInteger getId() { return id; }
|
public BigInteger getId() { return id; }
|
||||||
@@ -30,10 +27,4 @@ public class DatacenterCatalogMeta {
|
|||||||
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||||
public String getCatalogDesc() { return catalogDesc; }
|
public String getCatalogDesc() { return catalogDesc; }
|
||||||
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||||
public String getLogicalSchemaName() { return logicalSchemaName; }
|
|
||||||
public void setLogicalSchemaName(String logicalSchemaName) { this.logicalSchemaName = logicalSchemaName; }
|
|
||||||
public String getPhysicalCatalogName() { return physicalCatalogName; }
|
|
||||||
public void setPhysicalCatalogName(String physicalCatalogName) { this.physicalCatalogName = physicalCatalogName; }
|
|
||||||
public String getPhysicalSchemaName() { return physicalSchemaName; }
|
|
||||||
public void setPhysicalSchemaName(String physicalSchemaName) { this.physicalSchemaName = physicalSchemaName; }
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,85 +2,24 @@ package tech.easyflow.datacenter.meta.model;
|
|||||||
|
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
|
|
||||||
/**
|
|
||||||
* 字段说明与只读查询治理更新。
|
|
||||||
*/
|
|
||||||
public class DatacenterFieldDescriptionUpdate {
|
public class DatacenterFieldDescriptionUpdate {
|
||||||
|
|
||||||
private BigInteger fieldId;
|
private BigInteger fieldId;
|
||||||
private String fieldDesc;
|
private String fieldDesc;
|
||||||
private Integer queryable;
|
|
||||||
private String sensitivityLevel;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取字段 ID。
|
|
||||||
*
|
|
||||||
* @return 字段 ID
|
|
||||||
*/
|
|
||||||
public BigInteger getFieldId() {
|
public BigInteger getFieldId() {
|
||||||
return fieldId;
|
return fieldId;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 设置字段 ID。
|
|
||||||
*
|
|
||||||
* @param fieldId 字段 ID
|
|
||||||
*/
|
|
||||||
public void setFieldId(BigInteger fieldId) {
|
public void setFieldId(BigInteger fieldId) {
|
||||||
this.fieldId = fieldId;
|
this.fieldId = fieldId;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取字段说明。
|
|
||||||
*
|
|
||||||
* @return 字段说明
|
|
||||||
*/
|
|
||||||
public String getFieldDesc() {
|
public String getFieldDesc() {
|
||||||
return fieldDesc;
|
return fieldDesc;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 设置字段说明。
|
|
||||||
*
|
|
||||||
* @param fieldDesc 字段说明
|
|
||||||
*/
|
|
||||||
public void setFieldDesc(String fieldDesc) {
|
public void setFieldDesc(String fieldDesc) {
|
||||||
this.fieldDesc = fieldDesc;
|
this.fieldDesc = fieldDesc;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取是否允许只读查询。
|
|
||||||
*
|
|
||||||
* @return 1 为允许,0 为禁止,空表示保持原值
|
|
||||||
*/
|
|
||||||
public Integer getQueryable() {
|
|
||||||
return queryable;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 设置是否允许只读查询。
|
|
||||||
*
|
|
||||||
* @param queryable 1 为允许,0 为禁止,空表示保持原值
|
|
||||||
*/
|
|
||||||
public void setQueryable(Integer queryable) {
|
|
||||||
this.queryable = queryable;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取敏感级别。
|
|
||||||
*
|
|
||||||
* @return 敏感级别,空表示保持原值
|
|
||||||
*/
|
|
||||||
public String getSensitivityLevel() {
|
|
||||||
return sensitivityLevel;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 设置敏感级别。
|
|
||||||
*
|
|
||||||
* @param sensitivityLevel PUBLIC、INTERNAL、SENSITIVE 或 RESTRICTED
|
|
||||||
*/
|
|
||||||
public void setSensitivityLevel(String sensitivityLevel) {
|
|
||||||
this.sensitivityLevel = sensitivityLevel;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,24 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.meta.model;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 一次连接内读取的已纳管表元数据快照。
|
|
||||||
*
|
|
||||||
* @param details 当前仍存在的表详情
|
|
||||||
* @param missingTableNames 当前连接中未发现的表名
|
|
||||||
*/
|
|
||||||
public record DatacenterManagedMetadataSnapshot(
|
|
||||||
List<DatacenterTableDetailMeta> details,
|
|
||||||
Set<String> missingTableNames) {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 对返回集合做防御性复制。
|
|
||||||
*/
|
|
||||||
public DatacenterManagedMetadataSnapshot {
|
|
||||||
details = List.copyOf(details == null ? List.of() : details);
|
|
||||||
missingTableNames = Set.copyOf(
|
|
||||||
missingTableNames == null ? Set.of() : missingTableNames);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.meta.model;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 有界元数据分页结果。
|
|
||||||
*
|
|
||||||
* @param records 当前页记录
|
|
||||||
* @param pageNumber 页码,从 1 开始
|
|
||||||
* @param pageSize 每页记录上限
|
|
||||||
* @param hasMore 是否还有下一页
|
|
||||||
* @param <T> 元数据类型
|
|
||||||
*/
|
|
||||||
public record DatacenterMetadataPage<T>(
|
|
||||||
List<T> records,
|
|
||||||
long pageNumber,
|
|
||||||
long pageSize,
|
|
||||||
boolean hasMore) {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 防御性复制当前页记录。
|
|
||||||
*/
|
|
||||||
public DatacenterMetadataPage {
|
|
||||||
records = List.copyOf(records == null ? List.of() : records);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 从已加载记录中创建有界分页结果。
|
|
||||||
*
|
|
||||||
* @param records 全部候选记录
|
|
||||||
* @param pageNumber 页码
|
|
||||||
* @param pageSize 每页大小
|
|
||||||
* @param <T> 元数据类型
|
|
||||||
* @return 当前页结果
|
|
||||||
*/
|
|
||||||
public static <T> DatacenterMetadataPage<T> slice(
|
|
||||||
List<T> records,
|
|
||||||
long pageNumber,
|
|
||||||
long pageSize) {
|
|
||||||
List<T> safeRecords = records == null ? List.of() : records;
|
|
||||||
long safePage = Math.max(1L, pageNumber);
|
|
||||||
long safeSize = Math.max(1L, pageSize);
|
|
||||||
if (safePage - 1L > Long.MAX_VALUE / safeSize) {
|
|
||||||
return new DatacenterMetadataPage<>(List.of(), safePage, safeSize, false);
|
|
||||||
}
|
|
||||||
long offset = (safePage - 1L) * safeSize;
|
|
||||||
if (offset >= safeRecords.size()) {
|
|
||||||
return new DatacenterMetadataPage<>(List.of(), safePage, safeSize, false);
|
|
||||||
}
|
|
||||||
int from = Math.toIntExact(offset);
|
|
||||||
int to = (int) Math.min((long) safeRecords.size(), offset + safeSize);
|
|
||||||
return new DatacenterMetadataPage<>(
|
|
||||||
safeRecords.subList(from, to), safePage, safeSize, to < safeRecords.size());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -9,8 +9,6 @@ public class DatacenterSaveDescriptionsRequest {
|
|||||||
private BigInteger tableId;
|
private BigInteger tableId;
|
||||||
private String tableDesc;
|
private String tableDesc;
|
||||||
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
|
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
|
||||||
private Long fieldPageNumber;
|
|
||||||
private Long fieldPageSize;
|
|
||||||
|
|
||||||
public BigInteger getTableId() {
|
public BigInteger getTableId() {
|
||||||
return tableId;
|
return tableId;
|
||||||
@@ -35,24 +33,4 @@ public class DatacenterSaveDescriptionsRequest {
|
|||||||
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
|
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
|
||||||
this.fields = fields;
|
this.fields = fields;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return 保存后返回的字段页码 */
|
|
||||||
public Long getFieldPageNumber() {
|
|
||||||
return fieldPageNumber;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param fieldPageNumber 保存后返回的字段页码 */
|
|
||||||
public void setFieldPageNumber(Long fieldPageNumber) {
|
|
||||||
this.fieldPageNumber = fieldPageNumber;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return 保存后返回的字段页大小 */
|
|
||||||
public Long getFieldPageSize() {
|
|
||||||
return fieldPageSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param fieldPageSize 保存后返回的字段页大小 */
|
|
||||||
public void setFieldPageSize(Long fieldPageSize) {
|
|
||||||
this.fieldPageSize = fieldPageSize;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,26 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.meta.model;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 激活数据源并纳管选定对象的请求。
|
|
||||||
*
|
|
||||||
* @param sourceId 数据源 ID
|
|
||||||
* @param catalogName 物理 Catalog 或 Schema 名称
|
|
||||||
* @param tableNames 需要纳管的表或视图
|
|
||||||
* @param prewarm 是否立即预热节点本地连接池
|
|
||||||
*/
|
|
||||||
public record DatacenterSourceActivateRequest(
|
|
||||||
BigInteger sourceId,
|
|
||||||
String catalogName,
|
|
||||||
List<String> tableNames,
|
|
||||||
boolean prewarm) {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 防御性复制表名。
|
|
||||||
*/
|
|
||||||
public DatacenterSourceActivateRequest {
|
|
||||||
tableNames = List.copyOf(tableNames == null ? List.of() : tableNames);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user